From owner-svn-src-vendor@freebsd.org Sun Apr 9 20:53:04 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 1AEC8D36D3C; Sun, 9 Apr 2017 20:53:04 +0000 (UTC) (envelope-from cognet@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id E3EBA161E; Sun, 9 Apr 2017 20:53:03 +0000 (UTC) (envelope-from cognet@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v39Kr3FK036072; Sun, 9 Apr 2017 20:53:03 GMT (envelope-from cognet@FreeBSD.org) Received: (from cognet@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v39Kr2JO036070; Sun, 9 Apr 2017 20:53:02 GMT (envelope-from cognet@FreeBSD.org) Message-Id: <201704092053.v39Kr2JO036070@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: cognet set sender to cognet@FreeBSD.org using -f From: Olivier Houchard Date: Sun, 9 Apr 2017 20:53:02 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316662 - in vendor-sys/ck/dist: include src X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 09 Apr 2017 20:53:04 -0000 Author: cognet Date: Sun Apr 9 20:53:02 2017 New Revision: 316662 URL: https://svnweb.freebsd.org/changeset/base/316662 Log: Import CK as of commit 6b141c0bdd21ce8b3e14147af8f87f22b20ecf32 This brings us changes we needed in ck_epoch. Modified: vendor-sys/ck/dist/include/ck_epoch.h vendor-sys/ck/dist/src/ck_epoch.c Modified: vendor-sys/ck/dist/include/ck_epoch.h ============================================================================== --- vendor-sys/ck/dist/include/ck_epoch.h Sun Apr 9 20:41:00 2017 (r316661) +++ vendor-sys/ck/dist/include/ck_epoch.h Sun Apr 9 20:53:02 2017 (r316662) @@ -83,6 +83,7 @@ struct ck_epoch_ref { }; struct ck_epoch_record { + ck_stack_entry_t record_next; struct ck_epoch *global; unsigned int state; unsigned int epoch; @@ -92,17 +93,16 @@ struct ck_epoch_record { } local CK_CC_CACHELINE; unsigned int n_pending; unsigned int n_peak; - unsigned long n_dispatch; + unsigned int n_dispatch; + void *ct; ck_stack_t pending[CK_EPOCH_LENGTH]; - ck_stack_entry_t record_next; } CK_CC_CACHELINE; typedef struct ck_epoch_record ck_epoch_record_t; struct ck_epoch { unsigned int epoch; - char pad[CK_MD_CACHELINE - sizeof(unsigned int)]; - ck_stack_t records; unsigned int n_free; + ck_stack_t records; }; typedef struct ck_epoch ck_epoch_t; @@ -110,7 +110,14 @@ typedef struct ck_epoch ck_epoch_t; * Internal functions. */ void _ck_epoch_addref(ck_epoch_record_t *, ck_epoch_section_t *); -void _ck_epoch_delref(ck_epoch_record_t *, ck_epoch_section_t *); +bool _ck_epoch_delref(ck_epoch_record_t *, ck_epoch_section_t *); + +CK_CC_FORCE_INLINE static void * +ck_epoch_record_ct(const ck_epoch_record_t *record) +{ + + return ck_pr_load_ptr(&record->ct); +} /* * Marks the beginning of an epoch-protected section. @@ -160,9 +167,10 @@ ck_epoch_begin(ck_epoch_record_t *record } /* - * Marks the end of an epoch-protected section. + * Marks the end of an epoch-protected section. Returns true if no more + * sections exist for the caller. */ -CK_CC_FORCE_INLINE static void +CK_CC_FORCE_INLINE static bool ck_epoch_end(ck_epoch_record_t *record, ck_epoch_section_t *section) { @@ -170,15 +178,19 @@ ck_epoch_end(ck_epoch_record_t *record, ck_pr_store_uint(&record->active, record->active - 1); if (section != NULL) - _ck_epoch_delref(record, section); + return _ck_epoch_delref(record, section); - return; + return record->active == 0; } /* * Defers the execution of the function pointed to by the "cb" * argument until an epoch counter loop. This allows for a * non-blocking deferral. + * + * We can get away without a fence here due to the monotonic nature + * of the epoch counter. Worst case, this will result in some delays + * before object destruction. */ CK_CC_FORCE_INLINE static void ck_epoch_call(ck_epoch_record_t *record, @@ -195,13 +207,74 @@ ck_epoch_call(ck_epoch_record_t *record, return; } +/* + * Same as ck_epoch_call, but allows for records to be shared and is reentrant. + */ +CK_CC_FORCE_INLINE static void +ck_epoch_call_strict(ck_epoch_record_t *record, + ck_epoch_entry_t *entry, + ck_epoch_cb_t *function) +{ + struct ck_epoch *epoch = record->global; + unsigned int e = ck_pr_load_uint(&epoch->epoch); + unsigned int offset = e & (CK_EPOCH_LENGTH - 1); + + ck_pr_inc_uint(&record->n_pending); + entry->function = function; + + /* Store fence is implied by push operation. */ + ck_stack_push_upmc(&record->pending[offset], &entry->stack_entry); + return; +} + +/* + * This callback is used for synchronize_wait to allow for custom blocking + * behavior. + */ +typedef void ck_epoch_wait_cb_t(ck_epoch_t *, ck_epoch_record_t *, + void *); + +/* + * Return latest epoch value. This operation provides load ordering. + */ +CK_CC_FORCE_INLINE static unsigned int +ck_epoch_value(const ck_epoch_t *ep) +{ + + ck_pr_fence_load(); + return ck_pr_load_uint(&ep->epoch); +} + void ck_epoch_init(ck_epoch_t *); -ck_epoch_record_t *ck_epoch_recycle(ck_epoch_t *); -void ck_epoch_register(ck_epoch_t *, ck_epoch_record_t *); + +/* + * Attempts to recycle an unused epoch record. If one is successfully + * allocated, the record context pointer is also updated. + */ +ck_epoch_record_t *ck_epoch_recycle(ck_epoch_t *, void *); + +/* + * Registers an epoch record. An optional context pointer may be passed that + * is retrievable with ck_epoch_record_ct. + */ +void ck_epoch_register(ck_epoch_t *, ck_epoch_record_t *, void *); + +/* + * Marks a record as available for re-use by a subsequent recycle operation. + * Note that the record cannot be physically destroyed. + */ void ck_epoch_unregister(ck_epoch_record_t *); + bool ck_epoch_poll(ck_epoch_record_t *); void ck_epoch_synchronize(ck_epoch_record_t *); +void ck_epoch_synchronize_wait(ck_epoch_t *, ck_epoch_wait_cb_t *, void *); void ck_epoch_barrier(ck_epoch_record_t *); +void ck_epoch_barrier_wait(ck_epoch_record_t *, ck_epoch_wait_cb_t *, void *); + +/* + * Reclaim entries associated with a record. This is safe to call only on + * the caller's record or records that are using call_strict. + */ void ck_epoch_reclaim(ck_epoch_record_t *); #endif /* CK_EPOCH_H */ Modified: vendor-sys/ck/dist/src/ck_epoch.c ============================================================================== --- vendor-sys/ck/dist/src/ck_epoch.c Sun Apr 9 20:41:00 2017 (r316661) +++ vendor-sys/ck/dist/src/ck_epoch.c Sun Apr 9 20:53:02 2017 (r316662) @@ -139,7 +139,7 @@ CK_STACK_CONTAINER(struct ck_epoch_entry #define CK_EPOCH_SENSE_MASK (CK_EPOCH_SENSE - 1) -void +bool _ck_epoch_delref(struct ck_epoch_record *record, struct ck_epoch_section *section) { @@ -150,7 +150,7 @@ _ck_epoch_delref(struct ck_epoch_record current->count--; if (current->count > 0) - return; + return false; /* * If the current bucket no longer has any references, then @@ -161,8 +161,7 @@ _ck_epoch_delref(struct ck_epoch_record * If no other active bucket exists, then the record will go * inactive in order to allow for forward progress. */ - other = &record->local.bucket[(i + 1) & - CK_EPOCH_SENSE_MASK]; + other = &record->local.bucket[(i + 1) & CK_EPOCH_SENSE_MASK]; if (other->count > 0 && ((int)(current->epoch - other->epoch) < 0)) { /* @@ -172,7 +171,7 @@ _ck_epoch_delref(struct ck_epoch_record ck_pr_store_uint(&record->epoch, other->epoch); } - return; + return true; } void @@ -230,7 +229,7 @@ ck_epoch_init(struct ck_epoch *global) } struct ck_epoch_record * -ck_epoch_recycle(struct ck_epoch *global) +ck_epoch_recycle(struct ck_epoch *global, void *ct) { struct ck_epoch_record *record; ck_stack_entry_t *cursor; @@ -249,6 +248,12 @@ ck_epoch_recycle(struct ck_epoch *global CK_EPOCH_STATE_USED); if (state == CK_EPOCH_STATE_FREE) { ck_pr_dec_uint(&global->n_free); + ck_pr_store_ptr(&record->ct, ct); + + /* + * The context pointer is ordered by a + * subsequent protected section. + */ return record; } } @@ -258,7 +263,8 @@ ck_epoch_recycle(struct ck_epoch *global } void -ck_epoch_register(struct ck_epoch *global, struct ck_epoch_record *record) +ck_epoch_register(struct ck_epoch *global, struct ck_epoch_record *record, + void *ct) { size_t i; @@ -269,6 +275,7 @@ ck_epoch_register(struct ck_epoch *globa record->n_dispatch = 0; record->n_peak = 0; record->n_pending = 0; + record->ct = ct; memset(&record->local, 0, sizeof record->local); for (i = 0; i < CK_EPOCH_LENGTH; i++) @@ -295,6 +302,7 @@ ck_epoch_unregister(struct ck_epoch_reco for (i = 0; i < CK_EPOCH_LENGTH; i++) ck_stack_init(&record->pending[i]); + ck_pr_store_ptr(&record->ct, NULL); ck_pr_fence_store(); ck_pr_store_uint(&record->state, CK_EPOCH_STATE_FREE); ck_pr_inc_uint(&global->n_free); @@ -345,11 +353,10 @@ ck_epoch_dispatch(struct ck_epoch_record { unsigned int epoch = e & (CK_EPOCH_LENGTH - 1); ck_stack_entry_t *head, *next, *cursor; + unsigned int n_pending, n_peak; unsigned int i = 0; - head = CK_STACK_FIRST(&record->pending[epoch]); - ck_stack_init(&record->pending[epoch]); - + head = ck_stack_batch_pop_upmc(&record->pending[epoch]); for (cursor = head; cursor != NULL; cursor = next) { struct ck_epoch_entry *entry = ck_epoch_entry_container(cursor); @@ -359,11 +366,18 @@ ck_epoch_dispatch(struct ck_epoch_record i++; } - if (record->n_pending > record->n_peak) - record->n_peak = record->n_pending; + n_peak = ck_pr_load_uint(&record->n_peak); + n_pending = ck_pr_load_uint(&record->n_pending); + + /* We don't require accuracy around peak calculation. */ + if (n_pending > n_peak) + ck_pr_store_uint(&record->n_peak, n_peak); + + if (i > 0) { + ck_pr_add_uint(&record->n_dispatch, i); + ck_pr_sub_uint(&record->n_pending, i); + } - record->n_dispatch += i; - record->n_pending -= i; return; } @@ -381,13 +395,24 @@ ck_epoch_reclaim(struct ck_epoch_record return; } +CK_CC_FORCE_INLINE static void +epoch_block(struct ck_epoch *global, struct ck_epoch_record *cr, + ck_epoch_wait_cb_t *cb, void *ct) +{ + + if (cb != NULL) + cb(global, cr, ct); + + return; +} + /* * This function must not be called with-in read section. */ void -ck_epoch_synchronize(struct ck_epoch_record *record) +ck_epoch_synchronize_wait(struct ck_epoch *global, + ck_epoch_wait_cb_t *cb, void *ct) { - struct ck_epoch *global = record->global; struct ck_epoch_record *cr; unsigned int delta, epoch, goal, i; bool active; @@ -424,10 +449,27 @@ ck_epoch_synchronize(struct ck_epoch_rec * period. */ e_d = ck_pr_load_uint(&global->epoch); - if (e_d != delta) { - delta = e_d; - goto reload; + if (e_d == delta) { + epoch_block(global, cr, cb, ct); + continue; } + + /* + * If the epoch has been updated, we may have already + * met our goal. + */ + delta = e_d; + if ((goal > epoch) & (delta >= goal)) + goto leave; + + epoch_block(global, cr, cb, ct); + + /* + * If the epoch has been updated, then a grace period + * requires that all threads are observed idle at the + * same epoch. + */ + cr = NULL; } /* @@ -459,20 +501,6 @@ ck_epoch_synchronize(struct ck_epoch_rec * Otherwise, we have just acquired latest snapshot. */ delta = delta + r; - continue; - -reload: - if ((goal > epoch) & (delta >= goal)) { - /* - * Right now, epoch overflow is handled as an edge - * case. If we have already observed an epoch - * generation, then we can be sure no hazardous - * references exist to objects from this generation. We - * can actually avoid an addtional scan step at this - * point. - */ - break; - } } /* @@ -480,8 +508,16 @@ reload: * However, if non-temporal instructions are used, full barrier * semantics are necessary. */ +leave: ck_pr_fence_memory(); - record->epoch = delta; + return; +} + +void +ck_epoch_synchronize(struct ck_epoch_record *record) +{ + + ck_epoch_synchronize_wait(record->global, NULL, NULL); return; } @@ -494,6 +530,16 @@ ck_epoch_barrier(struct ck_epoch_record return; } +void +ck_epoch_barrier_wait(struct ck_epoch_record *record, ck_epoch_wait_cb_t *cb, + void *ct) +{ + + ck_epoch_synchronize_wait(record->global, cb, ct); + ck_epoch_reclaim(record); + return; +} + /* * It may be worth it to actually apply these deferral semantics to an epoch * that was observed at ck_epoch_call time. The problem is that the latter @@ -509,7 +555,6 @@ ck_epoch_poll(struct ck_epoch_record *re { bool active; unsigned int epoch; - unsigned int snapshot; struct ck_epoch_record *cr = NULL; struct ck_epoch *global = record->global; @@ -533,12 +578,7 @@ ck_epoch_poll(struct ck_epoch_record *re } /* If an active thread exists, rely on epoch observation. */ - if (ck_pr_cas_uint_value(&global->epoch, epoch, epoch + 1, - &snapshot) == false) { - record->epoch = snapshot; - } else { - record->epoch = epoch + 1; - } + (void)ck_pr_cas_uint(&global->epoch, epoch, epoch + 1); ck_epoch_dispatch(record, epoch + 1); return true; From owner-svn-src-vendor@freebsd.org Sun Apr 9 20:54:34 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id DA3BAD36DB1; Sun, 9 Apr 2017 20:54:34 +0000 (UTC) (envelope-from cognet@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 93BD31759; Sun, 9 Apr 2017 20:54:34 +0000 (UTC) (envelope-from cognet@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v39KsXRn036161; Sun, 9 Apr 2017 20:54:33 GMT (envelope-from cognet@FreeBSD.org) Received: (from cognet@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v39KsXUH036160; Sun, 9 Apr 2017 20:54:33 GMT (envelope-from cognet@FreeBSD.org) Message-Id: <201704092054.v39KsXUH036160@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: cognet set sender to cognet@FreeBSD.org using -f From: Olivier Houchard Date: Sun, 9 Apr 2017 20:54:33 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316663 - vendor-sys/ck/20170407 X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 09 Apr 2017 20:54:35 -0000 Author: cognet Date: Sun Apr 9 20:54:33 2017 New Revision: 316663 URL: https://svnweb.freebsd.org/changeset/base/316663 Log: Tag new CK import. Added: vendor-sys/ck/20170407/ - copied from r316662, vendor-sys/ck/dist/ From owner-svn-src-vendor@freebsd.org Mon Apr 10 22:46:35 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id C3E85D3788C; Mon, 10 Apr 2017 22:46:35 +0000 (UTC) (envelope-from pfg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 95B6FF10; Mon, 10 Apr 2017 22:46:35 +0000 (UTC) (envelope-from pfg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3AMkYMD087403; Mon, 10 Apr 2017 22:46:34 GMT (envelope-from pfg@FreeBSD.org) Received: (from pfg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3AMkYt0087402; Mon, 10 Apr 2017 22:46:34 GMT (envelope-from pfg@FreeBSD.org) Message-Id: <201704102246.v3AMkYt0087402@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: pfg set sender to pfg@FreeBSD.org using -f From: "Pedro F. Giffuni" Date: Mon, 10 Apr 2017 22:46:34 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316693 - vendor/illumos/dist/lib/libzfs/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 10 Apr 2017 22:46:35 -0000 Author: pfg Date: Mon Apr 10 22:46:34 2017 New Revision: 316693 URL: https://svnweb.freebsd.org/changeset/base/316693 Log: 8046 Let calloc() do the multiplication in libzfs_fru_refresh https://github.com/illumos/illumos-gate/commit/5697e03e6e3e2697f56ae341c7c8ce79680d6a2e https://www.illumos.org/issues/8046 Reviewed by: Matthew Ahrens Reviewed by: Paul Dagnelie Approved by: Robert Mustacchi Author: Pedro Giffuni Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_fru.c Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_fru.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_fru.c Mon Apr 10 21:49:35 2017 (r316692) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_fru.c Mon Apr 10 22:46:34 2017 (r316693) @@ -282,7 +282,7 @@ libzfs_fru_refresh(libzfs_handle_t *hdl) if (hdl->libzfs_fru_hash == NULL && (hdl->libzfs_fru_hash = - calloc(ZFS_FRU_HASH_SIZE * sizeof (void *), 1)) == NULL) + calloc(ZFS_FRU_HASH_SIZE, sizeof (void *))) == NULL) return; /* From owner-svn-src-vendor@freebsd.org Thu Apr 13 05:47:53 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id BAB44D3B7B8; Thu, 13 Apr 2017 05:47:53 +0000 (UTC) (envelope-from jpaetzel@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 89888C05; Thu, 13 Apr 2017 05:47:53 +0000 (UTC) (envelope-from jpaetzel@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3D5lqDP054511; Thu, 13 Apr 2017 05:47:52 GMT (envelope-from jpaetzel@FreeBSD.org) Received: (from jpaetzel@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3D5lqKj054510; Thu, 13 Apr 2017 05:47:52 GMT (envelope-from jpaetzel@FreeBSD.org) Message-Id: <201704130547.v3D5lqKj054510@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: jpaetzel set sender to jpaetzel@FreeBSD.org using -f From: Josh Paetzel Date: Thu, 13 Apr 2017 05:47:52 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316751 - vendor/illumos/dist/man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 13 Apr 2017 05:47:53 -0000 Author: jpaetzel Date: Thu Apr 13 05:47:52 2017 New Revision: 316751 URL: https://svnweb.freebsd.org/changeset/base/316751 Log: 5661 ZFS: "compression = on" should use lz4 if feature is enabled Reviewed by: Matthew Ahrens Reviewed by: Josef 'Jeff' Sipek Reviewed by: Xin LI Approved by: Robert Mustacchi Author: Justin T. Gibbs illumos/illumos-gate@db1741f555ec79def5e9846e6bfd132248514ff Modified: vendor/illumos/dist/man/man1m/zfs.1m Modified: vendor/illumos/dist/man/man1m/zfs.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zfs.1m Thu Apr 13 04:10:27 2017 (r316750) +++ vendor/illumos/dist/man/man1m/zfs.1m Thu Apr 13 05:47:52 2017 (r316751) @@ -987,20 +987,25 @@ Changing this property affects only newl .sp .ne 2 .na -\fB\fBcompression\fR=\fBon\fR | \fBoff\fR | \fBlzjb\fR | \fBgzip\fR | -\fBgzip-\fR\fIN\fR | \fBzle\fR\fR | \fBlz4\fR +\fB\fBcompression\fR=\fBon\fR | \fBoff\fR | \fBlzjb\fR | \fBlz4\fR | +\fBgzip\fR | \fBgzip-\fR\fIN\fR | \fBzle\fR\fR .ad .sp .6 .RS 4n -Controls the compression algorithm used for this dataset. The \fBlzjb\fR -compression algorithm is optimized for performance while providing decent data -compression. Setting compression to \fBon\fR uses the \fBlzjb\fR compression -algorithm. The \fBgzip\fR compression algorithm uses the same compression as -the \fBgzip\fR(1) command. You can specify the \fBgzip\fR level by using the -value \fBgzip-\fR\fIN\fR where \fIN\fR is an integer from 1 (fastest) to 9 -(best compression ratio). Currently, \fBgzip\fR is equivalent to \fBgzip-6\fR -(which is also the default for \fBgzip\fR(1)). The \fBzle\fR compression -algorithm compresses runs of zeros. +Controls the compression algorithm used for this dataset. +.sp +Setting compression to \fBon\fR indicates that the current default +compression algorithm should be used. The default balances compression +and decompression speed, with compression ratio and is expected to +work well on a wide variety of workloads. Unlike all other settings for +this property, \fBon\fR does not select a fixed compression type. As +new compression algorithms are added to ZFS and enabled on a pool, the +default compression algorithm may change. The current default compression +algorthm is either \fBlzjb\fR or, if the \fBlz4_compress\fR feature is +enabled, \fBlz4\fR. +.sp +The \fBlzjb\fR compression algorithm is optimized for performance while +providing decent data compression. .sp The \fBlz4\fR compression algorithm is a high-performance replacement for the \fBlzjb\fR algorithm. It features significantly faster @@ -1010,6 +1015,13 @@ the \fBlz4_compress\fR feature set to \f \fBzpool-features\fR(5) for details on ZFS feature flags and the \fBlz4_compress\fR feature. .sp +The \fBgzip\fR compression algorithm uses the same compression as +the \fBgzip\fR(1) command. You can specify the \fBgzip\fR level by using the +value \fBgzip-\fR\fIN\fR where \fIN\fR is an integer from 1 (fastest) to 9 +(best compression ratio). Currently, \fBgzip\fR is equivalent to \fBgzip-6\fR +(which is also the default for \fBgzip\fR(1)). The \fBzle\fR compression +algorithm compresses runs of zeros. +.sp This property can also be referred to by its shortened column name \fBcompress\fR. Changing this property affects only newly-written data. .RE From owner-svn-src-vendor@freebsd.org Thu Apr 13 05:55:38 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 1779DD3BAA3; Thu, 13 Apr 2017 05:55:38 +0000 (UTC) (envelope-from jpaetzel@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id D73C8266; Thu, 13 Apr 2017 05:55:37 +0000 (UTC) (envelope-from jpaetzel@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3D5tbb5058843; Thu, 13 Apr 2017 05:55:37 GMT (envelope-from jpaetzel@FreeBSD.org) Received: (from jpaetzel@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3D5tb42058842; Thu, 13 Apr 2017 05:55:37 GMT (envelope-from jpaetzel@FreeBSD.org) Message-Id: <201704130555.v3D5tb42058842@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: jpaetzel set sender to jpaetzel@FreeBSD.org using -f From: Josh Paetzel Date: Thu, 13 Apr 2017 05:55:37 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316752 - vendor/illumos/dist/man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 13 Apr 2017 05:55:38 -0000 Author: jpaetzel Date: Thu Apr 13 05:55:36 2017 New Revision: 316752 URL: https://svnweb.freebsd.org/changeset/base/316752 Log: 5409 Remove shareiscsi description and example from zfs(1M) illumos/illumos-gate@b3cff10cdd26674d8dc66e0b349fd185df709fad https://github.com/illumos/illumos-gate/commit/b3cff10cdd26674d8dc66e0b349fd185df709fad https://www.illumos.org/issues/5409 Reviewed by: Matthew Ahrens Approved by: Gordon Ross Modified: vendor/illumos/dist/man/man1m/zfs.1m Modified: vendor/illumos/dist/man/man1m/zfs.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zfs.1m Thu Apr 13 05:47:52 2017 (r316751) +++ vendor/illumos/dist/man/man1m/zfs.1m Thu Apr 13 05:55:36 2017 (r316752) @@ -1,4 +1,3 @@ -'\" t .\" .\" CDDL HEADER START .\" @@ -26,647 +25,574 @@ .\" Copyright (c) 2013 by Saso Kiselkov. All rights reserved. .\" Copyright (c) 2014, Joyent, Inc. All rights reserved. .\" Copyright (c) 2014 by Adam Stevko. All rights reserved. -.\" Copyright 2014 Nexenta Systems, Inc. All Rights Reserved. +.\" Copyright 2015 Nexenta Systems, Inc. All Rights Reserved. .\" -.TH ZFS 1M "November 11, 2014" -.SH NAME -zfs \- configures ZFS file systems -.SH SYNOPSIS -.LP -.nf -\fBzfs\fR [\fB-?\fR] -.fi - -.LP -.nf -\fBzfs\fR \fBcreate\fR [\fB-p\fR] [\fB-o\fR \fIproperty\fR=\fIvalue\fR]... \fIfilesystem\fR -.fi - -.LP -.nf -\fBzfs\fR \fBcreate\fR [\fB-ps\fR] [\fB-b\fR \fIblocksize\fR] [\fB-o\fR \fIproperty\fR=\fIvalue\fR]... \fB-V\fR \fIsize\fR \fIvolume\fR -.fi - -.LP -.nf -\fBzfs\fR \fBdestroy\fR [\fB-fnpRrv\fR] \fIfilesystem\fR|\fIvolume\fR -.fi - -.LP -.nf -\fBzfs\fR \fBdestroy\fR [\fB-dnpRrv\fR] \fIfilesystem\fR|\fIvolume\fR@\fIsnap\fR[%\fIsnap\fR][,\fIsnap\fR[%\fIsnap\fR]]... -.fi - -.LP -.nf -\fBzfs\fR \fBdestroy\fR \fIfilesystem\fR|\fIvolume\fR#\fIbookmark\fR -.fi - -.LP -.nf -\fBzfs\fR \fBsnapshot\fR [\fB-r\fR] [\fB-o\fR \fIproperty\fR=\fIvalue\fR]... - \fIfilesystem@snapname\fR|\fIvolume@snapname\fR... -.fi - -.LP -.nf -\fBzfs\fR \fBrollback\fR [\fB-rRf\fR] \fIsnapshot\fR -.fi - -.LP -.nf -\fBzfs\fR \fBclone\fR [\fB-p\fR] [\fB-o\fR \fIproperty\fR=\fIvalue\fR]... \fIsnapshot\fR \fIfilesystem\fR|\fIvolume\fR -.fi - -.LP -.nf -\fBzfs\fR \fBpromote\fR \fIclone-filesystem\fR -.fi - -.LP -.nf -\fBzfs\fR \fBrename\fR [\fB-f\fR] \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR - \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR -.fi - -.LP -.nf -\fBzfs\fR \fBrename\fR [\fB-fp\fR] \fIfilesystem\fR|\fIvolume\fR \fIfilesystem\fR|\fIvolume\fR -.fi - -.LP -.nf -\fBzfs\fR \fBrename\fR \fB-r\fR \fIsnapshot\fR \fIsnapshot\fR -.fi - -.LP -.nf -\fBzfs\fR \fBlist\fR [\fB-r\fR|\fB-d\fR \fIdepth\fR][\fB-Hp\fR][\fB-o\fR \fIproperty\fR[,\fIproperty\fR]...] [\fB-t\fR \fItype\fR[,\fItype\fR]...] - [\fB-s\fR \fIproperty\fR]... [\fB-S\fR \fIproperty\fR]... [\fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR]... -.fi - -.LP -.nf -\fBzfs\fR \fBset\fR \fIproperty\fR=\fIvalue\fR... \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR... -.fi - -.LP -.nf -\fBzfs\fR \fBget\fR [\fB-r\fR|\fB-d\fR \fIdepth\fR][\fB-Hp\fR][\fB-o\fR \fIfield\fR[,\fIfield\fR]...] [\fB-t\fR \fItype\fR[,\fItype\fR]...] - [\fB-s\fR \fIsource\fR[,\fIsource\fR]...] \fBall\fR | \fIproperty\fR[,\fIproperty\fR]... - \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR... -.fi - -.LP -.nf -\fBzfs\fR \fBinherit\fR [\fB-rS\fR] \fIproperty\fR \fIfilesystem\fR|\fIvolume|snapshot\fR... -.fi - -.LP -.nf -\fBzfs\fR \fBupgrade\fR [\fB-v\fR] -.fi - -.LP -.nf -\fBzfs\fR \fBupgrade\fR [\fB-r\fR] [\fB-V\fR \fIversion\fR] \fB-a\fR | \fIfilesystem\fR -.fi - -.LP -.nf -\fBzfs\fR \fBuserspace\fR [\fB-Hinp\fR] [\fB-o\fR \fIfield\fR[,\fIfield\fR]...] [\fB-s\fR \fIfield\fR]... - [\fB-S\fR \fIfield\fR]... [\fB-t\fR \fItype\fR[,\fItype\fR]...] \fIfilesystem\fR|\fIsnapshot\fR -.fi - -.LP -.nf -\fBzfs\fR \fBgroupspace\fR [\fB-Hinp\fR] [\fB-o\fR \fIfield\fR[,\fIfield\fR]...] [\fB-s\fR \fIfield\fR]... - [\fB-S\fR \fIfield\fR]... [\fB-t\fR \fItype\fR[,\fItype\fR]...] \fIfilesystem\fR|\fIsnapshot\fR -.fi - -.LP -.nf -\fBzfs\fR \fBmount\fR -.fi - -.LP -.nf -\fBzfs\fR \fBmount\fR [\fB-vO\fR] [\fB-o \fIoptions\fR\fR] \fB-a\fR | \fIfilesystem\fR -.fi - -.LP -.nf -\fBzfs\fR \fBunmount\fR [\fB-f\fR] \fB-a\fR | \fIfilesystem\fR|\fImountpoint\fR -.fi - -.LP -.nf -\fBzfs\fR \fBshare\fR \fB-a\fR | \fIfilesystem\fR -.fi - -.LP -.nf -\fBzfs\fR \fBunshare\fR \fB-a\fR \fIfilesystem\fR|\fImountpoint\fR -.fi - -.LP -.nf -\fBzfs\fR \fBbookmark\fR \fIsnapshot\fR \fIbookmark\fR -.fi - -.LP -.nf -\fBzfs\fR \fBsend\fR [\fB-DnPpRveL\fR] [\fB-\fR[\fBiI\fR] \fIsnapshot\fR] \fIsnapshot\fR -.fi - -.LP -.nf -\fBzfs\fR \fBsend\fR [\fB-eL\fR] [\fB-i \fIsnapshot\fR|\fIbookmark\fR]\fR \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR -.fi - -.LP -.nf -\fBzfs\fR \fBreceive\fR [\fB-vnFu\fR] \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR -.fi - -.LP -.nf -\fBzfs\fR \fBreceive\fR [\fB-vnFu\fR] [\fB-d\fR|\fB-e\fR] \fIfilesystem\fR -.fi - -.LP -.nf -\fBzfs\fR \fBallow\fR \fIfilesystem\fR|\fIvolume\fR -.fi - -.LP -.nf -\fBzfs\fR \fBallow\fR [\fB-ldug\fR] \fIuser\fR|\fIgroup\fR[,\fIuser\fR|\fIgroup\fR]... - \fIperm\fR|\fI@setname\fR[,\fIperm\fR|\fI@setname\fR]... \fIfilesystem\fR|\fIvolume\fR -.fi - -.LP -.nf -\fBzfs\fR \fBallow\fR [\fB-ld\fR] \fB-e\fR|\fBeveryone\fR \fIperm\fR|@\fIsetname\fR[,\fIperm\fR|\fI@setname\fR]... - \fIfilesystem\fR|\fIvolume\fR -.fi - -.LP -.nf -\fBzfs\fR \fBallow\fR \fB-c\fR \fIperm\fR|@\fIsetname\fR[,\fIperm\fR|\fI@setname\fR]... \fIfilesystem\fR|\fIvolume\fR -.fi - -.LP -.nf -\fBzfs\fR \fBallow\fR \fB-s\fR @\fIsetname\fR \fIperm\fR|@\fIsetname\fR[,\fIperm\fR|\fI@setname\fR]... \fIfilesystem\fR|\fIvolume\fR -.fi - -.LP -.nf -\fBzfs\fR \fBunallow\fR [\fB-rldug\fR] \fIuser\fR|\fIgroup\fR[,\fIuser\fR|\fIgroup\fR]... - [\fIperm\fR|@\fIsetname\fR[,\fIperm\fR|\fI@setname\fR]...] \fIfilesystem\fR|\fIvolume\fR -.fi - -.LP -.nf -\fBzfs\fR \fBunallow\fR [\fB-rld\fR] \fB-e\fR|\fBeveryone\fR [\fIperm\fR|@\fIsetname\fR[,\fIperm\fR|\fI@setname\fR]...] - \fIfilesystem\fR|\fIvolume\fR -.fi - -.LP -.nf -\fBzfs\fR \fBunallow\fR [\fB-r\fR] \fB-c\fR [\fIperm\fR|@\fIsetname\fR[,\fIperm\fR|\fI@setname\fR]...] \fIfilesystem\fR|\fIvolume\fR -.fi - -.LP -.nf -\fBzfs\fR \fBunallow\fR [\fB-r\fR] \fB-s\fR @\fIsetname\fR [\fIperm\fR|@\fIsetname\fR[,\fIperm\fR|\fI@setname\fR]...] - \fIfilesystem\fR|\fIvolume\fR -.fi - -.LP -.nf -\fBzfs\fR \fBhold\fR [\fB-r\fR] \fItag\fR \fIsnapshot\fR... -.fi - -.LP -.nf -\fBzfs\fR \fBholds\fR [\fB-r\fR] \fIsnapshot\fR... -.fi - -.LP -.nf -\fBzfs\fR \fBrelease\fR [\fB-r\fR] \fItag\fR \fIsnapshot\fR... -.fi - -.LP -.nf -\fBzfs\fR \fBdiff\fR [\fB-FHt\fR] \fIsnapshot\fR \fIsnapshot|filesystem\fR - -.SH DESCRIPTION -.LP -The \fBzfs\fR command configures \fBZFS\fR datasets within a \fBZFS\fR storage -pool, as described in \fBzpool\fR(1M). A dataset is identified by a unique path -within the \fBZFS\fR namespace. For example: -.sp -.in +2 -.nf +.Dd June 8, 2015 +.Dt ZFS 1M +.Os +.Sh NAME +.Nm zfs +.Nd configures ZFS file systems +.Sh SYNOPSIS +.Nm +.Op Fl \? +.Nm +.Cm create +.Op Fl p +.Oo Fl o Ar property Ns = Ns Ar value Oc Ns ... +.Ar filesystem +.Nm +.Cm create +.Op Fl ps +.Op Fl b Ar blocksize +.Oo Fl o Ar property Ns = Ns Ar value Oc Ns ... +.Fl V Ar size Ar volume +.Nm +.Cm destroy +.Op Fl Rfnprv +.Ar filesystem Ns | Ns Ar volume +.Nm +.Cm destroy +.Op Fl Rdnprv +.Ar filesystem Ns | Ns Ar volume Ns @ Ns Ar snap Ns +.Oo % Ns Ar snap Ns Oo , Ns Ar snap Ns Oo % Ns Ar snap Oc Oc Oc Ns ... +.Nm +.Cm destroy +.Ar filesystem Ns | Ns Ar volume Ns # Ns Ar bookmark +.Nm +.Cm snapshot +.Op Fl r +.Oo Fl o Ar property Ns = Ns value Oc Ns ... +.Ar filesystem Ns @ Ns Ar snapname Ns | Ns Ar volume Ns @ Ns Ar snapname Ns ... +.Nm +.Cm rollback +.Op Fl Rfr +.Ar snapshot +.Nm +.Cm clone +.Op Fl p +.Oo Fl o Ar property Ns = Ns Ar value Oc Ns ... +.Ar snapshot Ar filesystem Ns | Ns Ar volume +.Nm +.Cm promote +.Ar clone-filesystem +.Nm +.Cm rename +.Op Fl f +.Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot +.Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot +.Nm +.Cm rename +.Op Fl fp +.Ar filesystem Ns | Ns Ar volume +.Ar filesystem Ns | Ns Ar volume +.Nm +.Cm rename +.Fl r +.Ar snapshot Ar snapshot +.Nm +.Cm list +.Op Fl r Ns | Ns Fl d Ar depth +.Op Fl Hp +.Oo Fl o Ar property Ns Oo , Ns Ar property Oc Ns ... Oc +.Oo Fl s Ar property Oc Ns ... +.Oo Fl S Ar property Oc Ns ... +.Oo Fl t Ar type Ns Oo , Ns Ar type Oc Ns ... Oc +.Oo Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot Oc Ns ... +.Nm +.Cm set +.Ar property Ns = Ns Ar value Oo Ar property Ns = Ns Ar value Oc Ns ... +.Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot Ns ... +.Nm +.Cm get +.Op Fl r Ns | Ns Fl d Ar depth +.Op Fl Hp +.Oo Fl o Ar field Ns Oo , Ns Ar field Oc Ns ... Oc +.Oo Fl s Ar source Ns Oo , Ns Ar source Oc Ns ... Oc +.Oo Fl t Ar type Ns Oo , Ns Ar type Oc Ns ... Oc +.Cm all | Ar property Ns Oo , Ns Ar property Oc Ns ... +.Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot Ns ... +.Nm +.Cm inherit +.Op Fl rS +.Ar property Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot Ns ... +.Nm +.Cm upgrade +.Nm +.Cm upgrade +.Fl v +.Nm +.Cm upgrade +.Op Fl r +.Op Fl V Ar version +.Fl a | Ar filesystem +.Nm +.Cm userspace +.Op Fl Hinp +.Oo Fl o Ar field Ns Oo , Ns Ar field Oc Ns ... Oc +.Oo Fl s Ar field Oc Ns ... +.Oo Fl S Ar field Oc Ns ... +.Oo Fl t Ar type Ns Oo , Ns Ar type Oc Ns ... Oc +.Ar filesystem Ns | Ns Ar snapshot +.Nm +.Cm groupspace +.Op Fl Hinp +.Oo Fl o Ar field Ns Oo , Ns Ar field Oc Ns ... Oc +.Oo Fl s Ar field Oc Ns ... +.Oo Fl S Ar field Oc Ns ... +.Oo Fl t Ar type Ns Oo , Ns Ar type Oc Ns ... Oc +.Ar filesystem Ns | Ns Ar snapshot +.Nm +.Cm mount +.Nm +.Cm mount +.Op Fl Ov +.Op Fl o Ar options +.Fl a | Ar filesystem +.Nm +.Cm unmount +.Op Fl f +.Fl a | Ar filesystem Ns | Ns Ar mountpoint +.Nm +.Cm share +.Fl a | Ar filesystem +.Nm +.Cm unshare +.Fl a | Ar filesystem Ns | Ns Ar mountpoint +.Nm +.Cm bookmark +.Ar snapshot bookmark +.Nm +.Cm send +.Op Fl DLPRenpv +.Op Oo Fl I Ns | Ns Fl i Oc Ar snapshot +.Ar snapshot +.Nm +.Cm send +.Op Fl Le +.Op Fl i Ar snapshot Ns | Ns Ar bookmark +.Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot +.Nm +.Cm receive +.Op Fl Fnuv +.Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot +.Nm +.Cm receive +.Op Fl Fnuv +.Op Fl d Ns | Ns Fl e +.Ar filesystem +.Nm +.Cm allow +.Ar filesystem Ns | Ns Ar volume +.Nm +.Cm allow +.Op Fl dglu +.Ar user Ns | Ns Ar group Ns Oo , Ns Ar user Ns | Ns Ar group Oc Ns ... +.Ar perm Ns | Ns @ Ns Ar setname Ns Oo , Ns Ar perm Ns | Ns @ Ns +.Ar setname Oc Ns ... +.Ar filesystem Ns | Ns Ar volume +.Nm +.Cm allow +.Op Fl dl +.Fl e Ns | Ns Sy everyone +.Ar perm Ns | Ns @ Ns Ar setname Ns Oo , Ns Ar perm Ns | Ns @ Ns +.Ar setname Oc Ns ... +.Ar filesystem Ns | Ns Ar volume +.Nm +.Cm allow +.Fl c +.Ar perm Ns | Ns @ Ns Ar setname Ns Oo , Ns Ar perm Ns | Ns @ Ns +.Ar setname Oc Ns ... +.Ar filesystem Ns | Ns Ar volume +.Nm +.Cm allow +.Fl s No @ Ns Ar setname +.Ar perm Ns | Ns @ Ns Ar setname Ns Oo , Ns Ar perm Ns | Ns @ Ns +.Ar setname Oc Ns ... +.Ar filesystem Ns | Ns Ar volume +.Nm +.Cm unallow +.Op Fl dglru +.Ar user Ns | Ns Ar group Ns Oo , Ns Ar user Ns | Ns Ar group Oc Ns ... +.Oo Ar perm Ns | Ns @ Ns Ar setname Ns Oo , Ns Ar perm Ns | Ns @ Ns +.Ar setname Oc Ns ... Oc +.Ar filesystem Ns | Ns Ar volume +.Nm +.Cm unallow +.Op Fl dlr +.Fl e Ns | Ns Sy everyone +.Oo Ar perm Ns | Ns @ Ns Ar setname Ns Oo , Ns Ar perm Ns | Ns @ Ns +.Ar setname Oc Ns ... Oc +.Ar filesystem Ns | Ns Ar volume +.Nm +.Cm unallow +.Op Fl r +.Fl c +.Oo Ar perm Ns | Ns @ Ns Ar setname Ns Oo , Ns Ar perm Ns | Ns @ Ns +.Ar setname Oc Ns ... Oc +.Ar filesystem Ns | Ns Ar volume +.Nm +.Cm unallow +.Op Fl r +.Fl s @ Ns Ar setname +.Oo Ar perm Ns | Ns @ Ns Ar setname Ns Oo , Ns Ar perm Ns | Ns @ Ns +.Ar setname Oc Ns ... Oc +.Ar filesystem Ns | Ns Ar volume +.Nm +.Cm hold +.Op Fl r +.Ar tag Ar snapshot Ns ... +.Nm +.Cm holds +.Op Fl r +.Ar snapshot Ns ... +.Nm +.Cm release +.Op Fl r +.Ar tag Ar snapshot Ns ... +.Nm +.Cm diff +.Op Fl FHt +.Ar snapshot Ar snapshot Ns | Ns Ar filesystem +.Sh DESCRIPTION +The +.Nm +command configures ZFS datasets within a ZFS storage pool, as described in +.Xr zpool 1M . +A dataset is identified by a unique path within the ZFS namespace. For example: +.Bd -literal pool/{filesystem,volume,snapshot} -.fi -.in -2 -.sp - -.sp -.LP -where the maximum length of a dataset name is \fBMAXNAMELEN\fR (256 bytes). -.sp -.LP +.Ed +.Pp +where the maximum length of a dataset name is +.Dv MAXNAMELEN +.Pq 256 bytes . +.Pp A dataset can be one of the following: -.sp -.ne 2 -.na -\fB\fIfile system\fR\fR -.ad -.sp .6 -.RS 4n -A \fBZFS\fR dataset of type \fBfilesystem\fR can be mounted within the standard -system namespace and behaves like other file systems. While \fBZFS\fR file -systems are designed to be \fBPOSIX\fR compliant, known issues exist that -prevent compliance in some cases. Applications that depend on standards -conformance might fail due to nonstandard behavior when checking file system -free space. -.RE - -.sp -.ne 2 -.na -\fB\fIvolume\fR\fR -.ad -.sp .6 -.RS 4n +.Bl -tag -width "file system" +.It Sy file system +A ZFS dataset of type +.Sy filesystem +can be mounted within the standard system namespace and behaves like other file +systems. While ZFS file systems are designed to be POSIX compliant, known issues +exist that prevent compliance in some cases. Applications that depend on +standards conformance might fail due to non-standard behavior when checking file +system free space. +.It Sy volume A logical volume exported as a raw or block device. This type of dataset should only be used under special circumstances. File systems are typically used in most environments. -.RE - -.sp -.ne 2 -.na -\fB\fIsnapshot\fR\fR -.ad -.sp .6 -.RS 4n +.It Sy snapshot A read-only version of a file system or volume at a given point in time. It is -specified as \fIfilesystem@name\fR or \fIvolume@name\fR. -.RE - -.SS "ZFS File System Hierarchy" -.LP -A \fBZFS\fR storage pool is a logical collection of devices that provide space -for datasets. A storage pool is also the root of the \fBZFS\fR file system -hierarchy. -.sp -.LP +specified as +.Ar filesystem Ns @ Ns Ar name +or +.Ar volume Ns @ Ns Ar name . +.El +.Ss ZFS File System Hierarchy +A ZFS storage pool is a logical collection of devices that provide space for +datasets. A storage pool is also the root of the ZFS file system hierarchy. +.Pp The root of the pool can be accessed as a file system, such as mounting and unmounting, taking snapshots, and setting properties. The physical storage -characteristics, however, are managed by the \fBzpool\fR(1M) command. -.sp -.LP -See \fBzpool\fR(1M) for more information on creating and administering pools. -.SS "Snapshots" -.LP +characteristics, however, are managed by the +.Xr zpool 1M +command. +.Pp +See +.Xr zpool 1M +for more information on creating and administering pools. +.Ss Snapshots A snapshot is a read-only copy of a file system or volume. Snapshots can be created extremely quickly, and initially consume no additional space within the pool. As data within the active dataset changes, the snapshot consumes more data than would otherwise be shared with the active dataset. -.sp -.LP +.Pp Snapshots can have arbitrary names. Snapshots of volumes can be cloned or rolled back, but cannot be accessed independently. -.sp -.LP -File system snapshots can be accessed under the \fB\&.zfs/snapshot\fR directory -in the root of the file system. Snapshots are automatically mounted on demand -and may be unmounted at regular intervals. The visibility of the \fB\&.zfs\fR -directory can be controlled by the \fBsnapdir\fR property. -.SS "Clones" -.LP +.Pp +File system snapshots can be accessed under the +.Pa .zfs/snapshot +directory in the root of the file system. Snapshots are automatically mounted on +demand and may be unmounted at regular intervals. The visibility of the +.Pa .zfs +directory can be controlled by the +snapdir +property. +.Ss Clones A clone is a writable volume or file system whose initial contents are the same -as another dataset. As with snapshots, creating a clone is nearly -instantaneous, and initially consumes no additional space. -.sp -.LP +as another dataset. As with snapshots, creating a clone is nearly instantaneous, +and initially consumes no additional space. +.Pp Clones can only be created from a snapshot. When a snapshot is cloned, it creates an implicit dependency between the parent and child. Even though the clone is created somewhere else in the dataset hierarchy, the original snapshot -cannot be destroyed as long as a clone exists. The \fBorigin\fR property -exposes this dependency, and the \fBdestroy\fR command lists any such -dependencies, if they exist. -.sp -.LP +cannot be destroyed as long as a clone exists. The +.Sy origin +property exposes this dependency, and the +.Cm destroy +command lists any such dependencies, if they exist. +.Pp The clone parent-child dependency relationship can be reversed by using the -\fBpromote\fR subcommand. This causes the "origin" file system to become a -clone of the specified file system, which makes it possible to destroy the file -system that the clone was created from. -.SS "Mount Points" -.LP -Creating a \fBZFS\fR file system is a simple operation, so the number of file -systems per system is likely to be numerous. To cope with this, \fBZFS\fR -automatically manages mounting and unmounting file systems without the need to -edit the \fB/etc/vfstab\fR file. All automatically managed file systems are -mounted by \fBZFS\fR at boot time. -.sp -.LP -By default, file systems are mounted under \fB/\fIpath\fR\fR, where \fIpath\fR -is the name of the file system in the \fBZFS\fR namespace. Directories are -created and destroyed as needed. -.sp -.LP -A file system can also have a mount point set in the \fBmountpoint\fR property. -This directory is created as needed, and \fBZFS\fR automatically mounts the -file system when the \fBzfs mount -a\fR command is invoked (without editing -\fB/etc/vfstab\fR). The \fBmountpoint\fR property can be inherited, so if -\fBpool/home\fR has a mount point of \fB/export/stuff\fR, then -\fBpool/home/user\fR automatically inherits a mount point of -\fB/export/stuff/user\fR. -.sp -.LP -A file system \fBmountpoint\fR property of \fBnone\fR prevents the file system -from being mounted. -.sp -.LP -If needed, \fBZFS\fR file systems can also be managed with traditional tools -(\fBmount\fR, \fBumount\fR, \fB/etc/vfstab\fR). If a file system's mount point -is set to \fBlegacy\fR, \fBZFS\fR makes no attempt to manage the file system, -and the administrator is responsible for mounting and unmounting the file -system. -.SS "Zones" -.LP -A \fBZFS\fR file system can be added to a non-global zone by using the -\fBzonecfg\fR \fBadd fs\fR subcommand. A \fBZFS\fR file system that is added to -a non-global zone must have its \fBmountpoint\fR property set to \fBlegacy\fR. -.sp -.LP +.Cm promote +subcommand. This causes the +.Qq origin +file system to become a clone of the specified file system, which makes it +possible to destroy the file system that the clone was created from. +.Ss "Mount Points" +Creating a ZFS file system is a simple operation, so the number of file systems +per system is likely to be numerous. To cope with this, ZFS automatically +manages mounting and unmounting file systems without the need to edit the +.Pa /etc/vfstab +file. All automatically managed file systems are mounted by ZFS at boot time. +.Pp +By default, file systems are mounted under +.Pa /path , +where +.Ar path +is the name of the file system in the ZFS namespace. Directories are created and +destroyed as needed. +.Pp +A file system can also have a mount point set in the +.Sy mountpoint +property. This directory is created as needed, and ZFS automatically mounts the +file system when the +.Nm zfs Cm mount Fl a +command is invoked +.Po without editing +.Pa /etc/vfstab +.Pc . +The +.Sy mountpoint +property can be inherited, so if +.Em pool/home +has a mount point of +.Pa /export/stuff , +then +.Em pool/home/user +automatically inherits a mount point of +.Pa /export/stuff/user . +.Pp +A file system +.Sy mountpoint +property of +.Sy none +prevents the file system from being mounted. +.Pp +If needed, ZFS file systems can also be managed with traditional tools +.Po +.Nm mount , +.Nm umount , +.Pa /etc/vfstab +.Pc . +If a file system's mount point is set to +.Sy legacy , +ZFS makes no attempt to manage the file system, and the administrator is +responsible for mounting and unmounting the file system. +.Ss "Zones" +A ZFS file system can be added to a non-global zone by using the +.Nm zonecfg Cm add Sy fs +subcommand. A ZFS file system that is added to a non-global zone must have its +.Sy mountpoint +property set to +.Sy legacy . +.Pp The physical properties of an added file system are controlled by the global administrator. However, the zone administrator can create, modify, or destroy -files within the added file system, depending on how the file system is -mounted. -.sp -.LP -A dataset can also be delegated to a non-global zone by using the \fBzonecfg\fR -\fBadd dataset\fR subcommand. You cannot delegate a dataset to one zone and the -children of the same dataset to another zone. The zone administrator can change -properties of the dataset or any of its children. However, the \fBquota\fR, -\fBfilesystem_limit\fR and \fBsnapshot_limit\fR properties of the delegated -dataset can be modified only by the global administrator. -.sp -.LP -A \fBZFS\fR volume can be added as a device to a non-global zone by using the -\fBzonecfg\fR \fBadd device\fR subcommand. However, its physical properties can -be modified only by the global administrator. -.sp -.LP -For more information about \fBzonecfg\fR syntax, see \fBzonecfg\fR(1M). -.sp -.LP -After a dataset is delegated to a non-global zone, the \fBzoned\fR property is -automatically set. A zoned file system cannot be mounted in the global zone, -since the zone administrator might have to set the mount point to an -unacceptable value. -.sp -.LP -The global administrator can forcibly clear the \fBzoned\fR property, though -this should be done with extreme care. The global administrator should verify -that all the mount points are acceptable before clearing the property. -.SS "Native Properties" -.LP -Properties are divided into two types, native properties and user-defined (or -"user") properties. Native properties either export internal statistics or -control \fBZFS\fR behavior. In addition, native properties are either editable -or read-only. User properties have no effect on \fBZFS\fR behavior, but you can -use them to annotate datasets in a way that is meaningful in your environment. -For more information about user properties, see the "User Properties" section, -below. -.sp -.LP +files within the added file system, depending on how the file system is mounted. +.Pp +A dataset can also be delegated to a non-global zone by using the +.Nm zonecfg Cm add Sy dataset +subcommand. You cannot delegate a dataset to one zone and the children of the +same dataset to another zone. The zone administrator can change properties of +the dataset or any of its children. However, the +.Sy quota , +.Sy filesystem_limit +and +.Sy snapshot_limit +properties of the delegated dataset can be modified only by the global +administrator. +.Pp +A ZFS volume can be added as a device to a non-global zone by using the +.Nm zonecfg Cm add Sy device +subcommand. However, its physical properties can be modified only by the global +administrator. +.Pp +For more information about +.Nm zonecfg +syntax, see +.Xr zonecfg 1M . +.Pp +After a dataset is delegated to a non-global zone, the +.Sy zoned +property is automatically set. A zoned file system cannot be mounted in the +global zone, since the zone administrator might have to set the mount point to +an unacceptable value. +.Pp +The global administrator can forcibly clear the +.Sy zoned +property, though this should be done with extreme care. The global administrator +should verify that all the mount points are acceptable before clearing the +property. +.Ss Native Properties +Properties are divided into two types, native properties and user-defined +.Po or +.Qq user +.Pc +properties. Native properties either export internal statistics or control ZFS +behavior. In addition, native properties are either editable or read-only. User +properties have no effect on ZFS behavior, but you can use them to annotate +datasets in a way that is meaningful in your environment. For more information +about user properties, see the +.Sx User Properties +section, below. +.Pp Every dataset has a set of properties that export statistics about the dataset as well as control various behaviors. Properties are inherited from the parent unless overridden by the child. Some properties apply only to certain types of -datasets (file systems, volumes, or snapshots). -.sp -.LP +datasets +.Pq file systems, volumes, or snapshots . +.Pp The values of numeric properties can be specified using human-readable suffixes -(for example, \fBk\fR, \fBKB\fR, \fBM\fR, \fBGb\fR, and so forth, up to \fBZ\fR -for zettabyte). The following are all valid (and equal) specifications: -.sp -.in +2 -.nf -1536M, 1.5g, 1.50GB -.fi -.in -2 -.sp - -.sp -.LP +.Po for example, +.Sy k , +.Sy KB , +.Sy M , +.Sy Gb , +and so forth, up to +.Sy Z +for zettabyte +.Pc . +The following are all valid +.Pq and equal +specifications: +.Li 1536M, 1.5g, 1.50GB . +.Pp The values of non-numeric properties are case sensitive and must be lowercase, -except for \fBmountpoint\fR, \fBsharenfs\fR, and \fBsharesmb\fR. -.sp -.LP +except for +.Sy mountpoint , +.Sy sharenfs , +and +.Sy sharesmb . +.Pp The following native properties consist of read-only statistics about the dataset. These properties can be neither set, nor inherited. Native properties apply to all dataset types unless otherwise noted. -.sp -.ne 2 -.na -\fB\fBavailable\fR\fR -.ad -.sp .6 -.RS 4n -The amount of space available to the dataset and all its children, assuming -that there is no other activity in the pool. Because space is shared within a -pool, availability can be limited by any number of factors, including physical -pool size, quotas, reservations, or other datasets within the pool. -.sp +.Bl -tag -width "usedbyrefreservation" +.It Sy available +The amount of space available to the dataset and all its children, assuming that +there is no other activity in the pool. Because space is shared within a pool, +availability can be limited by any number of factors, including physical pool +size, quotas, reservations, or other datasets within the pool. +.Pp This property can also be referred to by its shortened column name, -\fBavail\fR. -.RE - -.sp -.ne 2 -.na -\fB\fBcompressratio\fR\fR -.ad -.sp .6 -.RS 4n -For non-snapshots, the compression ratio achieved for the \fBused\fR -space of this dataset, expressed as a multiplier. The \fBused\fR -property includes descendant datasets, and, for clones, does not include -the space shared with the origin snapshot. For snapshots, the -\fBcompressratio\fR is the same as the \fBrefcompressratio\fR property. -Compression can be turned on by running: \fBzfs set compression=on -\fIdataset\fR\fR. The default value is \fBoff\fR. -.RE - -.sp -.ne 2 -.na -\fB\fBcreation\fR\fR -.ad -.sp .6 -.RS 4n +.Sy avail . +.It Sy compressratio +For non-snapshots, the compression ratio achieved for the +.Sy used +space of this dataset, expressed as a multiplier. The +.Sy used +property includes descendant datasets, and, for clones, does not include the +space shared with the origin snapshot. For snapshots, the +.Sy compressratio +is the same as the +.Sy refcompressratio +property. Compression can be turned on by running: +.Nm zfs Cm set Sy compression Ns = Ns Sy on Ar dataset . +The default value is +.Sy off . +.It Sy creation The time this dataset was created. -.RE - -.sp -.ne 2 -.na -\fB\fBclones\fR\fR -.ad -.sp .6 -.RS 4n -For snapshots, this property is a comma-separated list of filesystems or -volumes which are clones of this snapshot. The clones' \fBorigin\fR property -is this snapshot. If the \fBclones\fR property is not empty, then this -snapshot can not be destroyed (even with the \fB-r\fR or \fB-f\fR options). -.RE - -.sp -.ne 2 -.na -\fB\fBdefer_destroy\fR\fR -.ad -.sp .6 -.RS 4n -This property is \fBon\fR if the snapshot has been marked for deferred destroy -by using the \fBzfs destroy\fR \fB-d\fR command. Otherwise, the property is -\fBoff\fR. -.RE - -.sp -.ne 2 -.na -\fB\fBfilesystem_count\fR -.ad -.sp .6 -.RS 4n -The total number of filesystems and volumes that exist under this location in the -dataset tree. This value is only available when a \fBfilesystem_limit\fR has -been set somewhere in the tree under which the dataset resides. -.RE - -.sp -.ne 2 -.na -\fB\fBlogicalreferenced\fR\fR -.ad -.sp .6 -.RS 4n -The amount of space that is "logically" accessible by this dataset. See -the \fBreferenced\fR property. The logical space ignores the effect of -the \fBcompression\fR and \fBcopies\fR properties, giving a quantity -closer to the amount of data that applications see. However, it does -include space consumed by metadata. -.sp +.It Sy clones +For snapshots, this property is a comma-separated list of filesystems or volumes *** DIFF OUTPUT TRUNCATED AT 1000 LINES *** From owner-svn-src-vendor@freebsd.org Fri Apr 14 16:18:54 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 3DC8FD3DC81; Fri, 14 Apr 2017 16:18:54 +0000 (UTC) (envelope-from asomers@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 0FAFB31D; Fri, 14 Apr 2017 16:18:53 +0000 (UTC) (envelope-from asomers@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EGIrwx013531; Fri, 14 Apr 2017 16:18:53 GMT (envelope-from asomers@FreeBSD.org) Received: (from asomers@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EGIrNX013530; Fri, 14 Apr 2017 16:18:53 GMT (envelope-from asomers@FreeBSD.org) Message-Id: <201704141618.v3EGIrNX013530@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: asomers set sender to asomers@FreeBSD.org using -f From: Alan Somers Date: Fri, 14 Apr 2017 16:18:53 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316855 - vendor/illumos/dist/cmd/zdb X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 16:18:54 -0000 Author: asomers Date: Fri Apr 14 16:18:53 2017 New Revision: 316855 URL: https://svnweb.freebsd.org/changeset/base/316855 Log: 7900 zdb shouldn't print the path of a znode at verbosity < 5 Reviewed by: Paul Dagnelie Reviewed by: Matt Ahrens Approved by: Dan McDonald Author: Alan Somers illumos/illumos-gate@e548d2fa41d1baa06662ed9abbb8bcec86e27dd9 https://www.illumos.org/issues/7900 Modified: vendor/illumos/dist/cmd/zdb/zdb.c Modified: vendor/illumos/dist/cmd/zdb/zdb.c ============================================================================== --- vendor/illumos/dist/cmd/zdb/zdb.c Fri Apr 14 15:35:07 2017 (r316854) +++ vendor/illumos/dist/cmd/zdb/zdb.c Fri Apr 14 16:18:53 2017 (r316855) @@ -1693,23 +1693,19 @@ dump_znode(objset_t *os, uint64_t object return; } - error = zfs_obj_to_path(os, object, path, sizeof (path)); - if (error != 0) { - (void) snprintf(path, sizeof (path), "\?\?\?", - (u_longlong_t)object); - } - if (dump_opt['d'] < 3) { - (void) printf("\t%s\n", path); - (void) sa_handle_destroy(hdl); - return; - } - z_crtime = (time_t)crtm[0]; z_atime = (time_t)acctm[0]; z_mtime = (time_t)modtm[0]; z_ctime = (time_t)chgtm[0]; - (void) printf("\tpath %s\n", path); + if (dump_opt['d'] > 4) { + error = zfs_obj_to_path(os, object, path, sizeof (path)); + if (error != 0) { + (void) snprintf(path, sizeof (path), + "\?\?\?", (u_longlong_t)object); + } + (void) printf("\tpath %s\n", path); + } dump_uidgid(os, uid, gid); (void) printf("\tatime %s", ctime(&z_atime)); (void) printf("\tmtime %s", ctime(&z_mtime)); From owner-svn-src-vendor@freebsd.org Fri Apr 14 16:53:35 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id E02FDD3E930; Fri, 14 Apr 2017 16:53:35 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 9CE886A4; Fri, 14 Apr 2017 16:53:35 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EGrY49030491; Fri, 14 Apr 2017 16:53:34 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EGrYEO030485; Fri, 14 Apr 2017 16:53:34 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141653.v3EGrYEO030485@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 16:53:34 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316858 - in vendor/illumos/dist: cmd/zdb cmd/ztest lib/libzpool/common lib/libzpool/common/sys man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 16:53:36 -0000 Author: avg Date: Fri Apr 14 16:53:34 2017 New Revision: 316858 URL: https://svnweb.freebsd.org/changeset/base/316858 Log: 7280 Allow changing global libzpool variables in zdb and ztest through command line illumos/illumos-gate@0e60744c982adecd0a1f146f5637475d07ab1069 https://github.com/illumos/illumos-gate/commit/0e60744c982adecd0a1f146f5637475d07ab1069 https://www.illumos.org/issues/7280 zdb is very handy for diagnosing problems with a pool in a safe and quick way. When a pool is in a bad shape, we often want to disable some fail-safes, or adjust some tunables in order to open them. In the kernel, this is done by changing public variables in mdb. The goal of this feature is to add the same capability to zdb and ztest, so that they can change libzpool tuneables from the command line. Reviewed by: Matthew Ahrens Reviewed by: Dan Kimmel Approved by: Robert Mustacchi Author: Pavel Zakharov Modified: vendor/illumos/dist/cmd/zdb/zdb.c vendor/illumos/dist/cmd/ztest/ztest.c vendor/illumos/dist/lib/libzpool/common/sys/zfs_context.h vendor/illumos/dist/lib/libzpool/common/util.c vendor/illumos/dist/man/man1m/zdb.1m Modified: vendor/illumos/dist/cmd/zdb/zdb.c ============================================================================== --- vendor/illumos/dist/cmd/zdb/zdb.c Fri Apr 14 16:40:10 2017 (r316857) +++ vendor/illumos/dist/cmd/zdb/zdb.c Fri Apr 14 16:53:34 2017 (r316858) @@ -118,7 +118,8 @@ usage(void) { (void) fprintf(stderr, "Usage: %s [-CumMdibcsDvhLXFPAG] [-t txg] [-e [-p path...]] " - "[-U config] [-I inflight I/Os] [-x dumpdir] poolname [object...]\n" + "[-U config] [-I inflight I/Os] [-x dumpdir] [-o var=value] " + "poolname [object...]\n" " %s [-divPA] [-e -p path...] [-U config] dataset " "[object...]\n" " %s -mM [-LXFPA] [-t txg] [-e [-p path...]] [-U config] " @@ -180,6 +181,8 @@ usage(void) "checksumming I/Os [default is 200]\n"); (void) fprintf(stderr, " -G dump zfs_dbgmsg buffer before " "exiting\n"); + (void) fprintf(stderr, " -o = set global " + "variable to an unsigned 32-bit integer value\n"); (void) fprintf(stderr, "Specify an option more than once (e.g. -bb) " "to make only that option verbose\n"); (void) fprintf(stderr, "Default is to dump everything non-verbosely\n"); @@ -3575,7 +3578,7 @@ main(int argc, char **argv) spa_config_path = spa_config_path_env; while ((c = getopt(argc, argv, - "bcdhilmMI:suCDRSAFLXx:evp:t:U:PG")) != -1) { + "bcdhilmMI:suCDRSAFLXx:evp:t:U:PGo:")) != -1) { switch (c) { case 'b': case 'c': @@ -3644,6 +3647,11 @@ main(int argc, char **argv) case 'x': vn_dumpdir = optarg; break; + case 'o': + error = set_global_var(optarg); + if (error != 0) + usage(); + break; default: usage(); break; Modified: vendor/illumos/dist/cmd/ztest/ztest.c ============================================================================== --- vendor/illumos/dist/cmd/ztest/ztest.c Fri Apr 14 16:40:10 2017 (r316857) +++ vendor/illumos/dist/cmd/ztest/ztest.c Fri Apr 14 16:53:34 2017 (r316858) @@ -579,6 +579,8 @@ usage(boolean_t requested) "\t[-F freezeloops (default: %llu)] max loops in spa_freeze()\n" "\t[-P passtime (default: %llu sec)] time per pass\n" "\t[-B alt_ztest (default: )] alternate ztest path\n" + "\t[-o variable=value] ... set global variable to an unsigned\n" + "\t 32-bit integer value\n" "\t[-h] (print help)\n" "", zo->zo_pool, @@ -614,7 +616,7 @@ process_options(int argc, char **argv) bcopy(&ztest_opts_defaults, zo, sizeof (*zo)); while ((opt = getopt(argc, argv, - "v:s:a:m:r:R:d:t:g:i:k:p:f:VET:P:hF:B:")) != EOF) { + "v:s:a:m:r:R:d:t:g:i:k:p:f:VET:P:hF:B:o:")) != EOF) { value = 0; switch (opt) { case 'v': @@ -701,6 +703,10 @@ process_options(int argc, char **argv) case 'B': (void) strlcpy(altdir, optarg, sizeof (altdir)); break; + case 'o': + if (set_global_var(optarg) != 0) + usage(B_FALSE); + break; case 'h': usage(B_TRUE); break; Modified: vendor/illumos/dist/lib/libzpool/common/sys/zfs_context.h ============================================================================== --- vendor/illumos/dist/lib/libzpool/common/sys/zfs_context.h Fri Apr 14 16:40:10 2017 (r316857) +++ vendor/illumos/dist/lib/libzpool/common/sys/zfs_context.h Fri Apr 14 16:53:34 2017 (r316858) @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2011 Nexenta Systems, Inc. All rights reserved. - * Copyright (c) 2012, 2015 by Delphix. All rights reserved. + * Copyright (c) 2012, 2016 by Delphix. All rights reserved. * Copyright (c) 2012, Joyent, Inc. All rights reserved. */ @@ -541,6 +541,7 @@ extern void kernel_fini(void); struct spa; extern void nicenum(uint64_t num, char *buf); extern void show_pool_stats(struct spa *); +extern int set_global_var(char *arg); typedef struct callb_cpr { kmutex_t *cc_lockp; Modified: vendor/illumos/dist/lib/libzpool/common/util.c ============================================================================== --- vendor/illumos/dist/lib/libzpool/common/util.c Fri Apr 14 16:40:10 2017 (r316857) +++ vendor/illumos/dist/lib/libzpool/common/util.c Fri Apr 14 16:53:34 2017 (r316858) @@ -20,6 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016 by Delphix. All rights reserved. */ #include @@ -31,6 +32,7 @@ #include #include #include +#include /* * Routines needed by more than one client of libzpool. @@ -153,3 +155,58 @@ show_pool_stats(spa_t *spa) nvlist_free(config); } + +/* + * Sets given global variable in libzpool to given unsigned 32-bit value. + * arg: "=" + */ +int +set_global_var(char *arg) +{ + void *zpoolhdl; + char *varname = arg, *varval; + u_longlong_t val; + +#ifndef _LITTLE_ENDIAN + /* + * On big endian systems changing a 64-bit variable would set the high + * 32 bits instead of the low 32 bits, which could cause unexpected + * results. + */ + fprintf(stderr, "Setting global variables is only supported on " + "little-endian systems\n", varname); + return (ENOTSUP); +#endif + if ((varval = strchr(arg, '=')) != NULL) { + *varval = '\0'; + varval++; + val = strtoull(varval, NULL, 0); + if (val > UINT32_MAX) { + fprintf(stderr, "Value for global variable '%s' must " + "be a 32-bit unsigned integer\n", varname); + return (EOVERFLOW); + } + } else { + return (EINVAL); + } + + zpoolhdl = dlopen("libzpool.so", RTLD_LAZY); + if (zpoolhdl != NULL) { + uint32_t *var; + var = dlsym(zpoolhdl, varname); + if (var == NULL) { + fprintf(stderr, "Global variable '%s' does not exist " + "in libzpool.so\n", varname); + return (EINVAL); + } + *var = (uint32_t)val; + + dlclose(zpoolhdl); + } else { + fprintf(stderr, "Failed to open libzpool.so to set global " + "variable\n"); + return (EIO); + } + + return (0); +} Modified: vendor/illumos/dist/man/man1m/zdb.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zdb.1m Fri Apr 14 16:40:10 2017 (r316857) +++ vendor/illumos/dist/man/man1m/zdb.1m Fri Apr 14 16:53:34 2017 (r316858) @@ -21,7 +21,7 @@ .SH "SYNOPSIS" \fBzdb\fR [-CumdibcsDvhLMXFPAG] [-e [-p \fIpath\fR...]] [-t \fItxg\fR] [-U \fIcache\fR] [-I \fIinflight I/Os\fR] [-x \fIdumpdir\fR] - [\fIpoolname\fR [\fIobject\fR ...]] + [-o \fIvar\fR=\fIvalue\fR] ... [\fIpoolname\fR [\fIobject\fR ...]] .P \fBzdb\fR [-divPA] [-e [-p \fIpath\fR...]] [-U \fIcache\fR] @@ -423,6 +423,18 @@ option. .sp .ne 2 .na +\fB-o \fIvar\fR=\fIvalue\fR ... \fR +.ad +.sp .6 +.RS 4n +Set the given global libzpool variable to the provided value. The value must +be an unsigned 32-bit integer. Currently only little-endian systems are +supported to avoid accidentally setting the high 32 bits of 64-bit variables. +.RE + +.sp +.ne 2 +.na \fB-P\fR .ad .sp .6 From owner-svn-src-vendor@freebsd.org Fri Apr 14 16:55:16 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id C05E9D3EA56; Fri, 14 Apr 2017 16:55:16 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 7653DA5B; Fri, 14 Apr 2017 16:55:16 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EGtF39030711; Fri, 14 Apr 2017 16:55:15 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EGtFIw030710; Fri, 14 Apr 2017 16:55:15 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141655.v3EGtFIw030710@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 16:55:15 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316860 - vendor/illumos/dist/cmd/zdb X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 16:55:16 -0000 Author: avg Date: Fri Apr 14 16:55:15 2017 New Revision: 316860 URL: https://svnweb.freebsd.org/changeset/base/316860 Log: 7545 zdb should disable reference tracking illumos/illumos-gate@4dd77f9e38ef05b39db128ff7608d926fd3218c6 https://github.com/illumos/illumos-gate/commit/4dd77f9e38ef05b39db128ff7608d926fd3218c6 https://www.illumos.org/issues/7545 When evicting from the ARC, we manipulate some refcount_t's, e.g. arcs_size. When using zdb to examine a large amount of data (e.g. zdb -bb on a large pool with small blocks), the ARC may have a large number of entries. If reference tracking is enabled, there will be ~1 reference for each block in the ARC. When evicting, we decrement the refcount and have to search all the references to find the one that we are removing, which is very slow. Since zdb is typically used to find problems with the on-disk format, and not with the code it is running, we should disable reference tracking in zdb. Reviewed by: Dan Kimmel Reviewed by: Steve Gonczi Reviewed by: George Wilson Approved by: Robert Mustacchi Author: Matthew Ahrens Modified: vendor/illumos/dist/cmd/zdb/zdb.c Modified: vendor/illumos/dist/cmd/zdb/zdb.c ============================================================================== --- vendor/illumos/dist/cmd/zdb/zdb.c Fri Apr 14 16:54:50 2017 (r316859) +++ vendor/illumos/dist/cmd/zdb/zdb.c Fri Apr 14 16:55:15 2017 (r316860) @@ -75,10 +75,12 @@ DMU_OT_ZAP_OTHER : DMU_OT_NUMTYPES)) #ifndef lint +extern int reference_tracking_enable; extern boolean_t zfs_recover; extern uint64_t zfs_arc_max, zfs_arc_meta_limit; extern int zfs_vdev_async_read_max_active; #else +int reference_tracking_enable; boolean_t zfs_recover; uint64_t zfs_arc_max, zfs_arc_meta_limit; int zfs_vdev_async_read_max_active; @@ -3676,6 +3678,11 @@ main(int argc, char **argv) */ zfs_vdev_async_read_max_active = 10; + /* + * Disable reference tracking for better performance. + */ + reference_tracking_enable = B_FALSE; + kernel_init(FREAD); g_zfs = libzfs_init(); ASSERT(g_zfs != NULL); From owner-svn-src-vendor@freebsd.org Fri Apr 14 16:57:07 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id A48E2D3EB6D; Fri, 14 Apr 2017 16:57:07 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 7C3E0C47; Fri, 14 Apr 2017 16:57:07 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EGv6u0030852; Fri, 14 Apr 2017 16:57:06 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EGv622030850; Fri, 14 Apr 2017 16:57:06 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141657.v3EGv622030850@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 16:57:06 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316861 - in vendor/illumos/dist: cmd/zdb man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 16:57:07 -0000 Author: avg Date: Fri Apr 14 16:57:06 2017 New Revision: 316861 URL: https://svnweb.freebsd.org/changeset/base/316861 Log: 6866 zdb -l should return non-zero if it fails to find any label illumos/illumos-gate@64723e361134b2a2c45341334174f9d34002f8d0 https://github.com/illumos/illumos-gate/commit/64723e361134b2a2c45341334174f9d34002f8d0 https://www.illumos.org/issues/6866 Need this for #6865. To be generally more scripting-friendly, overload this issue with adding '-q' option which should skip printing any label information. Reviewed by: Matthew Ahrens Reviewed by: John Kennedy Approved by: Robert Mustacchi Author: Yuri Pankov Modified: vendor/illumos/dist/cmd/zdb/zdb.c vendor/illumos/dist/man/man1m/zdb.1m Modified: vendor/illumos/dist/cmd/zdb/zdb.c ============================================================================== --- vendor/illumos/dist/cmd/zdb/zdb.c Fri Apr 14 16:55:15 2017 (r316860) +++ vendor/illumos/dist/cmd/zdb/zdb.c Fri Apr 14 16:57:06 2017 (r316861) @@ -23,6 +23,7 @@ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2011, 2016 by Delphix. All rights reserved. * Copyright (c) 2014 Integros [integros.com] + * Copyright 2016 Nexenta Systems, Inc. */ #include @@ -119,7 +120,7 @@ static void usage(void) { (void) fprintf(stderr, - "Usage: %s [-CumMdibcsDvhLXFPAG] [-t txg] [-e [-p path...]] " + "Usage: %s [-CmMdibcsDvhLXFPAG] [-t txg] [-e [-p path...]] " "[-U config] [-I inflight I/Os] [-x dumpdir] [-o var=value] " "poolname [object...]\n" " %s [-divPA] [-e -p path...] [-U config] dataset " @@ -129,7 +130,7 @@ usage(void) " %s -R [-A] [-e [-p path...]] poolname " "vdev:offset:size[:flags]\n" " %s -S [-PA] [-e [-p path...]] [-U config] poolname\n" - " %s -l [-uA] device\n" + " %s -l [-Aqu] device\n" " %s -C [-A] [-U config]\n\n", cmdname, cmdname, cmdname, cmdname, cmdname, cmdname, cmdname); @@ -140,7 +141,6 @@ usage(void) (void) fprintf(stderr, " If object numbers are specified, only " "those objects are dumped\n\n"); (void) fprintf(stderr, " Options to control amount of output:\n"); - (void) fprintf(stderr, " -u uberblock\n"); (void) fprintf(stderr, " -d dataset(s)\n"); (void) fprintf(stderr, " -i intent logs\n"); (void) fprintf(stderr, " -C config (or cachefile if alone)\n"); @@ -154,7 +154,7 @@ usage(void) (void) fprintf(stderr, " -D dedup statistics\n"); (void) fprintf(stderr, " -S simulate dedup to measure effect\n"); (void) fprintf(stderr, " -v verbose (applies to all others)\n"); - (void) fprintf(stderr, " -l dump label contents\n"); + (void) fprintf(stderr, " -l read label contents\n"); (void) fprintf(stderr, " -L disable leak tracking (do not " "load spacemaps)\n"); (void) fprintf(stderr, " -R read and display block from a " @@ -185,6 +185,8 @@ usage(void) "exiting\n"); (void) fprintf(stderr, " -o = set global " "variable to an unsigned 32-bit integer value\n"); + (void) fprintf(stderr, " -q don't print label contents\n"); + (void) fprintf(stderr, " -u uberblock\n"); (void) fprintf(stderr, "Specify an option more than once (e.g. -bb) " "to make only that option verbose\n"); (void) fprintf(stderr, "Default is to dump everything non-verbosely\n"); @@ -2133,45 +2135,48 @@ dump_label_uberblocks(vdev_label_t *lbl, } } -static void +static int dump_label(const char *dev) { int fd; vdev_label_t label; - char *path, *buf = label.vl_vdev_phys.vp_nvlist; + char path[MAXPATHLEN]; + char *buf = label.vl_vdev_phys.vp_nvlist; size_t buflen = sizeof (label.vl_vdev_phys.vp_nvlist); struct stat64 statbuf; uint64_t psize, ashift; - int len = strlen(dev) + 1; + boolean_t label_found = B_FALSE; - if (strncmp(dev, ZFS_DISK_ROOTD, strlen(ZFS_DISK_ROOTD)) == 0) { - len++; - path = malloc(len); - (void) snprintf(path, len, "%s%s", ZFS_RDISK_ROOTD, - dev + strlen(ZFS_DISK_ROOTD)); - } else { - path = strdup(dev); - } + (void) strlcpy(path, dev, sizeof (path)); + if (dev[0] == '/') { + if (strncmp(dev, ZFS_DISK_ROOTD, + strlen(ZFS_DISK_ROOTD)) == 0) { + (void) snprintf(path, sizeof (path), "%s%s", + ZFS_RDISK_ROOTD, dev + strlen(ZFS_DISK_ROOTD)); + } + } else if (stat64(path, &statbuf) != 0) { + char *s; - if ((fd = open64(path, O_RDONLY)) < 0) { - (void) printf("cannot open '%s': %s\n", path, strerror(errno)); - free(path); - exit(1); + (void) snprintf(path, sizeof (path), "%s%s", ZFS_RDISK_ROOTD, + dev); + if ((s = strrchr(dev, 's')) == NULL || !isdigit(*(s + 1))) + (void) strlcat(path, "s0", sizeof (path)); } - if (fstat64(fd, &statbuf) != 0) { + if (stat64(path, &statbuf) != 0) { (void) printf("failed to stat '%s': %s\n", path, strerror(errno)); - free(path); - (void) close(fd); exit(1); } if (S_ISBLK(statbuf.st_mode)) { (void) printf("cannot use '%s': character device required\n", path); - free(path); - (void) close(fd); + exit(1); + } + + if ((fd = open64(path, O_RDONLY)) < 0) { + (void) printf("cannot open '%s': %s\n", path, strerror(errno)); exit(1); } @@ -2181,36 +2186,43 @@ dump_label(const char *dev) for (int l = 0; l < VDEV_LABELS; l++) { nvlist_t *config = NULL; - (void) printf("--------------------------------------------\n"); - (void) printf("LABEL %d\n", l); - (void) printf("--------------------------------------------\n"); + if (!dump_opt['q']) { + (void) printf("------------------------------------\n"); + (void) printf("LABEL %d\n", l); + (void) printf("------------------------------------\n"); + } if (pread64(fd, &label, sizeof (label), vdev_label_offset(psize, l, 0)) != sizeof (label)) { - (void) printf("failed to read label %d\n", l); + if (!dump_opt['q']) + (void) printf("failed to read label %d\n", l); continue; } if (nvlist_unpack(buf, buflen, &config, 0) != 0) { - (void) printf("failed to unpack label %d\n", l); + if (!dump_opt['q']) + (void) printf("failed to unpack label %d\n", l); ashift = SPA_MINBLOCKSHIFT; } else { nvlist_t *vdev_tree = NULL; - dump_nvlist(config, 4); + if (!dump_opt['q']) + dump_nvlist(config, 4); if ((nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &vdev_tree) != 0) || (nvlist_lookup_uint64(vdev_tree, ZPOOL_CONFIG_ASHIFT, &ashift) != 0)) ashift = SPA_MINBLOCKSHIFT; nvlist_free(config); + label_found = B_TRUE; } if (dump_opt['u']) dump_label_uberblocks(&label, ashift); } - free(path); (void) close(fd); + + return (label_found ? 0 : 2); } static uint64_t dataset_feature_count[SPA_FEATURES]; @@ -3580,7 +3592,7 @@ main(int argc, char **argv) spa_config_path = spa_config_path_env; while ((c = getopt(argc, argv, - "bcdhilmMI:suCDRSAFLXx:evp:t:U:PGo:")) != -1) { + "bcdhilmMI:suCDRSAFLXx:evp:t:U:PGo:q")) != -1) { switch (c) { case 'b': case 'c': @@ -3606,6 +3618,7 @@ main(int argc, char **argv) case 'X': case 'e': case 'P': + case 'q': dump_opt[c]++; break; case 'I': @@ -3713,10 +3726,8 @@ main(int argc, char **argv) usage(); } - if (dump_opt['l']) { - dump_label(argv[0]); - return (0); - } + if (dump_opt['l']) + return (dump_label(argv[0])); if (dump_opt['X'] || dump_opt['F']) rewind = ZPOOL_DO_REWIND | Modified: vendor/illumos/dist/man/man1m/zdb.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zdb.1m Fri Apr 14 16:55:15 2017 (r316860) +++ vendor/illumos/dist/man/man1m/zdb.1m Fri Apr 14 16:57:06 2017 (r316861) @@ -12,14 +12,15 @@ .\" .\" Copyright 2012, Richard Lowe. .\" Copyright (c) 2012, 2016 by Delphix. All rights reserved. +.\" Copyright 2016 Nexenta Systems, Inc. .\" -.TH "ZDB" "1M" "Feb 4, 2016" "" "" +.TH "ZDB" "1M" "April 9, 2016" .SH "NAME" \fBzdb\fR - Display zpool debugging and consistency information .SH "SYNOPSIS" -\fBzdb\fR [-CumdibcsDvhLMXFPAG] [-e [-p \fIpath\fR...]] [-t \fItxg\fR] +\fBzdb\fR [-CmdibcsDvhLMXFPAG] [-e [-p \fIpath\fR...]] [-t \fItxg\fR] [-U \fIcache\fR] [-I \fIinflight I/Os\fR] [-x \fIdumpdir\fR] [-o \fIvar\fR=\fIvalue\fR] ... [\fIpoolname\fR [\fIobject\fR ...]] @@ -39,7 +40,7 @@ \fBzdb\fR -S [-AP] [-e [-p \fIpath\fR...]] [-U \fIcache\fR] \fIpoolname\fR .P -\fBzdb\fR -l [-uA] \fIdevice\fR +\fBzdb\fR -l [-Aqu] \fIdevice\fR .P \fBzdb\fR -C [-A] [-U \fIcache\fR] @@ -176,8 +177,13 @@ transaction type. .ad .sp .6 .RS 4n -Display the vdev labels from the specified device. If the \fB-u\fR option is -also specified, also display the uberblocks on this device. +Read the vdev labels from the specified device. \fBzdb -l\fR will return 0 if +valid label was found, 1 if error occured, and 2 if no valid labels were found. +.P +If the \fB-u\fR option is also specified, also display the uberblocks on this +device. +.P +If the \fB-q\fR option is also specified, don't print the labels. .RE .sp From owner-svn-src-vendor@freebsd.org Fri Apr 14 16:58:42 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 7F3E5D3EC54; Fri, 14 Apr 2017 16:58:42 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 4B751E40; Fri, 14 Apr 2017 16:58:42 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EGwf6f030992; Fri, 14 Apr 2017 16:58:41 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EGwfxU030989; Fri, 14 Apr 2017 16:58:41 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141658.v3EGwfxU030989@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 16:58:41 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316862 - in vendor/illumos/dist: cmd/zdb man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 16:58:42 -0000 Author: avg Date: Fri Apr 14 16:58:41 2017 New Revision: 316862 URL: https://svnweb.freebsd.org/changeset/base/316862 Log: 6410 teach zdb to perform object lookups by path illumos/illumos-gate@ed61ec1da9132e570b0853386d0f78a32f852cd2 https://github.com/illumos/illumos-gate/commit/ed61ec1da9132e570b0853386d0f78a32f852cd2 https://www.illumos.org/issues/6410 This is primarily intended to ease debugging & testing ZFS when one is only interested in things like the on-disk location of a specific object's blocks, but doesn't know their object id. This allows doing things like the following (FreeBSD-based example): # zpool create -f foo da0 # dd if=/dev/zero of=/foo/1 bs=1M count=4 >/dev/null 2>&1 # zpool export foo # zdb -vvvvv -o "ZFS plain file" foo /1 Object lvl iblk dblk dsize lsize %full type 8 2 16K 128K 3.99M 4M 100.00 ZFS plain file (K=inherit) (Z=inherit) 168 bonus System attributes dnode flags: USED_BYTES USERUSED_ACCOUNTED dnode maxblkid: 31 path /1 uid 0 gid 0 atime Thu Apr 23 22:45:32 2015 mtime Thu Apr 23 22:45:32 2015 ctime Thu Apr 23 22:45:32 2015 crtime Thu Apr 23 22:45:32 2015 gen 7 mode 100644 size 4194304 parent 4 links 1 pflags 40800000004 Indirect blocks: 0 L1 DVA[0]=<0:c19200:600> DVA[1]=<0:10800019200:600> [L1 ZFS plain file] fletcher4 lz4 LE contiguous unique double size=4000L/200P birth=7L/ Reviewed by: Pavel Zakharov Reviewed by: Matthew Ahrens Reviewed by: Will Andrews Approved by: Dan McDonald Author: Yuri Pankov Modified: vendor/illumos/dist/cmd/zdb/zdb.c vendor/illumos/dist/man/man1m/zdb.1m Modified: vendor/illumos/dist/cmd/zdb/zdb.c ============================================================================== --- vendor/illumos/dist/cmd/zdb/zdb.c Fri Apr 14 16:57:06 2017 (r316861) +++ vendor/illumos/dist/cmd/zdb/zdb.c Fri Apr 14 16:58:41 2017 (r316862) @@ -23,7 +23,7 @@ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2011, 2016 by Delphix. All rights reserved. * Copyright (c) 2014 Integros [integros.com] - * Copyright 2016 Nexenta Systems, Inc. + * Copyright 2017 Nexenta Systems, Inc. */ #include @@ -120,19 +120,22 @@ static void usage(void) { (void) fprintf(stderr, - "Usage: %s [-CmMdibcsDvhLXFPAG] [-t txg] [-e [-p path...]] " - "[-U config] [-I inflight I/Os] [-x dumpdir] [-o var=value] " - "poolname [object...]\n" - " %s [-divPA] [-e -p path...] [-U config] dataset " - "[object...]\n" - " %s -mM [-LXFPA] [-t txg] [-e [-p path...]] [-U config] " - "poolname [vdev [metaslab...]]\n" - " %s -R [-A] [-e [-p path...]] poolname " - "vdev:offset:size[:flags]\n" - " %s -S [-PA] [-e [-p path...]] [-U config] poolname\n" - " %s -l [-Aqu] device\n" - " %s -C [-A] [-U config]\n\n", - cmdname, cmdname, cmdname, cmdname, cmdname, cmdname, cmdname); + "Usage:\t%s [-AbcdDFGhiLMPsvX] [-e [-p ...]] " + "[-I ]\n" + "\t\t[-o =]... [-t ] [-U ] [-x ]\n" + "\t\t[ [ ...]]\n" + "\t%s [-AdiPv] [-e [-p ...]] [-U ] " + "[ ...]\n" + "\t%s -C [-A] [-U ]\n" + "\t%s -l [-Aqu] \n" + "\t%s -m [-AFLPX] [-e [-p ...]] [-t ] [-U ]\n" + "\t\t [ [ ...]]\n" + "\t%s -O \n" + "\t%s -R [-A] [-e [-p ...]] [-U ]\n" + "\t\t ::[:]\n" + "\t%s -S [-AP] [-e [-p ...]] [-U ] \n\n", + cmdname, cmdname, cmdname, cmdname, cmdname, cmdname, cmdname, + cmdname); (void) fprintf(stderr, " Dataset name must include at least one " "separator character '/' or '@'\n"); @@ -141,52 +144,54 @@ usage(void) (void) fprintf(stderr, " If object numbers are specified, only " "those objects are dumped\n\n"); (void) fprintf(stderr, " Options to control amount of output:\n"); - (void) fprintf(stderr, " -d dataset(s)\n"); - (void) fprintf(stderr, " -i intent logs\n"); - (void) fprintf(stderr, " -C config (or cachefile if alone)\n"); - (void) fprintf(stderr, " -h pool history\n"); (void) fprintf(stderr, " -b block statistics\n"); - (void) fprintf(stderr, " -m metaslabs\n"); - (void) fprintf(stderr, " -M metaslab groups\n"); (void) fprintf(stderr, " -c checksum all metadata (twice for " "all data) blocks\n"); - (void) fprintf(stderr, " -s report stats on zdb's I/O\n"); + (void) fprintf(stderr, " -C config (or cachefile if alone)\n"); + (void) fprintf(stderr, " -d dataset(s)\n"); (void) fprintf(stderr, " -D dedup statistics\n"); - (void) fprintf(stderr, " -S simulate dedup to measure effect\n"); - (void) fprintf(stderr, " -v verbose (applies to all others)\n"); + (void) fprintf(stderr, " -h pool history\n"); + (void) fprintf(stderr, " -i intent logs\n"); (void) fprintf(stderr, " -l read label contents\n"); (void) fprintf(stderr, " -L disable leak tracking (do not " "load spacemaps)\n"); + (void) fprintf(stderr, " -m metaslabs\n"); + (void) fprintf(stderr, " -M metaslab groups\n"); + (void) fprintf(stderr, " -O perform object lookups by path\n"); (void) fprintf(stderr, " -R read and display block from a " - "device\n\n"); + "device\n"); + (void) fprintf(stderr, " -s report stats on zdb's I/O\n"); + (void) fprintf(stderr, " -S simulate dedup to measure effect\n"); + (void) fprintf(stderr, " -v verbose (applies to all " + "others)\n\n"); (void) fprintf(stderr, " Below options are intended for use " "with other options:\n"); (void) fprintf(stderr, " -A ignore assertions (-A), enable " "panic recovery (-AA) or both (-AAA)\n"); - (void) fprintf(stderr, " -F attempt automatic rewind within " - "safe range of transaction groups\n"); - (void) fprintf(stderr, " -U -- use alternate " - "cachefile\n"); - (void) fprintf(stderr, " -X attempt extreme rewind (does not " - "work with dataset)\n"); (void) fprintf(stderr, " -e pool is exported/destroyed/" "has altroot/not in a cachefile\n"); - (void) fprintf(stderr, " -p -- use one or more with " - "-e to specify path to vdev dir\n"); - (void) fprintf(stderr, " -x -- " - "dump all read blocks into specified directory\n"); - (void) fprintf(stderr, " -P print numbers in parseable form\n"); - (void) fprintf(stderr, " -t -- highest txg to use when " - "searching for uberblocks\n"); + (void) fprintf(stderr, " -F attempt automatic rewind within " + "safe range of transaction groups\n"); + (void) fprintf(stderr, " -G dump zfs_dbgmsg buffer before " + "exiting\n"); (void) fprintf(stderr, " -I -- " "specify the maximum number of " "checksumming I/Os [default is 200]\n"); - (void) fprintf(stderr, " -G dump zfs_dbgmsg buffer before " - "exiting\n"); (void) fprintf(stderr, " -o = set global " "variable to an unsigned 32-bit integer value\n"); + (void) fprintf(stderr, " -p -- use one or more with " + "-e to specify path to vdev dir\n"); + (void) fprintf(stderr, " -P print numbers in parseable form\n"); (void) fprintf(stderr, " -q don't print label contents\n"); + (void) fprintf(stderr, " -t -- highest txg to use when " + "searching for uberblocks\n"); (void) fprintf(stderr, " -u uberblock\n"); + (void) fprintf(stderr, " -U -- use alternate " + "cachefile\n"); + (void) fprintf(stderr, " -x -- " + "dump all read blocks into specified directory\n"); + (void) fprintf(stderr, " -X attempt extreme rewind (does not " + "work with dataset)\n\n"); (void) fprintf(stderr, "Specify an option more than once (e.g. -bb) " "to make only that option verbose\n"); (void) fprintf(stderr, "Default is to dump everything non-verbosely\n"); @@ -1576,8 +1581,55 @@ dump_deadlist(dsl_deadlist_t *dl) static avl_tree_t idx_tree; static avl_tree_t domain_tree; static boolean_t fuid_table_loaded; -static boolean_t sa_loaded; -sa_attr_type_t *sa_attr_table; +static objset_t *sa_os = NULL; +static sa_attr_type_t *sa_attr_table = NULL; + +static int +open_objset(const char *path, dmu_objset_type_t type, void *tag, objset_t **osp) +{ + int err; + uint64_t sa_attrs = 0; + uint64_t version = 0; + + VERIFY3P(sa_os, ==, NULL); + err = dmu_objset_own(path, type, B_TRUE, tag, osp); + if (err != 0) { + (void) fprintf(stderr, "failed to own dataset '%s': %s\n", path, + strerror(err)); + return (err); + } + + if (dmu_objset_type(*osp) == DMU_OST_ZFS) { + (void) zap_lookup(*osp, MASTER_NODE_OBJ, ZPL_VERSION_STR, + 8, 1, &version); + if (version >= ZPL_VERSION_SA) { + (void) zap_lookup(*osp, MASTER_NODE_OBJ, ZFS_SA_ATTRS, + 8, 1, &sa_attrs); + } + err = sa_setup(*osp, sa_attrs, zfs_attr_table, ZPL_END, + &sa_attr_table); + if (err != 0) { + (void) fprintf(stderr, "sa_setup failed: %s\n", + strerror(err)); + dmu_objset_disown(*osp, tag); + *osp = NULL; + } + } + sa_os = *osp; + + return (0); +} + +static void +close_objset(objset_t *os, void *tag) +{ + VERIFY3P(os, ==, sa_os); + if (os->os_sa != NULL) + sa_tear_down(os); + dmu_objset_disown(os, tag); + sa_attr_table = NULL; + sa_os = NULL; +} static void fuid_table_destroy() @@ -1649,25 +1701,7 @@ dump_znode(objset_t *os, uint64_t object int idx = 0; int error; - if (!sa_loaded) { - uint64_t sa_attrs = 0; - uint64_t version; - - VERIFY(zap_lookup(os, MASTER_NODE_OBJ, ZPL_VERSION_STR, - 8, 1, &version) == 0); - if (version >= ZPL_VERSION_SA) { - VERIFY(zap_lookup(os, MASTER_NODE_OBJ, ZFS_SA_ATTRS, - 8, 1, &sa_attrs) == 0); - } - if ((error = sa_setup(os, sa_attrs, zfs_attr_table, - ZPL_END, &sa_attr_table)) != 0) { - (void) printf("sa_setup failed errno %d, can't " - "display znode contents\n", error); - return; - } - sa_loaded = B_TRUE; - } - + VERIFY3P(os, ==, sa_os); if (sa_handle_get(os, object, NULL, SA_HDL_PRIVATE, &hdl)) { (void) printf("Failed to get handle for SA znode\n"); return; @@ -2135,6 +2169,108 @@ dump_label_uberblocks(vdev_label_t *lbl, } } +static char curpath[PATH_MAX]; + +/* + * Iterate through the path components, recursively passing + * current one's obj and remaining path until we find the obj + * for the last one. + */ +static int +dump_path_impl(objset_t *os, uint64_t obj, char *name) +{ + int err; + int header = 1; + uint64_t child_obj; + char *s; + dmu_buf_t *db; + dmu_object_info_t doi; + + if ((s = strchr(name, '/')) != NULL) + *s = '\0'; + err = zap_lookup(os, obj, name, 8, 1, &child_obj); + + (void) strlcat(curpath, name, sizeof (curpath)); + + if (err != 0) { + (void) fprintf(stderr, "failed to lookup %s: %s\n", + curpath, strerror(err)); + return (err); + } + + child_obj = ZFS_DIRENT_OBJ(child_obj); + err = sa_buf_hold(os, child_obj, FTAG, &db); + if (err != 0) { + (void) fprintf(stderr, + "failed to get SA dbuf for obj %llu: %s\n", + (u_longlong_t)child_obj, strerror(err)); + return (EINVAL); + } + dmu_object_info_from_db(db, &doi); + sa_buf_rele(db, FTAG); + + if (doi.doi_bonus_type != DMU_OT_SA && + doi.doi_bonus_type != DMU_OT_ZNODE) { + (void) fprintf(stderr, "invalid bonus type %d for obj %llu\n", + doi.doi_bonus_type, (u_longlong_t)child_obj); + return (EINVAL); + } + + if (dump_opt['v'] > 6) { + (void) printf("obj=%llu %s type=%d bonustype=%d\n", + (u_longlong_t)child_obj, curpath, doi.doi_type, + doi.doi_bonus_type); + } + + (void) strlcat(curpath, "/", sizeof (curpath)); + + switch (doi.doi_type) { + case DMU_OT_DIRECTORY_CONTENTS: + if (s != NULL && *(s + 1) != '\0') + return (dump_path_impl(os, child_obj, s + 1)); + /*FALLTHROUGH*/ + case DMU_OT_PLAIN_FILE_CONTENTS: + dump_object(os, child_obj, dump_opt['v'], &header); + return (0); + default: + (void) fprintf(stderr, "object %llu has non-file/directory " + "type %d\n", (u_longlong_t)obj, doi.doi_type); + break; + } + + return (EINVAL); +} + +/* + * Dump the blocks for the object specified by path inside the dataset. + */ +static int +dump_path(char *ds, char *path) +{ + int err; + objset_t *os; + uint64_t root_obj; + + err = open_objset(ds, DMU_OST_ZFS, FTAG, &os); + if (err != 0) + return (err); + + err = zap_lookup(os, MASTER_NODE_OBJ, ZFS_ROOT_OBJ, 8, 1, &root_obj); + if (err != 0) { + (void) fprintf(stderr, "can't lookup root znode: %s\n", + strerror(err)); + dmu_objset_disown(os, FTAG); + return (EINVAL); + } + + (void) snprintf(curpath, sizeof (curpath), "dataset=%s path=/", ds); + + err = dump_path_impl(os, root_obj, path); + + close_objset(os, FTAG); + return (err); +} + static int dump_label(const char *dev) { @@ -2234,11 +2370,9 @@ dump_one_dir(const char *dsname, void *a int error; objset_t *os; - error = dmu_objset_own(dsname, DMU_OST_ANY, B_TRUE, FTAG, &os); - if (error) { - (void) printf("Could not open %s, error %d\n", dsname, error); + error = open_objset(dsname, DMU_OST_ANY, FTAG, &os); + if (error != 0) return (0); - } for (spa_feature_t f = 0; f < SPA_FEATURES; f++) { if (!dmu_objset_ds(os)->ds_feature_inuse[f]) @@ -2249,9 +2383,8 @@ dump_one_dir(const char *dsname, void *a } dump_dir(os); - dmu_objset_disown(os, FTAG); + close_objset(os, FTAG); fuid_table_destroy(); - sa_loaded = B_FALSE; return (0); } @@ -3592,35 +3725,37 @@ main(int argc, char **argv) spa_config_path = spa_config_path_env; while ((c = getopt(argc, argv, - "bcdhilmMI:suCDRSAFLXx:evp:t:U:PGo:q")) != -1) { + "AbcCdDeFGhiI:lLmMo:Op:PqRsSt:uU:vx:X")) != -1) { switch (c) { case 'b': case 'c': + case 'C': case 'd': + case 'D': + case 'G': case 'h': case 'i': case 'l': case 'm': - case 's': - case 'u': - case 'C': - case 'D': case 'M': + case 'O': case 'R': + case 's': case 'S': - case 'G': + case 'u': dump_opt[c]++; dump_all = 0; break; case 'A': + case 'e': case 'F': case 'L': - case 'X': - case 'e': case 'P': case 'q': + case 'X': dump_opt[c]++; break; + /* NB: Sort single match options below. */ case 'I': max_inflight = strtoull(optarg, NULL, 0); if (max_inflight == 0) { @@ -3630,6 +3765,11 @@ main(int argc, char **argv) usage(); } break; + case 'o': + error = set_global_var(optarg); + if (error != 0) + usage(); + break; case 'p': if (searchdirs == NULL) { searchdirs = umem_alloc(sizeof (char *), @@ -3662,11 +3802,6 @@ main(int argc, char **argv) case 'x': vn_dumpdir = optarg; break; - case 'o': - error = set_global_var(optarg); - if (error != 0) - usage(); - break; default: usage(); break; @@ -3704,7 +3839,7 @@ main(int argc, char **argv) verbose = MAX(verbose, 1); for (c = 0; c < 256; c++) { - if (dump_all && !strchr("elAFLRSXP", c)) + if (dump_all && strchr("AeFlLOPRSX", c) == NULL) dump_opt[c] = 1; if (dump_opt[c]) dump_opt[c] += verbose; @@ -3729,6 +3864,13 @@ main(int argc, char **argv) if (dump_opt['l']) return (dump_label(argv[0])); + if (dump_opt['O']) { + if (argc != 2) + usage(); + dump_opt['v'] = verbose + 3; + return (dump_path(argv[0], argv[1])); + } + if (dump_opt['X'] || dump_opt['F']) rewind = ZPOOL_DO_REWIND | (dump_opt['X'] ? ZPOOL_EXTREME_REWIND : 0); @@ -3803,8 +3945,7 @@ main(int argc, char **argv) } } } else { - error = dmu_objset_own(target, DMU_OST_ANY, - B_TRUE, FTAG, &os); + error = open_objset(target, DMU_OST_ANY, FTAG, &os); } } nvlist_free(policy); @@ -3847,10 +3988,12 @@ main(int argc, char **argv) zdb_read_block(argv[i], spa); } - (os != NULL) ? dmu_objset_disown(os, FTAG) : spa_close(spa, FTAG); + if (os != NULL) + close_objset(os, FTAG); + else + spa_close(spa, FTAG); fuid_table_destroy(); - sa_loaded = B_FALSE; dump_debug_buffer(); Modified: vendor/illumos/dist/man/man1m/zdb.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zdb.1m Fri Apr 14 16:57:06 2017 (r316861) +++ vendor/illumos/dist/man/man1m/zdb.1m Fri Apr 14 16:58:41 2017 (r316862) @@ -1,4 +1,3 @@ -'\" t .\" .\" This file and its contents are supplied under the terms of the .\" Common Development and Distribution License ("CDDL"), version 1.0. @@ -12,545 +11,372 @@ .\" .\" Copyright 2012, Richard Lowe. .\" Copyright (c) 2012, 2016 by Delphix. All rights reserved. -.\" Copyright 2016 Nexenta Systems, Inc. +.\" Copyright 2017 Nexenta Systems, Inc. .\" -.TH "ZDB" "1M" "April 9, 2016" - -.SH "NAME" -\fBzdb\fR - Display zpool debugging and consistency information - -.SH "SYNOPSIS" -\fBzdb\fR [-CmdibcsDvhLMXFPAG] [-e [-p \fIpath\fR...]] [-t \fItxg\fR] - [-U \fIcache\fR] [-I \fIinflight I/Os\fR] [-x \fIdumpdir\fR] - [-o \fIvar\fR=\fIvalue\fR] ... [\fIpoolname\fR [\fIobject\fR ...]] - -.P -\fBzdb\fR [-divPA] [-e [-p \fIpath\fR...]] [-U \fIcache\fR] - \fIdataset\fR [\fIobject\fR ...] - -.P -\fBzdb\fR -m [-MLXFPA] [-t \fItxg\fR] [-e [-p \fIpath\fR...]] [-U \fIcache\fR] - \fIpoolname\fR [\fIvdev\fR [\fImetaslab\fR ...]] - -.P -\fBzdb\fR -R [-A] [-e [-p \fIpath\fR...]] [-U \fIcache\fR] \fIpoolname\fR - \fIvdev\fR:\fIoffset\fR:\fIsize\fR[:\fIflags\fR] - -.P -\fBzdb\fR -S [-AP] [-e [-p \fIpath\fR...]] [-U \fIcache\fR] \fIpoolname\fR - -.P -\fBzdb\fR -l [-Aqu] \fIdevice\fR - -.P -\fBzdb\fR -C [-A] [-U \fIcache\fR] - -.SH "DESCRIPTION" -The \fBzdb\fR utility displays information about a ZFS pool useful for -debugging and performs some amount of consistency checking. It is a not a -general purpose tool and options (and facilities) may change. This is neither -a fsck(1M) nor an fsdb(1M) utility. - -.P +.Dd January 14, 2017 +.Dt ZDB 1M +.Os +.Sh NAME +.Nm zdb +.Nd display zpool debugging and consistency information +.Sh SYNOPSIS +.Nm +.Op Fl AbcdDFGhiLMPsvX +.Op Fl e Op Fl p Ar path ... +.Op Fl I Ar inflight I/Os +.Oo Fl o Ar var Ns = Ns Ar value Oc Ns ... +.Op Fl t Ar txg +.Op Fl U Ar cache +.Op Fl x Ar dumpdir +.Op Ar poolname Op Ar object ... +.Nm +.Op Fl AdiPv +.Op Fl e Op Fl p Ar path ... +.Op Fl U Ar cache +.Ar dataset Op Ar object ... +.Nm +.Fl C +.Op Fl A +.Op Fl U Ar cache +.Nm +.Fl l +.Op Fl Aqu +.Ar device +.Nm +.Fl m +.Op Fl AFLPX +.Op Fl t Ar txg +.Op Fl e Op Fl p Ar path ... +.Op Fl U Ar cache +.Ar poolname Op Ar vdev Op Ar metaslab ... +.Nm +.Fl O +.Ar dataset path +.Nm +.Fl R +.Op Fl A +.Op Fl e Op Fl p Ar path ... +.Op Fl U Ar cache +.Ar poolname vdev Ns : Ns Ar offset Ns : Ns Ar size Ns Op : Ns Ar flags +.Nm +.Fl S +.Op Fl AP +.Op Fl e Op Fl p Ar path ... +.Op Fl U Ar cache +.Ar poolname +.Sh DESCRIPTION +The +.Nm +utility displays information about a ZFS pool useful for debugging and performs +some amount of consistency checking. +It is a not a general purpose tool and options +.Pq and facilities +may change. +This is neither a +.Xr fsck 1M +nor an +.Xr fsdb 1M +utility. +.Pp The output of this command in general reflects the on-disk structure of a ZFS -pool, and is inherently unstable. The precise output of most invocations is -not documented, a knowledge of ZFS internals is assumed. - -.P -If the \fIdataset\fR argument does not contain any \fB/\fR or \fB@\fR -characters, it is interpreted as a pool name. The root dataset can be -specified as \fIpool\fB/\fR (pool name followed by a slash). - -.P +pool, and is inherently unstable. +The precise output of most invocations is not documented, a knowledge of ZFS +internals is assumed. +.Pp +If the +.Ar dataset +argument does not contain any +.Qq Sy / +or +.Qq Sy @ +characters, it is interpreted as a pool name. +The root dataset can be specified as +.Ar pool Ns / +.Pq pool name followed by a slash . +.Pp When operating on an imported and active pool it is possible, though unlikely, that zdb may interpret inconsistent pool data and behave erratically. - -.SH "OPTIONS" +.Sh OPTIONS Display options: - -.sp -.ne 2 -.na -\fB-b\fR -.ad -.sp .6 -.RS 4n -Display statistics regarding the number, size (logical, physical and -allocated) and deduplication of blocks. -.RE - -.sp -.ne 2 -.na -\fB-c\fR -.ad -.sp .6 -.RS 4n +.Bl -tag -width Ds +.It Fl b +Display statistics regarding the number, size +.Pq logical, physical and allocated +and deduplication of blocks. +.It Fl c Verify the checksum of all metadata blocks while printing block statistics -(see \fB-b\fR). -.sp +.Po see +.Fl b +.Pc . +.Pp If specified multiple times, verify the checksums of all blocks. -.RE - -.sp -.ne 2 -.na -\fB-C\fR -.ad -.sp .6 -.RS 4n -Display information about the configuration. If specified with no other -options, instead display information about the cache file -(\fB/etc/zfs/zpool.cache\fR). To specify the cache file to display, see -\fB-U\fR. -.P -If specified multiple times, and a pool name is also specified display both -the cached configuration and the on-disk configuration. If specified multiple -times with \fB-e\fR also display the configuration that would be used were the -pool to be imported. -.RE - -.sp -.ne 2 -.na -\fB-d\fR -.ad -.sp .6 -.RS 4n -Display information about datasets. Specified once, displays basic dataset -information: ID, create transaction, size, and object count. -.sp +.It Fl C +Display information about the configuration. +If specified with no other options, instead display information about the cache +file +.Pq Pa /etc/zfs/zpool.cache . +To specify the cache file to display, see +.Fl U . +.Pp +If specified multiple times, and a pool name is also specified display both the +cached configuration and the on-disk configuration. +If specified multiple times with +.Fl e +also display the configuration that would be used were the pool to be imported. +.It Fl d +Display information about datasets. +Specified once, displays basic dataset information: ID, create transaction, +size, and object count. +.Pp If specified multiple times provides greater and greater verbosity. -.sp -If object IDs are specified, display information about those specific objects only. -.RE - -.sp -.ne 2 -.na -\fB-D\fR -.ad -.sp .6 -.RS 4n -Display deduplication statistics, including the deduplication ratio (dedup), -compression ratio (compress), inflation due to the zfs copies property -(copies), and an overall effective ratio (dedup * compress / copies). -.sp -If specified twice, display a histogram of deduplication statistics, showing -the allocated (physically present on disk) and referenced (logically -referenced in the pool) block counts and sizes by reference count. -.sp -If specified a third time, display the statistics independently for each deduplication table. -.sp -If specified a fourth time, dump the contents of the deduplication tables describing duplicate blocks. -.sp -If specified a fifth time, also dump the contents of the deduplication tables describing unique blocks. -.RE - -.sp -.ne 2 -.na -\fB-h\fR -.ad -.sp .6 -.RS 4n -Display pool history similar to \fBzpool history\fR, but include internal -changes, transaction, and dataset information. -.RE - -.sp -.ne 2 -.na -\fB-i\fR -.ad -.sp .6 -.RS 4n -Display information about intent log (ZIL) entries relating to each -dataset. If specified multiple times, display counts of each intent log -transaction type. -.RE - -.sp -.ne 2 -.na -\fB-l\fR \fIdevice\fR -.ad -.sp .6 -.RS 4n -Read the vdev labels from the specified device. \fBzdb -l\fR will return 0 if -valid label was found, 1 if error occured, and 2 if no valid labels were found. -.P -If the \fB-u\fR option is also specified, also display the uberblocks on this -device. -.P -If the \fB-q\fR option is also specified, don't print the labels. -.RE - -.sp -.ne 2 -.na -\fB-L\fR -.ad -.sp .6 -.RS 4n -Disable leak tracing and the loading of space maps. By default, \fBzdb\fR +.Pp +If object IDs are specified, display information about those specific objects +only. +.It Fl D +Display deduplication statistics, including the deduplication ratio +.Pq Sy dedup , +compression ratio +.Pq Sy compress , +inflation due to the zfs copies property +.Pq Sy copies , +and an overall effective ratio +.Pq Sy dedup No * Sy compress No / Sy copies . +.It Fl DD +Display a histogram of deduplication statistics, showing the allocated +.Pq physically present on disk +and referenced +.Pq logically referenced in the pool +block counts and sizes by reference count. +.It Fl DDD +Display the statistics independently for each deduplication table. +.It Fl DDDD +Dump the contents of the deduplication tables describing duplicate blocks. +.It Fl DDDDD +Also dump the contents of the deduplication tables describing unique blocks. +.It Fl h +Display pool history similar to +.Nm zpool Cm history , +but include internal changes, transaction, and dataset information. +.It Fl i +Display information about intent log +.Pq ZIL +entries relating to each dataset. +If specified multiple times, display counts of each intent log transaction type. +.It Fl l Ar device +Read the vdev labels from the specified device. +.Nm Fl l +will return 0 if valid label was found, 1 if error occurred, and 2 if no valid +labels were found. +.Pp +If the +.Fl q +option is also specified, don't print the labels. +.Pp +If the +.Fl u +option is also specified, also display the uberblocks on this device. +.It Fl L +Disable leak tracing and the loading of space maps. +By default, +.Nm verifies that all non-free blocks are referenced, which can be very expensive. -.RE - -.sp -.ne 2 -.na -\fB-m\fR -.ad -.sp .6 -.RS 4n +.It Fl m Display the offset, spacemap, and free space of each metaslab. -When specified twice, also display information about the on-disk free -space histogram associated with each metaslab. When specified three time, -display the maximum contiguous free space, the in-core free space histogram, -and the percentage of free space in each space map. When specified -four times display every spacemap record. -.RE - -.sp -.ne 2 -.na -\fB-M\fR -.ad -.sp .6 -.RS 4n +.It Fl mm +Also display information about the on-disk free space histogram associated with +each metaslab. +.It Fl mmm +Display the maximum contiguous free space, the in-core free space histogram, and +the percentage of free space in each space map. +.It Fl mmmm +Display every spacemap record. +.It Fl M Display the offset, spacemap, and free space of each metaslab. -When specified twice, also display information about the maximum contiguous -free space and the percentage of free space in each space map. When specified -three times display every spacemap record. -.RE - -.sp -.ne 2 -.na -\fB-R\fR \fIpoolname\fR \fIvdev\fR:\fIoffset\fR:\fIsize\fR[:\fIflags\fR] -.ad -.sp .6 -.RS 4n -Read and display a block from the specified device. By default the block is -displayed as a hex dump, but see the description of the \'r\' flag, below. -.sp -The block is specified in terms of a colon-separated tuple \fIvdev\fR (an -integer vdev identifier) \fIoffset\fR (the offset within the vdev) \fIsize\fR -(the size of the block to read) and, optionally, \fIflags\fR (a set of flags, -described below). - -.sp -.ne 2 -.na -\fBb\fR \fIoffset\fR -.ad -.sp .6 -.RS 4n +.It Fl MM +Also display information about the maximum contiguous free space and the +percentage of free space in each space map. +.It Fl MMM +Display every spacemap record. +.It Fl O Ar dataset path +Look up the specified +.Ar path +inside of the +.Ar dataset +and display its metadata and indirect blocks. +Specified +.Ar path +must be relative to the root of +.Ar dataset . +This option can be combined with +.Fl v +for increasing verbosity. +.It Fl R Ar poolname vdev Ns : Ns Ar offset Ns : Ns Ar size Ns Op : Ns Ar flags +Read and display a block from the specified device. +By default the block is displayed as a hex dump, but see the description of the +.Sy r +flag, below. +.Pp +The block is specified in terms of a colon-separated tuple +.Ar vdev +.Pq an integer vdev identifier +.Ar offset +.Pq the offset within the vdev +.Ar size +.Pq the size of the block to read +and, optionally, +.Ar flags +.Pq a set of flags, described below . +.Pp +.Bl -tag -compact -width "b offset" +.It Sy b Ar offset Print block pointer -.RE - -.sp -.ne 2 -.na -\fBd\fR -.ad -.sp .6 -.RS 4n +.It Sy d Decompress the block -.RE - -.sp -.ne 2 -.na -\fBe\fR -.ad -.sp .6 -.RS 4n +.It Sy e Byte swap the block -.RE - -.sp -.ne 2 -.na -\fBg\fR -.ad -.sp .6 -.RS 4n +.It Sy g Dump gang block header -.RE - -.sp -.ne 2 -.na -\fBi\fR -.ad -.sp .6 -.RS 4n +.It Sy i Dump indirect block -.RE - -.sp -.ne 2 -.na -\fBr\fR -.ad -.sp .6 -.RS 4n +.It Sy r Dump raw uninterpreted block data -.RE -.RE - -.sp -.ne 2 -.na -\fB-s\fR -.ad -.sp .6 -.RS 4n -Report statistics on \fBzdb\fR\'s I/O. Display operation counts, bandwidth, -and error counts of I/O to the pool from \fBzdb\fR. -.RE - -.sp -.ne 2 -.na -\fB-S\fR -.ad -.sp .6 -.RS 4n +.El +.It Fl s +Report statistics on +.Nm zdb +I/O. +Display operation counts, bandwidth, and error counts of I/O to the pool from *** DIFF OUTPUT TRUNCATED AT 1000 LINES *** From owner-svn-src-vendor@freebsd.org Fri Apr 14 16:59:54 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id F27D7D3EDB3; Fri, 14 Apr 2017 16:59:54 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id C49AF1AC; Fri, 14 Apr 2017 16:59:54 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EGxraX031102; Fri, 14 Apr 2017 16:59:53 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EGxr3B031101; Fri, 14 Apr 2017 16:59:53 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141659.v3EGxr3B031101@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 16:59:53 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316863 - vendor/illumos/dist/cmd/zdb X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 16:59:55 -0000 Author: avg Date: Fri Apr 14 16:59:53 2017 New Revision: 316863 URL: https://svnweb.freebsd.org/changeset/base/316863 Log: 3871 fix issues introduced by 3604 illumos/illumos-gate@de05b58863498b10283637eb9ac85e92fe85150e https://github.com/illumos/illumos-gate/commit/de05b58863498b10283637eb9ac85e92fe85150e https://www.illumos.org/issues/3871 GCC 4.5.3 on Gentoo Linux did not like a few of the changes made in the issue 3604 patch. It printed an error and a couple of warnings: ../../cmd/zdb/zdb.c: In function 'dump_bpobj': ../../cmd/zdb/zdb.c:1257:3: error: 'for' loop initial declarations are only allowed in C99 mode ../../cmd/zdb/zdb.c:1257:3: note: use option -std=c99 or -std=gnu99 to compile your code ../../cmd/zdb/zdb.c: In function 'dump_deadlist': ../../cmd/zdb/zdb.c:1323:8: warning: too many arguments for format ../../cmd/zdb/zdb.c:1323:8: warning: too many arguments for format Reviewed by: Paul Dagnelie Reviewed by: Matthew Ahrens Reviewed by: Serapheim Dimitropoulos Approved by: Dan McDonald Author: Richard Yao Modified: vendor/illumos/dist/cmd/zdb/zdb.c Modified: vendor/illumos/dist/cmd/zdb/zdb.c ============================================================================== --- vendor/illumos/dist/cmd/zdb/zdb.c Fri Apr 14 16:58:41 2017 (r316862) +++ vendor/illumos/dist/cmd/zdb/zdb.c Fri Apr 14 16:59:53 2017 (r316863) @@ -1564,7 +1564,8 @@ dump_deadlist(dsl_deadlist_t *dl) dle = AVL_NEXT(&dl->dl_tree, dle)) { if (dump_opt['d'] >= 5) { char buf[128]; - (void) snprintf(buf, sizeof (buf), "mintxg %llu -> ", + (void) snprintf(buf, sizeof (buf), + "mintxg %llu -> obj %llu", (longlong_t)dle->dle_mintxg, (longlong_t)dle->dle_bpobj.bpo_object); From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:01:02 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 5002ED3EF85; Fri, 14 Apr 2017 17:01:02 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 13C493E1; Fri, 14 Apr 2017 17:01:02 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EH11kf031916; Fri, 14 Apr 2017 17:01:01 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EH11qq031914; Fri, 14 Apr 2017 17:01:01 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141701.v3EH11qq031914@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:01:01 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316864 - in vendor/illumos/dist: cmd/zdb man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:01:02 -0000 Author: avg Date: Fri Apr 14 17:01:00 2017 New Revision: 316864 URL: https://svnweb.freebsd.org/changeset/base/316864 Log: 6392 zdb: introduce -V for verbatim import illumos/illumos-gate@dfd5965f7e43b6a630e5ac86708ae76b4f02cc40 https://github.com/illumos/illumos-gate/commit/dfd5965f7e43b6a630e5ac86708ae76b4f02cc40 https://www.illumos.org/issues/6392 When given a pool name via -e, zdb would attempt an import. If it failed, then it would attempt a verbatim import. This behavior is not always desirable so a -V switch is added to zdb to control the behavior. When specified, a verbatim import is done. Otherwise, the behavior is as it was previously, except no verbatim import is done on failure. https://github.com/zfsonlinux/zfs/commit/a5778ea2427bd340e3b4f697d9b6e1452bd71909 Reviewed by: Brian Behlendorf Reviewed by: Matthew Ahrens Reviewed by: Yuri Pankov Reviewed by: Brian Behlendorf Reviewed by: Matt Ahrens Reviewed by: Pavel Zakharov Approved by: Dan McDonald Author: Richard Yao Modified: vendor/illumos/dist/cmd/zdb/zdb.c vendor/illumos/dist/man/man1m/zdb.1m Modified: vendor/illumos/dist/cmd/zdb/zdb.c ============================================================================== --- vendor/illumos/dist/cmd/zdb/zdb.c Fri Apr 14 16:59:53 2017 (r316863) +++ vendor/illumos/dist/cmd/zdb/zdb.c Fri Apr 14 17:01:00 2017 (r316864) @@ -120,20 +120,21 @@ static void usage(void) { (void) fprintf(stderr, - "Usage:\t%s [-AbcdDFGhiLMPsvX] [-e [-p ...]] " + "Usage:\t%s [-AbcdDFGhiLMPsvX] [-e [-V] [-p ...]] " "[-I ]\n" "\t\t[-o =]... [-t ] [-U ] [-x ]\n" "\t\t[ [ ...]]\n" - "\t%s [-AdiPv] [-e [-p ...]] [-U ] " + "\t%s [-AdiPv] [-e [-V] [-p ...]] [-U ] " "[ ...]\n" "\t%s -C [-A] [-U ]\n" "\t%s -l [-Aqu] \n" - "\t%s -m [-AFLPX] [-e [-p ...]] [-t ] [-U ]\n" - "\t\t [ [ ...]]\n" + "\t%s -m [-AFLPX] [-e [-V] [-p ...]] [-t ] " + "[-U ]\n\t\t [ [ ...]]\n" "\t%s -O \n" - "\t%s -R [-A] [-e [-p ...]] [-U ]\n" + "\t%s -R [-A] [-e [-V] [-p ...]] [-U ]\n" "\t\t ::[:]\n" - "\t%s -S [-AP] [-e [-p ...]] [-U ] \n\n", + "\t%s -S [-AP] [-e [-V] [-p ...]] [-U ] " + "\n\n", cmdname, cmdname, cmdname, cmdname, cmdname, cmdname, cmdname, cmdname); @@ -188,6 +189,7 @@ usage(void) (void) fprintf(stderr, " -u uberblock\n"); (void) fprintf(stderr, " -U -- use alternate " "cachefile\n"); + (void) fprintf(stderr, " -V do verbatim import\n"); (void) fprintf(stderr, " -x -- " "dump all read blocks into specified directory\n"); (void) fprintf(stderr, " -X attempt extreme rewind (does not " @@ -3707,6 +3709,7 @@ main(int argc, char **argv) char *target; nvlist_t *policy = NULL; uint64_t max_txg = UINT64_MAX; + int flags = ZFS_IMPORT_MISSING_LOG; int rewind = ZPOOL_NEVER_REWIND; char *spa_config_path_env; boolean_t target_is_spa = B_TRUE; @@ -3726,7 +3729,7 @@ main(int argc, char **argv) spa_config_path = spa_config_path_env; while ((c = getopt(argc, argv, - "AbcCdDeFGhiI:lLmMo:Op:PqRsSt:uU:vx:X")) != -1) { + "AbcCdDeFGhiI:lLmMo:Op:PqRsSt:uU:vVx:X")) != -1) { switch (c) { case 'b': case 'c': @@ -3800,6 +3803,9 @@ main(int argc, char **argv) case 'v': verbose++; break; + case 'V': + flags = ZFS_IMPORT_VERBATIM; + break; case 'x': vn_dumpdir = optarg; break; @@ -3899,11 +3905,7 @@ main(int argc, char **argv) fatal("can't open '%s': %s", target, strerror(ENOMEM)); } - if ((error = spa_import(name, cfg, NULL, - ZFS_IMPORT_MISSING_LOG)) != 0) { - error = spa_import(name, cfg, NULL, - ZFS_IMPORT_VERBATIM); - } + error = spa_import(name, cfg, NULL, flags); } } Modified: vendor/illumos/dist/man/man1m/zdb.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zdb.1m Fri Apr 14 16:59:53 2017 (r316863) +++ vendor/illumos/dist/man/man1m/zdb.1m Fri Apr 14 17:01:00 2017 (r316864) @@ -22,7 +22,7 @@ .Sh SYNOPSIS .Nm .Op Fl AbcdDFGhiLMPsvX -.Op Fl e Op Fl p Ar path ... +.Op Fl e Oo Fl V Oc Op Fl p Ar path ... .Op Fl I Ar inflight I/Os .Oo Fl o Ar var Ns = Ns Ar value Oc Ns ... .Op Fl t Ar txg @@ -31,7 +31,7 @@ .Op Ar poolname Op Ar object ... .Nm .Op Fl AdiPv -.Op Fl e Op Fl p Ar path ... +.Op Fl e Oo Fl V Oc Op Fl p Ar path ... .Op Fl U Ar cache .Ar dataset Op Ar object ... .Nm @@ -45,8 +45,8 @@ .Nm .Fl m .Op Fl AFLPX +.Op Fl e Oo Fl V Oc Op Fl p Ar path ... .Op Fl t Ar txg -.Op Fl e Op Fl p Ar path ... .Op Fl U Ar cache .Ar poolname Op Ar vdev Op Ar metaslab ... .Nm @@ -55,13 +55,13 @@ .Nm .Fl R .Op Fl A -.Op Fl e Op Fl p Ar path ... +.Op Fl e Oo Fl V Oc Op Fl p Ar path ... .Op Fl U Ar cache .Ar poolname vdev Ns : Ns Ar offset Ns : Ns Ar size Ns Op : Ns Ar flags .Nm .Fl S .Op Fl AP -.Op Fl e Op Fl p Ar path ... +.Op Fl e Oo Fl V Oc Op Fl p Ar path ... .Op Fl U Ar cache .Ar poolname .Sh DESCRIPTION @@ -316,6 +316,11 @@ Use a cache file other than .It Fl v Enable verbosity. Specify multiple times for increased verbosity. +.It Fl V +Attempt verbatim import. +This mimics the behavior of the kernel when loading a pool from a cachefile. +Only usable with +.Fl e . .It Fl X Attempt .Qq extreme From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:13:50 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 8CFF5D3DA35; Fri, 14 Apr 2017 17:13:50 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 696D3BBD; Fri, 14 Apr 2017 17:13:50 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EHDnnJ039354; Fri, 14 Apr 2017 17:13:49 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EHDna4039350; Fri, 14 Apr 2017 17:13:49 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141713.v3EHDna4039350@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:13:49 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316868 - in vendor-sys/illumos/dist/uts/common/fs/zfs: . sys X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:13:50 -0000 Author: avg Date: Fri Apr 14 17:13:49 2017 New Revision: 316868 URL: https://svnweb.freebsd.org/changeset/base/316868 Log: 7430 Backfill metadnode more intelligently illumos/illumos-gate@af346df58864e8fe897b1ff1a3a4c12f9294391b https://github.com/illumos/illumos-gate/commit/af346df58864e8fe897b1ff1a3a4c12f9294391b https://www.illumos.org/issues/7430 Description and patch from brought over from the following ZoL commit: https:// github.com/zfsonlinux/zfs/commit/68cbd56e182ab949f58d004778d463aeb3f595c6 Only attempt to backfill lower metadnode object numbers if at least 4096 objects have been freed since the last rescan, and at most once per transaction group. This avoids a pathology in dmu_object_alloc() that caused O(N^2) behavior for create-heavy workloads and substantially improves object creation rates. As summarized by @mahrens in #4636: "Normally, the object allocator simply checks to see if the next object is available. The slow calls happened when dmu_object_alloc() checks to see if it can backfill lower object numbers. This happens every time we move on to a new L1 indirect block (i.e. every 32 * 128 = 4096 objects). When re-checking lower object numbers, we use the on-disk fill count (blkptr_t:blk_fill) to quickly skip over indirect blocks that don’t have enough free dnodes (defined as an L2 with at least 393,216 of 524,288 dnodes free). Therefore, we may find that a block of dnodes has a low (or zero) fill count, and yet we can’t allocate any of its dnodes, because they've been allocated in memory but not yet written to disk. In this case we have to hold each of the dnodes and then notice that it has been allocated in memory. The end result is that allocating N objects in the same TXG can require CPU usage proportional to N^2." Add a tunable dmu_rescan_dnode_threshold to define the number of objects that must be freed before a rescan is performed. Don't bother to export this as a module option because testing doesn't show a compelling reason to change it. The vast majority of the performance gain comes from limit the rescan to at most once per TXG. Reviewed by: Alek Pinchuk Reviewed by: Brian Behlendorf Reviewed by: Matthew Ahrens Approved by: Gordon Ross Author: Ned Bass Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_object.c vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c vendor-sys/illumos/dist/uts/common/fs/zfs/dnode_sync.c vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu_objset.h Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_object.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_object.c Fri Apr 14 17:08:37 2017 (r316867) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_object.c Fri Apr 14 17:13:49 2017 (r316868) @@ -36,20 +36,22 @@ dmu_object_alloc(objset_t *os, dmu_objec dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx) { uint64_t object; - uint64_t L2_dnode_count = DNODES_PER_BLOCK << + uint64_t L1_dnode_count = DNODES_PER_BLOCK << (DMU_META_DNODE(os)->dn_indblkshift - SPA_BLKPTRSHIFT); dnode_t *dn = NULL; - int restarted = B_FALSE; mutex_enter(&os->os_obj_lock); for (;;) { object = os->os_obj_next; /* - * Each time we polish off an L2 bp worth of dnodes - * (2^13 objects), move to another L2 bp that's still - * reasonably sparse (at most 1/4 full). Look from the - * beginning once, but after that keep looking from here. - * If we can't find one, just keep going from here. + * Each time we polish off a L1 bp worth of dnodes (2^12 + * objects), move to another L1 bp that's still reasonably + * sparse (at most 1/4 full). Look from the beginning at most + * once per txg, but after that keep looking from here. + * os_scan_dnodes is set during txg sync if enough objects + * have been freed since the previous rescan to justify + * backfilling again. If we can't find a suitable block, just + * keep going from here. * * Note that dmu_traverse depends on the behavior that we use * multiple blocks of the dnode object before going back to @@ -57,12 +59,19 @@ dmu_object_alloc(objset_t *os, dmu_objec * that property or find another solution to the issues * described in traverse_visitbp. */ - if (P2PHASE(object, L2_dnode_count) == 0) { - uint64_t offset = restarted ? object << DNODE_SHIFT : 0; - int error = dnode_next_offset(DMU_META_DNODE(os), + + if (P2PHASE(object, L1_dnode_count) == 0) { + uint64_t offset; + int error; + if (os->os_rescan_dnodes) { + offset = 0; + os->os_rescan_dnodes = B_FALSE; + } else { + offset = object << DNODE_SHIFT; + } + error = dnode_next_offset(DMU_META_DNODE(os), DNODE_FIND_HOLE, &offset, 2, DNODES_PER_BLOCK >> 2, 0); - restarted = B_TRUE; if (error == 0) object = offset >> DNODE_SHIFT; } Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c Fri Apr 14 17:08:37 2017 (r316867) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c Fri Apr 14 17:13:49 2017 (r316868) @@ -67,6 +67,13 @@ krwlock_t os_lock; */ int dmu_find_threads = 0; +/* + * Backfill lower metadnode objects after this many have been freed. + * Backfilling negatively impacts object creation rates, so only do it + * if there are enough holes to fill. + */ +int dmu_rescan_dnode_threshold = 131072; + static void dmu_objset_find_dp_cb(void *arg); void @@ -1176,6 +1183,13 @@ dmu_objset_sync(objset_t *os, zio_t *pio if (dr->dr_zio) zio_nowait(dr->dr_zio); } + + /* Enable dnode backfill if enough objects have been freed. */ + if (os->os_freed_dnodes >= dmu_rescan_dnode_threshold) { + os->os_rescan_dnodes = B_TRUE; + os->os_freed_dnodes = 0; + } + /* * Free intent log blocks up to this tx. */ Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dnode_sync.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dnode_sync.c Fri Apr 14 17:08:37 2017 (r316867) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dnode_sync.c Fri Apr 14 17:13:49 2017 (r316868) @@ -672,6 +672,7 @@ dnode_sync(dnode_t *dn, dmu_tx_t *tx) } if (freeing_dnode) { + dn->dn_objset->os_freed_dnodes++; dnode_sync_free(dn, tx); return; } Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu_objset.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu_objset.h Fri Apr 14 17:08:37 2017 (r316867) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu_objset.h Fri Apr 14 17:13:49 2017 (r316868) @@ -112,6 +112,8 @@ struct objset { zil_header_t os_zil_header; list_t os_synced_dnodes; uint64_t os_flags; + uint64_t os_freed_dnodes; + boolean_t os_rescan_dnodes; /* Protected by os_obj_lock */ kmutex_t os_obj_lock; From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:15:12 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id A37DCD3DE89; Fri, 14 Apr 2017 17:15:12 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 7EA7F217; Fri, 14 Apr 2017 17:15:12 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EHFBoa039515; Fri, 14 Apr 2017 17:15:11 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EHFBfs039513; Fri, 14 Apr 2017 17:15:11 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141715.v3EHFBfs039513@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:15:11 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316870 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:15:12 -0000 Author: avg Date: Fri Apr 14 17:15:11 2017 New Revision: 316870 URL: https://svnweb.freebsd.org/changeset/base/316870 Log: 7448 ZFS doesn't notice when disk vdevs have no write cache illumos/illumos-gate@295438ba3230419314faaa889a2616f561658bd5 https://github.com/illumos/illumos-gate/commit/295438ba3230419314faaa889a2616f561658bd5 https://www.illumos.org/issues/7448 I built a SmartOS image with all the NVMe commits including 7372 (support NVMe volatile write cache) and repeated my dd testing: > #!/bin/bash > for i in `seq 1 1000`; do > dd if=/dev/zero of=file00 bs=1M count=102400 oflag=sync & > dd if=/dev/zero of=file01 bs=1M count=102400 oflag=sync & > wait > rm file00 file01 > done > Previously each dd command took ~145 seconds to finish, now it takes ~400 seconds. Eventually I figured out it is 7372 that causes unnecessary nvme_bd_sync() executions which wasted CPU cycles. If a NVMe device doesn't support a write cache, the nvme_bd_sync function will return ENOTSUP to indicate this to upper layers. It seems this returned value is ignored by ZFS, and as such this bug is not really specific to NVMe. In vdev_disk_io_start() ZFS sends the flush to the disk driver (blkdev) with a callback to vdev_disk_ioctl_done(). As nvme filled in the bd_sync_cache function pointer, blkdev will not return ENOTSUP, as the nvme driver in general does support cache flush. Instead it will issue an asynchronous flush to nvme and immediately return 0, and hence ZFS will not set vdev_nowritecache here. The nvme driver will at some point process the cache flush command, and if there is no write cache on the device it will return ENOTSUP, which will be delivered to the vdev_disk_ioctl_done() callback. This function will not check the error code and not set nowritecache. The right place to check the error code from the cache flush is in zio_vdev_io_assess(). This would catch both cases, synchronous and asynchronous cache flushes. This would also be independent of the implementation detail that some drivers can return ENOTSUP immediately. Reviewed by: Dan Fields Reviewed by: Alek Pinchuk Reviewed by: George Wilson Approved by: Dan McDonald Author: Hans Rosenfeld Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/vdev_disk.c vendor-sys/illumos/dist/uts/common/fs/zfs/zio.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/vdev_disk.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/vdev_disk.c Fri Apr 14 17:13:57 2017 (r316869) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/vdev_disk.c Fri Apr 14 17:15:11 2017 (r316870) @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, 2015 by Delphix. All rights reserved. - * Copyright 2013 Nexenta Systems, Inc. All rights reserved. + * Copyright 2016 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2013 Joyent, Inc. All rights reserved. */ @@ -743,16 +743,6 @@ vdev_disk_io_start(zio_t *zio) return; } - if (error == ENOTSUP || error == ENOTTY) { - /* - * If we get ENOTSUP or ENOTTY, we know that - * no future attempts will ever succeed. - * In this case we set a persistent bit so - * that we don't bother with the ioctl in the - * future. - */ - vd->vdev_nowritecache = B_TRUE; - } zio->io_error = error; break; Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zio.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zio.c Fri Apr 14 17:13:57 2017 (r316869) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zio.c Fri Apr 14 17:15:11 2017 (r316870) @@ -3126,6 +3126,16 @@ zio_vdev_io_assess(zio_t *zio) vd->vdev_cant_write = B_TRUE; } + /* + * If a cache flush returns ENOTSUP or ENOTTY, we know that no future + * attempts will ever succeed. In this case we set a persistent bit so + * that we don't bother with it in the future. + */ + if ((zio->io_error == ENOTSUP || zio->io_error == ENOTTY) && + zio->io_type == ZIO_TYPE_IOCTL && + zio->io_cmd == DKIOCFLUSHWRITECACHE && vd != NULL) + vd->vdev_nowritecache = B_TRUE; + if (zio->io_error) zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:15:51 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 32D60D3DF16; Fri, 14 Apr 2017 17:15:51 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id DD1F0392; Fri, 14 Apr 2017 17:15:50 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EHFoDR039632; Fri, 14 Apr 2017 17:15:50 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EHFovE039631; Fri, 14 Apr 2017 17:15:50 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141715.v3EHFovE039631@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:15:50 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316871 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:15:51 -0000 Author: avg Date: Fri Apr 14 17:15:49 2017 New Revision: 316871 URL: https://svnweb.freebsd.org/changeset/base/316871 Log: 7490 real checksum errors are silenced when zinject is on illumos/illumos-gate@6cedfc397d92d64e442f0aae4445ac507beaf58f https://github.com/illumos/illumos-gate/commit/6cedfc397d92d64e442f0aae4445ac507beaf58f https://www.illumos.org/issues/7490 When zinject is on, error codes from zfs_checksum_error() can be overwritten due to an incorrect and overly-complex if condition. Reviewed by: George Wilson Reviewed by: Paul Dagnelie Reviewed by: Dan Kimmel Reviewed by: Matthew Ahrens Approved by: Robert Mustacchi Author: Pavel Zakharov Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zio_checksum.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zio_checksum.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zio_checksum.c Fri Apr 14 17:15:11 2017 (r316870) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zio_checksum.c Fri Apr 14 17:15:49 2017 (r316871) @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2013, 2015 by Delphix. All rights reserved. + * Copyright (c) 2013, 2016 by Delphix. All rights reserved. * Copyright (c) 2013, Joyent, Inc. All rights reserved. * Copyright 2013 Saso Kiselkov. All rights reserved. */ @@ -388,12 +388,13 @@ zio_checksum_error(zio_t *zio, zio_bad_c error = zio_checksum_error_impl(spa, bp, checksum, data, size, offset, info); - if (error != 0 && zio_injection_enabled && !zio->io_error && - (error = zio_handle_fault_injection(zio, ECKSUM)) != 0) { - info->zbc_injected = 1; - return (error); + if (zio_injection_enabled && error == 0 && zio->io_error == 0) { + error = zio_handle_fault_injection(zio, ECKSUM); + if (error != 0) + info->zbc_injected = 1; } + return (error); } From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:20:27 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id B7599D3E1F8; Fri, 14 Apr 2017 17:20:27 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 843BCAF5; Fri, 14 Apr 2017 17:20:27 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EHKQYJ040081; Fri, 14 Apr 2017 17:20:26 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EHKQ5I040080; Fri, 14 Apr 2017 17:20:26 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141720.v3EHKQ5I040080@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:20:26 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316872 - vendor/illumos/dist/cmd/ztest X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:20:27 -0000 Author: avg Date: Fri Apr 14 17:20:26 2017 New Revision: 316872 URL: https://svnweb.freebsd.org/changeset/base/316872 Log: 7502 ztest should run zdb with -G (debug mode) illumos/illumos-gate@c3c65d17f7b6689bbd6568a1a1fcc0c4a3bac127 https://github.com/illumos/illumos-gate/commit/c3c65d17f7b6689bbd6568a1a1fcc0c4a3bac127 https://www.illumos.org/issues/7502 Right now ztest executes zdb without -G, so when it has errors, the messages are often not very helpful: Executing zdb -bccsv -d -U /rpool/tmp/zpool.cache ztest zdb: can't open 'ztest': Operation not supported ztest: '/usr/sbin/amd64/zdb -bccsv -d -U /rpool/tmp/zpool.cache ztest' exit code 1 With -G, we'd have: /usr/sbin/amd64/zdb -bccsv -d -U /rpool/tmp/zpool.cache -G ztest zdb: can't open 'ztest': Operation not supported ZFS_DBGMSG(zdb): spa_open_common: opening ztest spa_load(ztest): LOADING spa_load(ztest): FAILED: unable to parse config [error=48] spa_load(ztest): UNLOADING Which indicates where the error came from Reviewed by: Matthew Ahrens Reviewed by: Dan Kimmel Approved by: Gordon Ross Author: Pavel Zakharov Modified: vendor/illumos/dist/cmd/ztest/ztest.c Modified: vendor/illumos/dist/cmd/ztest/ztest.c ============================================================================== --- vendor/illumos/dist/cmd/ztest/ztest.c Fri Apr 14 17:15:49 2017 (r316871) +++ vendor/illumos/dist/cmd/ztest/ztest.c Fri Apr 14 17:20:26 2017 (r316872) @@ -5262,7 +5262,7 @@ ztest_run_zdb(char *pool) isa = strdup(isa); /* LINTED */ (void) sprintf(bin, - "/usr/sbin%.*s/zdb -bcc%s%s -d -U %s %s", + "/usr/sbin%.*s/zdb -bcc%s%s -G -d -U %s %s", isalen, isa, ztest_opts.zo_verbose >= 3 ? "s" : "", From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:22:56 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 37C1BD3E384; Fri, 14 Apr 2017 17:22:56 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id E19C6EC2; Fri, 14 Apr 2017 17:22:55 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EHMtfn043830; Fri, 14 Apr 2017 17:22:55 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EHMtS6043829; Fri, 14 Apr 2017 17:22:55 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141722.v3EHMtS6043829@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:22:55 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316873 - vendor/illumos/dist/lib/libzfs/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:22:56 -0000 Author: avg Date: Fri Apr 14 17:22:54 2017 New Revision: 316873 URL: https://svnweb.freebsd.org/changeset/base/316873 Log: 7233 dir_is_empty should open directory with CLOEXEC illumos/illumos-gate@d420209d9c807f782c1d31f5683be74798142198 https://github.com/illumos/illumos-gate/commit/d420209d9c807f782c1d31f5683be74798142198 https://www.illumos.org/issues/7233 This fixes a race where one thread is executing zfs_mount() while another thread forks and execs. If the fork occurs while the directory is open, the child process will inherit (but not necessarily close immediately) the open fd for the directory, preventing the mount. Reviewed by: Matthew Ahrens Reviewed by: Paul Dagnelie Approved by: Richard Lowe Author: Alex Reece Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c Fri Apr 14 17:20:26 2017 (r316872) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c Fri Apr 14 17:22:54 2017 (r316873) @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2014 by Delphix. All rights reserved. + * Copyright (c) 2014, 2015 by Delphix. All rights reserved. * Copyright 2016 Igor Kozhukhov */ @@ -64,6 +64,7 @@ #include #include #include +#include #include #include #include @@ -179,9 +180,16 @@ dir_is_empty(const char *dirname) { DIR *dirp; struct dirent64 *dp; + int dirfd; - if ((dirp = opendir(dirname)) == NULL) + if ((dirfd = openat(AT_FDCWD, dirname, + O_RDONLY | O_NDELAY | O_LARGEFILE | O_CLOEXEC, 0)) < 0) { return (B_TRUE); + } + + if ((dirp = fdopendir(dirfd)) == NULL) { + return (B_TRUE); + } while ((dp = readdir64(dirp)) != NULL) { From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:23:53 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id EC7C9D3E447; Fri, 14 Apr 2017 17:23:53 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id AFF2D1FA; Fri, 14 Apr 2017 17:23:53 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EHNq1N043959; Fri, 14 Apr 2017 17:23:52 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EHNqu3043958; Fri, 14 Apr 2017 17:23:52 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141723.v3EHNqu3043958@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:23:52 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316875 - vendor/illumos/dist/lib/libzfs/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:23:54 -0000 Author: avg Date: Fri Apr 14 17:23:52 2017 New Revision: 316875 URL: https://svnweb.freebsd.org/changeset/base/316875 Log: 7336 vfork and O_CLOEXEC causes zfs_mount EBUSY illumos/illumos-gate@873c4903a52d089cd8234b79d24f5a3fc3bccc82 https://github.com/illumos/illumos-gate/commit/873c4903a52d089cd8234b79d24f5a3fc3bccc82 https://www.illumos.org/issues/7336 We can run into a problem where we call into zfs_mount, which in turn calls is_dir_empty, which opens the directory to try and make sure it's empty. The issue with the current approach is that it holds the directory open while it traverses it with readdir, which, due to subtle interaction with the Java JVM, vfork, and exec can cause a tricky race condition resulting in zfs_mount failures. The approach to resolving the issue in this patch is to drop the usage of readdir altogether, and instead rely on the fact that ZFS stores the number of entries contained in a directory using the st_size field of the stat structure. Thus, if the directory in question is a ZFS directory, we can check to see if it's empty by calling stat() and inspecting the st_size field of structure returned. =============================================================================== The root cause appears to be an interesting race between vfork, exec, and zfs_mount's usage of O_CLOEXEC when calling openat. Here's what is going on: 1. We call zfs_mount, and this in turn calls openat to check if the directory is empty, which results in opening the directory we're trying to mount onto, and increment v_count. 2. As we're in the middle of reading the directory, vfork is called by the JVM and proceeds to exec the jspawnhelper utility. As a result of the vfork, we take an additional hold on the directory, which increments v_count a second time. The semantics of vfork mean the parent process will wait for the child process to exit or exec before the parent can continue; at this point the parent is in the middle of zfs_mount, reading the directory to determine if it's empty or not. 3. The child process exec-ing jspawnhelper gets to the relvm call within exec_args (which is called by exec_common). relvm is the function that releases the parent process, allowing the parent to proceed. The problem is, at this point of calling relvm, the child hasn't yet called close_exec which is responsible for closing the file descriptors inherited from the parent process Reviewed by: Matt Ahrens Reviewed by: Paul Dagnelie Reviewed by: Robert Mustacchi Approved by: Dan McDonald Author: Prakash Surya Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c Fri Apr 14 17:23:28 2017 (r316874) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c Fri Apr 14 17:23:52 2017 (r316875) @@ -75,6 +75,7 @@ #include #include #include +#include #include @@ -170,13 +171,32 @@ is_shared(libzfs_handle_t *hdl, const ch return (SHARED_NOT_SHARED); } -/* - * Returns true if the specified directory is empty. If we can't open the - * directory at all, return true so that the mount can fail with a more - * informative error message. - */ static boolean_t -dir_is_empty(const char *dirname) +dir_is_empty_stat(const char *dirname) +{ + struct stat st; + + /* + * We only want to return false if the given path is a non empty + * directory, all other errors are handled elsewhere. + */ + if (stat(dirname, &st) < 0 || !S_ISDIR(st.st_mode)) { + return (B_TRUE); + } + + /* + * An empty directory will still have two entries in it, one + * entry for each of "." and "..". + */ + if (st.st_size > 2) { + return (B_FALSE); + } + + return (B_TRUE); +} + +static boolean_t +dir_is_empty_readdir(const char *dirname) { DIR *dirp; struct dirent64 *dp; @@ -206,6 +226,42 @@ dir_is_empty(const char *dirname) } /* + * Returns true if the specified directory is empty. If we can't open the + * directory at all, return true so that the mount can fail with a more + * informative error message. + */ +static boolean_t +dir_is_empty(const char *dirname) +{ + struct statvfs64 st; + + /* + * If the statvfs call fails or the filesystem is not a ZFS + * filesystem, fall back to the slow path which uses readdir. + */ + if ((statvfs64(dirname, &st) != 0) || + (strcmp(st.f_basetype, "zfs") != 0)) { + return (dir_is_empty_readdir(dirname)); + } + + /* + * At this point, we know the provided path is on a ZFS + * filesystem, so we can use stat instead of readdir to + * determine if the directory is empty or not. We try to avoid + * using readdir because that requires opening "dirname"; this + * open file descriptor can potentially end up in a child + * process if there's a concurrent fork, thus preventing the + * zfs_mount() from otherwise succeeding (the open file + * descriptor inherited by the child process will cause the + * parent's mount to fail with EBUSY). The performance + * implications of replacing the open, read, and close with a + * single stat is nice; but is not the main motivation for the + * added complexity. + */ + return (dir_is_empty_stat(dirname)); +} + +/* * Checks to see if the mount is active. If the filesystem is mounted, we fill * in 'where' with the current mountpoint, and return 1. Otherwise, we return * 0. From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:24:24 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 613AED3E49B; Fri, 14 Apr 2017 17:24:24 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 2374D386; Fri, 14 Apr 2017 17:24:24 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EHONem044033; Fri, 14 Apr 2017 17:24:23 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EHONpS044031; Fri, 14 Apr 2017 17:24:23 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141724.v3EHONpS044031@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:24:23 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316876 - vendor/illumos/dist/lib/libzfs/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:24:24 -0000 Author: avg Date: Fri Apr 14 17:24:22 2017 New Revision: 316876 URL: https://svnweb.freebsd.org/changeset/base/316876 Log: 7542 zfs_unmount failed with EZFS_UNSHARENFSFAILED illumos/illumos-gate@09c9e6dc9b69d10b771bb87e01040ec320a0bfd3 https://github.com/illumos/illumos-gate/commit/09c9e6dc9b69d10b771bb87e01040ec320a0bfd3 https://www.illumos.org/issues/7542 libshare keeps a cached copy of the sharetab listing in memory, which can become out of date if shares are destroyed or created while leaving a libzfs handle open. This results in a spurious unmounting failure when an NFS share exists but isn't in the stale libshare cache. Reviewed by: Matthew Ahrens Reviewed by: Dan Kimmel Reviewed by: Matt Amdur Approved by: Robert Mustacchi Author: Chris Williamson Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_impl.h vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_impl.h ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_impl.h Fri Apr 14 17:23:52 2017 (r316875) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_impl.h Fri Apr 14 17:24:22 2017 (r316876) @@ -22,7 +22,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2011 Pawel Jakub Dawidek. All rights reserved. - * Copyright (c) 2011, 2015 by Delphix. All rights reserved. + * Copyright (c) 2011, 2016 by Delphix. All rights reserved. */ #ifndef _LIBZFS_IMPL_H @@ -72,7 +72,6 @@ struct libzfs_handle { int libzfs_printerr; int libzfs_storeerr; /* stuff error messages into buffer */ void *libzfs_sharehdl; /* libshare handle */ - uint_t libzfs_shareflags; boolean_t libzfs_mnttab_enable; avl_tree_t libzfs_mnttab_cache; int libzfs_pool_iter; @@ -82,8 +81,6 @@ struct libzfs_handle { char libzfs_chassis_id[256]; }; -#define ZFSSHARE_MISS 0x01 /* Didn't find entry in cache */ - struct zfs_handle { libzfs_handle_t *zfs_hdl; zpool_handle_t *zpool_hdl; Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c Fri Apr 14 17:23:52 2017 (r316875) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c Fri Apr 14 17:24:22 2017 (r316876) @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2014, 2015 by Delphix. All rights reserved. + * Copyright (c) 2014, 2016 by Delphix. All rights reserved. * Copyright 2016 Igor Kozhukhov */ @@ -657,35 +657,30 @@ _zfs_init_libshare(void) int zfs_init_libshare(libzfs_handle_t *zhandle, int service) { - int ret = SA_OK; - if (_sa_init == NULL) - ret = SA_CONFIG_ERR; + return (SA_CONFIG_ERR); - if (ret == SA_OK && zhandle->libzfs_shareflags & ZFSSHARE_MISS) { - /* - * We had a cache miss. Most likely it is a new ZFS - * dataset that was just created. We want to make sure - * so check timestamps to see if a different process - * has updated any of the configuration. If there was - * some non-ZFS change, we need to re-initialize the - * internal cache. - */ - zhandle->libzfs_shareflags &= ~ZFSSHARE_MISS; - if (_sa_needs_refresh != NULL && - _sa_needs_refresh(zhandle->libzfs_sharehdl)) { - zfs_uninit_libshare(zhandle); - zhandle->libzfs_sharehdl = _sa_init(service); - } + /* + * Attempt to refresh libshare. This is necessary if there was a cache + * miss for a new ZFS dataset that was just created, or if state of the + * sharetab file has changed since libshare was last initialized. We + * want to make sure so check timestamps to see if a different process + * has updated any of the configuration. If there was some non-ZFS + * change, we need to re-initialize the internal cache. + */ + if (_sa_needs_refresh != NULL && + _sa_needs_refresh(zhandle->libzfs_sharehdl)) { + zfs_uninit_libshare(zhandle); + zhandle->libzfs_sharehdl = _sa_init(service); } - if (ret == SA_OK && zhandle && zhandle->libzfs_sharehdl == NULL) + if (zhandle && zhandle->libzfs_sharehdl == NULL) zhandle->libzfs_sharehdl = _sa_init(service); - if (ret == SA_OK && zhandle->libzfs_sharehdl == NULL) - ret = SA_NO_MEMORY; + if (zhandle->libzfs_sharehdl == NULL) + return (SA_NO_MEMORY); - return (ret); + return (SA_OK); } /* @@ -831,7 +826,6 @@ zfs_share_proto(zfs_handle_t *zhp, zfs_s zfs_get_name(zhp)); return (-1); } - hdl->libzfs_shareflags |= ZFSSHARE_MISS; share = zfs_sa_find_share(hdl->libzfs_sharehdl, mountpoint); } From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:27:10 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 85137D3E548; Fri, 14 Apr 2017 17:27:10 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 3B6D076F; Fri, 14 Apr 2017 17:27:10 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EHR9fW044188; Fri, 14 Apr 2017 17:27:09 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EHR9Vt044187; Fri, 14 Apr 2017 17:27:09 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141727.v3EHR9Vt044187@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:27:09 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316877 - vendor/illumos/dist/lib/libzfs/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:27:10 -0000 Author: avg Date: Fri Apr 14 17:27:09 2017 New Revision: 316877 URL: https://svnweb.freebsd.org/changeset/base/316877 Log: 7571 non-present readonly numeric ZFS props do not have default value illumos/illumos-gate@ad2760acbd9c3b479bf632f05c6f03d89830799d https://github.com/illumos/illumos-gate/commit/ad2760acbd9c3b479bf632f05c6f03d89830799d https://www.illumos.org/issues/7571 ZFS displays the default value for non-present readonly numeric (and index) properties. However, these properties default values are not meaningful. Instead, we should display a "-", indicating that they are not present. For example, on a version-12 pool, the usedby* properties are not available, but they show up as the incorrect value "0": 1. zfs get all test12 ... test12 usedbysnapshots 0 - test12 usedbydataset 0 - test12 usedbychildren 0 - test12 usedbyrefreservation 0 - We will be introducing more sometimes-present numeric readonly properties, so it would be nice to fix this. Reviewed by: Dan Kimmel Reviewed by: George Wilson Approved by: Robert Mustacchi Author: Matthew Ahrens Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_dataset.c Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_dataset.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_dataset.c Fri Apr 14 17:24:22 2017 (r316876) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_dataset.c Fri Apr 14 17:27:09 2017 (r316877) @@ -22,7 +22,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2013, Joyent, Inc. All rights reserved. - * Copyright (c) 2011, 2015 by Delphix. All rights reserved. + * Copyright (c) 2011, 2016 by Delphix. All rights reserved. * Copyright (c) 2012 DEY Storage Systems, Inc. All rights reserved. * Copyright (c) 2011-2012 Pawel Jakub Dawidek. All rights reserved. * Copyright (c) 2013 Martin Matuska. All rights reserved. @@ -2038,6 +2038,7 @@ get_numeric_property(zfs_handle_t *zhp, if (zfs_prop_readonly(prop) && *source != NULL && (*source)[0] == '\0') { *source = NULL; + return (-1); } break; From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:43:53 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 23D56D3EAFD; Fri, 14 Apr 2017 17:43:53 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id E9C0F3B7; Fri, 14 Apr 2017 17:43:52 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EHhqEP052307; Fri, 14 Apr 2017 17:43:52 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EHhqIt052306; Fri, 14 Apr 2017 17:43:52 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141743.v3EHhqIt052306@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:43:52 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316879 - vendor/illumos/dist/man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:43:53 -0000 Author: avg Date: Fri Apr 14 17:43:51 2017 New Revision: 316879 URL: https://svnweb.freebsd.org/changeset/base/316879 Log: missing zfs.1m changes from r286704 Modified: vendor/illumos/dist/man/man1m/zfs.1m Modified: vendor/illumos/dist/man/man1m/zfs.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:38:43 2017 (r316878) +++ vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:43:51 2017 (r316879) @@ -175,11 +175,13 @@ .Nm .Cm receive .Op Fl Fnuv +.Op Fl o Sy origin Ns = Ns Ar snapshot .Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot .Nm .Cm receive .Op Fl Fnuv .Op Fl d Ns | Ns Fl e +.Op Fl o Sy origin Ns = Ns Ar snapshot .Ar filesystem .Nm .Cm allow @@ -2635,12 +2637,14 @@ origin, etc. .Nm .Cm receive .Op Fl Fnuv +.Op Fl o Sy origin Ns = Ns Ar snapshot .Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot .br .Nm .Cm receive .Op Fl Fnuv .Op Fl d Ns | Ns Fl e +.Op Fl o Sy origin Ns = Ns Ar snapshot .Ar filesystem .Xc Creates a snapshot whose contents are as specified in the stream provided on @@ -2730,6 +2734,10 @@ snapshot as described in the paragraph a Do not actually receive the stream. This can be useful in conjunction with the .Fl v option to verify the name the receive operation would use. +.It Fl o Sy origin Ns = Ns Ar snapshot +Forces the stream to be received as a clone of the given snapshot. +This is only valid if the stream is an incremental stream whose source +is the same as the provided origin. .It Fl u File system that is associated with the received stream is not mounted. .It Fl v From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:46:41 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 960FED3EB65; Fri, 14 Apr 2017 17:46:41 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 6661077F; Fri, 14 Apr 2017 17:46:41 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EHkeGV052541; Fri, 14 Apr 2017 17:46:40 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EHkeAu052540; Fri, 14 Apr 2017 17:46:40 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141746.v3EHkeAu052540@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:46:40 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316880 - vendor/illumos/dist/man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:46:41 -0000 Author: avg Date: Fri Apr 14 17:46:40 2017 New Revision: 316880 URL: https://svnweb.freebsd.org/changeset/base/316880 Log: missing zfs.1m changes from r289310 Modified: vendor/illumos/dist/man/man1m/zfs.1m Modified: vendor/illumos/dist/man/man1m/zfs.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:43:51 2017 (r316879) +++ vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:46:40 2017 (r316880) @@ -908,7 +908,8 @@ command. This property is not inherited. .It Xo .Sy checksum Ns = Ns Sy on Ns | Ns Sy off Ns | Ns Sy fletcher2 Ns | Ns -.Sy fletcher4 Ns | Ns Sy sha256 Ns | Ns Sy noparity +.Sy fletcher4 Ns | Ns Sy sha256 Ns | Ns Sy noparity Ns | Ns +.Sy sha512 Ns | Ns Sy skein Ns | Ns Sy edonr .Xc Controls the checksum used to verify data integrity. The default value is .Sy on , @@ -927,6 +928,16 @@ should not be used by any other dataset. .Sy NOT a recommended practice. .Pp +The +.Sy sha512 , +.Sy skein , +and +.Sy edonr +checksum algorithms require enabling the appropriate features on the +pool. Please see +.Xr zpool-features 5 +for more information on these algorithms. +.Pp Changing this property affects only newly-written data. .It Xo .Sy compression Ns = Ns Sy on Ns | Ns Sy off Ns | Ns Sy gzip Ns | Ns From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:48:46 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id A8931D3EBCF; Fri, 14 Apr 2017 17:48:46 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 85BDC8D1; Fri, 14 Apr 2017 17:48:46 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EHmj3P052750; Fri, 14 Apr 2017 17:48:45 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EHmjOA052749; Fri, 14 Apr 2017 17:48:45 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141748.v3EHmjOA052749@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:48:45 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316881 - vendor/illumos/dist/man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:48:46 -0000 Author: avg Date: Fri Apr 14 17:48:45 2017 New Revision: 316881 URL: https://svnweb.freebsd.org/changeset/base/316881 Log: missing zfs.1m changes from r289312 Modified: vendor/illumos/dist/man/man1m/zfs.1m Modified: vendor/illumos/dist/man/man1m/zfs.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:46:40 2017 (r316880) +++ vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:48:45 2017 (r316881) @@ -173,17 +173,25 @@ .Op Fl i Ar snapshot Ns | Ns Ar bookmark .Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot .Nm +.Cm send +.Op Fl Penv +.Fl t Ar receive_resume_token +.Nm .Cm receive -.Op Fl Fnuv +.Op Fl Fnsuv .Op Fl o Sy origin Ns = Ns Ar snapshot .Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot .Nm .Cm receive -.Op Fl Fnuv +.Op Fl Fnsuv .Op Fl d Ns | Ns Fl e .Op Fl o Sy origin Ns = Ns Ar snapshot .Ar filesystem .Nm +.Cm receive +.Fl A +.Ar filesystem Ns | Ns Ar volume +.Nm .Cm allow .Ar filesystem Ns | Ns Ar volume .Nm @@ -569,6 +577,13 @@ For cloned file systems or volumes, the created. See also the .Sy clones property. +.It Sy receive_resume_token +For filesystems or volumes which have saved partially-completed state from +.Sy zfs receive -s , +this opaque token can be provided to +.Sy zfs send -t +to resume and complete the +.Sy zfs receive . .It Sy referenced The amount of data that is accessible by this dataset, which may or may not be shared with other datasets in the pool. When a snapshot or clone is created, it @@ -2646,14 +2661,27 @@ origin, etc. .El .It Xo .Nm +.Cm send +.Op Fl Penv +.Fl t +.Ar receive_resume_token +.Xc +Creates a send stream which resumes an interrupted receive. The +.Ar receive_resume_token +is the value of this property on the filesystem +or volume that was being received into. See the documentation for +.Sy zfs receive -s +for more details. +.It Xo +.Nm .Cm receive -.Op Fl Fnuv +.Op Fl Fnsuv .Op Fl o Sy origin Ns = Ns Ar snapshot .Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot .br .Nm .Cm receive -.Op Fl Fnuv +.Op Fl Fnsuv .Op Fl d Ns | Ns Fl e .Op Fl o Sy origin Ns = Ns Ar snapshot .Ar filesystem @@ -2754,9 +2782,42 @@ File system that is associated with the .It Fl v Print verbose information about the stream and the time required to perform the receive operation. +.It Fl s +If the receive is interrupted, save the partially received state, rather +than deleting it. Interruption may be due to premature termination of +the stream +.Po e.g. due to network failure or failure of the remote system +if the stream is being read over a network connection +.Pc , +a checksum error in the stream, termination of the +.Nm zfs Cm receive +process, or unclean shutdown of the system. +.Pp +The receive can be resumed with a stream generated by +.Nm zfs Cm send Fl t Ar token , +where the +.Ar token +is the value of the +.Sy receive_resume_token +property of the filesystem or volume which is received into. +.Pp +To use this flag, the storage pool must have the +.Sy extensible_dataset +feature enabled. See +.Xr zpool-features 5 +for details on ZFS feature flags. .El .It Xo .Nm +.Cm receive +.Fl A +.Ar filesystem Ns | Ns Ar volume +.Xc +Abort an interrupted +.Nm zfs Cm receive Fl s , +deleting its saved partially received state. +.It Xo +.Nm .Cm allow .Ar filesystem Ns | Ns Ar volume .Xc From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:49:38 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id A7B34D3EC27; Fri, 14 Apr 2017 17:49:38 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 5E11A9F9; Fri, 14 Apr 2017 17:49:38 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EHnb4R052827; Fri, 14 Apr 2017 17:49:37 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EHnbUd052826; Fri, 14 Apr 2017 17:49:37 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141749.v3EHnbUd052826@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:49:37 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316882 - vendor/illumos/dist/man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:49:38 -0000 Author: avg Date: Fri Apr 14 17:49:37 2017 New Revision: 316882 URL: https://svnweb.freebsd.org/changeset/base/316882 Log: 6346 zfs(1M) has spurious comma illumos/illumos-gate@1058dba45ec5d6876a5bbb0b9174347e13e5748d https://github.com/illumos/illumos-gate/commit/1058dba45ec5d6876a5bbb0b9174347e13e5748d https://www.illumos.org/issues/6346 The xref to gzip(1) in the SEE ALSO puts the comma inside the parens, because a space is missing in the source .Xr gzip 1, should be .Xr gzip 1 , It'd be cool if the manual page checks in pbchk could catch this, too, but I'm not sure how easy that'd be. Reviewed by: Garrett D'Amore Reviewed by: Igor Kozhukhov Reviewed by: Robert Mustacchi Approved by: Gordon Ross Author: Yuri Pankov Modified: vendor/illumos/dist/man/man1m/zfs.1m Modified: vendor/illumos/dist/man/man1m/zfs.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:48:45 2017 (r316881) +++ vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:49:37 2017 (r316882) @@ -773,7 +773,7 @@ in which case it will be interpreted as this dataset. The .Em snapshot may be a full snapshot name -.No Po Em filesystem Ns @ Ns Em snapshot Pc , +.Po Em filesystem Ns @ Ns Em snapshot Pc , which for clones may be a snapshot in the origin's filesystem .Pq or the origin of the origin's filesystem, etc. .El @@ -1620,7 +1620,7 @@ administrators can use them to annotate .Pq file systems, volumes, and snapshots . .Pp User property names must contain a colon -.No Po Ns Sy \&: Ns Pc +.Pq Qq Sy \&: character to distinguish them from native properties. They may contain lowercase letters, numbers, and the following punctuation characters: colon .Pq Qq Sy \&: , @@ -3527,7 +3527,7 @@ M F /tank/test/modified .Sh INTERFACE STABILITY .Sy Commited . .Sh SEE ALSO -.Xr gzip 1, +.Xr gzip 1 , .Xr ssh 1 , .Xr mount 1M , .Xr share 1M , From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:52:48 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id B8353D3EFCD; Fri, 14 Apr 2017 17:52:48 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 88615EA8; Fri, 14 Apr 2017 17:52:48 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EHql6Y056721; Fri, 14 Apr 2017 17:52:47 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EHql2l056720; Fri, 14 Apr 2017 17:52:47 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141752.v3EHql2l056720@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:52:47 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316883 - vendor/illumos/dist/man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:52:48 -0000 Author: avg Date: Fri Apr 14 17:52:47 2017 New Revision: 316883 URL: https://svnweb.freebsd.org/changeset/base/316883 Log: missing zfs.1m changes from r294814 Modified: vendor/illumos/dist/man/man1m/zfs.1m Modified: vendor/illumos/dist/man/man1m/zfs.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:49:37 2017 (r316882) +++ vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:52:47 2017 (r316883) @@ -21,7 +21,7 @@ .\" .\" Copyright (c) 2009 Sun Microsystems, Inc. All Rights Reserved. .\" Copyright 2011 Joshua M. Clulow -.\" Copyright (c) 2011, 2014 by Delphix. All rights reserved. +.\" Copyright (c) 2011, 2015 by Delphix. All rights reserved. .\" Copyright (c) 2013 by Saso Kiselkov. All rights reserved. .\" Copyright (c) 2014, Joyent, Inc. All rights reserved. .\" Copyright (c) 2014 by Adam Stevko. All rights reserved. @@ -2775,8 +2775,11 @@ Do not actually receive the stream. This option to verify the name the receive operation would use. .It Fl o Sy origin Ns = Ns Ar snapshot Forces the stream to be received as a clone of the given snapshot. -This is only valid if the stream is an incremental stream whose source -is the same as the provided origin. +If the stream is a full send stream, this will create the filesystem +described by the stream as a clone of the specified snapshot. Which +snapshot was specified will not affect the success or failure of the +receive, as long as the snapshot does exist. If the stream is an +incremental send stream, all the normal verification will be performed. .It Fl u File system that is associated with the received stream is not mounted. .It Fl v From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:55:07 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 72F2ED3E039; Fri, 14 Apr 2017 17:55:07 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 44B6AFED; Fri, 14 Apr 2017 17:55:07 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EHt60R056898; Fri, 14 Apr 2017 17:55:06 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EHt68C056897; Fri, 14 Apr 2017 17:55:06 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141755.v3EHt68C056897@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:55:06 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316884 - vendor/illumos/dist/man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:55:07 -0000 Author: avg Date: Fri Apr 14 17:55:06 2017 New Revision: 316884 URL: https://svnweb.freebsd.org/changeset/base/316884 Log: missing zfs.1m changes from r296518 Modified: vendor/illumos/dist/man/man1m/zfs.1m Modified: vendor/illumos/dist/man/man1m/zfs.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:52:47 2017 (r316883) +++ vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:55:06 2017 (r316884) @@ -26,6 +26,7 @@ .\" Copyright (c) 2014, Joyent, Inc. All rights reserved. .\" Copyright (c) 2014 by Adam Stevko. All rights reserved. .\" Copyright 2015 Nexenta Systems, Inc. All Rights Reserved. +.\" Copyright (c) 2014 Integros [integros.com] .\" .Dd June 8, 2015 .Dt ZFS 1M From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:57:12 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 29B2BD3E0BF; Fri, 14 Apr 2017 17:57:12 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id E0179195; Fri, 14 Apr 2017 17:57:11 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EHvBJ6057025; Fri, 14 Apr 2017 17:57:11 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EHvBAb057024; Fri, 14 Apr 2017 17:57:11 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141757.v3EHvBAb057024@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:57:10 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316885 - vendor/illumos/dist/man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:57:12 -0000 Author: avg Date: Fri Apr 14 17:57:10 2017 New Revision: 316885 URL: https://svnweb.freebsd.org/changeset/base/316885 Log: missing zfs.1m changes from r299449 Modified: vendor/illumos/dist/man/man1m/zfs.1m Modified: vendor/illumos/dist/man/man1m/zfs.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:55:06 2017 (r316884) +++ vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:57:10 2017 (r316885) @@ -25,10 +25,10 @@ .\" Copyright (c) 2013 by Saso Kiselkov. All rights reserved. .\" Copyright (c) 2014, Joyent, Inc. All rights reserved. .\" Copyright (c) 2014 by Adam Stevko. All rights reserved. -.\" Copyright 2015 Nexenta Systems, Inc. All Rights Reserved. .\" Copyright (c) 2014 Integros [integros.com] +.\" Copyright 2016 Nexenta Systems, Inc. .\" -.Dd June 8, 2015 +.Dd March 20, 2016 .Dt ZFS 1M .Os .Sh NAME @@ -786,17 +786,12 @@ dataset. .Sy aclinherit Ns = Ns Sy discard Ns | Ns Sy noallow Ns | Ns .Sy restricted Ns | Ns Sy passthrough Ns | Ns Sy passthrough-x .Xc -Controls how -.Sy ACE Ns s -are inherited when files and directories are created. +Controls how ACEs are inherited when files and directories are created. .Bl -tag -width "passthrough-x" .It Sy discard -does not inherit any -.Sy ACE Ns s . +does not inherit any ACEs. .It Sy noallow -only inherits inheritable -.Sy ACE Ns s -that specify +only inherits inheritable ACEs that specify .Qq deny permissions. .It Sy restricted @@ -804,13 +799,9 @@ default, removes the .Sy write_acl and .Sy write_owner -permissions when the -.Sy ACE -is inherited. +permissions when the ACE is inherited. .It Sy passthrough -inherits all inheritable -.Sy ACE Ns s -without any modifications. +inherits all inheritable ACEs without any modifications. .It Sy passthrough-x same meaning as .Sy passthrough , @@ -819,69 +810,51 @@ except that the .Sy group@ , and .Sy everyone@ -.Sy ACE Ns s -inherit the execute permission only if the file creation mode also requests the -execute bit. +ACEs inherit the execute permission only if the file creation mode also requests +the execute bit. .El .Pp When the property value is set to .Sy passthrough , -files are created with a mode determined by the inheritable -.Sy ACE Ns s . -If no inheritable -.Sy ACE Ns s -exist that affect the mode, then the mode is set in accordance to the requested -mode from the application. +files are created with a mode determined by the inheritable ACEs. +If no inheritable ACEs exist that affect the mode, then the mode is set in +accordance to the requested mode from the application. .It Xo .Sy aclmode Ns = Ns Sy discard Ns | Ns Sy groupmask Ns | Ns .Sy passthrough Ns | Ns Sy restricted .Xc -Controls how an -.Sy ACL -is modified during -.Xr chmod 2 . +Controls how an ACL is modified during +.Xr chmod 2 +and how inherited ACEs are modified by the file creation mode. .Bl -tag -width "passthrough" .It Sy discard -default, deletes all -.Sy ACE Ns s -that do not represent the mode of the file. +default, deletes all ACEs except for those representing the mode of the file or +directory requested by +.Xr chmod 2 . .It Sy groupmask -reduces permissions granted in all +reduces permissions granted by all .Sy ALLOW -entries found in the -.Sy ACL -such that they are no greater than the group permissions specified by -.Xr chmod 2 . +entries found in the ACL such that they are no greater than the group +permissions specified by the mode. .It Sy passthrough -indicates that no changes are made to the -.Sy ACL -other than creating or updating the necessary -.Sy ACE Ns s -to represent the new mode of the file or directory. +indicates that no changes are made to the ACL other than creating or updating +the necessary ACEs to represent the new mode of the file or directory. .It Sy restricted causes the .Xr chmod 2 operation to return an error when used on any file or directory which has a -non-trivial -.Sy ACE Ns s -whose entries can not be represented by a mode. +non-trivial ACL, with entries in addition to those that represent the mode. .El .Pp .Xr chmod 2 -is required to change the set user ID, set group ID, or sticky bits on a file or -directory, as they do not have equivalent -.Sy ACE Ns s. -In order to use +is required to change the set user ID, set group ID, or sticky bit on a file or +directory, as they do not have equivalent ACEs. In order to use .Xr chmod 2 -on a file or directory with a non-trivial -.Sy ACL -when +on a file or directory with a non-trivial ACL when .Sy aclmode is set to .Sy restricted , -you must first remove all -.Sy ACE Ns s -which do not represent the current mode. +you must first remove all ACEs except for those that represent the current mode. .It Sy atime Ns = Ns Sy on Ns | Ns Sy off Controls whether the access time for files is updated when they are read. Turning this property off avoids producing write traffic when reading files and @@ -3437,9 +3410,7 @@ mount point permission is set to 755 by .Sy cindys will be unable to mount file systems under .Em tank/cindys . -Add an -.Sy ACE -similar to the following syntax to provide mount point access: +Add an ACE similar to the following syntax to provide mount point access: .Bd -literal # chmod A+user:cindys:add_subdirectory:allow /tank/cindys .Ed From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:58:01 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id C9802D3E11F; Fri, 14 Apr 2017 17:58:01 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 9B7362EC; Fri, 14 Apr 2017 17:58:01 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EHw0T0057112; Fri, 14 Apr 2017 17:58:00 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EHw0Gb057111; Fri, 14 Apr 2017 17:58:00 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141758.v3EHw0Gb057111@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:58:00 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316886 - vendor/illumos/dist/man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:58:01 -0000 Author: avg Date: Fri Apr 14 17:58:00 2017 New Revision: 316886 URL: https://svnweb.freebsd.org/changeset/base/316886 Log: 7041 Fix spelling mistakes in sections 1 and 1M illumos/illumos-gate@df23d905b96680e56379c5a5ecb4b363f36b9e74 https://github.com/illumos/illumos-gate/commit/df23d905b96680e56379c5a5ecb4b363f36b9e74 https://www.illumos.org/issues/7041 With the changes made in #7040, I found and fixed misspellings in sections 1 and 1M of the manual pages. Reviewed by: Marcel Telka Approved by: Dan McDonald Author: Cody Peter Mello Modified: vendor/illumos/dist/man/man1m/zfs.1m Modified: vendor/illumos/dist/man/man1m/zfs.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:57:10 2017 (r316885) +++ vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:58:00 2017 (r316886) @@ -28,7 +28,7 @@ .\" Copyright (c) 2014 Integros [integros.com] .\" Copyright 2016 Nexenta Systems, Inc. .\" -.Dd March 20, 2016 +.Dd April 9, 2016 .Dt ZFS 1M .Os .Sh NAME @@ -2459,7 +2459,7 @@ By default, a full stream is generated. .It Fl D Generate a deduplicated stream. Blocks which would have been sent multiple times in the send stream will only be sent once. The receiving system must also -support this feature to recieve a deduplicated stream. This flag can be used +support this feature to receive a deduplicated stream. This flag can be used regardless of the dataset's .Sy dedup property, but performance will be much better if the filesystem uses a @@ -3500,7 +3500,7 @@ M F /tank/test/modified .Ed .El .Sh INTERFACE STABILITY -.Sy Commited . +.Sy Committed . .Sh SEE ALSO .Xr gzip 1 , .Xr ssh 1 , From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:58:32 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id EB481D3E16F; Fri, 14 Apr 2017 17:58:32 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id A31E663C; Fri, 14 Apr 2017 17:58:32 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EHwVl8057175; Fri, 14 Apr 2017 17:58:31 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EHwVgV057174; Fri, 14 Apr 2017 17:58:31 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141758.v3EHwVgV057174@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:58:31 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316887 - vendor/illumos/dist/man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:58:33 -0000 Author: avg Date: Fri Apr 14 17:58:31 2017 New Revision: 316887 URL: https://svnweb.freebsd.org/changeset/base/316887 Log: 7257 zfs manpage user property length needs to be updated illumos/illumos-gate@3bc71695036246c52af58ab0ad58a23a0feb5a0c https://github.com/illumos/illumos-gate/commit/3bc71695036246c52af58ab0ad58a23a0feb5a0c https://www.illumos.org/issues/7257 The zfs.1m manpage says: User Properties ... Property values are limited to 1024 characters. Since zpool version 16, this limit is actually 8192 characters. Additionally, this limit is actually 8192 bytes, as it supports UTF-8. Reviewed by: Paul Dagnelie Reviewed by: Matthew Ahrens Reviewed by: Yuri Pankov Reviewed by: Igor Kozhukhov Reviewed by: Robert Mustacchi Approved by: Richard Lowe Author: Eli Rosenthal Modified: vendor/illumos/dist/man/man1m/zfs.1m Modified: vendor/illumos/dist/man/man1m/zfs.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:58:00 2017 (r316886) +++ vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:58:31 2017 (r316887) @@ -1629,9 +1629,8 @@ and so forth .Pc can be used to manipulate both native properties and user properties. Use the .Nm zfs Cm inherit -command to clear a user property . If the property is not defined in any parent -dataset, it is removed entirely. Property values are limited to 1024 -characters. +command to clear a user property. If the property is not defined in any parent +dataset, it is removed entirely. Property values are limited to 8192 bytes. .Ss ZFS Volumes as Swap or Dump Devices During an initial installation a swap device and dump device are created on ZFS volumes in the ZFS root pool. By default, the swap area size is based on 1/2 the From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:59:11 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id B2325D3E1C3; Fri, 14 Apr 2017 17:59:11 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 8A0A77E3; Fri, 14 Apr 2017 17:59:11 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EHxAc5057260; Fri, 14 Apr 2017 17:59:10 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EHxApZ057259; Fri, 14 Apr 2017 17:59:10 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141759.v3EHxApZ057259@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:59:10 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316888 - vendor/illumos/dist/man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:59:11 -0000 Author: avg Date: Fri Apr 14 17:59:10 2017 New Revision: 316888 URL: https://svnweb.freebsd.org/changeset/base/316888 Log: 7276 zfs(1m) manpage could better describe space properties illumos/illumos-gate@5749c35234ed96d3a61ee7c44d2ef37e043c7c59 https://github.com/illumos/illumos-gate/commit/5749c35234ed96d3a61ee7c44d2ef37e043c7c59 https://www.illumos.org/issues/7276 The "used" and "written" properties could be described better by the zfs.1m manpage. "written" could be better described as "The amount of space referenced by this dataset, that was written since the previous snapshot (i.e. that is not referenced by the previous snapshot)." The "used" section needs more work, but at a minimum it could say that the "used" space of a snapshot is the space unique to the snapshot (i.e. the space referenced only by this snapshot). The "used" space of a snapshot is a subset of the "written" space of the snapshot. Reviewed by: Dan Kimmel Reviewed by: Matthew Ahrens Reviewed by: Robert Mustacchi Approved by: Richard Lowe Author: Pavel Zakharov Modified: vendor/illumos/dist/man/man1m/zfs.1m Modified: vendor/illumos/dist/man/man1m/zfs.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:58:31 2017 (r316887) +++ vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:59:10 2017 (r316888) @@ -21,7 +21,7 @@ .\" .\" Copyright (c) 2009 Sun Microsystems, Inc. All Rights Reserved. .\" Copyright 2011 Joshua M. Clulow -.\" Copyright (c) 2011, 2015 by Delphix. All rights reserved. +.\" Copyright (c) 2011, 2016 by Delphix. All rights reserved. .\" Copyright (c) 2013 by Saso Kiselkov. All rights reserved. .\" Copyright (c) 2014, Joyent, Inc. All rights reserved. .\" Copyright (c) 2014 by Adam Stevko. All rights reserved. @@ -615,22 +615,27 @@ The amount of space consumed by this dat the value that is checked against this dataset's quota and reservation. The space used does not include this dataset's reservation, but does take into account the reservations of any descendent datasets. The amount of space that a -dataset consumes from its parent, as well as the amount of space that are freed +dataset consumes from its parent, as well as the amount of space that is freed if this dataset is recursively destroyed, is the greater of its space used and its reservation. .Pp -When snapshots +The used space of a snapshot .Po see the .Sx Snapshots section .Pc -are created, their space is initially shared between the snapshot and -the file system, and possibly with previous snapshots. As the file system -changes, space that was previously shared becomes unique to the snapshot, and -counted in the snapshot's space used. Additionally, deleting snapshots can -increase the amount of space unique to -.Pq and used by -other snapshots. +is space that is referenced exclusively by this snapshot. If this snapshot is +destroyed, the amount of +.Sy used +space will be freed. Space that is shared by multiple snapshots isn't accounted +for in this metric. When a snapshot is destroyed, space that was previously +shared with this snapshot can become unique to snapshots adjacent to it, thus +changing the used space of those snapshots. The used space of the latest +snapshot can also be affected by changes in the file system. Note that the +.Sy used +space of a snapshot is a subset of the +.Sy written +space of the snapshot. .Pp The amount of space used, available, or referenced does not take into account pending changes. Pending changes are generally accounted for within a few @@ -754,8 +759,10 @@ for volumes is 8 Kbytes. Any power of 2 This property can also be referred to by its shortened column name, .Sy volblock . .It Sy written -The amount of +The amount of space .Sy referenced +by this dataset, that was written since the previous snapshot +.Pq i.e. that is not referenced by the previous snapshot . space written to this dataset since the previous snapshot. .It Sy written Ns @ Ns Em snapshot The amount of From owner-svn-src-vendor@freebsd.org Fri Apr 14 17:59:57 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 6329DD3E231; Fri, 14 Apr 2017 17:59:57 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 192CD938; Fri, 14 Apr 2017 17:59:57 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EHxucR057337; Fri, 14 Apr 2017 17:59:56 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EHxuQ6057336; Fri, 14 Apr 2017 17:59:56 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141759.v3EHxuQ6057336@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 17:59:56 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316889 - vendor/illumos/dist/man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 17:59:57 -0000 Author: avg Date: Fri Apr 14 17:59:56 2017 New Revision: 316889 URL: https://svnweb.freebsd.org/changeset/base/316889 Log: 7345 zfs(1m): Description for -d depth appeared in -r illumos/illumos-gate@cb3c687bb9b8537283a9a4ce7309ff66fc266f2d https://github.com/illumos/illumos-gate/commit/cb3c687bb9b8537283a9a4ce7309ff66fc266f2d https://www.illumos.org/issues/7345 The following part of the zfs(1m) man page: -d depth Recursively display any children of the dataset, limiting the recursion to ... -r Recursively display any children of the dataset on the command line. depth. A depth of 1 will display only the dataset and its direct children. should be changed to: -d depth Recursively display any children of the dataset, limiting the recursion to depth. A depth of 1 will display only the dataset and its direct children. ... -r Recursively display any children of the dataset on the command line. Reviewed by: Yuri Pankov Approved by: Richard Lowe Author: Marcel Telka Modified: vendor/illumos/dist/man/man1m/zfs.1m Modified: vendor/illumos/dist/man/man1m/zfs.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:59:10 2017 (r316888) +++ vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:59:56 2017 (r316889) @@ -28,7 +28,7 @@ .\" Copyright (c) 2014 Integros [integros.com] .\" Copyright 2016 Nexenta Systems, Inc. .\" -.Dd April 9, 2016 +.Dd September 3, 2016 .Dt ZFS 1M .Os .Sh NAME @@ -2019,6 +2019,12 @@ Same as the option, but sorts by property in descending order. .It Fl d Ar depth Recursively display any children of the dataset, limiting the recursion to +.Ar depth . +A +.Ar depth +of +.Sy 1 +will display only the dataset and its direct children. .It Fl o Ar property A comma-separated list of properties to display. The property must be: .Bl -bullet @@ -2048,10 +2054,6 @@ Display numbers in parsable values. .It Fl r Recursively display any children of the dataset on the command line. -.Ar depth . -A depth of -.Sy 1 -will display only the dataset and its direct children. .It Fl s Ar property A property for sorting the output by column in ascending order based on the value of the property. The property must be one of the properties described in From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:00:44 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 09B63D3E2A8; Fri, 14 Apr 2017 18:00:44 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id CFFC6A9C; Fri, 14 Apr 2017 18:00:43 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EI0gag058128; Fri, 14 Apr 2017 18:00:42 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EI0gOD058127; Fri, 14 Apr 2017 18:00:42 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141800.v3EI0gOD058127@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:00:42 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316890 - vendor/illumos/dist/man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:00:44 -0000 Author: avg Date: Fri Apr 14 18:00:42 2017 New Revision: 316890 URL: https://svnweb.freebsd.org/changeset/base/316890 Log: 7276 zfs(1m) manpage could better describe space properties (remove extra line) illumos/illumos-gate@079d2996640860ff94d139e2c8cff6849c849af7 https://github.com/illumos/illumos-gate/commit/079d2996640860ff94d139e2c8cff6849c849af7 https://www.illumos.org/issues/7276 The "used" and "written" properties could be described better by the zfs.1m manpage. "written" could be better described as "The amount of space referenced by this dataset, that was written since the previous snapshot (i.e. that is not referenced by the previous snapshot)." The "used" section needs more work, but at a minimum it could say that the "used" space of a snapshot is the space unique to the snapshot (i.e. the space referenced only by this snapshot). The "used" space of a snapshot is a subset of the "written" space of the snapshot. Author: Matthew Ahrens Modified: vendor/illumos/dist/man/man1m/zfs.1m Modified: vendor/illumos/dist/man/man1m/zfs.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 17:59:56 2017 (r316889) +++ vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 18:00:42 2017 (r316890) @@ -763,7 +763,6 @@ The amount of space .Sy referenced by this dataset, that was written since the previous snapshot .Pq i.e. that is not referenced by the previous snapshot . -space written to this dataset since the previous snapshot. .It Sy written Ns @ Ns Em snapshot The amount of .Sy referenced From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:01:44 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 69000D3E30F; Fri, 14 Apr 2017 18:01:44 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 44858DB3; Fri, 14 Apr 2017 18:01:44 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EI1hh8060304; Fri, 14 Apr 2017 18:01:43 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EI1hIj060303; Fri, 14 Apr 2017 18:01:43 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141801.v3EI1hIj060303@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:01:43 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316891 - vendor-sys/illumos/dist/common/zfs vendor/illumos/dist/cmd/zfs vendor/illumos/dist/lib/libzfs/common vendor/illumos/dist/man/man1m X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:01:44 -0000 Author: avg Date: Fri Apr 14 18:01:43 2017 New Revision: 316891 URL: https://svnweb.freebsd.org/changeset/base/316891 Log: 7386 zfs get does not work properly with bookmarks illumos/illumos-gate@edb901aab9c738b5eb15aa55933e82b0f2f9d9a2 https://github.com/illumos/illumos-gate/commit/edb901aab9c738b5eb15aa55933e82b0f2f9d9a2 https://www.illumos.org/issues/7386 The zfs get command does not work with the bookmark parameter while it works properly with both filesystem and snapshot: # zfs get -t all -r creation rpool/test NAME PROPERTY VALUE SOURCE rpool/test creation Fri Sep 16 15:00 2016 - rpool/test@snap creation Fri Sep 16 15:00 2016 - rpool/test#bkmark creation Fri Sep 16 15:00 2016 - # zfs get -t all -r creation rpool/test@snap NAME PROPERTY VALUE SOURCE rpool/test@snap creation Fri Sep 16 15:00 2016 - # zfs get -t all -r creation rpool/test#bkmark cannot open 'rpool/test#bkmark': invalid dataset name # The zfs get command should be modified to work properly with bookmarks too. Reviewed by: Simon Klinkert Reviewed by: Paul Dagnelie Approved by: Matthew Ahrens Author: Marcel Telka Modified: vendor-sys/illumos/dist/common/zfs/zfs_namecheck.c vendor-sys/illumos/dist/common/zfs/zfs_namecheck.h Changes in other areas also in this revision: Modified: vendor/illumos/dist/cmd/zfs/zfs_main.c vendor/illumos/dist/lib/libzfs/common/libzfs_dataset.c vendor/illumos/dist/lib/libzfs/common/libzfs_pool.c vendor/illumos/dist/lib/libzfs/common/libzfs_util.c vendor/illumos/dist/man/man1m/zfs.1m Modified: vendor-sys/illumos/dist/common/zfs/zfs_namecheck.c ============================================================================== --- vendor-sys/illumos/dist/common/zfs/zfs_namecheck.c Fri Apr 14 18:00:42 2017 (r316890) +++ vendor-sys/illumos/dist/common/zfs/zfs_namecheck.c Fri Apr 14 18:01:43 2017 (r316891) @@ -120,9 +120,9 @@ permset_namecheck(const char *path, name } /* - * Dataset names must be of the following form: + * Entity names must be of the following form: * - * [component][/]*[component][@component] + * [component/]*[component][(@|#)component]? * * Where each component is made up of alphanumeric characters plus the following * characters: @@ -133,10 +133,10 @@ permset_namecheck(const char *path, name * names for temporary clones (for online recv). */ int -dataset_namecheck(const char *path, namecheck_err_t *why, char *what) +entity_namecheck(const char *path, namecheck_err_t *why, char *what) { - const char *loc, *end; - int found_snapshot; + const char *start, *end; + int found_delim; /* * Make sure the name is not too long. @@ -161,12 +161,13 @@ dataset_namecheck(const char *path, name return (-1); } - loc = path; - found_snapshot = 0; + start = path; + found_delim = 0; for (;;) { /* Find the end of this component */ - end = loc; - while (*end != '/' && *end != '@' && *end != '\0') + end = start; + while (*end != '/' && *end != '@' && *end != '#' && + *end != '\0') end++; if (*end == '\0' && end[-1] == '/') { @@ -176,25 +177,8 @@ dataset_namecheck(const char *path, name return (-1); } - /* Zero-length components are not allowed */ - if (loc == end) { - if (why) { - /* - * Make sure this is really a zero-length - * component and not a '@@'. - */ - if (*end == '@' && found_snapshot) { - *why = NAME_ERR_MULTIPLE_AT; - } else { - *why = NAME_ERR_EMPTY_COMPONENT; - } - } - - return (-1); - } - /* Validate the contents of this component */ - while (loc != end) { + for (const char *loc = start; loc != end; loc++) { if (!valid_char(*loc) && *loc != '%') { if (why) { *why = NAME_ERR_INVALCHAR; @@ -202,43 +186,64 @@ dataset_namecheck(const char *path, name } return (-1); } - loc++; } - /* If we've reached the end of the string, we're OK */ - if (*end == '\0') - return (0); - - if (*end == '@') { - /* - * If we've found an @ symbol, indicate that we're in - * the snapshot component, and report a second '@' - * character as an error. - */ - if (found_snapshot) { + /* Snapshot or bookmark delimiter found */ + if (*end == '@' || *end == '#') { + /* Multiple delimiters are not allowed */ + if (found_delim != 0) { if (why) - *why = NAME_ERR_MULTIPLE_AT; + *why = NAME_ERR_MULTIPLE_DELIMITERS; return (-1); } - found_snapshot = 1; + found_delim = 1; + } + + /* Zero-length components are not allowed */ + if (start == end) { + if (why) + *why = NAME_ERR_EMPTY_COMPONENT; + return (-1); } + /* If we've reached the end of the string, we're OK */ + if (*end == '\0') + return (0); + /* - * If there is a '/' in a snapshot name + * If there is a '/' in a snapshot or bookmark name * then report an error */ - if (*end == '/' && found_snapshot) { + if (*end == '/' && found_delim != 0) { if (why) *why = NAME_ERR_TRAILING_SLASH; return (-1); } /* Update to the next component */ - loc = end + 1; + start = end + 1; } } +/* + * Dataset is any entity, except bookmark + */ +int +dataset_namecheck(const char *path, namecheck_err_t *why, char *what) +{ + int ret = entity_namecheck(path, why, what); + + if (ret == 0 && strchr(path, '#') != NULL) { + if (why != NULL) { + *why = NAME_ERR_INVALCHAR; + *what = '#'; + } + return (-1); + } + + return (ret); +} /* * mountpoint names must be of the following form: Modified: vendor-sys/illumos/dist/common/zfs/zfs_namecheck.h ============================================================================== --- vendor-sys/illumos/dist/common/zfs/zfs_namecheck.h Fri Apr 14 18:00:42 2017 (r316890) +++ vendor-sys/illumos/dist/common/zfs/zfs_namecheck.h Fri Apr 14 18:01:43 2017 (r316891) @@ -38,7 +38,7 @@ typedef enum { NAME_ERR_EMPTY_COMPONENT, /* name contains an empty component */ NAME_ERR_TRAILING_SLASH, /* name ends with a slash */ NAME_ERR_INVALCHAR, /* invalid character found */ - NAME_ERR_MULTIPLE_AT, /* multiple '@' characters found */ + NAME_ERR_MULTIPLE_DELIMITERS, /* multiple '@'/'#' delimiters found */ NAME_ERR_NOLETTER, /* pool doesn't begin with a letter */ NAME_ERR_RESERVED, /* entire name is reserved */ NAME_ERR_DISKLIKE, /* reserved disk name (c[0-9].*) */ @@ -49,6 +49,7 @@ typedef enum { #define ZFS_PERMSET_MAXLEN 64 int pool_namecheck(const char *, namecheck_err_t *, char *); +int entity_namecheck(const char *, namecheck_err_t *, char *); int dataset_namecheck(const char *, namecheck_err_t *, char *); int mountpoint_namecheck(const char *, namecheck_err_t *); int zfs_component_namecheck(const char *, namecheck_err_t *, char *); From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:01:45 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 20BADD3E313; Fri, 14 Apr 2017 18:01:45 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id D8353DB6; Fri, 14 Apr 2017 18:01:44 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EI1iBs060314; Fri, 14 Apr 2017 18:01:44 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EI1h3X060309; Fri, 14 Apr 2017 18:01:43 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141801.v3EI1h3X060309@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:01:43 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316891 - vendor-sys/illumos/dist/common/zfs vendor/illumos/dist/cmd/zfs vendor/illumos/dist/lib/libzfs/common vendor/illumos/dist/man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:01:45 -0000 Author: avg Date: Fri Apr 14 18:01:43 2017 New Revision: 316891 URL: https://svnweb.freebsd.org/changeset/base/316891 Log: 7386 zfs get does not work properly with bookmarks illumos/illumos-gate@edb901aab9c738b5eb15aa55933e82b0f2f9d9a2 https://github.com/illumos/illumos-gate/commit/edb901aab9c738b5eb15aa55933e82b0f2f9d9a2 https://www.illumos.org/issues/7386 The zfs get command does not work with the bookmark parameter while it works properly with both filesystem and snapshot: # zfs get -t all -r creation rpool/test NAME PROPERTY VALUE SOURCE rpool/test creation Fri Sep 16 15:00 2016 - rpool/test@snap creation Fri Sep 16 15:00 2016 - rpool/test#bkmark creation Fri Sep 16 15:00 2016 - # zfs get -t all -r creation rpool/test@snap NAME PROPERTY VALUE SOURCE rpool/test@snap creation Fri Sep 16 15:00 2016 - # zfs get -t all -r creation rpool/test#bkmark cannot open 'rpool/test#bkmark': invalid dataset name # The zfs get command should be modified to work properly with bookmarks too. Reviewed by: Simon Klinkert Reviewed by: Paul Dagnelie Approved by: Matthew Ahrens Author: Marcel Telka Modified: vendor/illumos/dist/cmd/zfs/zfs_main.c vendor/illumos/dist/lib/libzfs/common/libzfs_dataset.c vendor/illumos/dist/lib/libzfs/common/libzfs_pool.c vendor/illumos/dist/lib/libzfs/common/libzfs_util.c vendor/illumos/dist/man/man1m/zfs.1m Changes in other areas also in this revision: Modified: vendor-sys/illumos/dist/common/zfs/zfs_namecheck.c vendor-sys/illumos/dist/common/zfs/zfs_namecheck.h Modified: vendor/illumos/dist/cmd/zfs/zfs_main.c ============================================================================== --- vendor/illumos/dist/cmd/zfs/zfs_main.c Fri Apr 14 18:00:42 2017 (r316890) +++ vendor/illumos/dist/cmd/zfs/zfs_main.c Fri Apr 14 18:01:43 2017 (r316891) @@ -232,7 +232,7 @@ get_usage(zfs_help_t idx) "[-o \"all\" | field[,...]]\n" "\t [-t type[,...]] [-s source[,...]]\n" "\t <\"all\" | property[,...]> " - "[filesystem|volume|snapshot] ...\n")); + "[filesystem|volume|snapshot|bookmark] ...\n")); case HELP_INHERIT: return (gettext("\tinherit [-rS] " " ...\n")); @@ -1594,7 +1594,7 @@ zfs_do_get(int argc, char **argv) { zprop_get_cbdata_t cb = { 0 }; int i, c, flags = ZFS_ITER_ARGS_CAN_BE_PATHS; - int types = ZFS_TYPE_DATASET; + int types = ZFS_TYPE_DATASET | ZFS_TYPE_BOOKMARK; char *value, *fields; int ret = 0; int limit = 0; Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_dataset.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_dataset.c Fri Apr 14 18:00:42 2017 (r316890) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_dataset.c Fri Apr 14 18:01:43 2017 (r316891) @@ -105,7 +105,7 @@ zfs_validate_name(libzfs_handle_t *hdl, char what; (void) zfs_prop_get_table(); - if (dataset_namecheck(path, &why, &what) != 0) { + if (entity_namecheck(path, &why, &what) != 0) { if (hdl != NULL) { switch (why) { case NAME_ERR_TOOLONG: @@ -134,9 +134,10 @@ zfs_validate_name(libzfs_handle_t *hdl, "'%c' in name"), what); break; - case NAME_ERR_MULTIPLE_AT: + case NAME_ERR_MULTIPLE_DELIMITERS: zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, - "multiple '@' delimiters in name")); + "multiple '@' and/or '#' delimiters in " + "name")); break; case NAME_ERR_NOLETTER: @@ -167,7 +168,7 @@ zfs_validate_name(libzfs_handle_t *hdl, if (!(type & ZFS_TYPE_SNAPSHOT) && strchr(path, '@') != NULL) { if (hdl != NULL) zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, - "snapshot delimiter '@' in filesystem name")); + "snapshot delimiter '@' is not expected here")); return (0); } @@ -178,6 +179,20 @@ zfs_validate_name(libzfs_handle_t *hdl, return (0); } + if (!(type & ZFS_TYPE_BOOKMARK) && strchr(path, '#') != NULL) { + if (hdl != NULL) + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "bookmark delimiter '#' is not expected here")); + return (0); + } + + if (type == ZFS_TYPE_BOOKMARK && strchr(path, '#') == NULL) { + if (hdl != NULL) + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "missing '#' delimiter in bookmark name")); + return (0); + } + if (modifying && strchr(path, '%') != NULL) { if (hdl != NULL) zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, @@ -615,8 +630,36 @@ make_bookmark_handle(zfs_handle_t *paren return (zhp); } +struct zfs_open_bookmarks_cb_data { + const char *path; + zfs_handle_t *zhp; +}; + +static int +zfs_open_bookmarks_cb(zfs_handle_t *zhp, void *data) +{ + struct zfs_open_bookmarks_cb_data *dp = data; + + /* + * Is it the one we are looking for? + */ + if (strcmp(dp->path, zfs_get_name(zhp)) == 0) { + /* + * We found it. Save it and let the caller know we are done. + */ + dp->zhp = zhp; + return (EEXIST); + } + + /* + * Not found. Close the handle and ask for another one. + */ + zfs_close(zhp); + return (0); +} + /* - * Opens the given snapshot, filesystem, or volume. The 'types' + * Opens the given snapshot, bookmark, filesystem, or volume. The 'types' * argument is a mask of acceptable types. The function will print an * appropriate error message and return NULL if it can't be opened. */ @@ -625,6 +668,7 @@ zfs_open(libzfs_handle_t *hdl, const cha { zfs_handle_t *zhp; char errbuf[1024]; + char *bookp; (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN, "cannot open '%s'"), path); @@ -632,20 +676,68 @@ zfs_open(libzfs_handle_t *hdl, const cha /* * Validate the name before we even try to open it. */ - if (!zfs_validate_name(hdl, path, ZFS_TYPE_DATASET, B_FALSE)) { - zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, - "invalid dataset name")); + if (!zfs_validate_name(hdl, path, types, B_FALSE)) { (void) zfs_error(hdl, EZFS_INVALIDNAME, errbuf); return (NULL); } /* - * Try to get stats for the dataset, which will tell us if it exists. + * Bookmarks needs to be handled separately. */ - errno = 0; - if ((zhp = make_dataset_handle(hdl, path)) == NULL) { - (void) zfs_standard_error(hdl, errno, errbuf); - return (NULL); + bookp = strchr(path, '#'); + if (bookp == NULL) { + /* + * Try to get stats for the dataset, which will tell us if it + * exists. + */ + errno = 0; + if ((zhp = make_dataset_handle(hdl, path)) == NULL) { + (void) zfs_standard_error(hdl, errno, errbuf); + return (NULL); + } + } else { + char dsname[ZFS_MAX_DATASET_NAME_LEN]; + zfs_handle_t *pzhp; + struct zfs_open_bookmarks_cb_data cb_data = {path, NULL}; + + /* + * We need to cut out '#' and everything after '#' + * to get the parent dataset name only. + */ + assert(bookp - path < sizeof (dsname)); + (void) strncpy(dsname, path, bookp - path); + dsname[bookp - path] = '\0'; + + /* + * Create handle for the parent dataset. + */ + errno = 0; + if ((pzhp = make_dataset_handle(hdl, dsname)) == NULL) { + (void) zfs_standard_error(hdl, errno, errbuf); + return (NULL); + } + + /* + * Iterate bookmarks to find the right one. + */ + errno = 0; + if ((zfs_iter_bookmarks(pzhp, zfs_open_bookmarks_cb, + &cb_data) == 0) && (cb_data.zhp == NULL)) { + (void) zfs_error(hdl, EZFS_NOENT, errbuf); + zfs_close(pzhp); + return (NULL); + } + if (cb_data.zhp == NULL) { + (void) zfs_standard_error(hdl, errno, errbuf); + zfs_close(pzhp); + return (NULL); + } + zhp = cb_data.zhp; + + /* + * Cleanup. + */ + zfs_close(pzhp); } if (!(types & zhp->zfs_type)) { Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_pool.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_pool.c Fri Apr 14 18:00:42 2017 (r316890) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_pool.c Fri Apr 14 18:01:43 2017 (r316891) @@ -946,9 +946,10 @@ zpool_name_valid(libzfs_handle_t *hdl, b "trailing slash in name")); break; - case NAME_ERR_MULTIPLE_AT: + case NAME_ERR_MULTIPLE_DELIMITERS: zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, - "multiple '@' delimiters in name")); + "multiple '@' and/or '#' delimiters in " + "name")); break; default: Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_util.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_util.c Fri Apr 14 18:00:42 2017 (r316890) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_util.c Fri Apr 14 18:01:43 2017 (r316891) @@ -689,7 +689,7 @@ zfs_get_pool_handle(const zfs_handle_t * * Given a name, determine whether or not it's a valid path * (starts with '/' or "./"). If so, walk the mnttab trying * to match the device number. If not, treat the path as an - * fs/vol/snap name. + * fs/vol/snap/bkmark name. */ zfs_handle_t * zfs_path_to_zhandle(libzfs_handle_t *hdl, char *path, zfs_type_t argtype) Modified: vendor/illumos/dist/man/man1m/zfs.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 18:00:42 2017 (r316890) +++ vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 18:01:43 2017 (r316891) @@ -28,7 +28,7 @@ .\" Copyright (c) 2014 Integros [integros.com] .\" Copyright 2016 Nexenta Systems, Inc. .\" -.Dd September 3, 2016 +.Dd September 16, 2016 .Dt ZFS 1M .Os .Sh NAME @@ -112,7 +112,7 @@ .Oo Fl s Ar source Ns Oo , Ns Ar source Oc Ns ... Oc .Oo Fl t Ar type Ns Oo , Ns Ar type Oc Ns ... Oc .Cm all | Ar property Ns Oo , Ns Ar property Oc Ns ... -.Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot Ns ... +.Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot Ns | Ns Ar bookmark Ns ... .Nm .Cm inherit .Op Fl rS @@ -2122,7 +2122,7 @@ section. .Oo Fl s Ar source Ns Oo , Ns Ar source Oc Ns ... Oc .Oo Fl t Ar type Ns Oo , Ns Ar type Oc Ns ... Oc .Cm all | Ar property Ns Oo , Ns Ar property Oc Ns ... -.Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot Ns ... +.Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot Ns | Ns Ar bookmark Ns ... .Xc Displays properties for the given datasets. If no datasets are specified, then the command displays properties for all datasets on the system. For each From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:02:27 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 2F66CD3E51C; Fri, 14 Apr 2017 18:02:27 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id F304A11A3; Fri, 14 Apr 2017 18:02:26 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EI2Qlq061225; Fri, 14 Apr 2017 18:02:26 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EI2QxN061224; Fri, 14 Apr 2017 18:02:26 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141802.v3EI2QxN061224@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:02:26 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316892 - vendor/illumos/dist/man/man1m X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:02:27 -0000 Author: avg Date: Fri Apr 14 18:02:25 2017 New Revision: 316892 URL: https://svnweb.freebsd.org/changeset/base/316892 Log: 7602 minor issues with zfs manpage illumos/illumos-gate@5c262fd00992208f65151758483eb8841166798b https://github.com/illumos/illumos-gate/commit/5c262fd00992208f65151758483eb8841166798b https://www.illumos.org/issues/7602 The line volblocksize=blocksize should just read volblocksize in the same rendering as the other properties in the same section. The zfs.1m man page renders one variant of unallow as zfs unallow [-r] -s -@setname [perm|@setname[,perm|@setname]...] filesystem|volume There is an extra dash preceeding @setname that should not be there. Reviewed by: Matthew Ahrens Reviewed by: Daniel Hoffman Approved by: Richard Lowe Author: Sara Hartse Modified: vendor/illumos/dist/man/man1m/zfs.1m Modified: vendor/illumos/dist/man/man1m/zfs.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 18:01:43 2017 (r316891) +++ vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 18:02:25 2017 (r316892) @@ -748,7 +748,7 @@ or a user who has been granted the privilege with .Nm zfs Cm allow , can access all groups' usage. -.It Sy volblocksize Ns = Ns Em blocksize +.It Sy volblocksize For volumes, specifies the block size of the volume. The .Sy blocksize cannot be changed once the volume has been written, so it should be set at @@ -3026,7 +3026,7 @@ Recursively remove the permissions from .Nm .Cm unallow .Op Fl r -.Fl s @ Ns Ar setname +.Fl s No @ Ns Ar setname .Oo Ar perm Ns | Ns @ Ns Ar setname Ns Oo , Ns Ar perm Ns | Ns @ Ns .Ar setname Oc Ns ... Oc .Ar filesystem Ns | Ns Ar volume From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:05:21 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 8A149D3E5CE; Fri, 14 Apr 2017 18:05:21 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 4D44713A2; Fri, 14 Apr 2017 18:05:21 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EI5KXi061397; Fri, 14 Apr 2017 18:05:20 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EI5Kdo061396; Fri, 14 Apr 2017 18:05:20 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141805.v3EI5Kdo061396@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:05:20 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316893 - vendor/illumos/dist/lib/libzfs/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:05:21 -0000 Author: avg Date: Fri Apr 14 18:05:20 2017 New Revision: 316893 URL: https://svnweb.freebsd.org/changeset/base/316893 Log: 7604 if volblocksize property is the default, it displays as "-" rather than 8K illumos/illumos-gate@4d86c0eab246bdfddc2dd52410ba808433bd6266 https://github.com/illumos/illumos-gate/commit/4d86c0eab246bdfddc2dd52410ba808433bd6266 https://www.illumos.org/issues/7604 If a zvol has the default setting for the "volblocksize" property, it is 8KB. However, it is displayed as "-" (not present), rather than "8K". The problem was introduced by: commit 25228e830e86924a41243343b1de9daf2d7dd43a Author: Matthew Ahrens <mahrens@delphix.com> Date: Thu Nov 17 14:37:24 2016 -0800 7571 non-present readonly numeric ZFS props do not have default value which changed changed get_numeric_property() to indicate that readonly default properties are not present. However, zfs_prop_readonly() returns TRUE for both readonly and set-once properties (e.g. volblocksize). Amusingly, that commit essentially reverted: 6900484 default volblocksize is no longer being reported correctly from November 2009. However, that change was not correct either; the correct solution is to only do this check for "truly readonly" (i.e. not setonce) properties. $ zfs list -t volume -o name,volblocksize NAME VOLBLOCK domain0/group-100/appdata_container-101/appdata_windows_timeflow-102/ archive - domain0/group-100/appdata_container-101/appdata_windows_timeflow-102/ datafile - domain0/group-100/appdata_container-101/appdata_windows_timeflow-102/ external - rpool/dump 128K rpool/swap 4K rpool/swap1 =============================================================================== Reviewed by: Pavel Zakharov Reviewed by: Paul Dagnelie Reviewed by: John Kennedy Reviewed by: George Wilson Approved by: Robert Mustacchi Author: Matthew Ahrens Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_dataset.c Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_dataset.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_dataset.c Fri Apr 14 18:02:25 2017 (r316892) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_dataset.c Fri Apr 14 18:05:20 2017 (r316893) @@ -2125,9 +2125,12 @@ get_numeric_property(zfs_handle_t *zhp, /* * If we tried to use a default value for a * readonly property, it means that it was not - * present. + * present. Note this only applies to "truly" + * readonly properties, not set-once properties + * like volblocksize. */ if (zfs_prop_readonly(prop) && + !zfs_prop_setonce(prop) && *source != NULL && (*source)[0] == '\0') { *source = NULL; return (-1); From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:07:45 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 672CED3E6AF; Fri, 14 Apr 2017 18:07:45 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 2F8FB1673; Fri, 14 Apr 2017 18:07:45 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EI7irV061736; Fri, 14 Apr 2017 18:07:44 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EI7hWe061729; Fri, 14 Apr 2017 18:07:43 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141807.v3EI7hWe061729@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:07:43 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316894 - vendor-sys/illumos/dist/uts/common/fs/zfs vendor-sys/illumos/dist/uts/common/fs/zfs/sys vendor/illumos/dist/cmd/zfs vendor/illumos/dist/cmd/zstreamdump vendor/illumos/dist/lib... X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:07:45 -0000 Author: avg Date: Fri Apr 14 18:07:43 2017 New Revision: 316894 URL: https://svnweb.freebsd.org/changeset/base/316894 Log: 7252 7628 compressed zfs send / receive illumos/illumos-gate@5602294fda888d923d57a78bafdaf48ae6223dea https://github.com/illumos/illumos-gate/commit/5602294fda888d923d57a78bafdaf48ae6223dea https://www.illumos.org/issues/7252 This feature includes code to allow a system with compressed ARC enabled to send data in its compressed form straight out of the ARC, and receive data in its compressed form directly into the ARC. https://www.illumos.org/issues/7628 We should have longer, more readable versions of the ZFS send / recv options. 7628 create long versions of ZFS send / receive options Reviewed by: George Wilson Reviewed by: John Kennedy Reviewed by: Matthew Ahrens Reviewed by: Paul Dagnelie Reviewed by: Pavel Zakharov Reviewed by: Sebastien Roy Reviewed by: David Quigley Reviewed by: Thomas Caputi Approved by: Dan McDonald Author: Dan Kimmel Modified: vendor/illumos/dist/cmd/zfs/zfs_main.c vendor/illumos/dist/cmd/zstreamdump/zstreamdump.c vendor/illumos/dist/lib/libzfs/common/libzfs.h vendor/illumos/dist/lib/libzfs/common/libzfs_sendrecv.c vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.c vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.h vendor/illumos/dist/man/man1m/zfs.1m Changes in other areas also in this revision: Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c vendor-sys/illumos/dist/uts/common/fs/zfs/dmu.c vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_send.c vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_dataset.c vendor-sys/illumos/dist/uts/common/fs/zfs/lz4.c vendor-sys/illumos/dist/uts/common/fs/zfs/sys/arc.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu_send.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dsl_dataset.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/refcount.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zfs_ioctl.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zio.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zio_compress.h vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ioctl.c vendor-sys/illumos/dist/uts/common/fs/zfs/zio.c Modified: vendor/illumos/dist/cmd/zfs/zfs_main.c ============================================================================== --- vendor/illumos/dist/cmd/zfs/zfs_main.c Fri Apr 14 18:05:20 2017 (r316893) +++ vendor/illumos/dist/cmd/zfs/zfs_main.c Fri Apr 14 18:07:43 2017 (r316894) @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -262,7 +263,7 @@ get_usage(zfs_help_t idx) case HELP_ROLLBACK: return (gettext("\trollback [-rRf] \n")); case HELP_SEND: - return (gettext("\tsend [-DnPpRvLe] [-[iI] snapshot] " + return (gettext("\tsend [-DnPpRvLec] [-[iI] snapshot] " "\n" "\tsend [-Le] [-i snapshot|bookmark] " "\n" @@ -3704,8 +3705,23 @@ zfs_do_send(int argc, char **argv) nvlist_t *dbgnv = NULL; boolean_t extraverbose = B_FALSE; + struct option long_options[] = { + {"replicate", no_argument, NULL, 'R'}, + {"props", no_argument, NULL, 'p'}, + {"parsable", no_argument, NULL, 'P'}, + {"dedup", no_argument, NULL, 'D'}, + {"verbose", no_argument, NULL, 'v'}, + {"dryrun", no_argument, NULL, 'n'}, + {"large-block", no_argument, NULL, 'L'}, + {"embed", no_argument, NULL, 'e'}, + {"resume", required_argument, NULL, 't'}, + {"compressed", no_argument, NULL, 'c'}, + {0, 0, 0, 0} + }; + /* check options */ - while ((c = getopt(argc, argv, ":i:I:RDpvnPLet:")) != -1) { + while ((c = getopt_long(argc, argv, ":i:I:RbDpvnPLet:c", long_options, + NULL)) != -1) { switch (c) { case 'i': if (fromname) @@ -3749,12 +3765,17 @@ zfs_do_send(int argc, char **argv) case 't': resume_token = optarg; break; + case 'c': + flags.compress = B_TRUE; + break; case ':': (void) fprintf(stderr, gettext("missing argument for " "'%c' option\n"), optopt); usage(B_FALSE); break; case '?': + /*FALLTHROUGH*/ + default: (void) fprintf(stderr, gettext("invalid option '%c'\n"), optopt); usage(B_FALSE); @@ -3825,6 +3846,8 @@ zfs_do_send(int argc, char **argv) lzc_flags |= LZC_SEND_FLAG_LARGE_BLOCK; if (flags.embed_data) lzc_flags |= LZC_SEND_FLAG_EMBED_DATA; + if (flags.compress) + lzc_flags |= LZC_SEND_FLAG_COMPRESS; if (fromname != NULL && (fromname[0] == '#' || fromname[0] == '@')) { Modified: vendor/illumos/dist/cmd/zstreamdump/zstreamdump.c ============================================================================== --- vendor/illumos/dist/cmd/zstreamdump/zstreamdump.c Fri Apr 14 18:05:20 2017 (r316893) +++ vendor/illumos/dist/cmd/zstreamdump/zstreamdump.c Fri Apr 14 18:07:43 2017 (r316894) @@ -25,8 +25,8 @@ */ /* - * Copyright (c) 2013, 2014 by Delphix. All rights reserved. * Copyright (c) 2014 Integros [integros.com] + * Copyright (c) 2013, 2015 by Delphix. All rights reserved. */ #include @@ -39,6 +39,7 @@ #include #include +#include #include /* @@ -251,6 +252,7 @@ main(int argc, char *argv[]) (void) fprintf(stderr, "invalid option '%c'\n", optopt); usage(); + break; } } @@ -453,38 +455,50 @@ main(int argc, char *argv[]) drrw->drr_object = BSWAP_64(drrw->drr_object); drrw->drr_type = BSWAP_32(drrw->drr_type); drrw->drr_offset = BSWAP_64(drrw->drr_offset); - drrw->drr_length = BSWAP_64(drrw->drr_length); + drrw->drr_logical_size = + BSWAP_64(drrw->drr_logical_size); drrw->drr_toguid = BSWAP_64(drrw->drr_toguid); drrw->drr_key.ddk_prop = BSWAP_64(drrw->drr_key.ddk_prop); + drrw->drr_compressed_size = + BSWAP_64(drrw->drr_compressed_size); } + + uint64_t payload_size = DRR_WRITE_PAYLOAD_SIZE(drrw); + /* * If this is verbose and/or dump output, * print info on the modified block */ if (verbose) { (void) printf("WRITE object = %llu type = %u " - "checksum type = %u\n" - " offset = %llu length = %llu " + "checksum type = %u compression type = %u\n" + " offset = %llu logical_size = %llu " + "compressed_size = %llu " + "payload_size = %llu " "props = %llx\n", (u_longlong_t)drrw->drr_object, drrw->drr_type, drrw->drr_checksumtype, + drrw->drr_compressiontype, (u_longlong_t)drrw->drr_offset, - (u_longlong_t)drrw->drr_length, + (u_longlong_t)drrw->drr_logical_size, + (u_longlong_t)drrw->drr_compressed_size, + (u_longlong_t)payload_size, (u_longlong_t)drrw->drr_key.ddk_prop); } + /* * Read the contents of the block in from STDIN to buf */ - (void) ssread(buf, drrw->drr_length, &zc); + (void) ssread(buf, payload_size, &zc); /* * If in dump mode */ if (dump) { - print_block(buf, drrw->drr_length); + print_block(buf, payload_size); } - total_write_size += drrw->drr_length; + total_write_size += payload_size; break; case DRR_WRITE_BYREF: Modified: vendor/illumos/dist/lib/libzfs/common/libzfs.h ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs.h Fri Apr 14 18:05:20 2017 (r316893) +++ vendor/illumos/dist/lib/libzfs/common/libzfs.h Fri Apr 14 18:07:43 2017 (r316894) @@ -600,6 +600,9 @@ typedef struct sendflags { /* WRITE_EMBEDDED records of type DATA are permitted */ boolean_t embed_data; + + /* compressed WRITE records are permitted */ + boolean_t compress; } sendflags_t; typedef boolean_t (snapfilter_cb_t)(zfs_handle_t *, void *); Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_sendrecv.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_sendrecv.c Fri Apr 14 18:05:20 2017 (r316893) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_sendrecv.c Fri Apr 14 18:07:43 2017 (r316894) @@ -347,8 +347,10 @@ cksummer(void *arg) { struct drr_write *drrw = &drr->drr_u.drr_write; dataref_t dataref; + uint64_t payload_size; - (void) ssread(buf, drrw->drr_length, ofp); + payload_size = DRR_WRITE_PAYLOAD_SIZE(drrw); + (void) ssread(buf, payload_size, ofp); /* * Use the existing checksum if it's dedup-capable, @@ -362,7 +364,7 @@ cksummer(void *arg) zio_cksum_t tmpsha256; SHA256Init(&ctx); - SHA256Update(&ctx, buf, drrw->drr_length); + SHA256Update(&ctx, buf, payload_size); SHA256Final(&tmpsha256, &ctx); drrw->drr_key.ddk_cksum.zc_word[0] = BE_64(tmpsha256.zc_word[0]); @@ -392,7 +394,7 @@ cksummer(void *arg) wbr_drrr->drr_object = drrw->drr_object; wbr_drrr->drr_offset = drrw->drr_offset; - wbr_drrr->drr_length = drrw->drr_length; + wbr_drrr->drr_length = drrw->drr_logical_size; wbr_drrr->drr_toguid = drrw->drr_toguid; wbr_drrr->drr_refguid = dataref.ref_guid; wbr_drrr->drr_refobject = @@ -414,7 +416,7 @@ cksummer(void *arg) goto out; } else { /* block not previously seen */ - if (dump_record(drr, buf, drrw->drr_length, + if (dump_record(drr, buf, payload_size, &stream_cksum, outfd) != 0) goto out; } @@ -917,7 +919,7 @@ typedef struct send_dump_data { uint64_t prevsnap_obj; boolean_t seenfrom, seento, replicate, doall, fromorigin; boolean_t verbose, dryrun, parsable, progress, embed_data, std_out; - boolean_t large_block; + boolean_t large_block, compress; int outfd; boolean_t err; nvlist_t *fss; @@ -933,7 +935,7 @@ typedef struct send_dump_data { static int estimate_ioctl(zfs_handle_t *zhp, uint64_t fromsnap_obj, - boolean_t fromorigin, uint64_t *sizep) + boolean_t fromorigin, enum lzc_send_flags flags, uint64_t *sizep) { zfs_cmd_t zc = { 0 }; libzfs_handle_t *hdl = zhp->zfs_hdl; @@ -946,6 +948,7 @@ estimate_ioctl(zfs_handle_t *zhp, uint64 zc.zc_sendobj = zfs_prop_get_int(zhp, ZFS_PROP_OBJSETID); zc.zc_fromobj = fromsnap_obj; zc.zc_guid = 1; /* estimate flag */ + zc.zc_flags = flags; if (zfs_ioctl(zhp->zfs_hdl, ZFS_IOC_SEND, &zc) != 0) { char errbuf[1024]; @@ -1184,6 +1187,7 @@ dump_snapshot(zfs_handle_t *zhp, void *a progress_arg_t pa = { 0 }; pthread_t tid; char *thissnap; + enum lzc_send_flags flags = 0; int err; boolean_t isfromsnap, istosnap, fromorigin; boolean_t exclude = B_FALSE; @@ -1212,6 +1216,13 @@ dump_snapshot(zfs_handle_t *zhp, void *a if (istosnap) sdd->seento = B_TRUE; + if (sdd->large_block) + flags |= LZC_SEND_FLAG_LARGE_BLOCK; + if (sdd->embed_data) + flags |= LZC_SEND_FLAG_EMBED_DATA; + if (sdd->compress) + flags |= LZC_SEND_FLAG_COMPRESS; + if (!sdd->doall && !isfromsnap && !istosnap) { if (sdd->replicate) { char *snapname; @@ -1258,7 +1269,7 @@ dump_snapshot(zfs_handle_t *zhp, void *a if (sdd->verbose) { uint64_t size = 0; (void) estimate_ioctl(zhp, sdd->prevsnap_obj, - fromorigin, &size); + fromorigin, flags, &size); send_print_verbose(fout, zhp->zfs_name, sdd->prevsnap[0] ? sdd->prevsnap : NULL, @@ -1283,12 +1294,6 @@ dump_snapshot(zfs_handle_t *zhp, void *a } } - enum lzc_send_flags flags = 0; - if (sdd->large_block) - flags |= LZC_SEND_FLAG_LARGE_BLOCK; - if (sdd->embed_data) - flags |= LZC_SEND_FLAG_EMBED_DATA; - err = dump_ioctl(zhp, sdd->prevsnap, sdd->prevsnap_obj, fromorigin, sdd->outfd, flags, sdd->debugnv); @@ -1594,8 +1599,12 @@ zfs_send_resume(libzfs_handle_t *hdl, se fromguid = 0; (void) nvlist_lookup_uint64(resume_nvl, "fromguid", &fromguid); + if (flags->largeblock || nvlist_exists(resume_nvl, "largeblockok")) + lzc_flags |= LZC_SEND_FLAG_LARGE_BLOCK; if (flags->embed_data || nvlist_exists(resume_nvl, "embedok")) lzc_flags |= LZC_SEND_FLAG_EMBED_DATA; + if (flags->compress || nvlist_exists(resume_nvl, "compressok")) + lzc_flags |= LZC_SEND_FLAG_COMPRESS; if (guid_to_name(hdl, toname, toguid, B_FALSE, name) != 0) { if (zfs_dataset_exists(hdl, toname, ZFS_TYPE_DATASET)) { @@ -1628,7 +1637,8 @@ zfs_send_resume(libzfs_handle_t *hdl, se if (flags->verbose) { uint64_t size = 0; - error = lzc_send_space(zhp->zfs_name, fromname, &size); + error = lzc_send_space(zhp->zfs_name, fromname, + lzc_flags, &size); if (error == 0) size = MAX(0, (int64_t)(size - bytes)); send_print_verbose(stderr, zhp->zfs_name, fromname, @@ -1856,6 +1866,7 @@ zfs_send(zfs_handle_t *zhp, const char * sdd.dryrun = flags->dryrun; sdd.large_block = flags->largeblock; sdd.embed_data = flags->embed_data; + sdd.compress = flags->compress; sdd.filter_cb = filter_func; sdd.filter_cb_arg = cb_arg; if (debugnvp) @@ -2921,11 +2932,17 @@ recv_skip(libzfs_handle_t *hdl, int fd, case DRR_WRITE: if (byteswap) { - drr->drr_u.drr_write.drr_length = - BSWAP_64(drr->drr_u.drr_write.drr_length); + drr->drr_u.drr_write.drr_logical_size = + BSWAP_64( + drr->drr_u.drr_write.drr_logical_size); + drr->drr_u.drr_write.drr_compressed_size = + BSWAP_64( + drr->drr_u.drr_write.drr_compressed_size); } + uint64_t payload_size = + DRR_WRITE_PAYLOAD_SIZE(&drr->drr_u.drr_write); (void) recv_read(hdl, fd, buf, - drr->drr_u.drr_write.drr_length, B_FALSE, NULL); + payload_size, B_FALSE, NULL); break; case DRR_SPILL: if (byteswap) { Modified: vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.c ============================================================================== --- vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.c Fri Apr 14 18:05:20 2017 (r316893) +++ vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.c Fri Apr 14 18:07:43 2017 (r316894) @@ -487,6 +487,8 @@ lzc_send_resume(const char *snapname, co fnvlist_add_boolean(args, "largeblockok"); if (flags & LZC_SEND_FLAG_EMBED_DATA) fnvlist_add_boolean(args, "embedok"); + if (flags & LZC_SEND_FLAG_COMPRESS) + fnvlist_add_boolean(args, "compressok"); if (resumeobj != 0 || resumeoff != 0) { fnvlist_add_uint64(args, "resume_object", resumeobj); fnvlist_add_uint64(args, "resume_offset", resumeoff); @@ -512,7 +514,8 @@ lzc_send_resume(const char *snapname, co * an equivalent snapshot. */ int -lzc_send_space(const char *snapname, const char *from, uint64_t *spacep) +lzc_send_space(const char *snapname, const char *from, + enum lzc_send_flags flags, uint64_t *spacep) { nvlist_t *args; nvlist_t *result; @@ -521,6 +524,12 @@ lzc_send_space(const char *snapname, con args = fnvlist_alloc(); if (from != NULL) fnvlist_add_string(args, "from", from); + if (flags & LZC_SEND_FLAG_LARGE_BLOCK) + fnvlist_add_boolean(args, "largeblockok"); + if (flags & LZC_SEND_FLAG_EMBED_DATA) + fnvlist_add_boolean(args, "embedok"); + if (flags & LZC_SEND_FLAG_COMPRESS) + fnvlist_add_boolean(args, "compressok"); err = lzc_ioctl(ZFS_IOC_SEND_SPACE, snapname, args, &result); nvlist_free(args); if (err == 0) Modified: vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.h ============================================================================== --- vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.h Fri Apr 14 18:05:20 2017 (r316893) +++ vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.h Fri Apr 14 18:07:43 2017 (r316894) @@ -62,13 +62,14 @@ int lzc_get_holds(const char *, nvlist_t enum lzc_send_flags { LZC_SEND_FLAG_EMBED_DATA = 1 << 0, - LZC_SEND_FLAG_LARGE_BLOCK = 1 << 1 + LZC_SEND_FLAG_LARGE_BLOCK = 1 << 1, + LZC_SEND_FLAG_COMPRESS = 1 << 2 }; int lzc_send(const char *, const char *, int, enum lzc_send_flags); int lzc_send_resume(const char *, const char *, int, enum lzc_send_flags, uint64_t, uint64_t); -int lzc_send_space(const char *, const char *, uint64_t *); +int lzc_send_space(const char *, const char *, enum lzc_send_flags, uint64_t *); struct dmu_replay_record; Modified: vendor/illumos/dist/man/man1m/zfs.1m ============================================================================== --- vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 18:05:20 2017 (r316893) +++ vendor/illumos/dist/man/man1m/zfs.1m Fri Apr 14 18:07:43 2017 (r316894) @@ -165,12 +165,12 @@ .Ar snapshot bookmark .Nm .Cm send -.Op Fl DLPRenpv +.Op Fl DLPRcenpv .Op Oo Fl I Ns | Ns Fl i Oc Ar snapshot .Ar snapshot .Nm .Cm send -.Op Fl Le +.Op Fl Lce .Op Fl i Ar snapshot Ns | Ns Ar bookmark .Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot .Nm @@ -2450,7 +2450,7 @@ feature. .It Xo .Nm .Cm send -.Op Fl DLPRenpv +.Op Fl DLPRcenpv .Op Oo Fl I Ns | Ns Fl i Oc Ar snapshot .Ar snapshot .Xc @@ -2463,7 +2463,7 @@ to a different system .Pc . By default, a full stream is generated. .Bl -tag -width "-D" -.It Fl D +.It Fl D, -dedup Generate a deduplicated stream. Blocks which would have been sent multiple times in the send stream will only be sent once. The receiving system must also support this feature to receive a deduplicated stream. This flag can be used @@ -2483,7 +2483,7 @@ is similar to The incremental source may be specified as with the .Fl i option. -.It Fl L +.It Fl L, -large-block Generate a stream which may contain blocks larger than 128KB. This flag has no effect if the .Sy large_blocks @@ -2497,9 +2497,9 @@ pool feature enabled as well. See for details on ZFS feature flags and the .Sy large_blocks feature. -.It Fl P +.It Fl P, -parsable Print machine-parsable verbose information about the stream package generated. -.It Fl R +.It Fl R, -replicate Generate a replication stream package, which will replicate the specified file system, and all descendent file systems, up to the named snapshot. When received, all properties, snapshots, descendent file systems, and clones are @@ -2517,7 +2517,7 @@ is received. If the .Fl F flag is specified when this stream is received, snapshots and file systems that do not exist on the sending side are destroyed. -.It Fl e +.It Fl e, -embed Generate a more compact stream by using .Sy WRITE_EMBEDDED records for blocks which are stored more compactly on disk by the @@ -2534,6 +2534,16 @@ that feature enabled as well. See for details on ZFS feature flags and the .Sy embedded_data feature. +.It Fl c, -compressed +Generate a more compact stream by using compressed WRITE records for blocks +which are compressed on disk and in memory (see the +.Sy compression No property for details). If the Sy lz4_compress No feature +is active on the sending system, then the receiving system must have that +feature enabled as well. If the +.Sy large_blocks No feature is enabled on the sending system but the Fl L +option is not supplied in conjunction with +.Fl c, No then the data will be decompressed before sending so it can be split +into smaller block sizes. .It Fl i Ar snapshot Generate an incremental stream from the first .Ar snapshot @@ -2556,7 +2566,7 @@ be fully specified not just .Em @origin .Pc . -.It Fl n +.It Fl n, -dryrun Do a dry-run .Pq Qq No-op send. Do not generate any actual send data. This is useful in conjunction with @@ -2569,11 +2579,11 @@ be written to standard output .Po contrast with a non-dry-run, where the stream is written to standard output and the verbose output goes to standard error .Pc . -.It Fl p +.It Fl p, -props Include the dataset's properties in the stream. This flag is implicit when .Fl R is specified. The receiving system must also support this feature. -.It Fl v +.It Fl v, -verbose Print verbose information about the stream package generated. This information includes a per-second report of how much data has been sent. .Pp @@ -2583,7 +2593,7 @@ on future versions of ZFS . .It Xo .Nm .Cm send -.Op Fl Le +.Op Fl Lce .Op Fl i Ar snapshot Ns | Ns Ar bookmark .Ar filesystem Ns | Ns Ar volume Ns | Ns Ar snapshot .Xc @@ -2593,7 +2603,7 @@ read-only, or the filesystem must not be a filesystem or volume is received, the default snapshot name will be .Qq --head-- . .Bl -tag -width "-L" -.It Fl L +.It Fl L, -large-block Generate a stream which may contain blocks larger than 128KB. This flag has no effect if the .Sy large_blocks @@ -2607,7 +2617,17 @@ pool feature enabled as well. See for details on ZFS feature flags and the .Sy large_blocks feature. -.It Fl e +.It Fl c, -compressed +Generate a more compact stream by using compressed WRITE records for blocks +which are compressed on disk and in memory (see the +.Sy compression No property for details). If the Sy lz4_compress No feature is +active on the sending system, then the receiving system must have that feature +enabled as well. If the +.Sy large_blocks No feature is enabled on the sending system but the Fl L +option is not supplied in conjunction with +.Fl c, No then the data will be decompressed before sending so it can be split +into smaller block sizes. +.It Fl e, -embed Generate a more compact stream by using .Sy WRITE_EMBEDDED records for blocks which are stored more compactly on disk by the From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:07:44 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id C43CFD3E6A8; Fri, 14 Apr 2017 18:07:44 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 61FC21671; Fri, 14 Apr 2017 18:07:44 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EI7hMB061724; Fri, 14 Apr 2017 18:07:43 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EI7h4L061722; Fri, 14 Apr 2017 18:07:43 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141807.v3EI7h4L061722@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:07:43 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316894 - vendor-sys/illumos/dist/uts/common/fs/zfs vendor-sys/illumos/dist/uts/common/fs/zfs/sys vendor/illumos/dist/cmd/zfs vendor/illumos/dist/cmd/zstreamdump vendor/illumos/dist/lib... X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:07:44 -0000 Author: avg Date: Fri Apr 14 18:07:43 2017 New Revision: 316894 URL: https://svnweb.freebsd.org/changeset/base/316894 Log: 7252 7628 compressed zfs send / receive illumos/illumos-gate@5602294fda888d923d57a78bafdaf48ae6223dea https://github.com/illumos/illumos-gate/commit/5602294fda888d923d57a78bafdaf48ae6223dea https://www.illumos.org/issues/7252 This feature includes code to allow a system with compressed ARC enabled to send data in its compressed form straight out of the ARC, and receive data in its compressed form directly into the ARC. https://www.illumos.org/issues/7628 We should have longer, more readable versions of the ZFS send / recv options. 7628 create long versions of ZFS send / receive options Reviewed by: George Wilson Reviewed by: John Kennedy Reviewed by: Matthew Ahrens Reviewed by: Paul Dagnelie Reviewed by: Pavel Zakharov Reviewed by: Sebastien Roy Reviewed by: David Quigley Reviewed by: Thomas Caputi Approved by: Dan McDonald Author: Dan Kimmel Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c vendor-sys/illumos/dist/uts/common/fs/zfs/dmu.c vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_send.c vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_dataset.c vendor-sys/illumos/dist/uts/common/fs/zfs/lz4.c vendor-sys/illumos/dist/uts/common/fs/zfs/sys/arc.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu_send.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dsl_dataset.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/refcount.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zfs_ioctl.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zio.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zio_compress.h vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ioctl.c vendor-sys/illumos/dist/uts/common/fs/zfs/zio.c Changes in other areas also in this revision: Modified: vendor/illumos/dist/cmd/zfs/zfs_main.c vendor/illumos/dist/cmd/zstreamdump/zstreamdump.c vendor/illumos/dist/lib/libzfs/common/libzfs.h vendor/illumos/dist/lib/libzfs/common/libzfs_sendrecv.c vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.c vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.h vendor/illumos/dist/man/man1m/zfs.1m Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c Fri Apr 14 18:05:20 2017 (r316893) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c Fri Apr 14 18:07:43 2017 (r316894) @@ -77,10 +77,10 @@ * A new reference to a cache buffer can be obtained in two * ways: 1) via a hash table lookup using the DVA as a key, * or 2) via one of the ARC lists. The arc_read() interface - * uses method 1, while the internal arc algorithms for + * uses method 1, while the internal ARC algorithms for * adjusting the cache use method 2. We therefore provide two * types of locks: 1) the hash table lock array, and 2) the - * arc list locks. + * ARC list locks. * * Buffers do not have their own mutexes, rather they rely on the * hash table mutexes for the bulk of their protection (i.e. most @@ -93,21 +93,12 @@ * buf_hash_remove() expects the appropriate hash mutex to be * already held before it is invoked. * - * Each arc state also has a mutex which is used to protect the + * Each ARC state also has a mutex which is used to protect the * buffer list associated with the state. When attempting to - * obtain a hash table lock while holding an arc list lock you + * obtain a hash table lock while holding an ARC list lock you * must use: mutex_tryenter() to avoid deadlock. Also note that * the active state mutex must be held before the ghost state mutex. * - * Arc buffers may have an associated eviction callback function. - * This function will be invoked prior to removing the buffer (e.g. - * in arc_do_user_evicts()). Note however that the data associated - * with the buffer may be evicted prior to the callback. The callback - * must be made with *no locks held* (to prevent deadlock). Additionally, - * the users of callbacks must ensure that their private data is - * protected from simultaneous callbacks from arc_clear_callback() - * and arc_do_user_evicts(). - * * Note that the majority of the performance stats are manipulated * with atomic operations. * @@ -136,67 +127,81 @@ * are cached in the L1ARC. The L1ARC (l1arc_buf_hdr_t) is a structure within * the arc_buf_hdr_t that will point to the data block in memory. A block can * only be read by a consumer if it has an l1arc_buf_hdr_t. The L1ARC - * caches data in two ways -- in a list of arc buffers (arc_buf_t) and + * caches data in two ways -- in a list of ARC buffers (arc_buf_t) and * also in the arc_buf_hdr_t's private physical data block pointer (b_pdata). - * Each arc buffer (arc_buf_t) is being actively accessed by a specific ARC - * consumer, and always contains uncompressed data. The ARC will provide - * references to this data and will keep it cached until it is no longer in - * use. Typically, the arc will try to cache only the L1ARC's physical data - * block and will aggressively evict any arc_buf_t that is no longer referenced. - * The amount of memory consumed by the arc_buf_t's can be seen via the - * "overhead_size" kstat. - * * - * arc_buf_hdr_t - * +-----------+ - * | | - * | | - * | | - * +-----------+ - * l2arc_buf_hdr_t| | - * | | - * +-----------+ - * l1arc_buf_hdr_t| | - * | | arc_buf_t - * | b_buf +------------>+---------+ arc_buf_t - * | | |b_next +---->+---------+ - * | b_pdata +-+ |---------| |b_next +-->NULL - * +-----------+ | | | +---------+ - * | |b_data +-+ | | - * | +---------+ | |b_data +-+ - * +->+------+ | +---------+ | - * (potentially) | | | | - * compressed | | | | - * data +------+ | v - * +->+------+ +------+ - * uncompressed | | | | - * data | | | | - * +------+ +------+ - * - * The L1ARC's data pointer, however, may or may not be uncompressed. The - * ARC has the ability to store the physical data (b_pdata) associated with - * the DVA of the arc_buf_hdr_t. Since the b_pdata is a copy of the on-disk - * physical block, it will match its on-disk compression characteristics. - * If the block on-disk is compressed, then the physical data block - * in the cache will also be compressed and vice-versa. This behavior - * can be disabled by setting 'zfs_compressed_arc_enabled' to B_FALSE. When the + * The L1ARC's data pointer may or may not be uncompressed. The ARC has the + * ability to store the physical data (b_pdata) associated with the DVA of the + * arc_buf_hdr_t. Since the b_pdata is a copy of the on-disk physical block, + * it will match its on-disk compression characteristics. This behavior can be + * disabled by setting 'zfs_compressed_arc_enabled' to B_FALSE. When the * compressed ARC functionality is disabled, the b_pdata will point to an * uncompressed version of the on-disk data. * + * Data in the L1ARC is not accessed by consumers of the ARC directly. Each + * arc_buf_hdr_t can have multiple ARC buffers (arc_buf_t) which reference it. + * Each ARC buffer (arc_buf_t) is being actively accessed by a specific ARC + * consumer. The ARC will provide references to this data and will keep it + * cached until it is no longer in use. The ARC caches only the L1ARC's physical + * data block and will evict any arc_buf_t that is no longer referenced. The + * amount of memory consumed by the arc_buf_ts' data buffers can be seen via the + * "overhead_size" kstat. + * + * Depending on the consumer, an arc_buf_t can be requested in uncompressed or + * compressed form. The typical case is that consumers will want uncompressed + * data, and when that happens a new data buffer is allocated where the data is + * decompressed for them to use. Currently the only consumer who wants + * compressed arc_buf_t's is "zfs send", when it streams data exactly as it + * exists on disk. When this happens, the arc_buf_t's data buffer is shared + * with the arc_buf_hdr_t. + * + * Here is a diagram showing an arc_buf_hdr_t referenced by two arc_buf_t's. The + * first one is owned by a compressed send consumer (and therefore references + * the same compressed data buffer as the arc_buf_hdr_t) and the second could be + * used by any other consumer (and has its own uncompressed copy of the data + * buffer). + * + * arc_buf_hdr_t + * +-----------+ + * | fields | + * | common to | + * | L1- and | + * | L2ARC | + * +-----------+ + * | l2arc_buf_hdr_t + * | | + * +-----------+ + * | l1arc_buf_hdr_t + * | | arc_buf_t + * | b_buf +------------>+-----------+ arc_buf_t + * | b_pdata +-+ |b_next +---->+-----------+ + * +-----------+ | |-----------| |b_next +-->NULL + * | |b_comp = T | +-----------+ + * | |b_data +-+ |b_comp = F | + * | +-----------+ | |b_data +-+ + * +->+------+ | +-----------+ | + * compressed | | | | + * data | |<--------------+ | uncompressed + * +------+ compressed, | data + * shared +-->+------+ + * data | | + * | | + * +------+ + * * When a consumer reads a block, the ARC must first look to see if the - * arc_buf_hdr_t is cached. If the hdr is cached and already has an arc_buf_t, - * then an additional arc_buf_t is allocated and the uncompressed data is - * bcopied from the existing arc_buf_t. If the hdr is cached but does not - * have an arc_buf_t, then the ARC allocates a new arc_buf_t and decompresses - * the b_pdata contents into the arc_buf_t's b_data. If the arc_buf_hdr_t's - * b_pdata is not compressed, then the block is shared with the newly - * allocated arc_buf_t. This block sharing only occurs with one arc_buf_t - * in the arc buffer chain. Sharing the block reduces the memory overhead - * required when the hdr is caching uncompressed blocks or the compressed - * arc functionality has been disabled via 'zfs_compressed_arc_enabled'. + * arc_buf_hdr_t is cached. If the hdr is cached then the ARC allocates a new + * arc_buf_t and either copies uncompressed data into a new data buffer from an + * existing uncompressed arc_buf_t, decompresses the hdr's b_pdata buffer into a + * new data buffer, or shares the hdr's b_pdata buffer, depending on whether the + * hdr is compressed and the desired compression characteristics of the + * arc_buf_t consumer. If the arc_buf_t ends up sharing data with the + * arc_buf_hdr_t and both of them are uncompressed then the arc_buf_t must be + * the last buffer in the hdr's b_buf list, however a shared compressed buf can + * be anywhere in the hdr's list. * * The diagram below shows an example of an uncompressed ARC hdr that is - * sharing its data with an arc_buf_t: + * sharing its data with an arc_buf_t (note that the shared uncompressed buf is + * the last element in the buf list): * * arc_buf_hdr_t * +-----------+ @@ -225,20 +230,24 @@ * | +------+ | * +---------------------------------+ * - * Writing to the arc requires that the ARC first discard the b_pdata + * Writing to the ARC requires that the ARC first discard the hdr's b_pdata * since the physical block is about to be rewritten. The new data contents - * will be contained in the arc_buf_t (uncompressed). As the I/O pipeline - * performs the write, it may compress the data before writing it to disk. - * The ARC will be called with the transformed data and will bcopy the - * transformed on-disk block into a newly allocated b_pdata. + * will be contained in the arc_buf_t. As the I/O pipeline performs the write, + * it may compress the data before writing it to disk. The ARC will be called + * with the transformed data and will bcopy the transformed on-disk block into + * a newly allocated b_pdata. Writes are always done into buffers which have + * either been loaned (and hence are new and don't have other readers) or + * buffers which have been released (and hence have their own hdr, if there + * were originally other readers of the buf's original hdr). This ensures that + * the ARC only needs to update a single buf and its hdr after a write occurs. * * When the L2ARC is in use, it will also take advantage of the b_pdata. The * L2ARC will always write the contents of b_pdata to the L2ARC. This means - * that when compressed arc is enabled that the L2ARC blocks are identical + * that when compressed ARC is enabled that the L2ARC blocks are identical * to the on-disk block in the main data pool. This provides a significant * advantage since the ARC can leverage the bp's checksum when reading from the * L2ARC to determine if the contents are valid. However, if the compressed - * arc is disabled, then the L2ARC's block must be transformed to look + * ARC is disabled, then the L2ARC's block must be transformed to look * like the physical block in the main data pool before comparing the * checksum and determining its validity. */ @@ -804,6 +813,7 @@ struct arc_callback { void *acb_private; arc_done_func_t *acb_done; arc_buf_t *acb_buf; + boolean_t acb_compressed; zio_t *acb_zio_dummy; arc_callback_t *acb_next; }; @@ -855,7 +865,7 @@ typedef struct l1arc_buf_hdr { zio_cksum_t *b_freeze_cksum; #ifdef ZFS_DEBUG /* - * used for debugging wtih kmem_flags - by allocating and freeing + * Used for debugging with kmem_flags - by allocating and freeing * b_thawed when the buffer is thawed, we get a record of the stack * trace that thawed it. */ @@ -970,6 +980,8 @@ struct arc_buf_hdr { HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS, (cmp)); #define ARC_BUF_LAST(buf) ((buf)->b_next == NULL) +#define ARC_BUF_SHARED(buf) ((buf)->b_flags & ARC_BUF_FLAG_SHARED) +#define ARC_BUF_COMPRESSED(buf) ((buf)->b_flags & ARC_BUF_FLAG_COMPRESSED) /* * Other sizes @@ -1064,7 +1076,7 @@ static kmutex_t l2arc_free_on_write_mtx; static uint64_t l2arc_ndev; /* number of devices */ typedef struct l2arc_read_callback { - arc_buf_hdr_t *l2rcb_hdr; /* read buffer */ + arc_buf_hdr_t *l2rcb_hdr; /* read header */ blkptr_t l2rcb_bp; /* original blkptr */ zbookmark_phys_t l2rcb_zb; /* original bookmark */ int l2rcb_flags; /* original flags */ @@ -1399,6 +1411,31 @@ retry: } } +/* + * This is the size that the buf occupies in memory. If the buf is compressed, + * it will correspond to the compressed size. You should use this method of + * getting the buf size unless you explicitly need the logical size. + */ +int32_t +arc_buf_size(arc_buf_t *buf) +{ + return (ARC_BUF_COMPRESSED(buf) ? + HDR_GET_PSIZE(buf->b_hdr) : HDR_GET_LSIZE(buf->b_hdr)); +} + +int32_t +arc_buf_lsize(arc_buf_t *buf) +{ + return (HDR_GET_LSIZE(buf->b_hdr)); +} + +enum zio_compress +arc_get_compression(arc_buf_t *buf) +{ + return (ARC_BUF_COMPRESSED(buf) ? + HDR_GET_COMPRESS(buf->b_hdr) : ZIO_COMPRESS_OFF); +} + #define ARC_MINTIME (hz>>4) /* 62 ms */ static inline boolean_t @@ -1407,9 +1444,21 @@ arc_buf_is_shared(arc_buf_t *buf) boolean_t shared = (buf->b_data != NULL && buf->b_data == buf->b_hdr->b_l1hdr.b_pdata); IMPLY(shared, HDR_SHARED_DATA(buf->b_hdr)); + IMPLY(shared, ARC_BUF_SHARED(buf)); + IMPLY(shared, ARC_BUF_COMPRESSED(buf) || ARC_BUF_LAST(buf)); + + /* + * It would be nice to assert arc_can_share() too, but the "hdr isn't + * already being shared" requirement prevents us from doing that. + */ + return (shared); } +/* + * Free the checksum associated with this header. If there is no checksum, this + * is a no-op. + */ static inline void arc_cksum_free(arc_buf_hdr_t *hdr) { @@ -1422,6 +1471,25 @@ arc_cksum_free(arc_buf_hdr_t *hdr) mutex_exit(&hdr->b_l1hdr.b_freeze_lock); } +/* + * Return true iff at least one of the bufs on hdr is not compressed. + */ +static boolean_t +arc_hdr_has_uncompressed_buf(arc_buf_hdr_t *hdr) +{ + for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) { + if (!ARC_BUF_COMPRESSED(b)) { + return (B_TRUE); + } + } + return (B_FALSE); +} + +/* + * If we've turned on the ZFS_DEBUG_MODIFY flag, verify that the buf's data + * matches the checksum that is stored in the hdr. If there is no checksum, + * or if the buf is compressed, this is a no-op. + */ static void arc_cksum_verify(arc_buf_t *buf) { @@ -1431,6 +1499,12 @@ arc_cksum_verify(arc_buf_t *buf) if (!(zfs_flags & ZFS_DEBUG_MODIFY)) return; + if (ARC_BUF_COMPRESSED(buf)) { + ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL || + arc_hdr_has_uncompressed_buf(hdr)); + return; + } + ASSERT(HDR_HAS_L1HDR(hdr)); mutex_enter(&hdr->b_l1hdr.b_freeze_lock); @@ -1438,7 +1512,8 @@ arc_cksum_verify(arc_buf_t *buf) mutex_exit(&hdr->b_l1hdr.b_freeze_lock); return; } - fletcher_2_native(buf->b_data, HDR_GET_LSIZE(hdr), NULL, &zc); + + fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL, &zc); if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc)) panic("buffer modified while frozen!"); mutex_exit(&hdr->b_l1hdr.b_freeze_lock); @@ -1512,6 +1587,12 @@ arc_cksum_is_equal(arc_buf_hdr_t *hdr, z return (valid_cksum); } +/* + * Given a buf full of data, if ZFS_DEBUG_MODIFY is enabled this computes a + * checksum and attaches it to the buf's hdr so that we can ensure that the buf + * isn't modified later on. If buf is compressed or there is already a checksum + * on the hdr, this is a no-op (we only checksum uncompressed bufs). + */ static void arc_cksum_compute(arc_buf_t *buf) { @@ -1521,14 +1602,21 @@ arc_cksum_compute(arc_buf_t *buf) return; ASSERT(HDR_HAS_L1HDR(hdr)); + mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock); if (hdr->b_l1hdr.b_freeze_cksum != NULL) { + ASSERT(arc_hdr_has_uncompressed_buf(hdr)); + mutex_exit(&hdr->b_l1hdr.b_freeze_lock); + return; + } else if (ARC_BUF_COMPRESSED(buf)) { mutex_exit(&hdr->b_l1hdr.b_freeze_lock); return; } + + ASSERT(!ARC_BUF_COMPRESSED(buf)); hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP); - fletcher_2_native(buf->b_data, HDR_GET_LSIZE(hdr), NULL, + fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL, hdr->b_l1hdr.b_freeze_cksum); mutex_exit(&hdr->b_l1hdr.b_freeze_lock); arc_buf_watch(buf); @@ -1569,7 +1657,7 @@ arc_buf_watch(arc_buf_t *buf) procctl_t ctl; ctl.cmd = PCWATCH; ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data; - ctl.prwatch.pr_size = HDR_GET_LSIZE(buf->b_hdr); + ctl.prwatch.pr_size = arc_buf_size(buf); ctl.prwatch.pr_wflags = WA_WRITE; result = write(arc_procfd, &ctl, sizeof (ctl)); ASSERT3U(result, ==, sizeof (ctl)); @@ -1590,6 +1678,12 @@ arc_buf_type(arc_buf_hdr_t *hdr) return (type); } +boolean_t +arc_is_metadata(arc_buf_t *buf) +{ + return (HDR_ISTYPE_METADATA(buf->b_hdr) != 0); +} + static uint32_t arc_bufc_to_flags(arc_buf_contents_t type) { @@ -1611,12 +1705,19 @@ arc_buf_thaw(arc_buf_t *buf) { arc_buf_hdr_t *hdr = buf->b_hdr; - if (zfs_flags & ZFS_DEBUG_MODIFY) { - if (hdr->b_l1hdr.b_state != arc_anon) - panic("modifying non-anon buffer!"); - if (HDR_IO_IN_PROGRESS(hdr)) - panic("modifying buffer while i/o in progress!"); - arc_cksum_verify(buf); + ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); + ASSERT(!HDR_IO_IN_PROGRESS(hdr)); + + arc_cksum_verify(buf); + + /* + * Compressed buffers do not manipulate the b_freeze_cksum or + * allocate b_thawed. + */ + if (ARC_BUF_COMPRESSED(buf)) { + ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL || + arc_hdr_has_uncompressed_buf(hdr)); + return; } ASSERT(HDR_HAS_L1HDR(hdr)); @@ -1645,6 +1746,12 @@ arc_buf_freeze(arc_buf_t *buf) if (!(zfs_flags & ZFS_DEBUG_MODIFY)) return; + if (ARC_BUF_COMPRESSED(buf)) { + ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL || + arc_hdr_has_uncompressed_buf(hdr)); + return; + } + hash_lock = HDR_LOCK(hdr); mutex_enter(hash_lock); @@ -1653,7 +1760,6 @@ arc_buf_freeze(arc_buf_t *buf) hdr->b_l1hdr.b_state == arc_anon); arc_cksum_compute(buf); mutex_exit(hash_lock); - } /* @@ -1710,47 +1816,157 @@ arc_hdr_set_compress(arc_buf_hdr_t *hdr, } } +/* + * Looks for another buf on the same hdr which has the data decompressed, copies + * from it, and returns true. If no such buf exists, returns false. + */ +static boolean_t +arc_buf_try_copy_decompressed_data(arc_buf_t *buf) +{ + arc_buf_hdr_t *hdr = buf->b_hdr; + boolean_t copied = B_FALSE; + + ASSERT(HDR_HAS_L1HDR(hdr)); + ASSERT3P(buf->b_data, !=, NULL); + ASSERT(!ARC_BUF_COMPRESSED(buf)); + + for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL; + from = from->b_next) { + /* can't use our own data buffer */ + if (from == buf) { + continue; + } + + if (!ARC_BUF_COMPRESSED(from)) { + bcopy(from->b_data, buf->b_data, arc_buf_size(buf)); + copied = B_TRUE; + break; + } + } + + /* + * There were no decompressed bufs, so there should not be a + * checksum on the hdr either. + */ + EQUIV(!copied, hdr->b_l1hdr.b_freeze_cksum == NULL); + + return (copied); +} + +/* + * Given a buf that has a data buffer attached to it, this function will + * efficiently fill the buf with data of the specified compression setting from + * the hdr and update the hdr's b_freeze_cksum if necessary. If the buf and hdr + * are already sharing a data buf, no copy is performed. + * + * If the buf is marked as compressed but uncompressed data was requested, this + * will allocate a new data buffer for the buf, remove that flag, and fill the + * buf with uncompressed data. You can't request a compressed buf on a hdr with + * uncompressed data, and (since we haven't added support for it yet) if you + * want compressed data your buf must already be marked as compressed and have + * the correct-sized data buffer. + */ static int -arc_decompress(arc_buf_t *buf) +arc_buf_fill(arc_buf_t *buf, boolean_t compressed) { arc_buf_hdr_t *hdr = buf->b_hdr; + boolean_t hdr_compressed = (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF); dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap; - int error; - if (arc_buf_is_shared(buf)) { - ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF); - } else if (HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF) { - /* - * The arc_buf_hdr_t is either not compressed or is - * associated with an embedded block or a hole in which - * case they remain anonymous. - */ - IMPLY(HDR_COMPRESSION_ENABLED(hdr), HDR_GET_PSIZE(hdr) == 0 || - HDR_GET_PSIZE(hdr) == HDR_GET_LSIZE(hdr)); - ASSERT(!HDR_SHARED_DATA(hdr)); - bcopy(hdr->b_l1hdr.b_pdata, buf->b_data, HDR_GET_LSIZE(hdr)); + ASSERT3P(buf->b_data, !=, NULL); + IMPLY(compressed, hdr_compressed); + IMPLY(compressed, ARC_BUF_COMPRESSED(buf)); + + if (hdr_compressed == compressed) { + if (!arc_buf_is_shared(buf)) { + bcopy(hdr->b_l1hdr.b_pdata, buf->b_data, + arc_buf_size(buf)); + } } else { - ASSERT(!HDR_SHARED_DATA(hdr)); + ASSERT(hdr_compressed); + ASSERT(!compressed); ASSERT3U(HDR_GET_LSIZE(hdr), !=, HDR_GET_PSIZE(hdr)); - error = zio_decompress_data(HDR_GET_COMPRESS(hdr), - hdr->b_l1hdr.b_pdata, buf->b_data, HDR_GET_PSIZE(hdr), - HDR_GET_LSIZE(hdr)); - if (error != 0) { - zfs_dbgmsg("hdr %p, compress %d, psize %d, lsize %d", - hdr, HDR_GET_COMPRESS(hdr), HDR_GET_PSIZE(hdr), - HDR_GET_LSIZE(hdr)); - return (SET_ERROR(EIO)); + + /* + * If the buf is sharing its data with the hdr, unlink it and + * allocate a new data buffer for the buf. + */ + if (arc_buf_is_shared(buf)) { + ASSERT(ARC_BUF_COMPRESSED(buf)); + + /* We need to give the buf it's own b_data */ + buf->b_flags &= ~ARC_BUF_FLAG_SHARED; + buf->b_data = + arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf); + arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); + + /* Previously overhead was 0; just add new overhead */ + ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr)); + } else if (ARC_BUF_COMPRESSED(buf)) { + /* We need to reallocate the buf's b_data */ + arc_free_data_buf(hdr, buf->b_data, HDR_GET_PSIZE(hdr), + buf); + buf->b_data = + arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf); + + /* We increased the size of b_data; update overhead */ + ARCSTAT_INCR(arcstat_overhead_size, + HDR_GET_LSIZE(hdr) - HDR_GET_PSIZE(hdr)); + } + + /* + * Regardless of the buf's previous compression settings, it + * should not be compressed at the end of this function. + */ + buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED; + + /* + * Try copying the data from another buf which already has a + * decompressed version. If that's not possible, it's time to + * bite the bullet and decompress the data from the hdr. + */ + if (arc_buf_try_copy_decompressed_data(buf)) { + /* Skip byteswapping and checksumming (already done) */ + ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, !=, NULL); + return (0); + } else { + int error = zio_decompress_data(HDR_GET_COMPRESS(hdr), + hdr->b_l1hdr.b_pdata, buf->b_data, + HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr)); + + /* + * Absent hardware errors or software bugs, this should + * be impossible, but log it anyway so we can debug it. + */ + if (error != 0) { + zfs_dbgmsg( + "hdr %p, compress %d, psize %d, lsize %d", + hdr, HDR_GET_COMPRESS(hdr), + HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr)); + return (SET_ERROR(EIO)); + } } } + + /* Byteswap the buf's data if necessary */ if (bswap != DMU_BSWAP_NUMFUNCS) { ASSERT(!HDR_SHARED_DATA(hdr)); ASSERT3U(bswap, <, DMU_BSWAP_NUMFUNCS); dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr)); } + + /* Compute the hdr's checksum if necessary */ arc_cksum_compute(buf); + return (0); } +int +arc_decompress(arc_buf_t *buf) +{ + return (arc_buf_fill(buf, B_FALSE)); +} + /* * Return the size of the block, b_pdata, that is stored in the arc_buf_hdr_t. */ @@ -1778,7 +1994,6 @@ static void arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state) { arc_buf_contents_t type = arc_buf_type(hdr); - uint64_t lsize = HDR_GET_LSIZE(hdr); ASSERT(HDR_HAS_L1HDR(hdr)); @@ -1786,7 +2001,8 @@ arc_evictable_space_increment(arc_buf_hd ASSERT0(hdr->b_l1hdr.b_bufcnt); ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL); - (void) refcount_add_many(&state->arcs_esize[type], lsize, hdr); + (void) refcount_add_many(&state->arcs_esize[type], + HDR_GET_LSIZE(hdr), hdr); return; } @@ -1797,11 +2013,10 @@ arc_evictable_space_increment(arc_buf_hd } for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; buf = buf->b_next) { - if (arc_buf_is_shared(buf)) { - ASSERT(ARC_BUF_LAST(buf)); + if (arc_buf_is_shared(buf)) continue; - } - (void) refcount_add_many(&state->arcs_esize[type], lsize, buf); + (void) refcount_add_many(&state->arcs_esize[type], + arc_buf_size(buf), buf); } } @@ -1811,10 +2026,9 @@ arc_evictable_space_increment(arc_buf_hd * so that we can add and remove them from the refcount individually. */ static void -arc_evitable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state) +arc_evictable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state) { arc_buf_contents_t type = arc_buf_type(hdr); - uint64_t lsize = HDR_GET_LSIZE(hdr); ASSERT(HDR_HAS_L1HDR(hdr)); @@ -1823,7 +2037,7 @@ arc_evitable_space_decrement(arc_buf_hdr ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL); (void) refcount_remove_many(&state->arcs_esize[type], - lsize, hdr); + HDR_GET_LSIZE(hdr), hdr); return; } @@ -1834,12 +2048,10 @@ arc_evitable_space_decrement(arc_buf_hdr } for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; buf = buf->b_next) { - if (arc_buf_is_shared(buf)) { - ASSERT(ARC_BUF_LAST(buf)); + if (arc_buf_is_shared(buf)) continue; - } (void) refcount_remove_many(&state->arcs_esize[type], - lsize, buf); + arc_buf_size(buf), buf); } } @@ -1867,7 +2079,7 @@ add_reference(arc_buf_hdr_t *hdr, void * if (state != arc_l2c_only) { multilist_remove(&state->arcs_list[arc_buf_type(hdr)], hdr); - arc_evitable_space_decrement(hdr, state); + arc_evictable_space_decrement(hdr, state); } /* remove the prefetch flag if we get a reference */ arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH); @@ -1955,7 +2167,7 @@ arc_change_state(arc_state_t *new_state, ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); update_old = B_TRUE; } - arc_evitable_space_decrement(hdr, old_state); + arc_evictable_space_decrement(hdr, old_state); } if (new_state != arc_anon && new_state != arc_l2c_only) { @@ -2018,13 +2230,11 @@ arc_change_state(arc_state_t *new_state, * add to the refcount if the arc_buf_t is * not shared. */ - if (arc_buf_is_shared(buf)) { - ASSERT(ARC_BUF_LAST(buf)); + if (arc_buf_is_shared(buf)) continue; - } (void) refcount_add_many(&new_state->arcs_size, - HDR_GET_LSIZE(hdr), buf); + arc_buf_size(buf), buf); } ASSERT3U(bufcnt, ==, buffers); @@ -2041,6 +2251,7 @@ arc_change_state(arc_state_t *new_state, ASSERT(HDR_HAS_L1HDR(hdr)); if (GHOST_STATE(old_state)) { ASSERT0(bufcnt); + ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL); /* * When moving a header off of a ghost state, @@ -2052,7 +2263,6 @@ arc_change_state(arc_state_t *new_state, (void) refcount_remove_many(&old_state->arcs_size, HDR_GET_LSIZE(hdr), hdr); - ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL); } else { uint32_t buffers = 0; @@ -2063,7 +2273,7 @@ arc_change_state(arc_state_t *new_state, */ for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; buf = buf->b_next) { - ASSERT3P(bufcnt, !=, 0); + ASSERT3U(bufcnt, !=, 0); buffers++; /* @@ -2073,13 +2283,11 @@ arc_change_state(arc_state_t *new_state, * add to the refcount if the arc_buf_t is * not shared. */ - if (arc_buf_is_shared(buf)) { - ASSERT(ARC_BUF_LAST(buf)); + if (arc_buf_is_shared(buf)) continue; - } (void) refcount_remove_many( - &old_state->arcs_size, HDR_GET_LSIZE(hdr), + &old_state->arcs_size, arc_buf_size(buf), buf); } ASSERT3U(bufcnt, ==, buffers); @@ -2164,11 +2372,50 @@ arc_space_return(uint64_t space, arc_spa } /* - * Allocate an initial buffer for this hdr, subsequent buffers will - * use arc_buf_clone(). + * Given a hdr and a buf, returns whether that buf can share its b_data buffer + * with the hdr's b_pdata. */ -static arc_buf_t * -arc_buf_alloc_impl(arc_buf_hdr_t *hdr, void *tag) +static boolean_t +arc_can_share(arc_buf_hdr_t *hdr, arc_buf_t *buf) +{ + /* + * The criteria for sharing a hdr's data are: + * 1. the hdr's compression matches the buf's compression + * 2. the hdr doesn't need to be byteswapped + * 3. the hdr isn't already being shared + * 4. the buf is either compressed or it is the last buf in the hdr list + * + * Criterion #4 maintains the invariant that shared uncompressed + * bufs must be the final buf in the hdr's b_buf list. Reading this, you + * might ask, "if a compressed buf is allocated first, won't that be the + * last thing in the list?", but in that case it's impossible to create + * a shared uncompressed buf anyway (because the hdr must be compressed + * to have the compressed buf). You might also think that #3 is + * sufficient to make this guarantee, however it's possible + * (specifically in the rare L2ARC write race mentioned in + * arc_buf_alloc_impl()) there will be an existing uncompressed buf that + * is sharable, but wasn't at the time of its allocation. Rather than + * allow a new shared uncompressed buf to be created and then shuffle + * the list around to make it the last element, this simply disallows + * sharing if the new buf isn't the first to be added. + */ + ASSERT3P(buf->b_hdr, ==, hdr); + boolean_t hdr_compressed = HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF; + boolean_t buf_compressed = ARC_BUF_COMPRESSED(buf) != 0; + return (buf_compressed == hdr_compressed && + hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS && + !HDR_SHARED_DATA(hdr) && + (ARC_BUF_LAST(buf) || ARC_BUF_COMPRESSED(buf))); +} + +/* + * Allocate a buf for this hdr. If you care about the data that's in the hdr, + * or if you want a compressed buffer, pass those flags in. Returns 0 if the + * copy was made successfully, or an error code otherwise. + */ +static int +arc_buf_alloc_impl(arc_buf_hdr_t *hdr, void *tag, boolean_t compressed, + boolean_t fill, arc_buf_t **ret) { arc_buf_t *buf; @@ -2176,15 +2423,14 @@ arc_buf_alloc_impl(arc_buf_hdr_t *hdr, v ASSERT3U(HDR_GET_LSIZE(hdr), >, 0); VERIFY(hdr->b_type == ARC_BUFC_DATA || hdr->b_type == ARC_BUFC_METADATA); + ASSERT3P(ret, !=, NULL); + ASSERT3P(*ret, ==, NULL); - ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); - ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); - ASSERT0(hdr->b_l1hdr.b_bufcnt); - - buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); + buf = *ret = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); buf->b_hdr = hdr; buf->b_data = NULL; - buf->b_next = NULL; + buf->b_next = hdr->b_l1hdr.b_buf; + buf->b_flags = 0; add_reference(hdr, tag); @@ -2195,58 +2441,63 @@ arc_buf_alloc_impl(arc_buf_hdr_t *hdr, v ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); /* - * If the hdr's data can be shared (no byteswapping, hdr is - * uncompressed, hdr's data is not currently being written to the - * L2ARC write) then we share the data buffer and set the appropriate - * bit in the hdr's b_flags to indicate the hdr is sharing it's - * b_pdata with the arc_buf_t. Otherwise, we allocate a new buffer to - * store the buf's data. + * Only honor requests for compressed bufs if the hdr is actually + * compressed. */ - if (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS && - HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF && !HDR_L2_WRITING(hdr)) { + if (compressed && HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) + buf->b_flags |= ARC_BUF_FLAG_COMPRESSED; + + /* + * If the hdr's data can be shared then we share the data buffer and + * set the appropriate bit in the hdr's b_flags to indicate the hdr is + * sharing it's b_pdata with the arc_buf_t. Otherwise, we allocate a new + * buffer to store the buf's data. + * + * There is one additional restriction here because we're sharing + * hdr -> buf instead of the usual buf -> hdr: the hdr can't be actively + * involved in an L2ARC write, because if this buf is used by an + * arc_write() then the hdr's data buffer will be released when the + * write completes, even though the L2ARC write might still be using it. + */ + boolean_t can_share = arc_can_share(hdr, buf) && !HDR_L2_WRITING(hdr); + + /* Set up b_data and sharing */ + if (can_share) { buf->b_data = hdr->b_l1hdr.b_pdata; + buf->b_flags |= ARC_BUF_FLAG_SHARED; arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA); } else { - buf->b_data = arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf); - ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr)); - arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); + buf->b_data = + arc_get_data_buf(hdr, arc_buf_size(buf), buf); + ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf)); } VERIFY3P(buf->b_data, !=, NULL); hdr->b_l1hdr.b_buf = buf; hdr->b_l1hdr.b_bufcnt += 1; - return (buf); -} + /* + * If the user wants the data from the hdr, we need to either copy or + * decompress the data. + */ + if (fill) { + return (arc_buf_fill(buf, ARC_BUF_COMPRESSED(buf) != 0)); + } -/* - * Used when allocating additional buffers. - */ -static arc_buf_t * -arc_buf_clone(arc_buf_t *from) -{ - arc_buf_t *buf; - arc_buf_hdr_t *hdr = from->b_hdr; - uint64_t size = HDR_GET_LSIZE(hdr); + return (0); +} - ASSERT(HDR_HAS_L1HDR(hdr)); - ASSERT(hdr->b_l1hdr.b_state != arc_anon); +static char *arc_onloan_tag = "onloan"; - buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); - buf->b_hdr = hdr; - buf->b_data = NULL; - buf->b_next = hdr->b_l1hdr.b_buf; - hdr->b_l1hdr.b_buf = buf; - buf->b_data = arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf); - bcopy(from->b_data, buf->b_data, size); - hdr->b_l1hdr.b_bufcnt += 1; +static inline void +arc_loaned_bytes_update(int64_t delta) +{ + atomic_add_64(&arc_loaned_bytes, delta); - ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr)); - return (buf); + /* assert that it did not wrap around */ + ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0); } -static char *arc_onloan_tag = "onloan"; - /* * Loan out an anonymous arc buffer. Loaned buffers are not counted as in * flight data by arc_tempreserve_space() until they are "returned". Loaned @@ -2254,16 +2505,29 @@ static char *arc_onloan_tag = "onloan"; * freed. */ arc_buf_t * -arc_loan_buf(spa_t *spa, int size) +arc_loan_buf(spa_t *spa, boolean_t is_metadata, int size) { - arc_buf_t *buf; + arc_buf_t *buf = arc_alloc_buf(spa, arc_onloan_tag, + is_metadata ? ARC_BUFC_METADATA : ARC_BUFC_DATA, size); - buf = arc_alloc_buf(spa, size, arc_onloan_tag, ARC_BUFC_DATA); + arc_loaned_bytes_update(size); - atomic_add_64(&arc_loaned_bytes, size); return (buf); } +arc_buf_t * +arc_loan_compressed_buf(spa_t *spa, uint64_t psize, uint64_t lsize, + enum zio_compress compression_type) +{ + arc_buf_t *buf = arc_alloc_compressed_buf(spa, arc_onloan_tag, + psize, lsize, compression_type); + + arc_loaned_bytes_update(psize); + + return (buf); +} + + /* * Return a loaned arc buffer to the arc. */ @@ -2277,7 +2541,7 @@ arc_return_buf(arc_buf_t *buf, void *tag (void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag); (void) refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag); - atomic_add_64(&arc_loaned_bytes, -HDR_GET_LSIZE(hdr)); + arc_loaned_bytes_update(-arc_buf_size(buf)); } /* Detach an arc_buf from a dbuf (tag) */ @@ -2291,7 +2555,7 @@ arc_loan_inuse_buf(arc_buf_t *buf, void (void) refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag); (void) refcount_remove(&hdr->b_l1hdr.b_refcnt, tag); - atomic_add_64(&arc_loaned_bytes, HDR_GET_LSIZE(hdr)); + arc_loaned_bytes_update(arc_buf_size(buf)); } *** DIFF OUTPUT TRUNCATED AT 1000 LINES *** From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:08:51 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id DAD62D3E7A6; Fri, 14 Apr 2017 18:08:51 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id B5EC61955; Fri, 14 Apr 2017 18:08:51 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EI8oek061972; Fri, 14 Apr 2017 18:08:50 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EI8ouF061971; Fri, 14 Apr 2017 18:08:50 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141808.v3EI8ouF061971@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:08:50 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316895 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:08:52 -0000 Author: avg Date: Fri Apr 14 18:08:50 2017 New Revision: 316895 URL: https://svnweb.freebsd.org/changeset/base/316895 Log: 7606 dmu_objset_find_dp() takes a long time while importing pool illumos/illumos-gate@7588687e6ba67c47bf7c9805086dec4a97fcac7b https://github.com/illumos/illumos-gate/commit/7588687e6ba67c47bf7c9805086dec4a97fcac7b https://www.illumos.org/issues/7606 When importing a pool with a large number of filesystems within the same parent filesystem, we see that dmu_objset_find_dp() takes a long time. It is called from 3 places: spa_check_logs(), spa_ld_claim_log_blocks(), and spa_load_verify(). There are several ways to improve performance here: 1. We don't really need to do spa_check_logs() or spa_ld_claim_log_blocks() if the pool was closed cleanly. 2. spa_load_verify() uses dmu_objset_find_dp() to check that no datasets have too long of names. 3. dmu_objset_find_dp() is slow because it's doing zap_value_search() (which is O(N sibling datasets)) to determine the name of each dsl_dir when it's opened. In this case we actually know the name when we are opening it, so we can provide it and avoid the lookup. This change implements fix #3 from the above list; i.e. make dmu_objset_find_dp() provide the name of the dataset so that we don't have to search for it. Reviewed by: Steve Gonczi Reviewed by: George Wilson Reviewed by: Prashanth Sreenivasa Approved by: Gordon Ross Author: Matthew Ahrens Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c Fri Apr 14 18:07:43 2017 (r316894) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c Fri Apr 14 18:08:50 2017 (r316895) @@ -1705,6 +1705,7 @@ typedef struct dmu_objset_find_ctx { taskq_t *dc_tq; dsl_pool_t *dc_dp; uint64_t dc_ddobj; + char *dc_ddname; /* last component of ddobj's name */ int (*dc_func)(dsl_pool_t *, dsl_dataset_t *, void *); void *dc_arg; int dc_flags; @@ -1716,7 +1717,6 @@ static void dmu_objset_find_dp_impl(dmu_objset_find_ctx_t *dcp) { dsl_pool_t *dp = dcp->dc_dp; - dmu_objset_find_ctx_t *child_dcp; dsl_dir_t *dd; dsl_dataset_t *ds; zap_cursor_t zc; @@ -1728,7 +1728,12 @@ dmu_objset_find_dp_impl(dmu_objset_find_ if (*dcp->dc_error != 0) goto out; - err = dsl_dir_hold_obj(dp, dcp->dc_ddobj, NULL, FTAG, &dd); + /* + * Note: passing the name (dc_ddname) here is optional, but it + * improves performance because we don't need to call + * zap_value_search() to determine the name. + */ + err = dsl_dir_hold_obj(dp, dcp->dc_ddobj, dcp->dc_ddname, FTAG, &dd); if (err != 0) goto out; @@ -1753,9 +1758,11 @@ dmu_objset_find_dp_impl(dmu_objset_find_ sizeof (uint64_t)); ASSERT3U(attr->za_num_integers, ==, 1); - child_dcp = kmem_alloc(sizeof (*child_dcp), KM_SLEEP); + dmu_objset_find_ctx_t *child_dcp = + kmem_alloc(sizeof (*child_dcp), KM_SLEEP); *child_dcp = *dcp; child_dcp->dc_ddobj = attr->za_first_integer; + child_dcp->dc_ddname = spa_strdup(attr->za_name); if (dcp->dc_tq != NULL) (void) taskq_dispatch(dcp->dc_tq, dmu_objset_find_dp_cb, child_dcp, TQ_SLEEP); @@ -1798,16 +1805,25 @@ dmu_objset_find_dp_impl(dmu_objset_find_ } } - dsl_dir_rele(dd, FTAG); kmem_free(attr, sizeof (zap_attribute_t)); - if (err != 0) + if (err != 0) { + dsl_dir_rele(dd, FTAG); goto out; + } /* * Apply to self. */ err = dsl_dataset_hold_obj(dp, thisobj, FTAG, &ds); + + /* + * Note: we hold the dir while calling dsl_dataset_hold_obj() so + * that the dir will remain cached, and we won't have to re-instantiate + * it (which could be expensive due to finding its name via + * zap_value_search()). + */ + dsl_dir_rele(dd, FTAG); if (err != 0) goto out; err = dcp->dc_func(dp, ds, dcp->dc_arg); @@ -1822,6 +1838,8 @@ out: mutex_exit(dcp->dc_error_lock); } + if (dcp->dc_ddname != NULL) + spa_strfree(dcp->dc_ddname); kmem_free(dcp, sizeof (*dcp)); } @@ -1866,6 +1884,7 @@ dmu_objset_find_dp(dsl_pool_t *dp, uint6 dcp->dc_tq = NULL; dcp->dc_dp = dp; dcp->dc_ddobj = ddobj; + dcp->dc_ddname = NULL; dcp->dc_func = func; dcp->dc_arg = arg; dcp->dc_flags = flags; From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:09:33 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 77490D3E82F; Fri, 14 Apr 2017 18:09:33 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 52AD41B6C; Fri, 14 Apr 2017 18:09:33 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EI9WJh062045; Fri, 14 Apr 2017 18:09:32 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EI9WM2062043; Fri, 14 Apr 2017 18:09:32 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141809.v3EI9WM2062043@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:09:32 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316896 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:09:33 -0000 Author: avg Date: Fri Apr 14 18:09:32 2017 New Revision: 316896 URL: https://svnweb.freebsd.org/changeset/base/316896 Log: 7580 ztest failure in dbuf_read_impl illumos/illumos-gate@1a01181fdc809f40c64d5c6881ae3e4521a9d9c7 https://github.com/illumos/illumos-gate/commit/1a01181fdc809f40c64d5c6881ae3e4521a9d9c7 https://www.illumos.org/issues/7580 We need to prevent any reader whenever we're about the zero out all the blkptrs. To do this we need to grab the dn_struct_rwlock as writer in dbuf_write_children_ready and free_children just prior to calling bzero. Reviewed by: Pavel Zakharov Reviewed by: Steve Gonczi Reviewed by: Matthew Ahrens Approved by: Dan McDonald Author: George Wilson Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c vendor-sys/illumos/dist/uts/common/fs/zfs/dnode_sync.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c Fri Apr 14 18:08:50 2017 (r316895) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c Fri Apr 14 18:09:32 2017 (r316896) @@ -3313,13 +3313,13 @@ dbuf_write_children_ready(zio_t *zio, ar dmu_buf_impl_t *db = vdb; dnode_t *dn; blkptr_t *bp; - uint64_t i; - int epbs; + unsigned int epbs, i; ASSERT3U(db->db_level, >, 0); DB_DNODE_ENTER(db); dn = DB_DNODE(db); epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; + ASSERT3U(epbs, <, 31); /* Determine if all our children are holes */ for (i = 0, bp = db->db.db_data; i < 1 << epbs; i++, bp++) { @@ -3332,8 +3332,14 @@ dbuf_write_children_ready(zio_t *zio, ar * we may get compressed away. */ if (i == 1 << epbs) { - /* didn't find any non-holes */ + /* + * We only found holes. Grab the rwlock to prevent + * anybody from reading the blocks we're about to + * zero out. + */ + rw_enter(&dn->dn_struct_rwlock, RW_WRITER); bzero(db->db.db_data, db->db.db_size); + rw_exit(&dn->dn_struct_rwlock); } DB_DNODE_EXIT(db); } Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dnode_sync.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dnode_sync.c Fri Apr 14 18:08:50 2017 (r316895) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dnode_sync.c Fri Apr 14 18:09:32 2017 (r316896) @@ -236,8 +236,8 @@ free_children(dmu_buf_impl_t *db, uint64 dnode_t *dn; blkptr_t *bp; dmu_buf_impl_t *subdb; - uint64_t start, end, dbstart, dbend, i; - int epbs, shift; + uint64_t start, end, dbstart, dbend; + unsigned int epbs, shift, i; /* * There is a small possibility that this block will not be cached: @@ -254,6 +254,7 @@ free_children(dmu_buf_impl_t *db, uint64 DB_DNODE_ENTER(db); dn = DB_DNODE(db); epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; + ASSERT3U(epbs, <, 31); shift = (db->db_level - 1) * epbs; dbstart = db->db_blkid << epbs; start = blkid >> shift; @@ -273,12 +274,12 @@ free_children(dmu_buf_impl_t *db, uint64 FREE_VERIFY(db, start, end, tx); free_blocks(dn, bp, end-start+1, tx); } else { - for (i = start; i <= end; i++, bp++) { + for (uint64_t id = start; id <= end; id++, bp++) { if (BP_IS_HOLE(bp)) continue; rw_enter(&dn->dn_struct_rwlock, RW_READER); VERIFY0(dbuf_hold_impl(dn, db->db_level - 1, - i, TRUE, FALSE, FTAG, &subdb)); + id, TRUE, FALSE, FTAG, &subdb)); rw_exit(&dn->dn_struct_rwlock); ASSERT3P(bp, ==, subdb->db_blkptr); @@ -293,8 +294,14 @@ free_children(dmu_buf_impl_t *db, uint64 break; } if (i == 1 << epbs) { - /* didn't find any non-holes */ + /* + * We only found holes. Grab the rwlock to prevent + * anybody from reading the blocks we're about to + * zero out. + */ + rw_enter(&dn->dn_struct_rwlock, RW_WRITER); bzero(db->db.db_data, db->db.db_size); + rw_exit(&dn->dn_struct_rwlock); free_blocks(dn, db->db_blkptr, 1, tx); } else { /* From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:10:25 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 1633FD3E894; Fri, 14 Apr 2017 18:10:25 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id E59821CD6; Fri, 14 Apr 2017 18:10:24 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIAO3V062145; Fri, 14 Apr 2017 18:10:24 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIANd1062143; Fri, 14 Apr 2017 18:10:23 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141810.v3EIANd1062143@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:10:23 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316897 - vendor-sys/illumos/dist/uts/common/fs/zfs/sys X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:10:25 -0000 Author: avg Date: Fri Apr 14 18:10:23 2017 New Revision: 316897 URL: https://svnweb.freebsd.org/changeset/base/316897 Log: 7586 remove #ifdef __lint hack from dmu.h illumos/illumos-gate@4ba5b9616327ef64e8abc737d29b3faabc6ae68c https://github.com/illumos/illumos-gate/commit/4ba5b9616327ef64e8abc737d29b3faabc6ae68c https://www.illumos.org/issues/7586 The #ifdef __lint in dmu.h is ugly, and it would be nice not to duplicate it if we add other inline functions into header files in ZFS, especially since it is difficult to make any other solution work across all compilation targets. We should switch to disabling the lint flags that are failing instead. Reviewed by: Matthew Ahrens Reviewed by: Pavel Zakharov Approved by: Dan McDonald Author: Dan Kimmel Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa.h Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu.h Fri Apr 14 18:09:32 2017 (r316896) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu.h Fri Apr 14 18:10:23 2017 (r316897) @@ -565,12 +565,7 @@ typedef struct dmu_buf_user { * NOTE: This function should only be called once on a given dmu_buf_user_t. * To allow enforcement of this, dbu must already be zeroed on entry. */ -#ifdef __lint -/* Very ugly, but it beats issuing suppression directives in many Makefiles. */ -extern void -dmu_buf_init_user(dmu_buf_user_t *dbu, dmu_buf_evict_func_t *evict_func, - dmu_buf_evict_func_t *evict_func_async, dmu_buf_t **clear_on_evict_dbufp); -#else /* __lint */ +/*ARGSUSED*/ inline void dmu_buf_init_user(dmu_buf_user_t *dbu, dmu_buf_evict_func_t *evict_func_sync, dmu_buf_evict_func_t *evict_func_async, dmu_buf_t **clear_on_evict_dbufp) @@ -586,7 +581,6 @@ dmu_buf_init_user(dmu_buf_user_t *dbu, d dbu->dbu_clear_on_evict_dbufp = clear_on_evict_dbufp; #endif } -#endif /* __lint */ /* * Attach user data to a dbuf and mark it for normal (when the dbuf's Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa.h Fri Apr 14 18:09:32 2017 (r316896) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa.h Fri Apr 14 18:10:23 2017 (r316897) @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2014 by Delphix. All rights reserved. + * Copyright (c) 2011, 2016 by Delphix. All rights reserved. * Copyright 2011 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. * Copyright 2013 Saso Kiselkov. All rights reserved. @@ -36,6 +36,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -595,8 +596,6 @@ _NOTE(CONSTCOND) } while (0) ASSERT(len < size); \ } -#include - #define BP_GET_BUFC_TYPE(bp) \ (((BP_GET_LEVEL(bp) > 0) || (DMU_OT_IS_METADATA(BP_GET_TYPE(bp)))) ? \ ARC_BUFC_METADATA : ARC_BUFC_DATA) From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:11:17 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 6BE0ED3E90F; Fri, 14 Apr 2017 18:11:17 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 380CD1E9A; Fri, 14 Apr 2017 18:11:17 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIBGF5062245; Fri, 14 Apr 2017 18:11:16 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIBGJO062243; Fri, 14 Apr 2017 18:11:16 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141811.v3EIBGJO062243@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:11:16 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316898 - in vendor-sys/illumos/dist/uts/common/fs/zfs: . sys X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:11:17 -0000 Author: avg Date: Fri Apr 14 18:11:16 2017 New Revision: 316898 URL: https://svnweb.freebsd.org/changeset/base/316898 Log: 7613 ms_freetree[4] is only used in syncing context illumos/illumos-gate@5f145778012b555e084eacc858ead9e1e42bd149 https://github.com/illumos/illumos-gate/commit/5f145778012b555e084eacc858ead9e1e42bd149 https://www.illumos.org/issues/7613 metaslab_t:ms_freetree[TXG_SIZE] is only used in syncing context. We should replace it with two trees: the freeing tree (ranges that we are freeing this syncing txg) and the freed tree (ranges which have been freed this txg). Reviewed by: George Wilson Reviewed by: Alex Reece Approved by: Dan McDonald Author: Matthew Ahrens Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/metaslab.c vendor-sys/illumos/dist/uts/common/fs/zfs/sys/metaslab_impl.h Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/metaslab.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/metaslab.c Fri Apr 14 18:10:23 2017 (r316897) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/metaslab.c Fri Apr 14 18:11:16 2017 (r316898) @@ -446,7 +446,6 @@ metaslab_verify_space(metaslab_t *msp, u { spa_t *spa = msp->ms_group->mg_vd->vdev_spa; uint64_t allocated = 0; - uint64_t freed = 0; uint64_t sm_free_space, msp_free_space; ASSERT(MUTEX_HELD(&msp->ms_lock)); @@ -476,10 +475,9 @@ metaslab_verify_space(metaslab_t *msp, u allocated += range_tree_space(msp->ms_alloctree[(txg + t) & TXG_MASK]); } - freed = range_tree_space(msp->ms_freetree[TXG_CLEAN(txg) & TXG_MASK]); msp_free_space = range_tree_space(msp->ms_tree) + allocated + - msp->ms_deferspace + freed; + msp->ms_deferspace + range_tree_space(msp->ms_freedtree); VERIFY3U(sm_free_space, ==, msp_free_space); } @@ -1410,7 +1408,7 @@ metaslab_init(metaslab_group_t *mg, uint /* * We create the main range tree here, but we don't create the - * alloctree and freetree until metaslab_sync_done(). This serves + * other range trees until metaslab_sync_done(). This serves * two purposes: it allows metaslab_sync_done() to detect the * addition of new space; and for debugging, it ensures that we'd * data fault on any attempt to use this metaslab before it's ready. @@ -1468,10 +1466,11 @@ metaslab_fini(metaslab_t *msp) metaslab_unload(msp); range_tree_destroy(msp->ms_tree); + range_tree_destroy(msp->ms_freeingtree); + range_tree_destroy(msp->ms_freedtree); for (int t = 0; t < TXG_SIZE; t++) { range_tree_destroy(msp->ms_alloctree[t]); - range_tree_destroy(msp->ms_freetree[t]); } for (int t = 0; t < TXG_DEFER_SIZE; t++) { @@ -2082,7 +2081,6 @@ static void metaslab_condense(metaslab_t *msp, uint64_t txg, dmu_tx_t *tx) { spa_t *spa = msp->ms_group->mg_vd->vdev_spa; - range_tree_t *freetree = msp->ms_freetree[txg & TXG_MASK]; range_tree_t *condense_tree; space_map_t *sm = msp->ms_sm; @@ -2113,9 +2111,9 @@ metaslab_condense(metaslab_t *msp, uint6 /* * Remove what's been freed in this txg from the condense_tree. * Since we're in sync_pass 1, we know that all the frees from - * this txg are in the freetree. + * this txg are in the freeingtree. */ - range_tree_walk(freetree, range_tree_remove, condense_tree); + range_tree_walk(msp->ms_freeingtree, range_tree_remove, condense_tree); for (int t = 0; t < TXG_DEFER_SIZE; t++) { range_tree_walk(msp->ms_defertree[t], @@ -2171,9 +2169,6 @@ metaslab_sync(metaslab_t *msp, uint64_t spa_t *spa = vd->vdev_spa; objset_t *mos = spa_meta_objset(spa); range_tree_t *alloctree = msp->ms_alloctree[txg & TXG_MASK]; - range_tree_t **freetree = &msp->ms_freetree[txg & TXG_MASK]; - range_tree_t **freed_tree = - &msp->ms_freetree[TXG_CLEAN(txg) & TXG_MASK]; dmu_tx_t *tx; uint64_t object = space_map_object(msp->ms_sm); @@ -2182,14 +2177,14 @@ metaslab_sync(metaslab_t *msp, uint64_t /* * This metaslab has just been added so there's no work to do now. */ - if (*freetree == NULL) { + if (msp->ms_freeingtree == NULL) { ASSERT3P(alloctree, ==, NULL); return; } ASSERT3P(alloctree, !=, NULL); - ASSERT3P(*freetree, !=, NULL); - ASSERT3P(*freed_tree, !=, NULL); + ASSERT3P(msp->ms_freeingtree, !=, NULL); + ASSERT3P(msp->ms_freedtree, !=, NULL); /* * Normally, we don't want to process a metaslab if there @@ -2197,14 +2192,14 @@ metaslab_sync(metaslab_t *msp, uint64_t * is being forced to condense we need to let it through. */ if (range_tree_space(alloctree) == 0 && - range_tree_space(*freetree) == 0 && + range_tree_space(msp->ms_freeingtree) == 0 && !msp->ms_condense_wanted) return; /* * The only state that can actually be changing concurrently with * metaslab_sync() is the metaslab's ms_tree. No other thread can - * be modifying this txg's alloctree, freetree, freed_tree, or + * be modifying this txg's alloctree, freeingtree, freedtree, or * space_map_phys_t. Therefore, we only hold ms_lock to satify * space map ASSERTs. We drop it whenever we call into the DMU, * because the DMU can call down to us (e.g. via zio_free()) at @@ -2241,7 +2236,7 @@ metaslab_sync(metaslab_t *msp, uint64_t metaslab_condense(msp, txg, tx); } else { space_map_write(msp->ms_sm, alloctree, SM_ALLOC, tx); - space_map_write(msp->ms_sm, *freetree, SM_FREE, tx); + space_map_write(msp->ms_sm, msp->ms_freeingtree, SM_FREE, tx); } if (msp->ms_loaded) { @@ -2261,7 +2256,7 @@ metaslab_sync(metaslab_t *msp, uint64_t * to accurately reflect all free space even if some space * is not yet available for allocation (i.e. deferred). */ - space_map_histogram_add(msp->ms_sm, *freed_tree, tx); + space_map_histogram_add(msp->ms_sm, msp->ms_freedtree, tx); /* * Add back any deferred free space that has not been @@ -2283,7 +2278,7 @@ metaslab_sync(metaslab_t *msp, uint64_t * then we will lose some accuracy but will correct it the next * time we load the space map. */ - space_map_histogram_add(msp->ms_sm, *freetree, tx); + space_map_histogram_add(msp->ms_sm, msp->ms_freeingtree, tx); metaslab_group_histogram_add(mg, msp); metaslab_group_histogram_verify(mg); @@ -2291,20 +2286,21 @@ metaslab_sync(metaslab_t *msp, uint64_t /* * For sync pass 1, we avoid traversing this txg's free range tree - * and instead will just swap the pointers for freetree and - * freed_tree. We can safely do this since the freed_tree is + * and instead will just swap the pointers for freeingtree and + * freedtree. We can safely do this since the freed_tree is * guaranteed to be empty on the initial pass. */ if (spa_sync_pass(spa) == 1) { - range_tree_swap(freetree, freed_tree); + range_tree_swap(&msp->ms_freeingtree, &msp->ms_freedtree); } else { - range_tree_vacate(*freetree, range_tree_add, *freed_tree); + range_tree_vacate(msp->ms_freeingtree, + range_tree_add, msp->ms_freedtree); } range_tree_vacate(alloctree, NULL, NULL); ASSERT0(range_tree_space(msp->ms_alloctree[txg & TXG_MASK])); ASSERT0(range_tree_space(msp->ms_alloctree[TXG_CLEAN(txg) & TXG_MASK])); - ASSERT0(range_tree_space(msp->ms_freetree[txg & TXG_MASK])); + ASSERT0(range_tree_space(msp->ms_freeingtree)); mutex_exit(&msp->ms_lock); @@ -2326,7 +2322,6 @@ metaslab_sync_done(metaslab_t *msp, uint metaslab_group_t *mg = msp->ms_group; vdev_t *vd = mg->mg_vd; spa_t *spa = vd->vdev_spa; - range_tree_t **freed_tree; range_tree_t **defer_tree; int64_t alloc_delta, defer_delta; boolean_t defer_allowed = B_TRUE; @@ -2337,20 +2332,24 @@ metaslab_sync_done(metaslab_t *msp, uint /* * If this metaslab is just becoming available, initialize its - * alloctrees, freetrees, and defertree and add its capacity to - * the vdev. + * range trees and add its capacity to the vdev. */ - if (msp->ms_freetree[TXG_CLEAN(txg) & TXG_MASK] == NULL) { + if (msp->ms_freedtree == NULL) { for (int t = 0; t < TXG_SIZE; t++) { ASSERT(msp->ms_alloctree[t] == NULL); - ASSERT(msp->ms_freetree[t] == NULL); msp->ms_alloctree[t] = range_tree_create(NULL, msp, &msp->ms_lock); - msp->ms_freetree[t] = range_tree_create(NULL, msp, - &msp->ms_lock); } + ASSERT3P(msp->ms_freeingtree, ==, NULL); + msp->ms_freeingtree = range_tree_create(NULL, msp, + &msp->ms_lock); + + ASSERT3P(msp->ms_freedtree, ==, NULL); + msp->ms_freedtree = range_tree_create(NULL, msp, + &msp->ms_lock); + for (int t = 0; t < TXG_DEFER_SIZE; t++) { ASSERT(msp->ms_defertree[t] == NULL); @@ -2361,7 +2360,6 @@ metaslab_sync_done(metaslab_t *msp, uint vdev_space_update(vd, 0, 0, msp->ms_size); } - freed_tree = &msp->ms_freetree[TXG_CLEAN(txg) & TXG_MASK]; defer_tree = &msp->ms_defertree[txg % TXG_DEFER_SIZE]; uint64_t free_space = metaslab_class_get_space(spa_normal_class(spa)) - @@ -2373,7 +2371,7 @@ metaslab_sync_done(metaslab_t *msp, uint defer_delta = 0; alloc_delta = space_map_alloc_delta(msp->ms_sm); if (defer_allowed) { - defer_delta = range_tree_space(*freed_tree) - + defer_delta = range_tree_space(msp->ms_freedtree) - range_tree_space(*defer_tree); } else { defer_delta -= range_tree_space(*defer_tree); @@ -2381,9 +2379,6 @@ metaslab_sync_done(metaslab_t *msp, uint vdev_space_update(vd, alloc_delta + defer_delta, defer_delta, 0); - ASSERT0(range_tree_space(msp->ms_alloctree[txg & TXG_MASK])); - ASSERT0(range_tree_space(msp->ms_freetree[txg & TXG_MASK])); - /* * If there's a metaslab_load() in progress, wait for it to complete * so that we have a consistent view of the in-core space map. @@ -2399,9 +2394,9 @@ metaslab_sync_done(metaslab_t *msp, uint range_tree_vacate(*defer_tree, msp->ms_loaded ? range_tree_add : NULL, msp->ms_tree); if (defer_allowed) { - range_tree_swap(freed_tree, defer_tree); + range_tree_swap(&msp->ms_freedtree, defer_tree); } else { - range_tree_vacate(*freed_tree, + range_tree_vacate(msp->ms_freedtree, msp->ms_loaded ? range_tree_add : NULL, msp->ms_tree); } @@ -3161,10 +3156,10 @@ metaslab_free_dva(spa_t *spa, const dva_ range_tree_add(msp->ms_tree, offset, size); msp->ms_max_size = metaslab_block_maxsize(msp); } else { - if (range_tree_space(msp->ms_freetree[txg & TXG_MASK]) == 0) + VERIFY3U(txg, ==, spa->spa_syncing_txg); + if (range_tree_space(msp->ms_freeingtree) == 0) vdev_dirty(vd, VDD_METASLAB, msp, txg); - range_tree_add(msp->ms_freetree[txg & TXG_MASK], - offset, size); + range_tree_add(msp->ms_freeingtree, offset, size); } mutex_exit(&msp->ms_lock); @@ -3396,8 +3391,8 @@ metaslab_check_free(spa_t *spa, const bl if (msp->ms_loaded) range_tree_verify(msp->ms_tree, offset, size); - for (int j = 0; j < TXG_SIZE; j++) - range_tree_verify(msp->ms_freetree[j], offset, size); + range_tree_verify(msp->ms_freeingtree, offset, size); + range_tree_verify(msp->ms_freedtree, offset, size); for (int j = 0; j < TXG_DEFER_SIZE; j++) range_tree_verify(msp->ms_defertree[j], offset, size); } Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/metaslab_impl.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/metaslab_impl.h Fri Apr 14 18:10:23 2017 (r316897) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/metaslab_impl.h Fri Apr 14 18:11:16 2017 (r316898) @@ -254,21 +254,24 @@ struct metaslab_group { #define MAX_LBAS 64 /* - * Each metaslab maintains a set of in-core trees to track metaslab operations. - * The in-core free tree (ms_tree) contains the current list of free segments. - * As blocks are allocated, the allocated segment are removed from the ms_tree - * and added to a per txg allocation tree (ms_alloctree). As blocks are freed, - * they are added to the per txg free tree (ms_freetree). These per txg - * trees allow us to process all allocations and frees in syncing context - * where it is safe to update the on-disk space maps. One additional in-core - * tree is maintained to track deferred frees (ms_defertree). Once a block - * is freed it will move from the ms_freetree to the ms_defertree. A deferred - * free means that a block has been freed but cannot be used by the pool - * until TXG_DEFER_SIZE transactions groups later. For example, a block - * that is freed in txg 50 will not be available for reallocation until - * txg 52 (50 + TXG_DEFER_SIZE). This provides a safety net for uberblock - * rollback. A pool could be safely rolled back TXG_DEFERS_SIZE - * transactions groups and ensure that no block has been reallocated. + * Each metaslab maintains a set of in-core trees to track metaslab + * operations. The in-core free tree (ms_tree) contains the list of + * free segments which are eligible for allocation. As blocks are + * allocated, the allocated segments are removed from the ms_tree and + * added to a per txg allocation tree (ms_alloctree). This allows us to + * process all allocations in syncing context where it is safe to update + * the on-disk space maps. Frees are also processed in syncing context. + * Most frees are generated from syncing context, and those that are not + * are held in the spa_free_bplist for processing in syncing context. + * An additional set of in-core trees is maintained to track deferred + * frees (ms_defertree). Once a block is freed it will move from the + * ms_freedtree to the ms_defertree. A deferred free means that a block + * has been freed but cannot be used by the pool until TXG_DEFER_SIZE + * transactions groups later. For example, a block that is freed in txg + * 50 will not be available for reallocation until txg 52 (50 + + * TXG_DEFER_SIZE). This provides a safety net for uberblock rollback. + * A pool could be safely rolled back TXG_DEFERS_SIZE transactions + * groups and ensure that no block has been reallocated. * * The simplified transition diagram looks like this: * @@ -276,33 +279,34 @@ struct metaslab_group { * ALLOCATE * | * V - * free segment (ms_tree) --------> ms_alloctree ----> (write to space map) + * free segment (ms_tree) -----> ms_alloctree[4] ----> (write to space map) * ^ - * | - * | ms_freetree <--- FREE - * | | + * | ms_freeingtree <--- FREE * | | + * | v + * | ms_freedtree * | | - * +----------- ms_defertree <-------+---------> (write to space map) + * +-------- ms_defertree[2] <-------+---------> (write to space map) * * * Each metaslab's space is tracked in a single space map in the MOS, - * which is only updated in syncing context. Each time we sync a txg, - * we append the allocs and frees from that txg to the space map. - * The pool space is only updated once all metaslabs have finished syncing. - * - * To load the in-core free tree we read the space map from disk. - * This object contains a series of alloc and free records that are - * combined to make up the list of all free segments in this metaslab. These + * which is only updated in syncing context. Each time we sync a txg, + * we append the allocs and frees from that txg to the space map. The + * pool space is only updated once all metaslabs have finished syncing. + * + * To load the in-core free tree we read the space map from disk. This + * object contains a series of alloc and free records that are combined + * to make up the list of all free segments in this metaslab. These * segments are represented in-core by the ms_tree and are stored in an * AVL tree. * * As the space map grows (as a result of the appends) it will - * eventually become space-inefficient. When the metaslab's in-core free tree - * is zfs_condense_pct/100 times the size of the minimal on-disk - * representation, we rewrite it in its minimized form. If a metaslab - * needs to condense then we must set the ms_condensing flag to ensure - * that allocations are not performed on the metaslab that is being written. + * eventually become space-inefficient. When the metaslab's in-core + * free tree is zfs_condense_pct/100 times the size of the minimal + * on-disk representation, we rewrite it in its minimized form. If a + * metaslab needs to condense then we must set the ms_condensing flag to + * ensure that allocations are not performed on the metaslab that is + * being written. */ struct metaslab { kmutex_t ms_lock; @@ -314,10 +318,17 @@ struct metaslab { uint64_t ms_fragmentation; range_tree_t *ms_alloctree[TXG_SIZE]; - range_tree_t *ms_freetree[TXG_SIZE]; - range_tree_t *ms_defertree[TXG_DEFER_SIZE]; range_tree_t *ms_tree; + /* + * The following range trees are accessed only from syncing context. + * ms_free*tree only have entries while syncing, and are empty + * between syncs. + */ + range_tree_t *ms_freeingtree; /* to free this syncing txg */ + range_tree_t *ms_freedtree; /* already freed this syncing txg */ + range_tree_t *ms_defertree[TXG_DEFER_SIZE]; + boolean_t ms_condensing; /* condensing? */ boolean_t ms_condense_wanted; From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:11:54 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id E6EC5D3E967; Fri, 14 Apr 2017 18:11:54 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id B8B9EA4; Fri, 14 Apr 2017 18:11:54 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIBrFX064587; Fri, 14 Apr 2017 18:11:53 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIBrrx064586; Fri, 14 Apr 2017 18:11:53 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141811.v3EIBrrx064586@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:11:53 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316899 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:11:55 -0000 Author: avg Date: Fri Apr 14 18:11:53 2017 New Revision: 316899 URL: https://svnweb.freebsd.org/changeset/base/316899 Log: 7659 Missing thread_exit() in dmu_send.c illumos/illumos-gate@f2c1e9bc48e4e20e8e9bd56203a75ff2e219b345 https://github.com/illumos/illumos-gate/commit/f2c1e9bc48e4e20e8e9bd56203a75ff2e219b345 https://www.illumos.org/issues/7659 Two threads send_traverse_thread() and receive_writer_thread() should end with thread_exit(); Mostly a cosmetic issue under IllumOS. https://github.com/openzfs/openzfs/pull/252 Reviewed by: Paul Dagnelie Reviewed by: Matt Ahrens Approved by: Richard Lowe Author: Jorgen Lundman Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_send.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_send.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_send.c Fri Apr 14 18:11:16 2017 (r316898) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_send.c Fri Apr 14 18:11:53 2017 (r316899) @@ -576,6 +576,7 @@ send_traverse_thread(void *arg) data = kmem_zalloc(sizeof (*data), KM_SLEEP); data->eos_marker = B_TRUE; bqueue_enqueue(&st_arg->q, data, 1); + thread_exit(); } /* @@ -2815,6 +2816,7 @@ receive_writer_thread(void *arg) rwa->done = B_TRUE; cv_signal(&rwa->cv); mutex_exit(&rwa->mutex); + thread_exit(); } static int From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:13:08 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 3967AD3EAD6; Fri, 14 Apr 2017 18:13:08 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 167A43C2; Fri, 14 Apr 2017 18:13:08 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EID7AA066066; Fri, 14 Apr 2017 18:13:07 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EID7Zi066064; Fri, 14 Apr 2017 18:13:07 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141813.v3EID7Zi066064@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:13:07 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316900 - in vendor-sys/illumos/dist/uts/common/fs/zfs: . sys X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:13:08 -0000 Author: avg Date: Fri Apr 14 18:13:06 2017 New Revision: 316900 URL: https://svnweb.freebsd.org/changeset/base/316900 Log: 7743 per-vdev-zaps have no initialize path on upgrade illumos/illumos-gate@555da5111b0f2552c42d057b211aba89c9c79f6c https://github.com/illumos/illumos-gate/commit/555da5111b0f2552c42d057b211aba89c9c79f6c https://www.illumos.org/issues/7743 When loading a pool that had been created before the existance of per-vdev zaps, on a system that knows about per-vdev zaps, the per-vdev zaps will not be allocated and initialized. This appears to be because the logic that would have done so, in spa_sync_config_object(), is not reached under normal operation. It is only reached if spa_config_dirty_list is non-empty. The fix is to add another `AVZ_ACTION_` enum that will allow this code to be reached when we detect that we're loading an old pool, even when there are no dirty configs. Reviewed by: Matt Ahrens Reviewed by: Pavel Zakharov Reviewed by: George Wilson Reviewed by: Don Brady Approved by: Robert Mustacchi Author: Paul Dagnelie Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/spa.c vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa_impl.h Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/spa.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/spa.c Fri Apr 14 18:11:53 2017 (r316899) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/spa.c Fri Apr 14 18:13:06 2017 (r316900) @@ -2657,10 +2657,14 @@ spa_load_impl(spa_t *spa, uint64_t pool_ error = spa_dir_prop(spa, DMU_POOL_VDEV_ZAP_MAP, &spa->spa_all_vdev_zaps); - if (error != ENOENT && error != 0) { + if (error == ENOENT) { + VERIFY(!nvlist_exists(mos_config, + ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)); + spa->spa_avz_action = AVZ_ACTION_INITIALIZE; + ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev)); + } else if (error != 0) { return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); - } else if (error == 0 && !nvlist_exists(mos_config, - ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)) { + } else if (!nvlist_exists(mos_config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)) { /* * An older version of ZFS overwrote the sentinel value, so * we have orphaned per-vdev ZAPs in the MOS. Defer their @@ -6156,6 +6160,7 @@ spa_sync_config_object(spa_t *spa, dmu_t spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); ASSERT(spa->spa_avz_action == AVZ_ACTION_NONE || + spa->spa_avz_action == AVZ_ACTION_INITIALIZE || spa->spa_all_vdev_zaps != 0); if (spa->spa_avz_action == AVZ_ACTION_REBUILD) { Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa_impl.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa_impl.h Fri Apr 14 18:11:53 2017 (r316899) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa_impl.h Fri Apr 14 18:13:06 2017 (r316900) @@ -119,7 +119,8 @@ typedef struct spa_taskqs { typedef enum spa_all_vdev_zap_action { AVZ_ACTION_NONE = 0, AVZ_ACTION_DESTROY, /* Destroy all per-vdev ZAPs and the AVZ. */ - AVZ_ACTION_REBUILD /* Populate the new AVZ, see spa_avz_rebuild */ + AVZ_ACTION_REBUILD, /* Populate the new AVZ, see spa_avz_rebuild */ + AVZ_ACTION_INITIALIZE } spa_avz_action_t; struct spa { From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:13:34 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id CED27D3EB24; Fri, 14 Apr 2017 18:13:34 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 84692759; Fri, 14 Apr 2017 18:13:34 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIDXBZ066125; Fri, 14 Apr 2017 18:13:33 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIDXiI066124; Fri, 14 Apr 2017 18:13:33 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141813.v3EIDXiI066124@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:13:33 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316901 - vendor/illumos/dist/lib/libzfs/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:13:34 -0000 Author: avg Date: Fri Apr 14 18:13:33 2017 New Revision: 316901 URL: https://svnweb.freebsd.org/changeset/base/316901 Log: 7730 libzfs`add_config() leaks config nvl when reading spare/l2cache devices illumos/illumos-gate@105686550ee9cbf5d033166a8a2a5a763667d436 https://github.com/illumos/illumos-gate/commit/105686550ee9cbf5d033166a8a2a5a763667d436 https://www.illumos.org/issues/7730 antares:root:~# mdb /usr/sbin/zpool > ::sysbp _exit > ::run import pool: data id: 2093977168778024605 state: ONLINE action: The pool can be imported using its name or numeric identifier. config: data ONLINE c6t0d0 ONLINE c6t1d0 ONLINE cache c6t2d0 mdb: stop on entry to _exit mdb: target stopped at: 0xfee556ba: nop mdb: You've got symbols! Loading modules: [ ld.so.1 libumem.so.1 libc.so.1 libtopo.so.1 libavl.so.1 libnvpair.so.1 ] > ::findleaks -d BYTES LEAKED VMEM_SEG CALLER 4096 10 fda7b000 MMAP 8192 1 fea8d000 MMAP 8192 1 fe76d000 MMAP 8192 1 fe66e000 MMAP 4096 1 fe570000 MMAP 8192 1 fe470000 MMAP 4096 1 fe372000 MMAP 4096 1 fe273000 MMAP Reviewed by: Matthew Ahrens Reviewed by: Serapheim Dimitropoulos Approved by: Robert Mustacchi Author: Yuri Pankov Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_import.c Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_import.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_import.c Fri Apr 14 18:13:06 2017 (r316900) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_import.c Fri Apr 14 18:13:33 2017 (r316901) @@ -237,9 +237,12 @@ add_config(libzfs_handle_t *hdl, pool_li free(ne); return (-1); } + ne->ne_guid = vdev_guid; ne->ne_next = pl->names; pl->names = ne; + + nvlist_free(config); return (0); } From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:14:04 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 8B12ED3EB91; Fri, 14 Apr 2017 18:14:04 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 4BC8C897; Fri, 14 Apr 2017 18:14:04 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIE3Fo066193; Fri, 14 Apr 2017 18:14:03 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIE3ea066190; Fri, 14 Apr 2017 18:14:03 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141814.v3EIE3ea066190@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:14:03 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316902 - in vendor/illumos/dist/lib: libzfs/common libzfs_core/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:14:04 -0000 Author: avg Date: Fri Apr 14 18:14:02 2017 New Revision: 316902 URL: https://svnweb.freebsd.org/changeset/base/316902 Log: 7745 print error if lzc_* is called before libzfs_core_init illumos/illumos-gate@7c13517fff71be473e47531ef4330160c042bedc https://github.com/illumos/illumos-gate/commit/7c13517fff71be473e47531ef4330160c042bedc https://www.illumos.org/issues/7745 The problem is that consumers of `libZFS_Core` that forget to call `libzfs_core_init()` before calling any other function of the library are having a hard time realizing their mistake. The library's internal file descriptor is declared as global static, which is ok, but it is not initialized explicitly; therefore, it defaults to 0, which is a valid file descriptor. If `libzfs_core_init()`, which explicitly initializes the correct fd, is skipped, the ioctl functions return errors that do not have anything to do with `libZFS_Core`, where the problem is actually located. Even though assertions for that existed within `libZFS_Core` for debug builds, they were never enabled because the `-DDEBUG` flag was missing from the compiler flags. This patch applies the following changes: 1. It adds `-DDEBUG` for debug builds of `libZFS_Core` and `libzfs`, to enable their assertions on debug builds. 2. It corrects an assertion within `libzfs`, where a function had been spelled incorrectly (`zpool_prop_unsupported()`) and nobody knew because the `-DDEBUG` flag was missing, and the preprocessor was taking that part of the code away. 3. The library's internal fd is initialized to `-1` and `VERIFY` assertions have been placed to check that the fd is not equal to `-1` before issuing any ioctl. It is important here to note, that the `VERIFY` assertions exist in both debug and non-debug builds. 4. In `libzfs_core_fini` we make sure to never increment the refcount of our fd below 0, and also reset the fd to `-1` when no one refers to it. The reason for this, is for the rare case that the consumer closes all references but then calls one of the library's functions without using `libzfs_core_init()` first, and in the mean time, a previous call to `open()` decided to reuse our previous fd. This scenario would have passed our assertion in Reviewed by: Pavel Zakharov Reviewed by: Matthew Ahrens Approved by: Dan McDonald Author: Serapheim Dimitropoulos Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_pool.c vendor/illumos/dist/lib/libzfs/common/libzfs_sendrecv.c vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.c Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_pool.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_pool.c Fri Apr 14 18:13:33 2017 (r316901) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_pool.c Fri Apr 14 18:14:02 2017 (r316902) @@ -818,7 +818,7 @@ zpool_prop_get_feature(zpool_handle_t *z const char *feature = strchr(propname, '@') + 1; supported = zpool_prop_feature(propname); - ASSERT(supported || zfs_prop_unsupported(propname)); + ASSERT(supported || zpool_prop_unsupported(propname)); /* * Convert from feature name to feature guid. This conversion is Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_sendrecv.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_sendrecv.c Fri Apr 14 18:13:33 2017 (r316901) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_sendrecv.c Fri Apr 14 18:14:02 2017 (r316902) @@ -266,6 +266,15 @@ cksummer(void *arg) ofp = fdopen(dda->inputfd, "r"); while (ssread(drr, sizeof (*drr), ofp) != 0) { + /* + * kernel filled in checksum, we are going to write same + * record, but need to regenerate checksum. + */ + if (drr->drr_type != DRR_BEGIN) { + bzero(&drr->drr_u.drr_checksum.drr_checksum, + sizeof (drr->drr_u.drr_checksum.drr_checksum)); + } + switch (drr->drr_type) { case DRR_BEGIN: { Modified: vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.c ============================================================================== --- vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.c Fri Apr 14 18:13:33 2017 (r316901) +++ vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.c Fri Apr 14 18:14:02 2017 (r316902) @@ -85,7 +85,7 @@ #include #include -static int g_fd; +static int g_fd = -1; static pthread_mutex_t g_lock = PTHREAD_MUTEX_INITIALIZER; static int g_refcount; @@ -110,9 +110,14 @@ libzfs_core_fini(void) { (void) pthread_mutex_lock(&g_lock); ASSERT3S(g_refcount, >, 0); - g_refcount--; - if (g_refcount == 0) + + if (g_refcount > 0) + g_refcount--; + + if (g_refcount == 0 && g_fd != -1) { (void) close(g_fd); + g_fd = -1; + } (void) pthread_mutex_unlock(&g_lock); } @@ -126,6 +131,7 @@ lzc_ioctl(zfs_ioc_t ioc, const char *nam size_t size; ASSERT3S(g_refcount, >, 0); + VERIFY3S(g_fd, !=, -1); (void) strlcpy(zc.zc_name, name, sizeof (zc.zc_name)); @@ -328,6 +334,9 @@ lzc_exists(const char *dataset) */ zfs_cmd_t zc = { 0 }; + ASSERT3S(g_refcount, >, 0); + VERIFY3S(g_fd, !=, -1); + (void) strlcpy(zc.zc_name, dataset, sizeof (zc.zc_name)); return (ioctl(g_fd, ZFS_IOC_OBJSET_STATS, &zc) == 0); } @@ -573,6 +582,7 @@ recv_impl(const char *snapname, nvlist_t int error; ASSERT3S(g_refcount, >, 0); + VERIFY3S(g_fd, !=, -1); /* zc_name is name of containing filesystem */ (void) strlcpy(zc.zc_name, snapname, sizeof (zc.zc_name)); From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:14:42 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id DB3DDD3EBEB; Fri, 14 Apr 2017 18:14:42 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id B6B079C4; Fri, 14 Apr 2017 18:14:42 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIEfCX066258; Fri, 14 Apr 2017 18:14:41 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIEfkS066256; Fri, 14 Apr 2017 18:14:41 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141814.v3EIEfkS066256@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:14:41 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316903 - in vendor-sys/illumos/dist/uts/common/fs/zfs: . sys X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:14:43 -0000 Author: avg Date: Fri Apr 14 18:14:41 2017 New Revision: 316903 URL: https://svnweb.freebsd.org/changeset/base/316903 Log: 7779 clean up unused definitions in zfs ctldir code illumos/illumos-gate@b7f9f60c8eb061c94244a6181a00288684164e1b https://github.com/illumos/illumos-gate/commit/b7f9f60c8eb061c94244a6181a00288684164e1b https://www.illumos.org/issues/7779 zfsctl_ops_shares_dir and ZFSCTL_INO_SHARES are essentially dead code. While there, fix the index range check in zfsctl_root_inode_cb. Reviewed by: Matt Ahrens Reviewed by: Serapheim Dimitropoulos Approved by: Dan McDonald Author: Andriy Gapon Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zfs_ctldir.h vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ctldir.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zfs_ctldir.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zfs_ctldir.h Fri Apr 14 18:14:02 2017 (r316902) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zfs_ctldir.h Fri Apr 14 18:14:41 2017 (r316903) @@ -64,7 +64,6 @@ int zfsctl_lookup_objset(vfs_t *vfsp, ui #define ZFSCTL_INO_ROOT 0x1 #define ZFSCTL_INO_SNAPDIR 0x2 -#define ZFSCTL_INO_SHARES 0x3 #ifdef __cplusplus } Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ctldir.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ctldir.c Fri Apr 14 18:14:02 2017 (r316902) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ctldir.c Fri Apr 14 18:14:41 2017 (r316903) @@ -117,7 +117,6 @@ vnodeops_t *zfsctl_ops_root; vnodeops_t *zfsctl_ops_snapdir; vnodeops_t *zfsctl_ops_snapshot; vnodeops_t *zfsctl_ops_shares; -vnodeops_t *zfsctl_ops_shares_dir; static const fs_operation_def_t zfsctl_tops_root[]; static const fs_operation_def_t zfsctl_tops_snapdir[]; @@ -133,8 +132,7 @@ static gfs_opsvec_t zfsctl_opsvec[] = { { ".zfs", zfsctl_tops_root, &zfsctl_ops_root }, { ".zfs/snapshot", zfsctl_tops_snapdir, &zfsctl_ops_snapdir }, { ".zfs/snapshot/vnode", zfsctl_tops_snapshot, &zfsctl_ops_snapshot }, - { ".zfs/shares", zfsctl_tops_shares, &zfsctl_ops_shares_dir }, - { ".zfs/shares/vnode", zfsctl_tops_shares, &zfsctl_ops_shares }, + { ".zfs/shares", zfsctl_tops_shares, &zfsctl_ops_shares }, { NULL } }; @@ -178,14 +176,11 @@ zfsctl_fini(void) vn_freevnodeops(zfsctl_ops_snapshot); if (zfsctl_ops_shares) vn_freevnodeops(zfsctl_ops_shares); - if (zfsctl_ops_shares_dir) - vn_freevnodeops(zfsctl_ops_shares_dir); zfsctl_ops_root = NULL; zfsctl_ops_snapdir = NULL; zfsctl_ops_snapshot = NULL; zfsctl_ops_shares = NULL; - zfsctl_ops_shares_dir = NULL; } boolean_t @@ -194,8 +189,7 @@ zfsctl_is_node(vnode_t *vp) return (vn_matchops(vp, zfsctl_ops_root) || vn_matchops(vp, zfsctl_ops_snapdir) || vn_matchops(vp, zfsctl_ops_snapshot) || - vn_matchops(vp, zfsctl_ops_shares) || - vn_matchops(vp, zfsctl_ops_shares_dir)); + vn_matchops(vp, zfsctl_ops_shares)); } @@ -209,7 +203,7 @@ zfsctl_root_inode_cb(vnode_t *vp, int in { zfsvfs_t *zfsvfs = vp->v_vfsp->vfs_data; - ASSERT(index <= 2); + ASSERT(index < 2); if (index == 0) return (ZFSCTL_INO_SNAPDIR); From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:15:14 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 9F142D3EC3E; Fri, 14 Apr 2017 18:15:14 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 70F9DAF0; Fri, 14 Apr 2017 18:15:14 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIFDF0066332; Fri, 14 Apr 2017 18:15:13 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIFD5J066331; Fri, 14 Apr 2017 18:15:13 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141815.v3EIFD5J066331@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:15:13 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316904 - vendor/illumos/dist/lib/libzfs_core/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:15:14 -0000 Author: avg Date: Fri Apr 14 18:15:13 2017 New Revision: 316904 URL: https://svnweb.freebsd.org/changeset/base/316904 Log: 7729 libzfs_core`lzc_rollback() leaks result nvl illumos/illumos-gate@ac428481f96be89add7a1edf43ae47dd71038553 https://github.com/illumos/illumos-gate/commit/ac428481f96be89add7a1edf43ae47dd71038553 https://www.illumos.org/issues/7729 libzfs_core`lzc_rollback() doesn't free the result nvl after lzc_ioctl() call. Reviewed by: Matthew Ahrens Reviewed by: Prakash Surya Approved by: Dan McDonald Author: Yuri Pankov Modified: vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.c Modified: vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.c ============================================================================== --- vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.c Fri Apr 14 18:14:41 2017 (r316903) +++ vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.c Fri Apr 14 18:15:13 2017 (r316904) @@ -722,6 +722,8 @@ lzc_rollback(const char *fsname, char *s const char *snapname = fnvlist_lookup_string(result, "target"); (void) strlcpy(snapnamebuf, snapname, snapnamelen); } + nvlist_free(result); + return (err); } From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:15:48 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id E93B3D3EC96; Fri, 14 Apr 2017 18:15:48 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id AAB33C34; Fri, 14 Apr 2017 18:15:48 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIFlKH066396; Fri, 14 Apr 2017 18:15:47 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIFlA4066394; Fri, 14 Apr 2017 18:15:47 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141815.v3EIFlA4066394@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:15:47 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316905 - in vendor-sys/illumos/dist/uts/common/fs/zfs: . sys X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:15:49 -0000 Author: avg Date: Fri Apr 14 18:15:47 2017 New Revision: 316905 URL: https://svnweb.freebsd.org/changeset/base/316905 Log: 7740 fix for 6513 only works in hole punching case, not truncation illumos/illumos-gate@7de35a3ed0c2e6d4256bd2fb05b48b3798aaf553 https://github.com/illumos/illumos-gate/commit/7de35a3ed0c2e6d4256bd2fb05b48b3798aaf553 https://www.illumos.org/issues/7740 The problem is that dbuf_findbp will return ENOENT if the block it's trying to find is beyond the end of the file. If that happens, we assume there is no birth time, and so we lose that information when we write out new blkptrs. We should teach dbuf_findbp to look for things that are beyond the current end, but not beyond the absolute end of the file. To verify, create a large file, truncate it to a short length, and then write beyond the end. Check with zdb to make sure that there are no holes with birth time zero (will appear as gaps). Reviewed by: Steve Gonczi Reviewed by: Matthew Ahrens Approved by: Dan McDonald Author: Paul Dagnelie Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dnode.h Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c Fri Apr 14 18:15:13 2017 (r316904) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c Fri Apr 14 18:15:47 2017 (r316905) @@ -2161,8 +2161,6 @@ static int dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, dmu_buf_impl_t **parentp, blkptr_t **bpp) { - int nlevels, epbs; - *parentp = NULL; *bpp = NULL; @@ -2181,17 +2179,35 @@ dbuf_findbp(dnode_t *dn, int level, uint return (0); } - if (dn->dn_phys->dn_nlevels == 0) - nlevels = 1; - else - nlevels = dn->dn_phys->dn_nlevels; - - epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; + int nlevels = + (dn->dn_phys->dn_nlevels == 0) ? 1 : dn->dn_phys->dn_nlevels; + int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; ASSERT3U(level * epbs, <, 64); ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); + /* + * This assertion shouldn't trip as long as the max indirect block size + * is less than 1M. The reason for this is that up to that point, + * the number of levels required to address an entire object with blocks + * of size SPA_MINBLOCKSIZE satisfies nlevels * epbs + 1 <= 64. In + * other words, if N * epbs + 1 > 64, then if (N-1) * epbs + 1 > 55 + * (i.e. we can address the entire object), objects will all use at most + * N-1 levels and the assertion won't overflow. However, once epbs is + * 13, 4 * 13 + 1 = 53, but 5 * 13 + 1 = 66. Then, 4 levels will not be + * enough to address an entire object, so objects will have 5 levels, + * but then this assertion will overflow. + * + * All this is to say that if we ever increase DN_MAX_INDBLKSHIFT, we + * need to redo this logic to handle overflows. + */ + ASSERT(level >= nlevels || + ((nlevels - level - 1) * epbs) + + highbit64(dn->dn_phys->dn_nblkptr) <= 64); if (level >= nlevels || - (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { + blkid >= ((uint64_t)dn->dn_phys->dn_nblkptr << + ((nlevels - level - 1) * epbs)) || + (fail_sparse && + blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { /* the buffer has no parent yet */ return (SET_ERROR(ENOENT)); } else if (level < nlevels-1) { @@ -2209,6 +2225,8 @@ dbuf_findbp(dnode_t *dn, int level, uint } *bpp = ((blkptr_t *)(*parentp)->db.db_data) + (blkid & ((1ULL << epbs) - 1)); + if (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs))) + ASSERT(BP_IS_HOLE(*bpp)); return (0); } else { /* the block is referenced from the dnode */ Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dnode.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dnode.h Fri Apr 14 18:15:13 2017 (r316904) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dnode.h Fri Apr 14 18:15:47 2017 (r316905) @@ -58,6 +58,12 @@ extern "C" { */ #define DNODE_SHIFT 9 /* 512 bytes */ #define DN_MIN_INDBLKSHIFT 12 /* 4k */ +/* + * If we ever increase this value beyond 20, we need to revisit all logic that + * does x << level * ebps to handle overflow. With a 1M indirect block size, + * 4 levels of indirect blocks would not be able to guarantee addressing an + * entire object, so 5 levels will be used, but 5 * (20 - 7) = 65. + */ #define DN_MAX_INDBLKSHIFT 17 /* 128k */ #define DNODE_BLOCK_SHIFT 14 /* 16k */ #define DNODE_CORE_SIZE 64 /* 64 bytes for dnode sans blkptrs */ From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:19:50 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 5EFF6D3ECF1; Fri, 14 Apr 2017 18:19:50 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 2E38FD88; Fri, 14 Apr 2017 18:19:50 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIJnOw066590; Fri, 14 Apr 2017 18:19:49 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIJnoY066587; Fri, 14 Apr 2017 18:19:49 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141819.v3EIJnoY066587@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:19:49 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316906 - in vendor-sys/illumos/dist/uts/common: fs fs/zfs sys X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:19:50 -0000 Author: avg Date: Fri Apr 14 18:19:48 2017 New Revision: 316906 URL: https://svnweb.freebsd.org/changeset/base/316906 Log: 4242 file rename event fires before the rename happens illumos/illumos-gate@54207fd2e1e7ed01d0416da8cf296dbef920fbfc https://github.com/illumos/illumos-gate/commit/54207fd2e1e7ed01d0416da8cf296dbef920fbfc https://www.illumos.org/issues/4242 From Joyent's OS-2557: So we're basically just doing a check here that after we got a 'rename' event for our file, the file has actually been moved out of the way. What I've seen in bh1-stage2 is that this happens as you'd expect for all zones but many times over the last week it's failed because when we do the fs.exists () check here, the file that we got the rename event for, still exists. I've confirmed what's happening using the following dtrace script: #!/usr/sbin/dtrace -s #pragma D option quiet #pragma D option bufsize=256k syscall::open:entry, syscall::open64:entry /copyinstr(arg0) == "/var/svc/provisioning" || (strlen(copyinstr(arg0)) == 69 && substr(copyinstr(arg0), 48) == "/var/svc/provisioning")/ { this->watching_open = 1; printf("%d zone %s process %s(%d) [%s] open(%s)\\n", timestamp, zonename, execname, pid, curpsinfo->pr_psargs, copyinstr(arg0)); } syscall::open:return, syscall::open64:return /this->watching_open == 1/ Reviewed by: Robert Mustacchi Reviewed by: Marcel Telka Approved by: Dan McDonald Author: Jerry Jelinek Modified: vendor-sys/illumos/dist/uts/common/fs/vnode.c vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vnops.c vendor-sys/illumos/dist/uts/common/sys/vnode.h Modified: vendor-sys/illumos/dist/uts/common/fs/vnode.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/vnode.c Fri Apr 14 18:15:47 2017 (r316905) +++ vendor-sys/illumos/dist/uts/common/fs/vnode.c Fri Apr 14 18:19:48 2017 (r316906) @@ -2557,6 +2557,36 @@ vnevent_rmdir(vnode_t *vp, vnode_t *dvp, } void +vnevent_pre_rename_src(vnode_t *vp, vnode_t *dvp, char *name, + caller_context_t *ct) +{ + if (vp == NULL || vp->v_femhead == NULL) { + return; + } + (void) VOP_VNEVENT(vp, VE_PRE_RENAME_SRC, dvp, name, ct); +} + +void +vnevent_pre_rename_dest(vnode_t *vp, vnode_t *dvp, char *name, + caller_context_t *ct) +{ + if (vp == NULL || vp->v_femhead == NULL) { + return; + } + (void) VOP_VNEVENT(vp, VE_PRE_RENAME_DEST, dvp, name, ct); +} + +void +vnevent_pre_rename_dest_dir(vnode_t *vp, vnode_t *nvp, char *name, + caller_context_t *ct) +{ + if (vp == NULL || vp->v_femhead == NULL) { + return; + } + (void) VOP_VNEVENT(vp, VE_PRE_RENAME_DEST_DIR, nvp, name, ct); +} + +void vnevent_create(vnode_t *vp, caller_context_t *ct) { if (vp == NULL || vp->v_femhead == NULL) { Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vnops.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vnops.c Fri Apr 14 18:15:47 2017 (r316905) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vnops.c Fri Apr 14 18:19:48 2017 (r316906) @@ -23,6 +23,7 @@ * Copyright (c) 2012, 2015 by Delphix. All rights reserved. * Copyright 2014 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2014 Integros [integros.com] + * Copyright 2015 Joyent, Inc. */ /* Portions Copyright 2007 Jeremy Teo */ @@ -3447,7 +3448,7 @@ zfs_rename(vnode_t *sdvp, char *snm, vno dmu_tx_t *tx; zfs_zlock_t *zl; int cmp, serr, terr; - int error = 0; + int error = 0, rm_err = 0; int zflg = 0; boolean_t waited = B_FALSE; @@ -3659,16 +3660,16 @@ top: } } - vnevent_rename_src(ZTOV(szp), sdvp, snm, ct); + vnevent_pre_rename_src(ZTOV(szp), sdvp, snm, ct); if (tzp) - vnevent_rename_dest(ZTOV(tzp), tdvp, tnm, ct); + vnevent_pre_rename_dest(ZTOV(tzp), tdvp, tnm, ct); /* * notify the target directory if it is not the same * as source directory. */ if (tdvp != sdvp) { - vnevent_rename_dest_dir(tdvp, ct); + vnevent_pre_rename_dest_dir(tdvp, ZTOV(szp), tnm, ct); } tx = dmu_tx_create(zfsvfs->z_os); @@ -3712,7 +3713,7 @@ top: } if (tzp) /* Attempt to remove the existing target */ - error = zfs_link_destroy(tdl, tzp, tx, zflg, NULL); + error = rm_err = zfs_link_destroy(tdl, tzp, tx, zflg, NULL); if (error == 0) { error = zfs_link_create(tdl, szp, tx, ZRENAMING); @@ -3754,6 +3755,16 @@ top: } dmu_tx_commit(tx); + + if (tzp && rm_err == 0) + vnevent_rename_dest(ZTOV(tzp), tdvp, tnm, ct); + + if (error == 0) { + vnevent_rename_src(ZTOV(szp), sdvp, snm, ct); + /* notify the target dir if it is not the same as source dir */ + if (tdvp != sdvp) + vnevent_rename_dest_dir(tdvp, ct); + } out: if (zl != NULL) zfs_rename_unlock(&zl); Modified: vendor-sys/illumos/dist/uts/common/sys/vnode.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/sys/vnode.h Fri Apr 14 18:15:47 2017 (r316905) +++ vendor-sys/illumos/dist/uts/common/sys/vnode.h Fri Apr 14 18:19:48 2017 (r316906) @@ -724,7 +724,12 @@ typedef enum symfollow symfollow_t; typedef enum vcexcl vcexcl_t; typedef enum create create_t; -/* Vnode Events - Used by VOP_VNEVENT */ +/* + * Vnode Events - Used by VOP_VNEVENT + * The VE_PRE_RENAME_* events fire before the rename operation and are + * primarily used for specialized applications, such as NFSv4 delegation, which + * need to know about rename before it occurs. + */ typedef enum vnevent { VE_SUPPORT = 0, /* Query */ VE_RENAME_SRC = 1, /* Rename, with vnode as source */ @@ -735,7 +740,10 @@ typedef enum vnevent { VE_LINK = 6, /* Link with vnode's name as source */ VE_RENAME_DEST_DIR = 7, /* Rename with vnode as target dir */ VE_MOUNTEDOVER = 8, /* File or Filesystem got mounted over vnode */ - VE_TRUNCATE = 9 /* Truncate */ + VE_TRUNCATE = 9, /* Truncate */ + VE_PRE_RENAME_SRC = 10, /* Pre-rename, with vnode as source */ + VE_PRE_RENAME_DEST = 11, /* Pre-rename, with vnode as target/dest. */ + VE_PRE_RENAME_DEST_DIR = 12 /* Pre-rename with vnode as target dir */ } vnevent_t; /* @@ -1294,6 +1302,12 @@ void vnevent_rename_dest_dir(vnode_t *, void vnevent_mountedover(vnode_t *, caller_context_t *); void vnevent_truncate(vnode_t *, caller_context_t *); int vnevent_support(vnode_t *, caller_context_t *); +void vnevent_pre_rename_src(vnode_t *, vnode_t *, char *, + caller_context_t *); +void vnevent_pre_rename_dest(vnode_t *, vnode_t *, char *, + caller_context_t *); +void vnevent_pre_rename_dest_dir(vnode_t *, vnode_t *, char *, + caller_context_t *); /* Vnode specific data */ void vsd_create(uint_t *, void (*)(void *)); From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:20:23 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 01EDBD3ED5F; Fri, 14 Apr 2017 18:20:23 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id C495CEC2; Fri, 14 Apr 2017 18:20:22 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIKM7R066686; Fri, 14 Apr 2017 18:20:22 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIKL4E066677; Fri, 14 Apr 2017 18:20:21 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141820.v3EIKL4E066677@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:20:21 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316907 - in vendor-sys/illumos/dist/uts/common/fs/zfs: . sys X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:20:23 -0000 Author: avg Date: Fri Apr 14 18:20:20 2017 New Revision: 316907 URL: https://svnweb.freebsd.org/changeset/base/316907 Log: 1300 filename normalization doesn't work for removes illumos/illumos-gate@1c17160ac558f98048951327f4e9248d8f46acc0 https://github.com/illumos/illumos-gate/commit/1c17160ac558f98048951327f4e9248d8f46acc0 https://www.illumos.org/issues/1300 Problem: We can create invisible file in ZFS. How to reproduce: 0. Prepare normalization formD ZFS. # cat cat /etc/release cat: cat: No such file or directory OpenIndiana Development oi_151 X86 (powered by illumos) Copyright 2011 Oracle and/or its affiliates. All rights reserved. Use is subject to license terms. Assembled 28 April 2011 # mkfile 100M 100M # zpool create -O utf8only=on -O normalization=formD test_pool $( pwd )/ 100M # zpool upgrade This system is currently running ZFS pool version 28. All pools are formatted using this version. # zfs get normalization test_pool NAME PROPERTY VALUE SOURCE test_pool normalization formD - # chmod 777 /test_pool 1. Create a NFD file. $ cd /test_pool/ $ cp /etc/release $( echo "\x75\xcc\x88" ) $ ls -la total 4 drwxrwxrwx 2 root root 3 2011-07-29 08:53 . drwxr-xr-x 25 root root 26 2011-07-29 08:53 .. -r--r--r-- 1 test1 staff 251 2011-07-29 08:53 u? Reviewed by: Yuri Pankov Reviewed by: Pavel Zakharov Reviewed by: Matt Ahrens Approved by: Dan McDonald Author: Kevin Crowe Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_bookmark.c vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_dataset.c vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zap.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zap_impl.h vendor-sys/illumos/dist/uts/common/fs/zfs/zap_leaf.c vendor-sys/illumos/dist/uts/common/fs/zfs/zap_micro.c vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_dir.c vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vnops.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c Fri Apr 14 18:19:48 2017 (r316906) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c Fri Apr 14 18:20:20 2017 (r316907) @@ -18,15 +18,16 @@ * * CDDL HEADER END */ + /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, 2016 by Delphix. All rights reserved. * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. * Copyright (c) 2013, Joyent, Inc. All rights reserved. * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. - * Copyright 2015 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2015, STRATO AG, Inc. All rights reserved. * Copyright (c) 2014 Integros [integros.com] + * Copyright 2017 Nexenta Systems, Inc. */ /* Portions Copyright 2010 Robert Milkowski */ @@ -1622,7 +1623,7 @@ dmu_snapshot_realname(objset_t *os, char return (zap_lookup_norm(ds->ds_dir->dd_pool->dp_meta_objset, dsl_dataset_phys(ds)->ds_snapnames_zapobj, name, 8, 1, &ignored, - MT_FIRST, real, maxlen, conflict)); + MT_NORMALIZE, real, maxlen, conflict)); } int Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_bookmark.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_bookmark.c Fri Apr 14 18:19:48 2017 (r316906) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_bookmark.c Fri Apr 14 18:20:20 2017 (r316907) @@ -12,8 +12,10 @@ * * CDDL HEADER END */ + /* * Copyright (c) 2013, 2014 by Delphix. All rights reserved. + * Copyright 2017 Nexenta Systems, Inc. */ #include @@ -59,16 +61,14 @@ dsl_dataset_bmark_lookup(dsl_dataset_t * { objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset; uint64_t bmark_zapobj = ds->ds_bookmarks; - matchtype_t mt; + matchtype_t mt = 0; int err; if (bmark_zapobj == 0) return (SET_ERROR(ESRCH)); if (dsl_dataset_phys(ds)->ds_flags & DS_FLAG_CI_DATASET) - mt = MT_FIRST; - else - mt = MT_EXACT; + mt = MT_NORMALIZE; err = zap_lookup_norm(mos, bmark_zapobj, shortname, sizeof (uint64_t), sizeof (*bmark_phys) / sizeof (uint64_t), bmark_phys, mt, @@ -339,12 +339,10 @@ dsl_dataset_bookmark_remove(dsl_dataset_ { objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset; uint64_t bmark_zapobj = ds->ds_bookmarks; - matchtype_t mt; + matchtype_t mt = 0; if (dsl_dataset_phys(ds)->ds_flags & DS_FLAG_CI_DATASET) - mt = MT_FIRST; - else - mt = MT_EXACT; + mt = MT_NORMALIZE; return (zap_remove_norm(mos, bmark_zapobj, name, mt, tx)); } Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_dataset.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_dataset.c Fri Apr 14 18:19:48 2017 (r316906) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_dataset.c Fri Apr 14 18:20:20 2017 (r316907) @@ -18,6 +18,7 @@ * * CDDL HEADER END */ + /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2011, 2016 by Delphix. All rights reserved. @@ -26,6 +27,7 @@ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. * Copyright (c) 2014 Integros [integros.com] * Copyright 2016, OmniTI Computer Consulting, Inc. All rights reserved. + * Copyright 2017 Nexenta Systems, Inc. */ #include @@ -353,17 +355,15 @@ dsl_dataset_snap_lookup(dsl_dataset_t *d { objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset; uint64_t snapobj = dsl_dataset_phys(ds)->ds_snapnames_zapobj; - matchtype_t mt; + matchtype_t mt = 0; int err; if (dsl_dataset_phys(ds)->ds_flags & DS_FLAG_CI_DATASET) - mt = MT_FIRST; - else - mt = MT_EXACT; + mt = MT_NORMALIZE; err = zap_lookup_norm(mos, snapobj, name, 8, 1, value, mt, NULL, 0, NULL); - if (err == ENOTSUP && mt == MT_FIRST) + if (err == ENOTSUP && (mt & MT_NORMALIZE)) err = zap_lookup(mos, snapobj, name, 8, 1, value); return (err); } @@ -374,18 +374,16 @@ dsl_dataset_snap_remove(dsl_dataset_t *d { objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset; uint64_t snapobj = dsl_dataset_phys(ds)->ds_snapnames_zapobj; - matchtype_t mt; + matchtype_t mt = 0; int err; dsl_dir_snap_cmtime_update(ds->ds_dir); if (dsl_dataset_phys(ds)->ds_flags & DS_FLAG_CI_DATASET) - mt = MT_FIRST; - else - mt = MT_EXACT; + mt = MT_NORMALIZE; err = zap_remove_norm(mos, snapobj, name, mt, tx); - if (err == ENOTSUP && mt == MT_FIRST) + if (err == ENOTSUP && (mt & MT_NORMALIZE)) err = zap_remove(mos, snapobj, name, tx); if (err == 0 && adj_cnt) Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zap.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zap.h Fri Apr 14 18:19:48 2017 (r316906) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zap.h Fri Apr 14 18:20:20 2017 (r316907) @@ -18,9 +18,11 @@ * * CDDL HEADER END */ + /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, 2016 by Delphix. All rights reserved. + * Copyright 2017 Nexenta Systems, Inc. */ #ifndef _SYS_ZAP_H @@ -88,22 +90,15 @@ extern "C" { /* * Specifies matching criteria for ZAP lookups. - */ -typedef enum matchtype -{ - /* Only find an exact match (non-normalized) */ - MT_EXACT, - /* - * If there is an exact match, find that, otherwise find the - * first normalized match. - */ - MT_BEST, - /* - * Find the "first" normalized (case and Unicode form) match; - * the designated "first" match will not change as long as the - * set of entries with this normalization doesn't change. - */ - MT_FIRST + * MT_NORMALIZE Use ZAP normalization flags, which can include both + * unicode normalization and case-insensitivity. + * MT_MATCH_CASE Do case-sensitive lookups even if MT_NORMALIZE is + * specified and ZAP normalization flags include + * U8_TEXTPREP_TOUPPER. + */ +typedef enum matchtype { + MT_NORMALIZE = 1 << 0, + MT_MATCH_CASE = 1 << 1, } matchtype_t; typedef enum zap_flags { @@ -120,16 +115,6 @@ typedef enum zap_flags { /* * Create a new zapobj with no attributes and return its object number. - * MT_EXACT will cause the zap object to only support MT_EXACT lookups, - * otherwise any matchtype can be used for lookups. - * - * normflags specifies what normalization will be done. values are: - * 0: no normalization (legacy on-disk format, supports MT_EXACT matching - * only) - * U8_TEXTPREP_TOLOWER: case normalization will be performed. - * MT_FIRST/MT_BEST matching will find entries that match without - * regard to case (eg. looking for "foo" can find an entry "Foo"). - * Eventually, other flags will permit unicode normalization as well. */ uint64_t zap_create(objset_t *ds, dmu_object_type_t ot, dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zap_impl.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zap_impl.h Fri Apr 14 18:19:48 2017 (r316906) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zap_impl.h Fri Apr 14 18:20:20 2017 (r316907) @@ -18,11 +18,13 @@ * * CDDL HEADER END */ + /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2013, 2016 by Delphix. All rights reserved. * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. * Copyright (c) 2014 Integros [integros.com] + * Copyright 2017 Nexenta Systems, Inc. */ #ifndef _SYS_ZAP_IMPL_H @@ -189,6 +191,7 @@ typedef struct zap_name { int zn_key_norm_numints; uint64_t zn_hash; matchtype_t zn_matchtype; + int zn_normflags; char zn_normbuf[ZAP_MAXNAMELEN]; } zap_name_t; Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zap_leaf.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zap_leaf.c Fri Apr 14 18:19:48 2017 (r316906) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zap_leaf.c Fri Apr 14 18:20:20 2017 (r316907) @@ -18,9 +18,11 @@ * * CDDL HEADER END */ + /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2013, 2015 by Delphix. All rights reserved. + * Copyright 2017 Nexenta Systems, Inc. */ /* @@ -361,7 +363,7 @@ zap_leaf_array_match(zap_leaf_t *l, zap_ } ASSERT(zn->zn_key_intlen == 1); - if (zn->zn_matchtype == MT_FIRST) { + if (zn->zn_matchtype & MT_NORMALIZE) { char *thisname = kmem_alloc(array_numints, KM_SLEEP); boolean_t match; @@ -403,7 +405,6 @@ zap_leaf_lookup(zap_leaf_t *l, zap_name_ ASSERT3U(zap_leaf_phys(l)->l_hdr.lh_magic, ==, ZAP_LEAF_MAGIC); -again: for (chunkp = LEAF_HASH_ENTPTR(l, zn->zn_hash); *chunkp != CHAIN_END; chunkp = &le->le_next) { uint16_t chunk = *chunkp; @@ -418,9 +419,9 @@ again: /* * NB: the entry chain is always sorted by cd on * normalized zap objects, so this will find the - * lowest-cd match for MT_FIRST. + * lowest-cd match for MT_NORMALIZE. */ - ASSERT(zn->zn_matchtype == MT_EXACT || + ASSERT((zn->zn_matchtype == 0) || (zap_leaf_phys(l)->l_hdr.lh_flags & ZLF_ENTRIES_CDSORTED)); if (zap_leaf_array_match(l, zn, le->le_name_chunk, le->le_name_numints)) { @@ -434,15 +435,6 @@ again: } } - /* - * NB: we could of course do this in one pass, but that would be - * a pain. We'll see if MT_BEST is even used much. - */ - if (zn->zn_matchtype == MT_BEST) { - zn->zn_matchtype = MT_FIRST; - goto again; - } - return (SET_ERROR(ENOENT)); } @@ -697,7 +689,7 @@ zap_entry_normalization_conflict(zap_ent continue; if (zn == NULL) { - zn = zap_name_alloc(zap, name, MT_FIRST); + zn = zap_name_alloc(zap, name, MT_NORMALIZE); allocdzn = B_TRUE; } if (zap_leaf_array_match(zeh->zeh_leaf, zn, Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zap_micro.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zap_micro.c Fri Apr 14 18:19:48 2017 (r316906) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zap_micro.c Fri Apr 14 18:20:20 2017 (r316907) @@ -18,11 +18,13 @@ * * CDDL HEADER END */ + /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2011, 2016 by Delphix. All rights reserved. * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. * Copyright (c) 2014 Integros [integros.com] + * Copyright 2017 Nexenta Systems, Inc. */ #include @@ -133,7 +135,7 @@ zap_hash(zap_name_t *zn) } static int -zap_normalize(zap_t *zap, const char *name, char *namenorm) +zap_normalize(zap_t *zap, const char *name, char *namenorm, int normflags) { size_t inlen, outlen; int err; @@ -145,8 +147,8 @@ zap_normalize(zap_t *zap, const char *na err = 0; (void) u8_textprep_str((char *)name, &inlen, namenorm, &outlen, - zap->zap_normflags | U8_TEXTPREP_IGNORE_NULL | - U8_TEXTPREP_IGNORE_INVALID, U8_UNICODE_LATEST, &err); + normflags | U8_TEXTPREP_IGNORE_NULL | U8_TEXTPREP_IGNORE_INVALID, + U8_UNICODE_LATEST, &err); return (err); } @@ -156,15 +158,15 @@ zap_match(zap_name_t *zn, const char *ma { ASSERT(!(zap_getflags(zn->zn_zap) & ZAP_FLAG_UINT64_KEY)); - if (zn->zn_matchtype == MT_FIRST) { + if (zn->zn_matchtype & MT_NORMALIZE) { char norm[ZAP_MAXNAMELEN]; - if (zap_normalize(zn->zn_zap, matchname, norm) != 0) + if (zap_normalize(zn->zn_zap, matchname, norm, + zn->zn_normflags) != 0) return (B_FALSE); return (strcmp(zn->zn_key_norm, norm) == 0); } else { - /* MT_BEST or MT_EXACT */ return (strcmp(zn->zn_key_orig, matchname) == 0); } } @@ -185,15 +187,30 @@ zap_name_alloc(zap_t *zap, const char *k zn->zn_key_orig = key; zn->zn_key_orig_numints = strlen(zn->zn_key_orig) + 1; zn->zn_matchtype = mt; + zn->zn_normflags = zap->zap_normflags; + + /* + * If we're dealing with a case sensitive lookup on a mixed or + * insensitive fs, remove U8_TEXTPREP_TOUPPER or the lookup + * will fold case to all caps overriding the lookup request. + */ + if (mt & MT_MATCH_CASE) + zn->zn_normflags &= ~U8_TEXTPREP_TOUPPER; + if (zap->zap_normflags) { - if (zap_normalize(zap, key, zn->zn_normbuf) != 0) { + /* + * We *must* use zap_normflags because this normalization is + * what the hash is computed from. + */ + if (zap_normalize(zap, key, zn->zn_normbuf, + zap->zap_normflags) != 0) { zap_name_free(zn); return (NULL); } zn->zn_key_norm = zn->zn_normbuf; zn->zn_key_norm_numints = strlen(zn->zn_key_norm) + 1; } else { - if (mt != MT_EXACT) { + if (mt != 0) { zap_name_free(zn); return (NULL); } @@ -202,6 +219,20 @@ zap_name_alloc(zap_t *zap, const char *k } zn->zn_hash = zap_hash(zn); + + if (zap->zap_normflags != zn->zn_normflags) { + /* + * We *must* use zn_normflags because this normalization is + * what the matching is based on. (Not the hash!) + */ + if (zap_normalize(zap, key, zn->zn_normbuf, + zn->zn_normflags) != 0) { + zap_name_free(zn); + return (NULL); + } + zn->zn_key_norm_numints = strlen(zn->zn_key_norm) + 1; + } + return (zn); } @@ -215,7 +246,7 @@ zap_name_alloc_uint64(zap_t *zap, const zn->zn_key_intlen = sizeof (*key); zn->zn_key_orig = zn->zn_key_norm = key; zn->zn_key_orig_numints = zn->zn_key_norm_numints = numints; - zn->zn_matchtype = MT_EXACT; + zn->zn_matchtype = 0; zn->zn_hash = zap_hash(zn); return (zn); @@ -299,7 +330,6 @@ mze_find(zap_name_t *zn) mze_tofind.mze_hash = zn->zn_hash; mze_tofind.mze_cd = 0; -again: mze = avl_find(avl, &mze_tofind, &idx); if (mze == NULL) mze = avl_nearest(avl, idx, AVL_AFTER); @@ -308,10 +338,7 @@ again: if (zap_match(zn, MZE_PHYS(zn->zn_zap, mze)->mze_name)) return (mze); } - if (zn->zn_matchtype == MT_BEST) { - zn->zn_matchtype = MT_FIRST; - goto again; - } + return (NULL); } @@ -417,8 +444,7 @@ mzap_open(objset_t *os, uint64_t obj, dm zap_name_t *zn; zap->zap_m.zap_num_entries++; - zn = zap_name_alloc(zap, mze->mze_name, - MT_EXACT); + zn = zap_name_alloc(zap, mze->mze_name, 0); mze_insert(zap, i, zn->zn_hash); zap_name_free(zn); } @@ -618,7 +644,7 @@ mzap_upgrade(zap_t **zapp, void *tag, dm continue; dprintf("adding %s=%llu\n", mze->mze_name, mze->mze_value); - zn = zap_name_alloc(zap, mze->mze_name, MT_EXACT); + zn = zap_name_alloc(zap, mze->mze_name, 0); err = fzap_add_cd(zn, 8, 1, &mze->mze_value, mze->mze_cd, tag, tx); zap = zn->zn_zap; /* fzap_add_cd() may change zap */ @@ -631,6 +657,23 @@ mzap_upgrade(zap_t **zapp, void *tag, dm return (err); } +/* + * The "normflags" determine the behavior of the matchtype_t which is + * passed to zap_lookup_norm(). Names which have the same normalized + * version will be stored with the same hash value, and therefore we can + * perform normalization-insensitive lookups. We can be Unicode form- + * insensitive and/or case-insensitive. The following flags are valid for + * "normflags": + * + * U8_TEXTPREP_NFC + * U8_TEXTPREP_NFD + * U8_TEXTPREP_NFKC + * U8_TEXTPREP_NFKD + * U8_TEXTPREP_TOUPPER + * + * The *_NF* (Normalization Form) flags are mutually exclusive; at most one + * of them may be supplied. + */ void mzap_create_impl(objset_t *os, uint64_t obj, int normflags, zap_flags_t flags, dmu_tx_t *tx) @@ -789,7 +832,7 @@ again: if (zn == NULL) { zn = zap_name_alloc(zap, MZE_PHYS(zap, mze)->mze_name, - MT_FIRST); + MT_NORMALIZE); allocdzn = B_TRUE; } if (zap_match(zn, MZE_PHYS(zap, other)->mze_name)) { @@ -818,7 +861,7 @@ zap_lookup(objset_t *os, uint64_t zapobj uint64_t integer_size, uint64_t num_integers, void *buf) { return (zap_lookup_norm(os, zapobj, name, integer_size, - num_integers, buf, MT_EXACT, NULL, 0, NULL)); + num_integers, buf, 0, NULL, 0, NULL)); } static int @@ -886,7 +929,7 @@ zap_lookup_by_dnode(dnode_t *dn, const c uint64_t integer_size, uint64_t num_integers, void *buf) { return (zap_lookup_norm_by_dnode(dn, name, integer_size, - num_integers, buf, MT_EXACT, NULL, 0, NULL)); + num_integers, buf, 0, NULL, 0, NULL)); } int @@ -959,7 +1002,7 @@ int zap_contains(objset_t *os, uint64_t zapobj, const char *name) { int err = zap_lookup_norm(os, zapobj, name, 0, - 0, NULL, MT_EXACT, NULL, 0, NULL); + 0, NULL, 0, NULL, 0, NULL); if (err == EOVERFLOW || err == EINVAL) err = 0; /* found, but skipped reading the value */ return (err); @@ -977,7 +1020,7 @@ zap_length(objset_t *os, uint64_t zapobj err = zap_lockdir(os, zapobj, NULL, RW_READER, TRUE, FALSE, FTAG, &zap); if (err) return (err); - zn = zap_name_alloc(zap, name, MT_EXACT); + zn = zap_name_alloc(zap, name, 0); if (zn == NULL) { zap_unlockdir(zap, FTAG); return (SET_ERROR(ENOTSUP)); @@ -1080,7 +1123,7 @@ zap_add(objset_t *os, uint64_t zapobj, c err = zap_lockdir(os, zapobj, tx, RW_WRITER, TRUE, TRUE, FTAG, &zap); if (err) return (err); - zn = zap_name_alloc(zap, key, MT_EXACT); + zn = zap_name_alloc(zap, key, 0); if (zn == NULL) { zap_unlockdir(zap, FTAG); return (SET_ERROR(ENOTSUP)); @@ -1159,7 +1202,7 @@ zap_update(objset_t *os, uint64_t zapobj err = zap_lockdir(os, zapobj, tx, RW_WRITER, TRUE, TRUE, FTAG, &zap); if (err) return (err); - zn = zap_name_alloc(zap, name, MT_EXACT); + zn = zap_name_alloc(zap, name, 0); if (zn == NULL) { zap_unlockdir(zap, FTAG); return (SET_ERROR(ENOTSUP)); @@ -1222,7 +1265,7 @@ zap_update_uint64(objset_t *os, uint64_t int zap_remove(objset_t *os, uint64_t zapobj, const char *name, dmu_tx_t *tx) { - return (zap_remove_norm(os, zapobj, name, MT_EXACT, tx)); + return (zap_remove_norm(os, zapobj, name, 0, tx)); } int @@ -1474,7 +1517,7 @@ zap_count_write_by_dnode(dnode_t *dn, co return (err); if (!zap->zap_ismicro) { - zap_name_t *zn = zap_name_alloc(zap, name, MT_EXACT); + zap_name_t *zn = zap_name_alloc(zap, name, 0); if (zn) { err = fzap_count_write(zn, add, towrite, tooverwrite); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_dir.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_dir.c Fri Apr 14 18:19:48 2017 (r316906) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_dir.c Fri Apr 14 18:20:20 2017 (r316907) @@ -18,9 +18,11 @@ * * CDDL HEADER END */ + /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2013, 2015 by Delphix. All rights reserved. + * Copyright 2017 Nexenta Systems, Inc. */ #include @@ -62,13 +64,12 @@ * of names after deciding which is the appropriate lookup interface. */ static int -zfs_match_find(zfsvfs_t *zfsvfs, znode_t *dzp, char *name, boolean_t exact, +zfs_match_find(zfsvfs_t *zfsvfs, znode_t *dzp, char *name, matchtype_t mt, boolean_t update, int *deflags, pathname_t *rpnp, uint64_t *zoid) { int error; if (zfsvfs->z_norm) { - matchtype_t mt = MT_FIRST; boolean_t conflict = B_FALSE; size_t bufsz = 0; char *buf = NULL; @@ -77,8 +78,7 @@ zfs_match_find(zfsvfs_t *zfsvfs, znode_t buf = rpnp->pn_buf; bufsz = rpnp->pn_bufsize; } - if (exact) - mt = MT_EXACT; + /* * In the non-mixed case we only expect there would ever * be one match, but we need to use the normalizing lookup. @@ -140,7 +140,7 @@ zfs_dirent_lock(zfs_dirlock_t **dlpp, zn zfsvfs_t *zfsvfs = dzp->z_zfsvfs; zfs_dirlock_t *dl; boolean_t update; - boolean_t exact; + matchtype_t mt = 0; uint64_t zoid; vnode_t *vp = NULL; int error = 0; @@ -175,13 +175,29 @@ zfs_dirent_lock(zfs_dirlock_t **dlpp, zn */ /* - * Decide if exact matches should be requested when performing - * a zap lookup on file systems supporting case-insensitive - * access. + * When matching we may need to normalize & change case according to + * FS settings. + * + * Note that a normalized match is necessary for a case insensitive + * filesystem when the lookup request is not exact because normalization + * can fold case independent of normalizing code point sequences. + * + * See the table above zfs_dropname(). */ - exact = - ((zfsvfs->z_case == ZFS_CASE_INSENSITIVE) && (flag & ZCIEXACT)) || - ((zfsvfs->z_case == ZFS_CASE_MIXED) && !(flag & ZCILOOK)); + if (zfsvfs->z_norm != 0) { + mt = MT_NORMALIZE; + + /* + * Determine if the match needs to honor the case specified in + * lookup, and if so keep track of that so that during + * normalization we don't fold case. + */ + if ((zfsvfs->z_case == ZFS_CASE_INSENSITIVE && + (flag & ZCIEXACT)) || + (zfsvfs->z_case == ZFS_CASE_MIXED && !(flag & ZCILOOK))) { + mt |= MT_MATCH_CASE; + } + } /* * Only look in or update the DNLC if we are looking for the @@ -194,7 +210,7 @@ zfs_dirent_lock(zfs_dirlock_t **dlpp, zn * case for performance improvement? */ update = !zfsvfs->z_norm || - ((zfsvfs->z_case == ZFS_CASE_MIXED) && + (zfsvfs->z_case == ZFS_CASE_MIXED && !(zfsvfs->z_norm & ~U8_TEXTPREP_TOUPPER) && !(flag & ZCILOOK)); /* @@ -308,7 +324,7 @@ zfs_dirent_lock(zfs_dirlock_t **dlpp, zn *zpp = VTOZ(vp); return (0); } else { - error = zfs_match_find(zfsvfs, dzp, name, exact, + error = zfs_match_find(zfsvfs, dzp, name, mt, update, direntflags, realpnp, &zoid); } } @@ -774,6 +790,28 @@ zfs_link_create(zfs_dirlock_t *dl, znode return (0); } +/* + * The match type in the code for this function should conform to: + * + * ------------------------------------------------------------------------ + * fs type | z_norm | lookup type | match type + * ---------|-------------|-------------|---------------------------------- + * CS !norm | 0 | 0 | 0 (exact) + * CS norm | formX | 0 | MT_NORMALIZE + * CI !norm | upper | !ZCIEXACT | MT_NORMALIZE + * CI !norm | upper | ZCIEXACT | MT_NORMALIZE | MT_MATCH_CASE + * CI norm | upper|formX | !ZCIEXACT | MT_NORMALIZE + * CI norm | upper|formX | ZCIEXACT | MT_NORMALIZE | MT_MATCH_CASE + * CM !norm | upper | !ZCILOOK | MT_NORMALIZE | MT_MATCH_CASE + * CM !norm | upper | ZCILOOK | MT_NORMALIZE + * CM norm | upper|formX | !ZCILOOK | MT_NORMALIZE | MT_MATCH_CASE + * CM norm | upper|formX | ZCILOOK | MT_NORMALIZE + * + * Abbreviations: + * CS = Case Sensitive, CI = Case Insensitive, CM = Case Mixed + * upper = case folding set by fs type on creation (U8_TEXTPREP_TOUPPER) + * formX = unicode normalization form set on fs creation + */ static int zfs_dropname(zfs_dirlock_t *dl, znode_t *zp, znode_t *dzp, dmu_tx_t *tx, int flag) @@ -781,18 +819,20 @@ zfs_dropname(zfs_dirlock_t *dl, znode_t int error; if (zp->z_zfsvfs->z_norm) { - if (((zp->z_zfsvfs->z_case == ZFS_CASE_INSENSITIVE) && + matchtype_t mt = MT_NORMALIZE; + + if ((zp->z_zfsvfs->z_case == ZFS_CASE_INSENSITIVE && (flag & ZCIEXACT)) || - ((zp->z_zfsvfs->z_case == ZFS_CASE_MIXED) && - !(flag & ZCILOOK))) - error = zap_remove_norm(zp->z_zfsvfs->z_os, - dzp->z_id, dl->dl_name, MT_EXACT, tx); - else - error = zap_remove_norm(zp->z_zfsvfs->z_os, - dzp->z_id, dl->dl_name, MT_FIRST, tx); + (zp->z_zfsvfs->z_case == ZFS_CASE_MIXED && + !(flag & ZCILOOK))) { + mt |= MT_MATCH_CASE; + } + + error = zap_remove_norm(zp->z_zfsvfs->z_os, dzp->z_id, + dl->dl_name, mt, tx); } else { - error = zap_remove(zp->z_zfsvfs->z_os, - dzp->z_id, dl->dl_name, tx); + error = zap_remove(zp->z_zfsvfs->z_os, dzp->z_id, dl->dl_name, + tx); } return (error); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vnops.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vnops.c Fri Apr 14 18:19:48 2017 (r316906) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vnops.c Fri Apr 14 18:20:20 2017 (r316907) @@ -18,12 +18,13 @@ * * CDDL HEADER END */ + /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, 2015 by Delphix. All rights reserved. - * Copyright 2014 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2014 Integros [integros.com] * Copyright 2015 Joyent, Inc. + * Copyright 2017 Nexenta Systems, Inc. */ /* Portions Copyright 2007 Jeremy Teo */ @@ -1237,7 +1238,15 @@ zfs_lookup(vnode_t *dvp, char *nm, vnode zfsvfs_t *zfsvfs = zdp->z_zfsvfs; int error = 0; - /* fast path */ + /* + * Fast path lookup, however we must skip DNLC lookup + * for case folding or normalizing lookups because the + * DNLC code only stores the passed in name. This means + * creating 'a' and removing 'A' on a case insensitive + * file system would work, but DNLC still thinks 'a' + * exists and won't let you create it again on the next + * pass through fast path. + */ if (!(flags & (LOOKUP_XATTR | FIGNORECASE))) { if (dvp->v_type != VDIR) { @@ -1254,7 +1263,9 @@ zfs_lookup(vnode_t *dvp, char *nm, vnode return (0); } return (error); - } else { + } else if (!zdp->z_zfsvfs->z_norm && + (zdp->z_zfsvfs->z_case == ZFS_CASE_SENSITIVE)) { + vnode_t *tvp = dnlc_lookup(dvp, nm); if (tvp) { From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:20:57 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id D6B34D3EDB6; Fri, 14 Apr 2017 18:20:57 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 9A1B080; Fri, 14 Apr 2017 18:20:57 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIKuKi067419; Fri, 14 Apr 2017 18:20:56 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIKuUq067417; Fri, 14 Apr 2017 18:20:56 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141820.v3EIKuUq067417@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:20:56 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316908 - vendor/illumos/dist/lib/libzfs/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:20:57 -0000 Author: avg Date: Fri Apr 14 18:20:56 2017 New Revision: 316908 URL: https://svnweb.freebsd.org/changeset/base/316908 Log: 7541 zpool import/tryimport ioctl returns ENOMEM because provided buffer is too small for config illumos/illumos-gate@8b65a70b763232c90a91f31eb2010314c02ed338 https://github.com/illumos/illumos-gate/commit/8b65a70b763232c90a91f31eb2010314c02ed338 https://www.illumos.org/issues/7541 When calling zpool import, zpool does a few ioctls to ZFS. zpool allocates a buffer in userland and passes it to the kernel so that ZFS can copy info into it. ZFS will use it to put the nvlist that describes the pool configuration. If the allocated buffer is too small, ZFS will return ENOMEM and the call will have to be redone. This wastes CPU time and slows down the import process. This happens very often for the ZFS_IOC_POOL_TRYIMPORT call. Reviewed by: Matthew Ahrens Reviewed by: Dan Kimmel Approved by: Dan McDonald Author: Pavel Zakharov Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_impl.h vendor/illumos/dist/lib/libzfs/common/libzfs_import.c Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_impl.h ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_impl.h Fri Apr 14 18:20:20 2017 (r316907) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_impl.h Fri Apr 14 18:20:56 2017 (r316908) @@ -130,6 +130,8 @@ typedef enum { SHARED_SMB = 0x4 } zfs_share_type_t; +#define CONFIG_BUF_MINSIZE 65536 + int zfs_error(libzfs_handle_t *, int, const char *); int zfs_error_fmt(libzfs_handle_t *, int, const char *, ...); void zfs_error_aux(libzfs_handle_t *, const char *, ...); Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_import.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_import.c Fri Apr 14 18:20:20 2017 (r316907) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_import.c Fri Apr 14 18:20:56 2017 (r316908) @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2015 by Delphix. All rights reserved. + * Copyright (c) 2012, 2016 by Delphix. All rights reserved. * Copyright 2015 RackTop Systems. * Copyright 2016 Nexenta Systems, Inc. */ @@ -380,13 +380,14 @@ refresh_config(libzfs_handle_t *hdl, nvl { nvlist_t *nvl; zfs_cmd_t zc = { 0 }; - int err; + int err, dstbuf_size; if (zcmd_write_conf_nvlist(hdl, &zc, config) != 0) return (NULL); - if (zcmd_alloc_dst_nvlist(hdl, &zc, - zc.zc_nvlist_conf_size * 2) != 0) { + dstbuf_size = MAX(CONFIG_BUF_MINSIZE, zc.zc_nvlist_conf_size * 4); + + if (zcmd_alloc_dst_nvlist(hdl, &zc, dstbuf_size) != 0) { zcmd_free_nvlists(&zc); return (NULL); } From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:21:59 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id B5C9CD3EF14; Fri, 14 Apr 2017 18:21:59 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 68E4861E; Fri, 14 Apr 2017 18:21:59 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EILwMJ070317; Fri, 14 Apr 2017 18:21:58 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EILwpH070316; Fri, 14 Apr 2017 18:21:58 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141821.v3EILwpH070316@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:21:58 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316909 - vendor/illumos/dist/lib/libzfs/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:21:59 -0000 Author: avg Date: Fri Apr 14 18:21:58 2017 New Revision: 316909 URL: https://svnweb.freebsd.org/changeset/base/316909 Log: 7803 want devid_str_from_path(3devid) illumos/illumos-gate@46d46cd4fa12218e54fa3d73a9be825ed504cabb https://github.com/illumos/illumos-gate/commit/46d46cd4fa12218e54fa3d73a9be825ed504cabb https://www.illumos.org/issues/7803 Make get_devid() from libzfs a public function in libdevid, as its pretty usable in other places and duplicating all the logic required to get string encoded devid from path seems counter-productive. Reviewed by: Dan McDonald Reviewed by: Jason King Reviewed by: Marcel Telka Reviewed by: Prakash Surya Approved by: Robert Mustacchi Author: Yuri Pankov Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_import.c Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_import.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_import.c Fri Apr 14 18:20:56 2017 (r316908) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_import.c Fri Apr 14 18:21:58 2017 (r316909) @@ -23,7 +23,7 @@ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, 2016 by Delphix. All rights reserved. * Copyright 2015 RackTop Systems. - * Copyright 2016 Nexenta Systems, Inc. + * Copyright 2017 Nexenta Systems, Inc. */ /* @@ -95,31 +95,6 @@ typedef struct pool_list { name_entry_t *names; } pool_list_t; -static char * -get_devid(const char *path) -{ - int fd; - ddi_devid_t devid; - char *minor, *ret; - - if ((fd = open(path, O_RDONLY)) < 0) - return (NULL); - - minor = NULL; - ret = NULL; - if (devid_get(fd, &devid) == 0) { - if (devid_get_minor_name(fd, &minor) == 0) - ret = devid_str_encode(devid, minor); - if (minor != NULL) - devid_str_free(minor); - devid_free(devid); - } - (void) close(fd); - - return (ret); -} - - /* * Go through and fix up any path and/or devid information for the given vdev * configuration. @@ -195,7 +170,7 @@ fix_paths(nvlist_t *nv, name_entry_t *na if (nvlist_add_string(nv, ZPOOL_CONFIG_PATH, best->ne_name) != 0) return (-1); - if ((devid = get_devid(best->ne_name)) == NULL) { + if ((devid = devid_str_from_path(best->ne_name)) == NULL) { (void) nvlist_remove_all(nv, ZPOOL_CONFIG_DEVID); } else { if (nvlist_add_string(nv, ZPOOL_CONFIG_DEVID, devid) != 0) { From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:22:43 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 743B1D3EF7C; Fri, 14 Apr 2017 18:22:43 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 4FA22838; Fri, 14 Apr 2017 18:22:43 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIMgrj070417; Fri, 14 Apr 2017 18:22:42 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIMgVp070414; Fri, 14 Apr 2017 18:22:42 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141822.v3EIMgVp070414@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:22:42 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316910 - in vendor-sys/illumos/dist: common/nvpair uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:22:43 -0000 Author: avg Date: Fri Apr 14 18:22:42 2017 New Revision: 316910 URL: https://svnweb.freebsd.org/changeset/base/316910 Log: 7812 Remove gender specific language illumos/illumos-gate@48bbca816818409505a6e214d0911fda44e622e3 https://github.com/illumos/illumos-gate/commit/48bbca816818409505a6e214d0911fda44e622e3 https://www.illumos.org/issues/7812 This change removes all gendered language that did not refer specifically to an individual person or pet. The convention taken was to use variations on "they" when referring to users and/or human beings, while using "it" when referring to code, functions, and/or libraries. Additionally, we took the liberty to fix up any whitespace issues that were found in any files that were already being modified. Reviewed by: Matt Ahrens Reviewed by: Prakash Surya Reviewed by: Steve Gonczi Reviewed by: Chris Williamson Reviewed by: George Wilson Reviewed by: Igor Kozhukhov Reviewed by: Dan McDonald Reviewed by: Robert Mustacchi Approved by: Richard Lowe Author: Daniel Hoffman Modified: vendor-sys/illumos/dist/common/nvpair/nvpair.c vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_dir.c vendor-sys/illumos/dist/uts/common/fs/zfs/zio.c Modified: vendor-sys/illumos/dist/common/nvpair/nvpair.c ============================================================================== --- vendor-sys/illumos/dist/common/nvpair/nvpair.c Fri Apr 14 18:21:58 2017 (r316909) +++ vendor-sys/illumos/dist/common/nvpair/nvpair.c Fri Apr 14 18:22:42 2017 (r316910) @@ -2375,7 +2375,7 @@ nvlist_xpack(nvlist_t *nvl, char **bufp, * 1. The nvlist has fixed allocator properties. * All other nvlist routines (like nvlist_add_*, ...) use * these properties. - * 2. When using nvlist_pack() the user can specify his own + * 2. When using nvlist_pack() the user can specify their own * allocator properties (e.g. by using KM_NOSLEEP). * * We use the user specified properties (2). A clearer solution Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_dir.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_dir.c Fri Apr 14 18:21:58 2017 (r316909) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_dir.c Fri Apr 14 18:22:42 2017 (r316910) @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2013, 2015 by Delphix. All rights reserved. + * Copyright (c) 2013, 2016 by Delphix. All rights reserved. * Copyright 2017 Nexenta Systems, Inc. */ @@ -287,8 +287,8 @@ zfs_dirent_lock(zfs_dirlock_t **dlpp, zn * dl_name in case the first thread goes away before we do. * Note that we initialize the new name before storing its * pointer into dl_name, because the first thread may load - * dl->dl_name at any time. He'll either see the old value, - * which is his, or the new shared copy; either is OK. + * dl->dl_name at any time. It'll either see the old value, + * which belongs to it, or the new shared copy; either is OK. */ dl->dl_namesize = strlen(dl->dl_name) + 1; name = kmem_alloc(dl->dl_namesize, KM_SLEEP); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zio.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zio.c Fri Apr 14 18:21:58 2017 (r316909) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zio.c Fri Apr 14 18:22:42 2017 (r316910) @@ -1679,7 +1679,7 @@ zio_reexecute(zio_t *pio) /* * Now that all children have been reexecuted, execute the parent. * We don't reexecute "The Godfather" I/O here as it's the - * responsibility of the caller to wait on him. + * responsibility of the caller to wait on it. */ if (!(pio->io_flags & ZIO_FLAG_GODFATHER)) { pio->io_queued_timestamp = gethrtime(); From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:23:19 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 63CE3D3EFDD; Fri, 14 Apr 2017 18:23:19 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 2C63D96A; Fri, 14 Apr 2017 18:23:19 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EINI81070488; Fri, 14 Apr 2017 18:23:18 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EINIcB070487; Fri, 14 Apr 2017 18:23:18 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141823.v3EINIcB070487@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:23:18 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316911 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:23:19 -0000 Author: avg Date: Fri Apr 14 18:23:18 2017 New Revision: 316911 URL: https://svnweb.freebsd.org/changeset/base/316911 Log: 7816 remove static unused variable in zfs_vfsops.c illumos/illumos-gate@2e972bf18f2d3dc8a060f336db39dc8432ee887c https://github.com/illumos/illumos-gate/commit/2e972bf18f2d3dc8a060f336db39dc8432ee887c https://www.illumos.org/issues/7816 found by gcc6 build unused static variable: -static const fs_operation_def_t zfs_vfsops_eio_template[] = { - VFSNAME_FREEVFS, { .vfs_freevfs = zfs_freevfs }, - NULL, NULL -}; Reviewed by: Matt Ahrens Reviewed by: Andy Stormont astormont@racktopsystems.com Reviewed by: Pavel Zakharov Approved by: Dan McDonald Author: Igor Kozhukhov Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vfsops.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vfsops.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vfsops.c Fri Apr 14 18:22:42 2017 (r316910) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vfsops.c Fri Apr 14 18:23:18 2017 (r316911) @@ -96,11 +96,6 @@ static const fs_operation_def_t zfs_vfso NULL, NULL }; -static const fs_operation_def_t zfs_vfsops_eio_template[] = { - VFSNAME_FREEVFS, { .vfs_freevfs = zfs_freevfs }, - NULL, NULL -}; - /* * We need to keep a count of active fs's. * This is necessary to prevent our module From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:24:04 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 9A1FFD3E042; Fri, 14 Apr 2017 18:24:04 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 39F79A95; Fri, 14 Apr 2017 18:24:04 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIO30k070562; Fri, 14 Apr 2017 18:24:03 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIO3bb070558; Fri, 14 Apr 2017 18:24:03 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141824.v3EIO3bb070558@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:24:03 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316912 - in vendor-sys/illumos/dist/uts/common/fs/zfs: . sys X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:24:04 -0000 Author: avg Date: Fri Apr 14 18:24:03 2017 New Revision: 316912 URL: https://svnweb.freebsd.org/changeset/base/316912 Log: 7793 ztest fails assertion in dmu_tx_willuse_space illumos/illumos-gate@61e255ce7267b52208af9daf434b77d37fb75622 https://github.com/illumos/illumos-gate/commit/61e255ce7267b52208af9daf434b77d37fb75622 https://www.illumos.org/issues/7793 Background information: This assertion about tx_space_* verifies that we are not dirtying more stuff than we thought we would. We “need” to know how much we will dirty so that we can check if we should fail this transaction with ENOSPC/EDQUOT, in dmu_tx_assign(). While the transaction is open (i.e. between dmu_tx_assign() and dmu_tx_commit() — typically less than a millisecond), we call dbuf_dirty() on the exact blocks that will be modified. Once this happens, the temporary accounting in tx_space_* is unnecessary, because we know exactly what blocks are newly dirtied; we call dnode_willuse_space() to track this more exact accounting. The fundamental problem causing this bug is that dmu_tx_hold_*() relies on the current state in the DMU (e.g. dn_nlevels) to predict how much will be dirtied by this transaction, but this state can change before we actually perform the transaction (i.e. call dbuf_dirty()). This bug will be fixed by removing the assertion that the tx_space_* accounting is perfectly accurate (i.e. we never dirty more than was predicted by dmu_tx_hold_*()). By removing the requirement that this accounting be perfectly accurate, we can also vastly simplify it, e.g. removing most of the logic in dmu_tx_count_*(). The new tx space accounting will be very approximate, and may be more or less than what is actually dirtied. It will still be used to determine if this transaction will put us over quota. Transactions that are marked by dmu_tx_mark_netfree() will be excepted from this check. We won’t make an attempt to determine how much space will be freed by the transaction — this was rarely accurate enough to determine if a transaction should be permitted when we are over quota, which is why dmu_tx_mark_netfree() was introduced in 2014. We also won’t attempt to give “credit” when overwriting existing blocks, if those blocks may be freed. This allows us to remove the do_free_accounting logic in dbuf_dirty(), and associated routines. This Reviewed by: Steve Gonczi Reviewed by: George Wilson Reviewed by: Pavel Zakharov Reviewed by: Brian Behlendorf Approved by: Robert Mustacchi Author: Matthew Ahrens Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_tx.c vendor-sys/illumos/dist/uts/common/fs/zfs/dnode.c vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_dataset.c vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_dir.c vendor-sys/illumos/dist/uts/common/fs/zfs/spa_misc.c vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu_impl.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu_objset.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu_tx.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dnode.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dsl_dataset.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dsl_dir.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zap_impl.h vendor-sys/illumos/dist/uts/common/fs/zfs/zap.c vendor-sys/illumos/dist/uts/common/fs/zfs/zap_micro.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c Fri Apr 14 18:23:18 2017 (r316911) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c Fri Apr 14 18:24:03 2017 (r316912) @@ -1346,41 +1346,6 @@ dbuf_free_range(dnode_t *dn, uint64_t st mutex_exit(&dn->dn_dbufs_mtx); } -static int -dbuf_block_freeable(dmu_buf_impl_t *db) -{ - dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; - uint64_t birth_txg = 0; - - /* - * We don't need any locking to protect db_blkptr: - * If it's syncing, then db_last_dirty will be set - * so we'll ignore db_blkptr. - * - * This logic ensures that only block births for - * filled blocks are considered. - */ - ASSERT(MUTEX_HELD(&db->db_mtx)); - if (db->db_last_dirty && (db->db_blkptr == NULL || - !BP_IS_HOLE(db->db_blkptr))) { - birth_txg = db->db_last_dirty->dr_txg; - } else if (db->db_blkptr != NULL && !BP_IS_HOLE(db->db_blkptr)) { - birth_txg = db->db_blkptr->blk_birth; - } - - /* - * If this block don't exist or is in a snapshot, it can't be freed. - * Don't pass the bp to dsl_dataset_block_freeable() since we - * are holding the db_mtx lock and might deadlock if we are - * prefetching a dedup-ed block. - */ - if (birth_txg != 0) - return (ds == NULL || - dsl_dataset_block_freeable(ds, NULL, birth_txg)); - else - return (B_FALSE); -} - void dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) { @@ -1430,7 +1395,7 @@ dbuf_new_size(dmu_buf_impl_t *db, int si } mutex_exit(&db->db_mtx); - dnode_willuse_space(dn, size-osize, tx); + dmu_objset_willuse_space(dn->dn_objset, size - osize, tx); DB_DNODE_EXIT(db); } @@ -1480,7 +1445,6 @@ dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t objset_t *os; dbuf_dirty_record_t **drp, *dr; int drop_struct_lock = FALSE; - boolean_t do_free_accounting = B_FALSE; int txgoff = tx->tx_txg & TXG_MASK; ASSERT(tx->tx_txg != 0); @@ -1602,15 +1566,7 @@ dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); if (db->db_blkid != DMU_BONUS_BLKID) { - /* - * Update the accounting. - * Note: we delay "free accounting" until after we drop - * the db_mtx. This keeps us from grabbing other locks - * (and possibly deadlocking) in bp_get_dsize() while - * also holding the db_mtx. - */ - dnode_willuse_space(dn, db->db.db_size, tx); - do_free_accounting = dbuf_block_freeable(db); + dmu_objset_willuse_space(os, db->db.db_size, tx); } /* @@ -1703,21 +1659,13 @@ dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t drop_struct_lock = TRUE; } - if (do_free_accounting) { - blkptr_t *bp = db->db_blkptr; - int64_t willfree = (bp && !BP_IS_HOLE(bp)) ? - bp_get_dsize(os->os_spa, bp) : db->db.db_size; - /* - * This is only a guess -- if the dbuf is dirty - * in a previous txg, we don't know how much - * space it will use on disk yet. We should - * really have the struct_rwlock to access - * db_blkptr, but since this is just a guess, - * it's OK if we get an odd answer. - */ - ddt_prefetch(os->os_spa, bp); - dnode_willuse_space(dn, -willfree, tx); - } + /* + * If we are overwriting a dedup BP, then unless it is snapshotted, + * when we get to syncing context we will need to decrement its + * refcount in the DDT. Prefetch the relevant DDT block so that + * syncing context won't have to wait for the i/o. + */ + ddt_prefetch(os->os_spa, db->db_blkptr); if (db->db_level == 0) { dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock); @@ -2927,19 +2875,6 @@ dmu_buf_user_evict_wait() taskq_wait(dbu_evict_taskq); } -boolean_t -dmu_buf_freeable(dmu_buf_t *dbuf) -{ - boolean_t res = B_FALSE; - dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; - - if (db->db_blkptr) - res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset, - db->db_blkptr, db->db_blkptr->blk_birth); - - return (res); -} - blkptr_t * dmu_buf_get_blkptr(dmu_buf_t *db) { Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c Fri Apr 14 18:23:18 2017 (r316911) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c Fri Apr 14 18:24:03 2017 (r316912) @@ -2103,3 +2103,20 @@ dmu_fsname(const char *snapname, char *b (void) strlcpy(buf, snapname, atp - snapname + 1); return (0); } + +/* + * Call when we think we're going to write/free space in open context to track + * the amount of dirty data in the open txg, which is also the amount + * of memory that can not be evicted until this txg syncs. + */ +void +dmu_objset_willuse_space(objset_t *os, int64_t space, dmu_tx_t *tx) +{ + dsl_dataset_t *ds = os->os_dsl_dataset; + int64_t aspace = spa_get_worst_case_asize(os->os_spa, space); + + if (ds != NULL) { + dsl_dir_willuse_space(ds->ds_dir, aspace, tx); + dsl_pool_dirty_space(dmu_tx_pool(tx), space, tx); + } +} Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_tx.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_tx.c Fri Apr 14 18:23:18 2017 (r316911) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_tx.c Fri Apr 14 18:24:03 2017 (r316912) @@ -30,10 +30,10 @@ #include #include #include -#include /* for dsl_dataset_block_freeable() */ -#include /* for dsl_dir_tempreserve_*() */ +#include +#include #include -#include /* for fzap_default_block_shift */ +#include #include #include #include @@ -56,10 +56,6 @@ dmu_tx_create_dd(dsl_dir_t *dd) list_create(&tx->tx_callbacks, sizeof (dmu_tx_callback_t), offsetof(dmu_tx_callback_t, dcb_node)); tx->tx_start = gethrtime(); -#ifdef ZFS_DEBUG - refcount_create(&tx->tx_space_written); - refcount_create(&tx->tx_space_freed); -#endif return (tx); } @@ -68,7 +64,6 @@ dmu_tx_create(objset_t *os) { dmu_tx_t *tx = dmu_tx_create_dd(os->os_dsl_dataset->ds_dir); tx->tx_objset = os; - tx->tx_lastsnap_txg = dsl_dataset_prev_snap_txg(os->os_dsl_dataset); return (tx); } @@ -130,16 +125,10 @@ dmu_tx_hold_object_impl(dmu_tx_t *tx, ob txh->txh_tx = tx; txh->txh_dnode = dn; refcount_create(&txh->txh_space_towrite); - refcount_create(&txh->txh_space_tofree); - refcount_create(&txh->txh_space_tooverwrite); - refcount_create(&txh->txh_space_tounref); refcount_create(&txh->txh_memory_tohold); - refcount_create(&txh->txh_fudge); -#ifdef ZFS_DEBUG txh->txh_type = type; txh->txh_arg1 = arg1; txh->txh_arg2 = arg2; -#endif list_insert_tail(&tx->tx_holds, txh); return (txh); @@ -158,6 +147,34 @@ dmu_tx_add_new_object(dmu_tx_t *tx, objs } } +/* + * This function reads specified data from disk. The specified data will + * be needed to perform the transaction -- i.e, it will be read after + * we do dmu_tx_assign(). There are two reasons that we read the data now + * (before dmu_tx_assign()): + * + * 1. Reading it now has potentially better performance. The transaction + * has not yet been assigned, so the TXG is not held open, and also the + * caller typically has less locks held when calling dmu_tx_hold_*() than + * after the transaction has been assigned. This reduces the lock (and txg) + * hold times, thus reducing lock contention. + * + * 2. It is easier for callers (primarily the ZPL) to handle i/o errors + * that are detected before they start making changes to the DMU state + * (i.e. now). Once the transaction has been assigned, and some DMU + * state has been changed, it can be difficult to recover from an i/o + * error (e.g. to undo the changes already made in memory at the DMU + * layer). Typically code to do so does not exist in the caller -- it + * assumes that the data has already been cached and thus i/o errors are + * not possible. + * + * It has been observed that the i/o initiated here can be a performance + * problem, and it appears to be optional, because we don't look at the + * data which is read. However, removing this read would only serve to + * move the work elsewhere (after the dmu_tx_assign()), where it may + * have a greater impact on performance (in addition to the impact on + * fault tolerance noted above). + */ static int dmu_tx_check_ioerr(zio_t *zio, dnode_t *dn, int level, uint64_t blkid) { @@ -174,259 +191,84 @@ dmu_tx_check_ioerr(zio_t *zio, dnode_t * return (err); } -static void -dmu_tx_count_twig(dmu_tx_hold_t *txh, dnode_t *dn, dmu_buf_impl_t *db, - int level, uint64_t blkid, boolean_t freeable, uint64_t *history) -{ - objset_t *os = dn->dn_objset; - dsl_dataset_t *ds = os->os_dsl_dataset; - int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; - dmu_buf_impl_t *parent = NULL; - blkptr_t *bp = NULL; - uint64_t space; - - if (level >= dn->dn_nlevels || history[level] == blkid) - return; - - history[level] = blkid; - - space = (level == 0) ? dn->dn_datablksz : (1ULL << dn->dn_indblkshift); - - if (db == NULL || db == dn->dn_dbuf) { - ASSERT(level != 0); - db = NULL; - } else { - ASSERT(DB_DNODE(db) == dn); - ASSERT(db->db_level == level); - ASSERT(db->db.db_size == space); - ASSERT(db->db_blkid == blkid); - bp = db->db_blkptr; - parent = db->db_parent; - } - - freeable = (bp && (freeable || - dsl_dataset_block_freeable(ds, bp, bp->blk_birth))); - - if (freeable) { - (void) refcount_add_many(&txh->txh_space_tooverwrite, - space, FTAG); - } else { - (void) refcount_add_many(&txh->txh_space_towrite, - space, FTAG); - } - - if (bp) { - (void) refcount_add_many(&txh->txh_space_tounref, - bp_get_dsize(os->os_spa, bp), FTAG); - } - - dmu_tx_count_twig(txh, dn, parent, level + 1, - blkid >> epbs, freeable, history); -} - /* ARGSUSED */ static void dmu_tx_count_write(dmu_tx_hold_t *txh, uint64_t off, uint64_t len) { dnode_t *dn = txh->txh_dnode; - uint64_t start, end, i; - int min_bs, max_bs, min_ibs, max_ibs, epbs, bits; int err = 0; if (len == 0) return; - min_bs = SPA_MINBLOCKSHIFT; - max_bs = highbit64(txh->txh_tx->tx_objset->os_recordsize) - 1; - min_ibs = DN_MIN_INDBLKSHIFT; - max_ibs = DN_MAX_INDBLKSHIFT; - - if (dn) { - uint64_t history[DN_MAX_LEVELS]; - int nlvls = dn->dn_nlevels; - int delta; + (void) refcount_add_many(&txh->txh_space_towrite, len, FTAG); - /* - * For i/o error checking, read the first and last level-0 - * blocks (if they are not aligned), and all the level-1 blocks. - */ - if (dn->dn_maxblkid == 0) { - delta = dn->dn_datablksz; - start = (off < dn->dn_datablksz) ? 0 : 1; - end = (off+len <= dn->dn_datablksz) ? 0 : 1; - if (start == 0 && (off > 0 || len < dn->dn_datablksz)) { - err = dmu_tx_check_ioerr(NULL, dn, 0, 0); - if (err) - goto out; - delta -= off; - } - } else { - zio_t *zio = zio_root(dn->dn_objset->os_spa, - NULL, NULL, ZIO_FLAG_CANFAIL); - - /* first level-0 block */ - start = off >> dn->dn_datablkshift; - if (P2PHASE(off, dn->dn_datablksz) || - len < dn->dn_datablksz) { - err = dmu_tx_check_ioerr(zio, dn, 0, start); - if (err) - goto out; - } + if (refcount_count(&txh->txh_space_towrite) > 2 * DMU_MAX_ACCESS) + err = SET_ERROR(EFBIG); - /* last level-0 block */ - end = (off+len-1) >> dn->dn_datablkshift; - if (end != start && end <= dn->dn_maxblkid && - P2PHASE(off+len, dn->dn_datablksz)) { - err = dmu_tx_check_ioerr(zio, dn, 0, end); - if (err) - goto out; - } + if (dn == NULL) + return; - /* level-1 blocks */ - if (nlvls > 1) { - int shft = dn->dn_indblkshift - SPA_BLKPTRSHIFT; - for (i = (start>>shft)+1; i < end>>shft; i++) { - err = dmu_tx_check_ioerr(zio, dn, 1, i); - if (err) - goto out; - } + /* + * For i/o error checking, read the blocks that will be needed + * to perform the write: the first and last level-0 blocks (if + * they are not aligned, i.e. if they are partial-block writes), + * and all the level-1 blocks. + */ + if (dn->dn_maxblkid == 0) { + if (off < dn->dn_datablksz && + (off > 0 || len < dn->dn_datablksz)) { + err = dmu_tx_check_ioerr(NULL, dn, 0, 0); + if (err != 0) { + txh->txh_tx->tx_err = err; } - - err = zio_wait(zio); - if (err) - goto out; - delta = P2NPHASE(off, dn->dn_datablksz); - } - - min_ibs = max_ibs = dn->dn_indblkshift; - if (dn->dn_maxblkid > 0) { - /* - * The blocksize can't change, - * so we can make a more precise estimate. - */ - ASSERT(dn->dn_datablkshift != 0); - min_bs = max_bs = dn->dn_datablkshift; - } else { - /* - * The blocksize can increase up to the recordsize, - * or if it is already more than the recordsize, - * up to the next power of 2. - */ - min_bs = highbit64(dn->dn_datablksz - 1); - max_bs = MAX(max_bs, highbit64(dn->dn_datablksz - 1)); } + } else { + zio_t *zio = zio_root(dn->dn_objset->os_spa, + NULL, NULL, ZIO_FLAG_CANFAIL); - /* - * If this write is not off the end of the file - * we need to account for overwrites/unref. - */ - if (start <= dn->dn_maxblkid) { - for (int l = 0; l < DN_MAX_LEVELS; l++) - history[l] = -1ULL; + /* first level-0 block */ + uint64_t start = off >> dn->dn_datablkshift; + if (P2PHASE(off, dn->dn_datablksz) || len < dn->dn_datablksz) { + err = dmu_tx_check_ioerr(zio, dn, 0, start); + if (err != 0) { + txh->txh_tx->tx_err = err; + } } - while (start <= dn->dn_maxblkid) { - dmu_buf_impl_t *db; - - rw_enter(&dn->dn_struct_rwlock, RW_READER); - err = dbuf_hold_impl(dn, 0, start, - FALSE, FALSE, FTAG, &db); - rw_exit(&dn->dn_struct_rwlock); - if (err) { + /* last level-0 block */ + uint64_t end = (off + len - 1) >> dn->dn_datablkshift; + if (end != start && end <= dn->dn_maxblkid && + P2PHASE(off + len, dn->dn_datablksz)) { + err = dmu_tx_check_ioerr(zio, dn, 0, end); + if (err != 0) { txh->txh_tx->tx_err = err; - return; } + } - dmu_tx_count_twig(txh, dn, db, 0, start, B_FALSE, - history); - dbuf_rele(db, FTAG); - if (++start > end) { - /* - * Account for new indirects appearing - * before this IO gets assigned into a txg. - */ - bits = 64 - min_bs; - epbs = min_ibs - SPA_BLKPTRSHIFT; - for (bits -= epbs * (nlvls - 1); - bits >= 0; bits -= epbs) { - (void) refcount_add_many( - &txh->txh_fudge, - 1ULL << max_ibs, FTAG); + /* level-1 blocks */ + if (dn->dn_nlevels > 1) { + int shft = dn->dn_indblkshift - SPA_BLKPTRSHIFT; + for (uint64_t i = (start >> shft) + 1; + i < end >> shft; i++) { + err = dmu_tx_check_ioerr(zio, dn, 1, i); + if (err != 0) { + txh->txh_tx->tx_err = err; } - goto out; } - off += delta; - if (len >= delta) - len -= delta; - delta = dn->dn_datablksz; } - } - - /* - * 'end' is the last thing we will access, not one past. - * This way we won't overflow when accessing the last byte. - */ - start = P2ALIGN(off, 1ULL << max_bs); - end = P2ROUNDUP(off + len, 1ULL << max_bs) - 1; - (void) refcount_add_many(&txh->txh_space_towrite, - end - start + 1, FTAG); - - start >>= min_bs; - end >>= min_bs; - epbs = min_ibs - SPA_BLKPTRSHIFT; - - /* - * The object contains at most 2^(64 - min_bs) blocks, - * and each indirect level maps 2^epbs. - */ - for (bits = 64 - min_bs; bits >= 0; bits -= epbs) { - start >>= epbs; - end >>= epbs; - ASSERT3U(end, >=, start); - (void) refcount_add_many(&txh->txh_space_towrite, - (end - start + 1) << max_ibs, FTAG); - if (start != 0) { - /* - * We also need a new blkid=0 indirect block - * to reference any existing file data. - */ - (void) refcount_add_many(&txh->txh_space_towrite, - 1ULL << max_ibs, FTAG); + err = zio_wait(zio); + if (err != 0) { + txh->txh_tx->tx_err = err; } } - -out: - if (refcount_count(&txh->txh_space_towrite) + - refcount_count(&txh->txh_space_tooverwrite) > - 2 * DMU_MAX_ACCESS) - err = SET_ERROR(EFBIG); - - if (err) - txh->txh_tx->tx_err = err; } static void dmu_tx_count_dnode(dmu_tx_hold_t *txh) { - dnode_t *dn = txh->txh_dnode; - dnode_t *mdn = DMU_META_DNODE(txh->txh_tx->tx_objset); - uint64_t space = mdn->dn_datablksz + - ((mdn->dn_nlevels-1) << mdn->dn_indblkshift); - - if (dn && dn->dn_dbuf->db_blkptr && - dsl_dataset_block_freeable(dn->dn_objset->os_dsl_dataset, - dn->dn_dbuf->db_blkptr, dn->dn_dbuf->db_blkptr->blk_birth)) { - (void) refcount_add_many(&txh->txh_space_tooverwrite, - space, FTAG); - (void) refcount_add_many(&txh->txh_space_tounref, space, FTAG); - } else { - (void) refcount_add_many(&txh->txh_space_towrite, space, FTAG); - if (dn && dn->dn_dbuf->db_blkptr) { - (void) refcount_add_many(&txh->txh_space_tounref, - space, FTAG); - } - } + (void) refcount_add_many(&txh->txh_space_towrite, DNODE_SIZE, FTAG); } void @@ -434,8 +276,8 @@ dmu_tx_hold_write(dmu_tx_t *tx, uint64_t { dmu_tx_hold_t *txh; - ASSERT(tx->tx_txg == 0); - ASSERT(len < DMU_MAX_ACCESS); + ASSERT0(tx->tx_txg); + ASSERT3U(len, <=, DMU_MAX_ACCESS); ASSERT(len == 0 || UINT64_MAX - off >= len - 1); txh = dmu_tx_hold_object_impl(tx, tx->tx_objset, @@ -447,179 +289,6 @@ dmu_tx_hold_write(dmu_tx_t *tx, uint64_t dmu_tx_count_dnode(txh); } -static void -dmu_tx_count_free(dmu_tx_hold_t *txh, uint64_t off, uint64_t len) -{ - uint64_t blkid, nblks, lastblk; - uint64_t space = 0, unref = 0, skipped = 0; - dnode_t *dn = txh->txh_dnode; - dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; - spa_t *spa = txh->txh_tx->tx_pool->dp_spa; - int epbs; - uint64_t l0span = 0, nl1blks = 0; - - if (dn->dn_nlevels == 0) - return; - - /* - * The struct_rwlock protects us against dn_nlevels - * changing, in case (against all odds) we manage to dirty & - * sync out the changes after we check for being dirty. - * Also, dbuf_hold_impl() wants us to have the struct_rwlock. - */ - rw_enter(&dn->dn_struct_rwlock, RW_READER); - epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; - if (dn->dn_maxblkid == 0) { - if (off == 0 && len >= dn->dn_datablksz) { - blkid = 0; - nblks = 1; - } else { - rw_exit(&dn->dn_struct_rwlock); - return; - } - } else { - blkid = off >> dn->dn_datablkshift; - nblks = (len + dn->dn_datablksz - 1) >> dn->dn_datablkshift; - - if (blkid > dn->dn_maxblkid) { - rw_exit(&dn->dn_struct_rwlock); - return; - } - if (blkid + nblks > dn->dn_maxblkid) - nblks = dn->dn_maxblkid - blkid + 1; - - } - l0span = nblks; /* save for later use to calc level > 1 overhead */ - if (dn->dn_nlevels == 1) { - int i; - for (i = 0; i < nblks; i++) { - blkptr_t *bp = dn->dn_phys->dn_blkptr; - ASSERT3U(blkid + i, <, dn->dn_nblkptr); - bp += blkid + i; - if (dsl_dataset_block_freeable(ds, bp, bp->blk_birth)) { - dprintf_bp(bp, "can free old%s", ""); - space += bp_get_dsize(spa, bp); - } - unref += BP_GET_ASIZE(bp); - } - nl1blks = 1; - nblks = 0; - } - - lastblk = blkid + nblks - 1; - while (nblks) { - dmu_buf_impl_t *dbuf; - uint64_t ibyte, new_blkid; - int epb = 1 << epbs; - int err, i, blkoff, tochk; - blkptr_t *bp; - - ibyte = blkid << dn->dn_datablkshift; - err = dnode_next_offset(dn, - DNODE_FIND_HAVELOCK, &ibyte, 2, 1, 0); - new_blkid = ibyte >> dn->dn_datablkshift; - if (err == ESRCH) { - skipped += (lastblk >> epbs) - (blkid >> epbs) + 1; - break; - } - if (err) { - txh->txh_tx->tx_err = err; - break; - } - if (new_blkid > lastblk) { - skipped += (lastblk >> epbs) - (blkid >> epbs) + 1; - break; - } - - if (new_blkid > blkid) { - ASSERT((new_blkid >> epbs) > (blkid >> epbs)); - skipped += (new_blkid >> epbs) - (blkid >> epbs) - 1; - nblks -= new_blkid - blkid; - blkid = new_blkid; - } - blkoff = P2PHASE(blkid, epb); - tochk = MIN(epb - blkoff, nblks); - - err = dbuf_hold_impl(dn, 1, blkid >> epbs, - FALSE, FALSE, FTAG, &dbuf); - if (err) { - txh->txh_tx->tx_err = err; - break; - } - - (void) refcount_add_many(&txh->txh_memory_tohold, - dbuf->db.db_size, FTAG); - - /* - * We don't check memory_tohold against DMU_MAX_ACCESS because - * memory_tohold is an over-estimation (especially the >L1 - * indirect blocks), so it could fail. Callers should have - * already verified that they will not be holding too much - * memory. - */ - - err = dbuf_read(dbuf, NULL, DB_RF_HAVESTRUCT | DB_RF_CANFAIL); - if (err != 0) { - txh->txh_tx->tx_err = err; - dbuf_rele(dbuf, FTAG); - break; - } - - bp = dbuf->db.db_data; - bp += blkoff; - - for (i = 0; i < tochk; i++) { - if (dsl_dataset_block_freeable(ds, &bp[i], - bp[i].blk_birth)) { - dprintf_bp(&bp[i], "can free old%s", ""); - space += bp_get_dsize(spa, &bp[i]); - } - unref += BP_GET_ASIZE(bp); - } - dbuf_rele(dbuf, FTAG); - - ++nl1blks; - blkid += tochk; - nblks -= tochk; - } - rw_exit(&dn->dn_struct_rwlock); - - /* - * Add in memory requirements of higher-level indirects. - * This assumes a worst-possible scenario for dn_nlevels and a - * worst-possible distribution of l1-blocks over the region to free. - */ - { - uint64_t blkcnt = 1 + ((l0span >> epbs) >> epbs); - int level = 2; - /* - * Here we don't use DN_MAX_LEVEL, but calculate it with the - * given datablkshift and indblkshift. This makes the - * difference between 19 and 8 on large files. - */ - int maxlevel = 2 + (DN_MAX_OFFSET_SHIFT - dn->dn_datablkshift) / - (dn->dn_indblkshift - SPA_BLKPTRSHIFT); - - while (level++ < maxlevel) { - (void) refcount_add_many(&txh->txh_memory_tohold, - MAX(MIN(blkcnt, nl1blks), 1) << dn->dn_indblkshift, - FTAG); - blkcnt = 1 + (blkcnt >> epbs); - } - } - - /* account for new level 1 indirect blocks that might show up */ - if (skipped > 0) { - (void) refcount_add_many(&txh->txh_fudge, - skipped << dn->dn_indblkshift, FTAG); - skipped = MIN(skipped, DMU_MAX_DELETEBLKCNT >> epbs); - (void) refcount_add_many(&txh->txh_memory_tohold, - skipped << dn->dn_indblkshift, FTAG); - } - (void) refcount_add_many(&txh->txh_space_tofree, space, FTAG); - (void) refcount_add_many(&txh->txh_space_tounref, unref, FTAG); -} - /* * This function marks the transaction as being a "net free". The end * result is that refquotas will be disabled for this transaction, and @@ -631,45 +300,27 @@ dmu_tx_count_free(dmu_tx_hold_t *txh, ui void dmu_tx_mark_netfree(dmu_tx_t *tx) { - dmu_tx_hold_t *txh; - - txh = dmu_tx_hold_object_impl(tx, tx->tx_objset, - DMU_NEW_OBJECT, THT_FREE, 0, 0); - - /* - * Pretend that this operation will free 1GB of space. This - * should be large enough to cancel out the largest write. - * We don't want to use something like UINT64_MAX, because that would - * cause overflows when doing math with these values (e.g. in - * dmu_tx_try_assign()). - */ - (void) refcount_add_many(&txh->txh_space_tofree, - 1024 * 1024 * 1024, FTAG); - (void) refcount_add_many(&txh->txh_space_tounref, - 1024 * 1024 * 1024, FTAG); + tx->tx_netfree = B_TRUE; } void dmu_tx_hold_free(dmu_tx_t *tx, uint64_t object, uint64_t off, uint64_t len) { - dmu_tx_hold_t *txh; - dnode_t *dn; int err; - zio_t *zio; ASSERT(tx->tx_txg == 0); - txh = dmu_tx_hold_object_impl(tx, tx->tx_objset, + dmu_tx_hold_t *txh = dmu_tx_hold_object_impl(tx, tx->tx_objset, object, THT_FREE, off, len); if (txh == NULL) return; - dn = txh->txh_dnode; + dnode_t *dn = txh->txh_dnode; dmu_tx_count_dnode(txh); - if (off >= (dn->dn_maxblkid+1) * dn->dn_datablksz) + if (off >= (dn->dn_maxblkid + 1) * dn->dn_datablksz) return; if (len == DMU_OBJECT_END) - len = (dn->dn_maxblkid+1) * dn->dn_datablksz - off; + len = (dn->dn_maxblkid + 1) * dn->dn_datablksz - off; /* * For i/o error checking, we read the first and last level-0 @@ -689,7 +340,7 @@ dmu_tx_hold_free(dmu_tx_t *tx, uint64_t dmu_tx_count_write(txh, off, 1); /* last block will be modified if it is not aligned */ if (!IS_P2ALIGNED(off + len, 1 << dn->dn_datablkshift)) - dmu_tx_count_write(txh, off+len, 1); + dmu_tx_count_write(txh, off + len, 1); } /* @@ -711,7 +362,7 @@ dmu_tx_hold_free(dmu_tx_t *tx, uint64_t if (dn->dn_datablkshift == 0) start = end = 0; - zio = zio_root(tx->tx_pool->dp_spa, + zio_t *zio = zio_root(tx->tx_pool->dp_spa, NULL, NULL, ZIO_FLAG_CANFAIL); for (uint64_t i = start; i <= end; i++) { uint64_t ibyte = i << shift; @@ -719,127 +370,80 @@ dmu_tx_hold_free(dmu_tx_t *tx, uint64_t i = ibyte >> shift; if (err == ESRCH || i > end) break; - if (err) { + if (err != 0) { tx->tx_err = err; + (void) zio_wait(zio); return; } + (void) refcount_add_many(&txh->txh_memory_tohold, + 1 << dn->dn_indblkshift, FTAG); + err = dmu_tx_check_ioerr(zio, dn, 1, i); - if (err) { + if (err != 0) { tx->tx_err = err; + (void) zio_wait(zio); return; } } err = zio_wait(zio); - if (err) { + if (err != 0) { tx->tx_err = err; return; } } - - dmu_tx_count_free(txh, off, len); } void dmu_tx_hold_zap(dmu_tx_t *tx, uint64_t object, int add, const char *name) { - dmu_tx_hold_t *txh; - dnode_t *dn; int err; ASSERT(tx->tx_txg == 0); - txh = dmu_tx_hold_object_impl(tx, tx->tx_objset, + dmu_tx_hold_t *txh = dmu_tx_hold_object_impl(tx, tx->tx_objset, object, THT_ZAP, add, (uintptr_t)name); if (txh == NULL) return; - dn = txh->txh_dnode; + dnode_t *dn = txh->txh_dnode; dmu_tx_count_dnode(txh); - if (dn == NULL) { - /* - * We will be able to fit a new object's entries into one leaf - * block. So there will be at most 2 blocks total, - * including the header block. - */ - dmu_tx_count_write(txh, 0, 2 << fzap_default_block_shift); + /* + * Modifying a almost-full microzap is around the worst case (128KB) + * + * If it is a fat zap, the worst case would be 7*16KB=112KB: + * - 3 blocks overwritten: target leaf, ptrtbl block, header block + * - 4 new blocks written if adding: + * - 2 blocks for possibly split leaves, + * - 2 grown ptrtbl blocks + */ + (void) refcount_add_many(&txh->txh_space_towrite, + MZAP_MAX_BLKSZ, FTAG); + + if (dn == NULL) return; - } ASSERT3P(DMU_OT_BYTESWAP(dn->dn_type), ==, DMU_BSWAP_ZAP); - if (dn->dn_maxblkid == 0 && !add) { - blkptr_t *bp; - + if (dn->dn_maxblkid == 0 || name == NULL) { /* - * If there is only one block (i.e. this is a micro-zap) - * and we are not adding anything, the accounting is simple. + * This is a microzap (only one block), or we don't know + * the name. Check the first block for i/o errors. */ err = dmu_tx_check_ioerr(NULL, dn, 0, 0); - if (err) { + if (err != 0) { tx->tx_err = err; - return; - } - - /* - * Use max block size here, since we don't know how much - * the size will change between now and the dbuf dirty call. - */ - bp = &dn->dn_phys->dn_blkptr[0]; - if (dsl_dataset_block_freeable(dn->dn_objset->os_dsl_dataset, - bp, bp->blk_birth)) { - (void) refcount_add_many(&txh->txh_space_tooverwrite, - MZAP_MAX_BLKSZ, FTAG); - } else { - (void) refcount_add_many(&txh->txh_space_towrite, - MZAP_MAX_BLKSZ, FTAG); } - if (!BP_IS_HOLE(bp)) { - (void) refcount_add_many(&txh->txh_space_tounref, - MZAP_MAX_BLKSZ, FTAG); - } - return; - } - - if (dn->dn_maxblkid > 0 && name) { + } else { /* - * access the name in this fat-zap so that we'll check - * for i/o errors to the leaf blocks, etc. + * Access the name so that we'll check for i/o errors to + * the leaf blocks, etc. We ignore ENOENT, as this name + * may not yet exist. */ err = zap_lookup_by_dnode(dn, name, 8, 0, NULL); - if (err == EIO) { + if (err == EIO || err == ECKSUM || err == ENXIO) { tx->tx_err = err; - return; - } - } - - err = zap_count_write_by_dnode(dn, name, add, - &txh->txh_space_towrite, &txh->txh_space_tooverwrite); - - /* - * If the modified blocks are scattered to the four winds, - * we'll have to modify an indirect twig for each. We can make - * modifications at up to 3 locations: - * - header block at the beginning of the object - * - target leaf block - * - end of the object, where we might need to write: - * - a new leaf block if the target block needs to be split - * - the new pointer table, if it is growing - * - the new cookie table, if it is growing - */ - int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; - dsl_dataset_phys_t *ds_phys = - dsl_dataset_phys(dn->dn_objset->os_dsl_dataset); - for (int lvl = 1; lvl < dn->dn_nlevels; lvl++) { - uint64_t num_indirects = 1 + (dn->dn_maxblkid >> (epbs * lvl)); - uint64_t spc = MIN(3, num_indirects) << dn->dn_indblkshift; - if (ds_phys->ds_prev_snap_obj != 0) { - (void) refcount_add_many(&txh->txh_space_towrite, - spc, FTAG); - } else { - (void) refcount_add_many(&txh->txh_space_tooverwrite, - spc, FTAG); } } } @@ -869,42 +473,15 @@ dmu_tx_hold_space(dmu_tx_t *tx, uint64_t (void) refcount_add_many(&txh->txh_space_towrite, space, FTAG); } -int -dmu_tx_holds(dmu_tx_t *tx, uint64_t object) -{ - dmu_tx_hold_t *txh; - int holds = 0; - - /* - * By asserting that the tx is assigned, we're counting the - * number of dn_tx_holds, which is the same as the number of - * dn_holds. Otherwise, we'd be counting dn_holds, but - * dn_tx_holds could be 0. - */ - ASSERT(tx->tx_txg != 0); - - /* if (tx->tx_anyobj == TRUE) */ - /* return (0); */ - - for (txh = list_head(&tx->tx_holds); txh; - txh = list_next(&tx->tx_holds, txh)) { - if (txh->txh_dnode && txh->txh_dnode->dn_object == object) - holds++; - } - - return (holds); -} - #ifdef ZFS_DEBUG void dmu_tx_dirty_buf(dmu_tx_t *tx, dmu_buf_impl_t *db) { - dmu_tx_hold_t *txh; - int match_object = FALSE, match_offset = FALSE; - dnode_t *dn; *** DIFF OUTPUT TRUNCATED AT 1000 LINES *** From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:24:39 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id A0B29D3E0A0; Fri, 14 Apr 2017 18:24:39 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 5EBA1BE4; Fri, 14 Apr 2017 18:24:39 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIOcUR070624; Fri, 14 Apr 2017 18:24:38 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIOc5T070622; Fri, 14 Apr 2017 18:24:38 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141824.v3EIOc5T070622@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:24:38 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316913 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:24:39 -0000 Author: avg Date: Fri Apr 14 18:24:38 2017 New Revision: 316913 URL: https://svnweb.freebsd.org/changeset/base/316913 Log: 7869 panic in bpobj_space(): null pointer dereference illumos/illumos-gate@a3905a45920de250d181b66ac0b6b71bd200d9ef https://github.com/illumos/illumos-gate/commit/a3905a45920de250d181b66ac0b6b71bd200d9ef https://www.illumos.org/issues/7869 The issue fixed by this patch is a race condition in the deadlist code. A thread executing an administrative command that uses `dsl_deadlist_space_range()` holds the lock of the whole `deadlist_t` to protect the access of all its entries that the deadlist contains in an avl tree. Sync threads trying to insert a new entry in the deadlist (through `dsl_deadlist_insert()` -> `dle_enqueue()`) do not hold the deadlist lock at that moment. If the `dle_bpobj` is the empty bpobj (our sentinel value), we close and reopen it. Between these two operations, it is possible for the `dsl_deadlist_space_range()` thread to dereference that bpobj which is `NULL` during that window. Threads should hold the a deadlist's `dl_lock` when they manipulate its internal data so scenarios like the one above are avoided. In addition, threads should also hold the bpobj lock whenever they are allocating the subobj list of a bpobj, and not just when they actually insert the subobj to the list. This way we can avoid potential memory leaks. Reviewed by: Matt Ahrens Reviewed by: Dan Kimmel Reviewed by: Steve Gonczi Reviewed by: John Kennedy Reviewed by: George Melikov Reviewed by: Brian Behlendorf Approved by: Dan McDonald Author: Serapheim Dimitropoulos Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_deadlist.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c Fri Apr 14 18:24:03 2017 (r316912) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c Fri Apr 14 18:24:38 2017 (r316913) @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2014 by Delphix. All rights reserved. + * Copyright (c) 2011, 2016 by Delphix. All rights reserved. * Copyright (c) 2014 Integros [integros.com] */ @@ -393,6 +393,7 @@ bpobj_enqueue_subobj(bpobj_t *bpo, uint6 return; } + mutex_enter(&bpo->bpo_lock); dmu_buf_will_dirty(bpo->bpo_dbuf, tx); if (bpo->bpo_phys->bpo_subobjs == 0) { bpo->bpo_phys->bpo_subobjs = dmu_object_alloc(bpo->bpo_os, @@ -404,7 +405,6 @@ bpobj_enqueue_subobj(bpobj_t *bpo, uint6 ASSERT0(dmu_object_info(bpo->bpo_os, bpo->bpo_phys->bpo_subobjs, &doi)); ASSERT3U(doi.doi_type, ==, DMU_OT_BPOBJ_SUBOBJ); - mutex_enter(&bpo->bpo_lock); dmu_write(bpo->bpo_os, bpo->bpo_phys->bpo_subobjs, bpo->bpo_phys->bpo_num_subobjs * sizeof (subobj), sizeof (subobj), &subobj, tx); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_deadlist.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_deadlist.c Fri Apr 14 18:24:03 2017 (r316912) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_deadlist.c Fri Apr 14 18:24:38 2017 (r316913) @@ -72,6 +72,8 @@ dsl_deadlist_load_tree(dsl_deadlist_t *d zap_cursor_t zc; zap_attribute_t za; + ASSERT(MUTEX_HELD(&dl->dl_lock)); + ASSERT(!dl->dl_oldfmt); if (dl->dl_havetree) return; @@ -182,6 +184,7 @@ static void dle_enqueue(dsl_deadlist_t *dl, dsl_deadlist_entry_t *dle, const blkptr_t *bp, dmu_tx_t *tx) { + ASSERT(MUTEX_HELD(&dl->dl_lock)); if (dle->dle_bpobj.bpo_object == dmu_objset_pool(dl->dl_os)->dp_empty_bpobj) { uint64_t obj = bpobj_alloc(dl->dl_os, SPA_OLD_MAXBLOCKSIZE, tx); @@ -198,6 +201,7 @@ static void dle_enqueue_subobj(dsl_deadlist_t *dl, dsl_deadlist_entry_t *dle, uint64_t obj, dmu_tx_t *tx) { + ASSERT(MUTEX_HELD(&dl->dl_lock)); if (dle->dle_bpobj.bpo_object != dmu_objset_pool(dl->dl_os)->dp_empty_bpobj) { bpobj_enqueue_subobj(&dle->dle_bpobj, obj, tx); @@ -222,15 +226,14 @@ dsl_deadlist_insert(dsl_deadlist_t *dl, return; } + mutex_enter(&dl->dl_lock); dsl_deadlist_load_tree(dl); dmu_buf_will_dirty(dl->dl_dbuf, tx); - mutex_enter(&dl->dl_lock); dl->dl_phys->dl_used += bp_get_dsize_sync(dmu_objset_spa(dl->dl_os), bp); dl->dl_phys->dl_comp += BP_GET_PSIZE(bp); dl->dl_phys->dl_uncomp += BP_GET_UCSIZE(bp); - mutex_exit(&dl->dl_lock); dle_tofind.dle_mintxg = bp->blk_birth; dle = avl_find(&dl->dl_tree, &dle_tofind, &where); @@ -239,6 +242,7 @@ dsl_deadlist_insert(dsl_deadlist_t *dl, else dle = AVL_PREV(&dl->dl_tree, dle); dle_enqueue(dl, dle, bp, tx); + mutex_exit(&dl->dl_lock); } /* @@ -254,16 +258,19 @@ dsl_deadlist_add_key(dsl_deadlist_t *dl, if (dl->dl_oldfmt) return; - dsl_deadlist_load_tree(dl); - dle = kmem_alloc(sizeof (*dle), KM_SLEEP); dle->dle_mintxg = mintxg; + + mutex_enter(&dl->dl_lock); + dsl_deadlist_load_tree(dl); + obj = bpobj_alloc_empty(dl->dl_os, SPA_OLD_MAXBLOCKSIZE, tx); VERIFY3U(0, ==, bpobj_open(&dle->dle_bpobj, dl->dl_os, obj)); avl_add(&dl->dl_tree, dle); VERIFY3U(0, ==, zap_add_int_key(dl->dl_os, dl->dl_object, mintxg, obj, tx)); + mutex_exit(&dl->dl_lock); } /* @@ -278,6 +285,7 @@ dsl_deadlist_remove_key(dsl_deadlist_t * if (dl->dl_oldfmt) return; + mutex_enter(&dl->dl_lock); dsl_deadlist_load_tree(dl); dle_tofind.dle_mintxg = mintxg; @@ -291,6 +299,7 @@ dsl_deadlist_remove_key(dsl_deadlist_t * kmem_free(dle, sizeof (*dle)); VERIFY3U(0, ==, zap_remove_int(dl->dl_os, dl->dl_object, mintxg, tx)); + mutex_exit(&dl->dl_lock); } /* @@ -334,6 +343,7 @@ dsl_deadlist_clone(dsl_deadlist_t *dl, u return (newobj); } + mutex_enter(&dl->dl_lock); dsl_deadlist_load_tree(dl); for (dle = avl_first(&dl->dl_tree); dle; @@ -347,6 +357,7 @@ dsl_deadlist_clone(dsl_deadlist_t *dl, u VERIFY3U(0, ==, zap_add_int_key(dl->dl_os, newobj, dle->dle_mintxg, obj, tx)); } + mutex_exit(&dl->dl_lock); return (newobj); } @@ -424,6 +435,8 @@ dsl_deadlist_insert_bpobj(dsl_deadlist_t uint64_t used, comp, uncomp; bpobj_t bpo; + ASSERT(MUTEX_HELD(&dl->dl_lock)); + VERIFY3U(0, ==, bpobj_open(&bpo, dl->dl_os, obj)); VERIFY3U(0, ==, bpobj_space(&bpo, &used, &comp, &uncomp)); bpobj_close(&bpo); @@ -431,11 +444,9 @@ dsl_deadlist_insert_bpobj(dsl_deadlist_t dsl_deadlist_load_tree(dl); dmu_buf_will_dirty(dl->dl_dbuf, tx); - mutex_enter(&dl->dl_lock); dl->dl_phys->dl_used += used; dl->dl_phys->dl_comp += comp; dl->dl_phys->dl_uncomp += uncomp; - mutex_exit(&dl->dl_lock); dle_tofind.dle_mintxg = birth; dle = avl_find(&dl->dl_tree, &dle_tofind, &where); @@ -475,6 +486,7 @@ dsl_deadlist_merge(dsl_deadlist_t *dl, u return; } + mutex_enter(&dl->dl_lock); for (zap_cursor_init(&zc, dl->dl_os, obj); zap_cursor_retrieve(&zc, &za) == 0; zap_cursor_advance(&zc)) { @@ -489,6 +501,7 @@ dsl_deadlist_merge(dsl_deadlist_t *dl, u dmu_buf_will_dirty(bonus, tx); bzero(dlp, sizeof (*dlp)); dmu_buf_rele(bonus, FTAG); + mutex_exit(&dl->dl_lock); } /* @@ -503,6 +516,8 @@ dsl_deadlist_move_bpobj(dsl_deadlist_t * avl_index_t where; ASSERT(!dl->dl_oldfmt); + + mutex_enter(&dl->dl_lock); dmu_buf_will_dirty(dl->dl_dbuf, tx); dsl_deadlist_load_tree(dl); @@ -518,14 +533,12 @@ dsl_deadlist_move_bpobj(dsl_deadlist_t * VERIFY3U(0, ==, bpobj_space(&dle->dle_bpobj, &used, &comp, &uncomp)); - mutex_enter(&dl->dl_lock); ASSERT3U(dl->dl_phys->dl_used, >=, used); ASSERT3U(dl->dl_phys->dl_comp, >=, comp); ASSERT3U(dl->dl_phys->dl_uncomp, >=, uncomp); dl->dl_phys->dl_used -= used; dl->dl_phys->dl_comp -= comp; dl->dl_phys->dl_uncomp -= uncomp; - mutex_exit(&dl->dl_lock); VERIFY3U(0, ==, zap_remove_int(dl->dl_os, dl->dl_object, dle->dle_mintxg, tx)); @@ -536,4 +549,5 @@ dsl_deadlist_move_bpobj(dsl_deadlist_t * kmem_free(dle, sizeof (*dle)); dle = dle_next; } + mutex_exit(&dl->dl_lock); } From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:25:04 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 8076BD3E110; Fri, 14 Apr 2017 18:25:04 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 4CC38D76; Fri, 14 Apr 2017 18:25:04 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIP3ra070705; Fri, 14 Apr 2017 18:25:03 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIP2s3070697; Fri, 14 Apr 2017 18:25:02 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141825.v3EIP2s3070697@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:25:02 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316914 - in vendor-sys/illumos/dist/uts/common/fs/zfs: . sys X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:25:04 -0000 Author: avg Date: Fri Apr 14 18:25:02 2017 New Revision: 316914 URL: https://svnweb.freebsd.org/changeset/base/316914 Log: 7801 add more by-dnode routines illumos/illumos-gate@b0c42cd4706ba01ce158bd2bb1004f7e59eca5fe https://github.com/illumos/illumos-gate/commit/b0c42cd4706ba01ce158bd2bb1004f7e59eca5fe https://www.illumos.org/issues/7801 Add *_by_dnode() routines for accessing objects given their dnode_t *, this is more efficient than accessing the object by (objset_t *, uint64_t object). This change converts some but not all of the existing consumers. As performance-sensitive code paths are discovered they should be converted to use these routines. Ported from: https://github.com/zfsonlinux/zfs/commit/ 0eef1bde31d67091d3deed23fe2394f5a8bf2276 Reviewed by: Matthew Ahrens Reviewed by: Brian Behlendorf Reviewed by: Pavel Zakharov Approved by: Robert Mustacchi Author: bzzz77 Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu.c vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_object.c vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_tx.c vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu_tx.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zap.h vendor-sys/illumos/dist/uts/common/fs/zfs/zap_micro.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dmu.c Fri Apr 14 18:24:38 2017 (r316913) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dmu.c Fri Apr 14 18:25:02 2017 (r316914) @@ -871,17 +871,12 @@ dmu_free_range(objset_t *os, uint64_t ob return (0); } -int -dmu_read(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, +static int +dmu_read_impl(dnode_t *dn, uint64_t offset, uint64_t size, void *buf, uint32_t flags) { - dnode_t *dn; dmu_buf_t **dbp; - int numbufs, err; - - err = dnode_hold(os, object, FTAG, &dn); - if (err) - return (err); + int numbufs, err = 0; /* * Deal with odd block sizes, where there can't be data past the first @@ -926,22 +921,37 @@ dmu_read(objset_t *os, uint64_t object, } dmu_buf_rele_array(dbp, numbufs, FTAG); } - dnode_rele(dn, FTAG); return (err); } -void -dmu_write(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, - const void *buf, dmu_tx_t *tx) +int +dmu_read(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, + void *buf, uint32_t flags) { - dmu_buf_t **dbp; - int numbufs, i; + dnode_t *dn; + int err; - if (size == 0) - return; + err = dnode_hold(os, object, FTAG, &dn); + if (err != 0) + return (err); - VERIFY(0 == dmu_buf_hold_array(os, object, offset, size, - FALSE, FTAG, &numbufs, &dbp)); + err = dmu_read_impl(dn, offset, size, buf, flags); + dnode_rele(dn, FTAG); + return (err); +} + +int +dmu_read_by_dnode(dnode_t *dn, uint64_t offset, uint64_t size, void *buf, + uint32_t flags) +{ + return (dmu_read_impl(dn, offset, size, buf, flags)); +} + +static void +dmu_write_impl(dmu_buf_t **dbp, int numbufs, uint64_t offset, uint64_t size, + const void *buf, dmu_tx_t *tx) +{ + int i; for (i = 0; i < numbufs; i++) { int tocpy; @@ -969,6 +979,37 @@ dmu_write(objset_t *os, uint64_t object, size -= tocpy; buf = (char *)buf + tocpy; } +} + +void +dmu_write(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, + const void *buf, dmu_tx_t *tx) +{ + dmu_buf_t **dbp; + int numbufs; + + if (size == 0) + return; + + VERIFY0(dmu_buf_hold_array(os, object, offset, size, + FALSE, FTAG, &numbufs, &dbp)); + dmu_write_impl(dbp, numbufs, offset, size, buf, tx); + dmu_buf_rele_array(dbp, numbufs, FTAG); +} + +void +dmu_write_by_dnode(dnode_t *dn, uint64_t offset, uint64_t size, + const void *buf, dmu_tx_t *tx) +{ + dmu_buf_t **dbp; + int numbufs; + + if (size == 0) + return; + + VERIFY0(dmu_buf_hold_array_by_dnode(dn, offset, size, + FALSE, FTAG, &numbufs, &dbp, DMU_READ_PREFETCH)); + dmu_write_impl(dbp, numbufs, offset, size, buf, tx); dmu_buf_rele_array(dbp, numbufs, FTAG); } Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_object.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_object.c Fri Apr 14 18:24:38 2017 (r316913) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_object.c Fri Apr 14 18:25:02 2017 (r316914) @@ -93,11 +93,11 @@ dmu_object_alloc(objset_t *os, dmu_objec } dnode_allocate(dn, ot, blocksize, 0, bonustype, bonuslen, tx); - dnode_rele(dn, FTAG); - mutex_exit(&os->os_obj_lock); - dmu_tx_add_new_object(tx, os, object); + dmu_tx_add_new_object(tx, dn); + dnode_rele(dn, FTAG); + return (object); } @@ -115,9 +115,10 @@ dmu_object_claim(objset_t *os, uint64_t if (err) return (err); dnode_allocate(dn, ot, blocksize, 0, bonustype, bonuslen, tx); + dmu_tx_add_new_object(tx, dn); + dnode_rele(dn, FTAG); - dmu_tx_add_new_object(tx, os, object); return (0); } Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_tx.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_tx.c Fri Apr 14 18:24:38 2017 (r316913) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_tx.c Fri Apr 14 18:25:02 2017 (r316914) @@ -93,21 +93,14 @@ dmu_tx_private_ok(dmu_tx_t *tx) } static dmu_tx_hold_t * -dmu_tx_hold_object_impl(dmu_tx_t *tx, objset_t *os, uint64_t object, - enum dmu_tx_hold_type type, uint64_t arg1, uint64_t arg2) +dmu_tx_hold_dnode_impl(dmu_tx_t *tx, dnode_t *dn, enum dmu_tx_hold_type type, + uint64_t arg1, uint64_t arg2) { dmu_tx_hold_t *txh; - dnode_t *dn = NULL; - int err; - - if (object != DMU_NEW_OBJECT) { - err = dnode_hold(os, object, tx, &dn); - if (err) { - tx->tx_err = err; - return (NULL); - } - if (err == 0 && tx->tx_txg != 0) { + if (dn != NULL) { + (void) refcount_add(&dn->dn_holds, tx); + if (tx->tx_txg != 0) { mutex_enter(&dn->dn_mtx); /* * dn->dn_assigned_txg == tx->tx_txg doesn't pose a @@ -134,17 +127,36 @@ dmu_tx_hold_object_impl(dmu_tx_t *tx, ob return (txh); } +static dmu_tx_hold_t * +dmu_tx_hold_object_impl(dmu_tx_t *tx, objset_t *os, uint64_t object, + enum dmu_tx_hold_type type, uint64_t arg1, uint64_t arg2) +{ + dnode_t *dn = NULL; + dmu_tx_hold_t *txh; + int err; + + if (object != DMU_NEW_OBJECT) { + err = dnode_hold(os, object, FTAG, &dn); + if (err != 0) { + tx->tx_err = err; + return (NULL); + } + } + txh = dmu_tx_hold_dnode_impl(tx, dn, type, arg1, arg2); + if (dn != NULL) + dnode_rele(dn, FTAG); + return (txh); +} + void -dmu_tx_add_new_object(dmu_tx_t *tx, objset_t *os, uint64_t object) +dmu_tx_add_new_object(dmu_tx_t *tx, dnode_t *dn) { /* * If we're syncing, they can manipulate any object anyhow, and * the hold on the dnode_t can cause problems. */ - if (!dmu_tx_is_syncing(tx)) { - (void) dmu_tx_hold_object_impl(tx, os, - object, THT_NEWOBJECT, 0, 0); - } + if (!dmu_tx_is_syncing(tx)) + (void) dmu_tx_hold_dnode_impl(tx, dn, THT_NEWOBJECT, 0, 0); } /* @@ -282,11 +294,26 @@ dmu_tx_hold_write(dmu_tx_t *tx, uint64_t txh = dmu_tx_hold_object_impl(tx, tx->tx_objset, object, THT_WRITE, off, len); - if (txh == NULL) - return; + if (txh != NULL) { + dmu_tx_count_write(txh, off, len); + dmu_tx_count_dnode(txh); + } +} - dmu_tx_count_write(txh, off, len); - dmu_tx_count_dnode(txh); +void +dmu_tx_hold_write_by_dnode(dmu_tx_t *tx, dnode_t *dn, uint64_t off, int len) +{ + dmu_tx_hold_t *txh; + + ASSERT0(tx->tx_txg); + ASSERT3U(len, <=, DMU_MAX_ACCESS); + ASSERT(len == 0 || UINT64_MAX - off >= len - 1); + + txh = dmu_tx_hold_dnode_impl(tx, dn, THT_WRITE, off, len); + if (txh != NULL) { + dmu_tx_count_write(txh, off, len); + dmu_tx_count_dnode(txh); + } } /* @@ -303,18 +330,18 @@ dmu_tx_mark_netfree(dmu_tx_t *tx) tx->tx_netfree = B_TRUE; } -void -dmu_tx_hold_free(dmu_tx_t *tx, uint64_t object, uint64_t off, uint64_t len) +static void +dmu_tx_hold_free_impl(dmu_tx_hold_t *txh, uint64_t off, uint64_t len) { + dmu_tx_t *tx; + dnode_t *dn; int err; + zio_t *zio; + tx = txh->txh_tx; ASSERT(tx->tx_txg == 0); - dmu_tx_hold_t *txh = dmu_tx_hold_object_impl(tx, tx->tx_objset, - object, THT_FREE, off, len); - if (txh == NULL) - return; - dnode_t *dn = txh->txh_dnode; + dn = txh->txh_dnode; dmu_tx_count_dnode(txh); if (off >= (dn->dn_maxblkid + 1) * dn->dn_datablksz) @@ -395,17 +422,36 @@ dmu_tx_hold_free(dmu_tx_t *tx, uint64_t } void -dmu_tx_hold_zap(dmu_tx_t *tx, uint64_t object, int add, const char *name) +dmu_tx_hold_free(dmu_tx_t *tx, uint64_t object, uint64_t off, uint64_t len) +{ + dmu_tx_hold_t *txh; + + txh = dmu_tx_hold_object_impl(tx, tx->tx_objset, + object, THT_FREE, off, len); + if (txh != NULL) + (void) dmu_tx_hold_free_impl(txh, off, len); +} + +void +dmu_tx_hold_free_by_dnode(dmu_tx_t *tx, dnode_t *dn, uint64_t off, uint64_t len) +{ + dmu_tx_hold_t *txh; + + txh = dmu_tx_hold_dnode_impl(tx, dn, THT_FREE, off, len); + if (txh != NULL) + (void) dmu_tx_hold_free_impl(txh, off, len); +} + +static void +dmu_tx_hold_zap_impl(dmu_tx_hold_t *txh, int add, const char *name) { + dmu_tx_t *tx = txh->txh_tx; + dnode_t *dn; int err; ASSERT(tx->tx_txg == 0); - dmu_tx_hold_t *txh = dmu_tx_hold_object_impl(tx, tx->tx_objset, - object, THT_ZAP, add, (uintptr_t)name); - if (txh == NULL) - return; - dnode_t *dn = txh->txh_dnode; + dn = txh->txh_dnode; dmu_tx_count_dnode(txh); @@ -449,6 +495,32 @@ dmu_tx_hold_zap(dmu_tx_t *tx, uint64_t o } void +dmu_tx_hold_zap(dmu_tx_t *tx, uint64_t object, int add, const char *name) +{ + dmu_tx_hold_t *txh; + + ASSERT0(tx->tx_txg); + + txh = dmu_tx_hold_object_impl(tx, tx->tx_objset, + object, THT_ZAP, add, (uintptr_t)name); + if (txh != NULL) + dmu_tx_hold_zap_impl(txh, add, name); +} + +void +dmu_tx_hold_zap_by_dnode(dmu_tx_t *tx, dnode_t *dn, int add, const char *name) +{ + dmu_tx_hold_t *txh; + + ASSERT0(tx->tx_txg); + ASSERT(dn != NULL); + + txh = dmu_tx_hold_dnode_impl(tx, dn, THT_ZAP, add, (uintptr_t)name); + if (txh != NULL) + dmu_tx_hold_zap_impl(txh, add, name); +} + +void dmu_tx_hold_bonus(dmu_tx_t *tx, uint64_t object) { dmu_tx_hold_t *txh; @@ -462,6 +534,18 @@ dmu_tx_hold_bonus(dmu_tx_t *tx, uint64_t } void +dmu_tx_hold_bonus_by_dnode(dmu_tx_t *tx, dnode_t *dn) +{ + dmu_tx_hold_t *txh; + + ASSERT0(tx->tx_txg); + + txh = dmu_tx_hold_dnode_impl(tx, dn, THT_BONUS, 0, 0); + if (txh) + dmu_tx_count_dnode(txh); +} + +void dmu_tx_hold_space(dmu_tx_t *tx, uint64_t space) { dmu_tx_hold_t *txh; Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu.h Fri Apr 14 18:24:38 2017 (r316913) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu.h Fri Apr 14 18:25:02 2017 (r316914) @@ -667,10 +667,17 @@ void dmu_buf_will_dirty(dmu_buf_t *db, d dmu_tx_t *dmu_tx_create(objset_t *os); void dmu_tx_hold_write(dmu_tx_t *tx, uint64_t object, uint64_t off, int len); +void dmu_tx_hold_write_by_dnode(dmu_tx_t *tx, dnode_t *dn, uint64_t off, + int len); void dmu_tx_hold_free(dmu_tx_t *tx, uint64_t object, uint64_t off, uint64_t len); +void dmu_tx_hold_free_by_dnode(dmu_tx_t *tx, dnode_t *dn, uint64_t off, + uint64_t len); void dmu_tx_hold_zap(dmu_tx_t *tx, uint64_t object, int add, const char *name); +void dmu_tx_hold_zap_by_dnode(dmu_tx_t *tx, dnode_t *dn, int add, + const char *name); void dmu_tx_hold_bonus(dmu_tx_t *tx, uint64_t object); +void dmu_tx_hold_bonus_by_dnode(dmu_tx_t *tx, dnode_t *dn); void dmu_tx_hold_spill(dmu_tx_t *tx, uint64_t object); void dmu_tx_hold_sa(dmu_tx_t *tx, struct sa_handle *hdl, boolean_t may_grow); void dmu_tx_hold_sa_create(dmu_tx_t *tx, int total_size); @@ -720,8 +727,12 @@ int dmu_free_long_object(objset_t *os, u #define DMU_READ_NO_PREFETCH 1 /* don't prefetch */ int dmu_read(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, void *buf, uint32_t flags); +int dmu_read_by_dnode(dnode_t *dn, uint64_t offset, uint64_t size, void *buf, + uint32_t flags); void dmu_write(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, const void *buf, dmu_tx_t *tx); +void dmu_write_by_dnode(dnode_t *dn, uint64_t offset, uint64_t size, + const void *buf, dmu_tx_t *tx); void dmu_prealloc(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, dmu_tx_t *tx); int dmu_read_uio(objset_t *os, uint64_t object, struct uio *uio, uint64_t size); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu_tx.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu_tx.h Fri Apr 14 18:24:38 2017 (r316913) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu_tx.h Fri Apr 14 18:25:02 2017 (r316914) @@ -136,7 +136,7 @@ extern dmu_tx_t *dmu_tx_create_assigned( dmu_tx_t *dmu_tx_create_dd(dsl_dir_t *dd); int dmu_tx_is_syncing(dmu_tx_t *tx); int dmu_tx_private_ok(dmu_tx_t *tx); -void dmu_tx_add_new_object(dmu_tx_t *tx, objset_t *os, uint64_t object); +void dmu_tx_add_new_object(dmu_tx_t *tx, dnode_t *dn); void dmu_tx_dirty_buf(dmu_tx_t *tx, struct dmu_buf_impl *db); void dmu_tx_hold_space(dmu_tx_t *tx, uint64_t space); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zap.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zap.h Fri Apr 14 18:24:38 2017 (r316913) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zap.h Fri Apr 14 18:25:02 2017 (r316914) @@ -220,6 +220,9 @@ int zap_count_write_by_dnode(dnode_t *dn int zap_add(objset_t *ds, uint64_t zapobj, const char *key, int integer_size, uint64_t num_integers, const void *val, dmu_tx_t *tx); +int zap_add_by_dnode(dnode_t *dn, const char *key, + int integer_size, uint64_t num_integers, + const void *val, dmu_tx_t *tx); int zap_add_uint64(objset_t *ds, uint64_t zapobj, const uint64_t *key, int key_numints, int integer_size, uint64_t num_integers, const void *val, dmu_tx_t *tx); @@ -259,6 +262,7 @@ int zap_length_uint64(objset_t *os, uint int zap_remove(objset_t *ds, uint64_t zapobj, const char *name, dmu_tx_t *tx); int zap_remove_norm(objset_t *ds, uint64_t zapobj, const char *name, matchtype_t mt, dmu_tx_t *tx); +int zap_remove_by_dnode(dnode_t *dn, const char *name, dmu_tx_t *tx); int zap_remove_uint64(objset_t *os, uint64_t zapobj, const uint64_t *key, int key_numints, dmu_tx_t *tx); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zap_micro.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zap_micro.c Fri Apr 14 18:24:38 2017 (r316913) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zap_micro.c Fri Apr 14 18:25:02 2017 (r316914) @@ -1109,34 +1109,30 @@ again: ASSERT(!"out of entries!"); } -int -zap_add(objset_t *os, uint64_t zapobj, const char *key, +static int +zap_add_impl(zap_t *zap, const char *key, int integer_size, uint64_t num_integers, - const void *val, dmu_tx_t *tx) + const void *val, dmu_tx_t *tx, void *tag) { - zap_t *zap; - int err; + int err = 0; mzap_ent_t *mze; const uint64_t *intval = val; zap_name_t *zn; - err = zap_lockdir(os, zapobj, tx, RW_WRITER, TRUE, TRUE, FTAG, &zap); - if (err) - return (err); zn = zap_name_alloc(zap, key, 0); if (zn == NULL) { - zap_unlockdir(zap, FTAG); + zap_unlockdir(zap, tag); return (SET_ERROR(ENOTSUP)); } if (!zap->zap_ismicro) { - err = fzap_add(zn, integer_size, num_integers, val, FTAG, tx); + err = fzap_add(zn, integer_size, num_integers, val, tag, tx); zap = zn->zn_zap; /* fzap_add() may change zap */ } else if (integer_size != 8 || num_integers != 1 || strlen(key) >= MZAP_NAME_LEN) { - err = mzap_upgrade(&zn->zn_zap, FTAG, tx, 0); + err = mzap_upgrade(&zn->zn_zap, tag, tx, 0); if (err == 0) { err = fzap_add(zn, integer_size, num_integers, val, - FTAG, tx); + tag, tx); } zap = zn->zn_zap; /* fzap_add() may change zap */ } else { @@ -1150,7 +1146,39 @@ zap_add(objset_t *os, uint64_t zapobj, c ASSERT(zap == zn->zn_zap); zap_name_free(zn); if (zap != NULL) /* may be NULL if fzap_add() failed */ - zap_unlockdir(zap, FTAG); + zap_unlockdir(zap, tag); + return (err); +} + +int +zap_add(objset_t *os, uint64_t zapobj, const char *key, + int integer_size, uint64_t num_integers, + const void *val, dmu_tx_t *tx) +{ + zap_t *zap; + int err; + + err = zap_lockdir(os, zapobj, tx, RW_WRITER, TRUE, TRUE, FTAG, &zap); + if (err != 0) + return (err); + err = zap_add_impl(zap, key, integer_size, num_integers, val, tx, FTAG); + /* zap_add_impl() calls zap_unlockdir() */ + return (err); +} + +int +zap_add_by_dnode(dnode_t *dn, const char *key, + int integer_size, uint64_t num_integers, + const void *val, dmu_tx_t *tx) +{ + zap_t *zap; + int err; + + err = zap_lockdir_by_dnode(dn, tx, RW_WRITER, TRUE, TRUE, FTAG, &zap); + if (err != 0) + return (err); + err = zap_add_impl(zap, key, integer_size, num_integers, val, tx, FTAG); + /* zap_add_impl() calls zap_unlockdir() */ return (err); } @@ -1268,23 +1296,17 @@ zap_remove(objset_t *os, uint64_t zapobj return (zap_remove_norm(os, zapobj, name, 0, tx)); } -int -zap_remove_norm(objset_t *os, uint64_t zapobj, const char *name, +static int +zap_remove_impl(zap_t *zap, const char *name, matchtype_t mt, dmu_tx_t *tx) { - zap_t *zap; - int err; mzap_ent_t *mze; zap_name_t *zn; + int err = 0; - err = zap_lockdir(os, zapobj, tx, RW_WRITER, TRUE, FALSE, FTAG, &zap); - if (err) - return (err); zn = zap_name_alloc(zap, name, mt); - if (zn == NULL) { - zap_unlockdir(zap, FTAG); + if (zn == NULL) return (SET_ERROR(ENOTSUP)); - } if (!zap->zap_ismicro) { err = fzap_remove(zn, tx); } else { @@ -1299,6 +1321,34 @@ zap_remove_norm(objset_t *os, uint64_t z } } zap_name_free(zn); + return (err); +} + +int +zap_remove_norm(objset_t *os, uint64_t zapobj, const char *name, + matchtype_t mt, dmu_tx_t *tx) +{ + zap_t *zap; + int err; + + err = zap_lockdir(os, zapobj, tx, RW_WRITER, TRUE, FALSE, FTAG, &zap); + if (err) + return (err); + err = zap_remove_impl(zap, name, mt, tx); + zap_unlockdir(zap, FTAG); + return (err); +} + +int +zap_remove_by_dnode(dnode_t *dn, const char *name, dmu_tx_t *tx) +{ + zap_t *zap; + int err; + + err = zap_lockdir_by_dnode(dn, tx, RW_WRITER, TRUE, FALSE, FTAG, &zap); + if (err) + return (err); + err = zap_remove_impl(zap, name, 0, tx); zap_unlockdir(zap, FTAG); return (err); } From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:25:23 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id D4554D3E1B8; Fri, 14 Apr 2017 18:25:23 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 86A1FEFB; Fri, 14 Apr 2017 18:25:23 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIPMhs070758; Fri, 14 Apr 2017 18:25:22 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIPME6070757; Fri, 14 Apr 2017 18:25:22 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141825.v3EIPME6070757@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:25:22 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316915 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:25:24 -0000 Author: avg Date: Fri Apr 14 18:25:22 2017 New Revision: 316915 URL: https://svnweb.freebsd.org/changeset/base/316915 Log: 7801 add more by-dnode routines (lint) illumos/illumos-gate@411be58a6e030a3b606f1afcc7f2e2459ffda844 https://github.com/illumos/illumos-gate/commit/411be58a6e030a3b606f1afcc7f2e2459ffda844 https://www.illumos.org/issues/7801 Add *_by_dnode() routines for accessing objects given their dnode_t *, this is more efficient than accessing the object by (objset_t *, uint64_t object). This change converts some but not all of the existing consumers. As performance-sensitive code paths are discovered they should be converted to use these routines. Ported from: https://github.com/zfsonlinux/zfs/commit/ 0eef1bde31d67091d3deed23fe2394f5a8bf2276 Author: Matthew Ahrens Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_tx.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_tx.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_tx.c Fri Apr 14 18:25:02 2017 (r316914) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_tx.c Fri Apr 14 18:25:22 2017 (r316915) @@ -336,7 +336,6 @@ dmu_tx_hold_free_impl(dmu_tx_hold_t *txh dmu_tx_t *tx; dnode_t *dn; int err; - zio_t *zio; tx = txh->txh_tx; ASSERT(tx->tx_txg == 0); @@ -443,7 +442,7 @@ dmu_tx_hold_free_by_dnode(dmu_tx_t *tx, } static void -dmu_tx_hold_zap_impl(dmu_tx_hold_t *txh, int add, const char *name) +dmu_tx_hold_zap_impl(dmu_tx_hold_t *txh, const char *name) { dmu_tx_t *tx = txh->txh_tx; dnode_t *dn; @@ -504,7 +503,7 @@ dmu_tx_hold_zap(dmu_tx_t *tx, uint64_t o txh = dmu_tx_hold_object_impl(tx, tx->tx_objset, object, THT_ZAP, add, (uintptr_t)name); if (txh != NULL) - dmu_tx_hold_zap_impl(txh, add, name); + dmu_tx_hold_zap_impl(txh, name); } void @@ -517,7 +516,7 @@ dmu_tx_hold_zap_by_dnode(dmu_tx_t *tx, d txh = dmu_tx_hold_dnode_impl(tx, dn, THT_ZAP, add, (uintptr_t)name); if (txh != NULL) - dmu_tx_hold_zap_impl(txh, add, name); + dmu_tx_hold_zap_impl(txh, name); } void From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:25:51 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 8BCD5D3E249; Fri, 14 Apr 2017 18:25:51 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 63AB8C0; Fri, 14 Apr 2017 18:25:51 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIPoSD070825; Fri, 14 Apr 2017 18:25:50 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIPo2v070820; Fri, 14 Apr 2017 18:25:50 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141825.v3EIPo2v070820@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:25:50 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316916 - in vendor-sys/illumos/dist/uts/common/fs/zfs: . sys X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:25:51 -0000 Author: avg Date: Fri Apr 14 18:25:49 2017 New Revision: 316916 URL: https://svnweb.freebsd.org/changeset/base/316916 Log: 7970 zfs_arc_num_sublists_per_state should be common to all multilists illumos/illumos-gate@10fbdecb05f411234920f8d3c92c148d39106d7e https://github.com/illumos/illumos-gate/commit/10fbdecb05f411234920f8d3c92c148d39106d7e https://www.illumos.org/issues/7970 The global tunable zfs_arc_num_sublists_per_state is used by the ARC and the dbuf cache, and other users are planned. We should change this tunable to be common to all multilists. Reviewed by: Pavel Zakharov Reviewed by: Brad Lewis Reviewed by: Saso Kiselkov Reviewed by: Brian Behlendorf Approved by: Dan McDonald Author: Matthew Ahrens Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c vendor-sys/illumos/dist/uts/common/fs/zfs/multilist.c vendor-sys/illumos/dist/uts/common/fs/zfs/sys/arc.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/multilist.h Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c Fri Apr 14 18:25:22 2017 (r316915) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c Fri Apr 14 18:25:49 2017 (r316916) @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, Joyent, Inc. All rights reserved. - * Copyright (c) 2011, 2016 by Delphix. All rights reserved. + * Copyright (c) 2011, 2017 by Delphix. All rights reserved. * Copyright (c) 2014 by Saso Kiselkov. All rights reserved. * Copyright 2015 Nexenta Systems, Inc. All rights reserved. */ @@ -296,13 +296,6 @@ uint_t arc_reduce_dnlc_percent = 3; */ int zfs_arc_evict_batch_limit = 10; -/* - * The number of sublists used for each of the arc state lists. If this - * is not set to a suitable value by the user, it will be configured to - * the number of CPUs on the system in arc_init(). - */ -int zfs_arc_num_sublists_per_state = 0; - /* number of seconds before growing cache again */ static int arc_grow_retry = 60; @@ -5775,43 +5768,43 @@ arc_state_init(void) multilist_create(&arc_mru->arcs_list[ARC_BUFC_METADATA], sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - zfs_arc_num_sublists_per_state, arc_state_multilist_index_func); + arc_state_multilist_index_func); multilist_create(&arc_mru->arcs_list[ARC_BUFC_DATA], sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - zfs_arc_num_sublists_per_state, arc_state_multilist_index_func); + arc_state_multilist_index_func); multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA], sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - zfs_arc_num_sublists_per_state, arc_state_multilist_index_func); + arc_state_multilist_index_func); multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA], sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - zfs_arc_num_sublists_per_state, arc_state_multilist_index_func); + arc_state_multilist_index_func); multilist_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA], sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - zfs_arc_num_sublists_per_state, arc_state_multilist_index_func); + arc_state_multilist_index_func); multilist_create(&arc_mfu->arcs_list[ARC_BUFC_DATA], sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - zfs_arc_num_sublists_per_state, arc_state_multilist_index_func); + arc_state_multilist_index_func); multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA], sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - zfs_arc_num_sublists_per_state, arc_state_multilist_index_func); + arc_state_multilist_index_func); multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA], sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - zfs_arc_num_sublists_per_state, arc_state_multilist_index_func); + arc_state_multilist_index_func); multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA], sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - zfs_arc_num_sublists_per_state, arc_state_multilist_index_func); + arc_state_multilist_index_func); multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA], sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), - zfs_arc_num_sublists_per_state, arc_state_multilist_index_func); + arc_state_multilist_index_func); refcount_create(&arc_anon->arcs_esize[ARC_BUFC_METADATA]); refcount_create(&arc_anon->arcs_esize[ARC_BUFC_DATA]); @@ -5969,9 +5962,6 @@ arc_init(void) if (zfs_arc_p_min_shift > 0) arc_p_min_shift = zfs_arc_p_min_shift; - if (zfs_arc_num_sublists_per_state < 1) - zfs_arc_num_sublists_per_state = MAX(boot_ncpus, 1); - /* if kmem_flags are set, lets try to use less memory */ if (kmem_debugging()) arc_c = arc_c / 2; Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c Fri Apr 14 18:25:22 2017 (r316915) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c Fri Apr 14 18:25:49 2017 (r316916) @@ -623,7 +623,6 @@ retry: multilist_create(&dbuf_cache, sizeof (dmu_buf_impl_t), offsetof(dmu_buf_impl_t, db_cache_link), - zfs_arc_num_sublists_per_state, dbuf_cache_multilist_index_func); refcount_create(&dbuf_cache_size); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/multilist.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/multilist.c Fri Apr 14 18:25:22 2017 (r316915) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/multilist.c Fri Apr 14 18:25:49 2017 (r316916) @@ -13,7 +13,7 @@ * CDDL HEADER END */ /* - * Copyright (c) 2013, 2014 by Delphix. All rights reserved. + * Copyright (c) 2013, 2017 by Delphix. All rights reserved. */ #include @@ -23,6 +23,12 @@ #include /* + * This overrides the number of sublists in each multilist_t, which defaults + * to the number of CPUs in the system (see multilist_create()). + */ +int zfs_multilist_num_sublists = 0; + +/* * Given the object contained on the list, return a pointer to the * object's multilist_node_t structure it contains. */ @@ -59,9 +65,9 @@ multilist_d2l(multilist_t *ml, void *obj * requirement, but a general rule of thumb in order to garner the * best multi-threaded performance out of the data structure. */ -void -multilist_create(multilist_t *ml, size_t size, size_t offset, unsigned int num, - multilist_sublist_index_func_t *index_func) +static void +multilist_create_impl(multilist_t *ml, size_t size, size_t offset, + unsigned int num, multilist_sublist_index_func_t *index_func) { ASSERT3P(ml, !=, NULL); ASSERT3U(size, >, 0); @@ -86,6 +92,26 @@ multilist_create(multilist_t *ml, size_t } /* + * Initialize a new sublist, using the default number of sublists + * (the number of CPUs, or at least 4, or the tunable + * zfs_multilist_num_sublists). + */ +void +multilist_create(multilist_t *ml, size_t size, size_t offset, + multilist_sublist_index_func_t *index_func) +{ + int num_sublists; + + if (zfs_multilist_num_sublists > 0) { + num_sublists = zfs_multilist_num_sublists; + } else { + num_sublists = MAX(boot_ncpus, 4); + } + + multilist_create_impl(ml, size, offset, num_sublists, index_func); +} + +/* * Destroy the given multilist object, and free up any memory it holds. */ void Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/arc.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/arc.h Fri Apr 14 18:25:22 2017 (r316915) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/arc.h Fri Apr 14 18:25:49 2017 (r316916) @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2016 by Delphix. All rights reserved. + * Copyright (c) 2012, 2017 by Delphix. All rights reserved. * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. */ @@ -64,8 +64,6 @@ typedef void arc_done_func_t(zio_t *zio, arc_done_func_t arc_bcopy_func; arc_done_func_t arc_getbuf_func; -extern int zfs_arc_num_sublists_per_state; - typedef enum arc_flags { /* Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/multilist.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/multilist.h Fri Apr 14 18:25:22 2017 (r316915) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/multilist.h Fri Apr 14 18:25:49 2017 (r316916) @@ -13,7 +13,7 @@ * CDDL HEADER END */ /* - * Copyright (c) 2013, 2014 by Delphix. All rights reserved. + * Copyright (c) 2013, 2017 by Delphix. All rights reserved. */ #ifndef _SYS_MULTILIST_H @@ -73,7 +73,7 @@ struct multilist { }; void multilist_destroy(multilist_t *); -void multilist_create(multilist_t *, size_t, size_t, unsigned int, +void multilist_create(multilist_t *, size_t, size_t, multilist_sublist_index_func_t *); void multilist_insert(multilist_t *, void *); From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:26:27 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 17978D3E2BE; Fri, 14 Apr 2017 18:26:27 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id AD3201FF; Fri, 14 Apr 2017 18:26:26 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIQPfS070909; Fri, 14 Apr 2017 18:26:25 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIQOT8070893; Fri, 14 Apr 2017 18:26:24 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141826.v3EIQOT8070893@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:26:24 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316917 - in vendor-sys/illumos/dist/uts/common/fs/zfs: . sys X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:26:27 -0000 Author: avg Date: Fri Apr 14 18:26:24 2017 New Revision: 316917 URL: https://svnweb.freebsd.org/changeset/base/316917 Log: 7968 multi-threaded spa_sync() illumos/illumos-gate@94c2d0eb22e9624151ee84a7edbf7178e1bf4087 https://github.com/illumos/illumos-gate/commit/94c2d0eb22e9624151ee84a7edbf7178e1bf4087 https://www.illumos.org/issues/7968 spa_sync() iterates over all the dirty dnodes and processes each of them by calling dnode_sync(). If there are many dirty dnodes (e.g. because we created or removed a lot of files), the single thread of spa_sync() calling dnode_sync () can become a bottleneck. Additionally, if many dnodes are dirtied concurrently in open context (e.g. due to concurrent file creation), the os_lock will experience lock contention via dnode_setdirty(). The solution is to track dirty dnodes on a multilist_t, and for spa_sync() to use separate threads to process each of the sublists in the multilist. On the concurrent file creation microbenchmark, the performance improvement from dnode_setdirty() is up to 7%. Additionally, the wall clock time spent in spa_sync() is reduced to 15%-40% of the single-threaded case. In terms of cost/ reward, once the other bottlenecks are addressed, fixing this bug will provide a medium-large performance gain and require a medium amount of effort to implement. Reviewed by: Pavel Zakharov Reviewed by: Brad Lewis Reviewed by: Saso Kiselkov Reviewed by: Brian Behlendorf Approved by: Dan McDonald Author: Matthew Ahrens Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c vendor-sys/illumos/dist/uts/common/fs/zfs/dnode.c vendor-sys/illumos/dist/uts/common/fs/zfs/dnode_sync.c vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_dataset.c vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_pool.c vendor-sys/illumos/dist/uts/common/fs/zfs/multilist.c vendor-sys/illumos/dist/uts/common/fs/zfs/spa_misc.c vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu_objset.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dnode.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dsl_pool.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/multilist.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zio.h vendor-sys/illumos/dist/uts/common/fs/zfs/zio.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c Fri Apr 14 18:25:49 2017 (r316916) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c Fri Apr 14 18:26:24 2017 (r316917) @@ -393,7 +393,7 @@ typedef struct arc_state { /* * list of evictable buffers */ - multilist_t arcs_list[ARC_BUFC_NUMTYPES]; + multilist_t *arcs_list[ARC_BUFC_NUMTYPES]; /* * total amount of evictable data in this state */ @@ -2070,7 +2070,7 @@ add_reference(arc_buf_hdr_t *hdr, void * (state != arc_anon)) { /* We don't use the L2-only state list. */ if (state != arc_l2c_only) { - multilist_remove(&state->arcs_list[arc_buf_type(hdr)], + multilist_remove(state->arcs_list[arc_buf_type(hdr)], hdr); arc_evictable_space_decrement(hdr, state); } @@ -2100,7 +2100,7 @@ remove_reference(arc_buf_hdr_t *hdr, kmu */ if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) && (state != arc_anon)) { - multilist_insert(&state->arcs_list[arc_buf_type(hdr)], hdr); + multilist_insert(state->arcs_list[arc_buf_type(hdr)], hdr); ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0); arc_evictable_space_increment(hdr, state); } @@ -2153,7 +2153,7 @@ arc_change_state(arc_state_t *new_state, if (refcnt == 0) { if (old_state != arc_anon && old_state != arc_l2c_only) { ASSERT(HDR_HAS_L1HDR(hdr)); - multilist_remove(&old_state->arcs_list[buftype], hdr); + multilist_remove(old_state->arcs_list[buftype], hdr); if (GHOST_STATE(old_state)) { ASSERT0(bufcnt); @@ -2171,7 +2171,7 @@ arc_change_state(arc_state_t *new_state, * beforehand. */ ASSERT(HDR_HAS_L1HDR(hdr)); - multilist_insert(&new_state->arcs_list[buftype], hdr); + multilist_insert(new_state->arcs_list[buftype], hdr); if (GHOST_STATE(new_state)) { ASSERT0(bufcnt); @@ -2297,8 +2297,8 @@ arc_change_state(arc_state_t *new_state, * L2 headers should never be on the L2 state list since they don't * have L1 headers allocated. */ - ASSERT(multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]) && - multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA])); + ASSERT(multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_DATA]) && + multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_METADATA])); } void @@ -3378,7 +3378,7 @@ arc_evict_state(arc_state_t *state, uint arc_buf_contents_t type) { uint64_t total_evicted = 0; - multilist_t *ml = &state->arcs_list[type]; + multilist_t *ml = state->arcs_list[type]; int num_sublists; arc_buf_hdr_t **markers; @@ -3582,8 +3582,8 @@ arc_adjust_meta(void) static arc_buf_contents_t arc_adjust_type(arc_state_t *state) { - multilist_t *data_ml = &state->arcs_list[ARC_BUFC_DATA]; - multilist_t *meta_ml = &state->arcs_list[ARC_BUFC_METADATA]; + multilist_t *data_ml = state->arcs_list[ARC_BUFC_DATA]; + multilist_t *meta_ml = state->arcs_list[ARC_BUFC_METADATA]; int data_idx = multilist_get_random_index(data_ml); int meta_idx = multilist_get_random_index(meta_ml); multilist_sublist_t *data_mls; @@ -5765,44 +5765,44 @@ arc_state_init(void) arc_mfu_ghost = &ARC_mfu_ghost; arc_l2c_only = &ARC_l2c_only; - multilist_create(&arc_mru->arcs_list[ARC_BUFC_METADATA], - sizeof (arc_buf_hdr_t), + arc_mru->arcs_list[ARC_BUFC_METADATA] = + multilist_create(sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), arc_state_multilist_index_func); - multilist_create(&arc_mru->arcs_list[ARC_BUFC_DATA], - sizeof (arc_buf_hdr_t), + arc_mru->arcs_list[ARC_BUFC_DATA] = + multilist_create(sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), arc_state_multilist_index_func); - multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA], - sizeof (arc_buf_hdr_t), + arc_mru_ghost->arcs_list[ARC_BUFC_METADATA] = + multilist_create(sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), arc_state_multilist_index_func); - multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA], - sizeof (arc_buf_hdr_t), + arc_mru_ghost->arcs_list[ARC_BUFC_DATA] = + multilist_create(sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), arc_state_multilist_index_func); - multilist_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA], - sizeof (arc_buf_hdr_t), + arc_mfu->arcs_list[ARC_BUFC_METADATA] = + multilist_create(sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), arc_state_multilist_index_func); - multilist_create(&arc_mfu->arcs_list[ARC_BUFC_DATA], - sizeof (arc_buf_hdr_t), + arc_mfu->arcs_list[ARC_BUFC_DATA] = + multilist_create(sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), arc_state_multilist_index_func); - multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA], - sizeof (arc_buf_hdr_t), + arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA] = + multilist_create(sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), arc_state_multilist_index_func); - multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA], - sizeof (arc_buf_hdr_t), + arc_mfu_ghost->arcs_list[ARC_BUFC_DATA] = + multilist_create(sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), arc_state_multilist_index_func); - multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA], - sizeof (arc_buf_hdr_t), + arc_l2c_only->arcs_list[ARC_BUFC_METADATA] = + multilist_create(sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), arc_state_multilist_index_func); - multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA], - sizeof (arc_buf_hdr_t), + arc_l2c_only->arcs_list[ARC_BUFC_DATA] = + multilist_create(sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), arc_state_multilist_index_func); @@ -5850,14 +5850,14 @@ arc_state_fini(void) refcount_destroy(&arc_mfu_ghost->arcs_size); refcount_destroy(&arc_l2c_only->arcs_size); - multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]); - multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]); - multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]); - multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]); - multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]); - multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]); - multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]); - multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]); + multilist_destroy(arc_mru->arcs_list[ARC_BUFC_METADATA]); + multilist_destroy(arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]); + multilist_destroy(arc_mfu->arcs_list[ARC_BUFC_METADATA]); + multilist_destroy(arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]); + multilist_destroy(arc_mru->arcs_list[ARC_BUFC_DATA]); + multilist_destroy(arc_mru_ghost->arcs_list[ARC_BUFC_DATA]); + multilist_destroy(arc_mfu->arcs_list[ARC_BUFC_DATA]); + multilist_destroy(arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]); } uint64_t @@ -6546,16 +6546,16 @@ l2arc_sublist_lock(int list_num) switch (list_num) { case 0: - ml = &arc_mfu->arcs_list[ARC_BUFC_METADATA]; + ml = arc_mfu->arcs_list[ARC_BUFC_METADATA]; break; case 1: - ml = &arc_mru->arcs_list[ARC_BUFC_METADATA]; + ml = arc_mru->arcs_list[ARC_BUFC_METADATA]; break; case 2: - ml = &arc_mfu->arcs_list[ARC_BUFC_DATA]; + ml = arc_mfu->arcs_list[ARC_BUFC_DATA]; break; case 3: - ml = &arc_mru->arcs_list[ARC_BUFC_DATA]; + ml = arc_mru->arcs_list[ARC_BUFC_DATA]; break; } Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c Fri Apr 14 18:25:49 2017 (r316916) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c Fri Apr 14 18:26:24 2017 (r316917) @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2011 Nexenta Systems, Inc. All rights reserved. - * Copyright (c) 2012, 2016 by Delphix. All rights reserved. + * Copyright (c) 2012, 2017 by Delphix. All rights reserved. * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. * Copyright (c) 2013, Joyent, Inc. All rights reserved. * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. @@ -80,7 +80,7 @@ static boolean_t dbuf_evict_thread_exit; * Dbufs that are aged out of the cache will be immediately destroyed and * become eligible for arc eviction. */ -static multilist_t dbuf_cache; +static multilist_t *dbuf_cache; static refcount_t dbuf_cache_size; uint64_t dbuf_cache_max_bytes = 100 * 1024 * 1024; @@ -454,8 +454,8 @@ dbuf_cache_above_lowater(void) static void dbuf_evict_one(void) { - int idx = multilist_get_random_index(&dbuf_cache); - multilist_sublist_t *mls = multilist_sublist_lock(&dbuf_cache, idx); + int idx = multilist_get_random_index(dbuf_cache); + multilist_sublist_t *mls = multilist_sublist_lock(dbuf_cache, idx); ASSERT(!MUTEX_HELD(&dbuf_evict_lock)); @@ -621,7 +621,7 @@ retry: */ dbu_evict_taskq = taskq_create("dbu_evict", 1, minclsyspri, 0, 0, 0); - multilist_create(&dbuf_cache, sizeof (dmu_buf_impl_t), + dbuf_cache = multilist_create(sizeof (dmu_buf_impl_t), offsetof(dmu_buf_impl_t, db_cache_link), dbuf_cache_multilist_index_func); refcount_create(&dbuf_cache_size); @@ -659,7 +659,7 @@ dbuf_fini(void) cv_destroy(&dbuf_evict_cv); refcount_destroy(&dbuf_cache_size); - multilist_destroy(&dbuf_cache); + multilist_destroy(dbuf_cache); } /* @@ -2029,7 +2029,7 @@ dbuf_destroy(dmu_buf_impl_t *db) dbuf_clear_data(db); if (multilist_link_active(&db->db_cache_link)) { - multilist_remove(&dbuf_cache, db); + multilist_remove(dbuf_cache, db); (void) refcount_remove_many(&dbuf_cache_size, db->db.db_size, db); } @@ -2577,7 +2577,7 @@ top: if (multilist_link_active(&db->db_cache_link)) { ASSERT(refcount_is_zero(&db->db_holds)); - multilist_remove(&dbuf_cache, db); + multilist_remove(dbuf_cache, db); (void) refcount_remove_many(&dbuf_cache_size, db->db.db_size, db); } @@ -2796,7 +2796,7 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, db->db_pending_evict) { dbuf_destroy(db); } else if (!multilist_link_active(&db->db_cache_link)) { - multilist_insert(&dbuf_cache, db); + multilist_insert(dbuf_cache, db); (void) refcount_add_many(&dbuf_cache_size, db->db.db_size, db); mutex_exit(&db->db_mtx); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c Fri Apr 14 18:25:49 2017 (r316916) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c Fri Apr 14 18:26:24 2017 (r316917) @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2016 by Delphix. All rights reserved. + * Copyright (c) 2012, 2017 by Delphix. All rights reserved. * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. * Copyright (c) 2013, Joyent, Inc. All rights reserved. * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. @@ -303,6 +303,42 @@ dmu_objset_byteswap(void *buf, size_t si } } +/* + * The hash is a CRC-based hash of the objset_t pointer and the object number. + */ +static uint64_t +dnode_hash(const objset_t *os, uint64_t obj) +{ + uintptr_t osv = (uintptr_t)os; + uint64_t crc = -1ULL; + + ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); + /* + * The low 6 bits of the pointer don't have much entropy, because + * the objset_t is larger than 2^6 bytes long. + */ + crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; + crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; + crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; + crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 16)) & 0xFF]; + + crc ^= (osv>>14) ^ (obj>>24); + + return (crc); +} + +unsigned int +dnode_multilist_index_func(multilist_t *ml, void *obj) +{ + dnode_t *dn = obj; + return (dnode_hash(dn->dn_objset, dn->dn_object) % + multilist_get_num_sublists(ml)); +} + +/* + * Instantiates the objset_t in-memory structure corresponding to the + * objset_phys_t that's pointed to by the specified blkptr_t. + */ int dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp, objset_t **osp) @@ -454,10 +490,9 @@ dmu_objset_open_impl(spa_t *spa, dsl_dat os->os_zil = zil_alloc(os, &os->os_zil_header); for (i = 0; i < TXG_SIZE; i++) { - list_create(&os->os_dirty_dnodes[i], sizeof (dnode_t), - offsetof(dnode_t, dn_dirty_link[i])); - list_create(&os->os_free_dnodes[i], sizeof (dnode_t), - offsetof(dnode_t, dn_dirty_link[i])); + os->os_dirty_dnodes[i] = multilist_create(sizeof (dnode_t), + offsetof(dnode_t, dn_dirty_link[i]), + dnode_multilist_index_func); } list_create(&os->os_dnodes, sizeof (dnode_t), offsetof(dnode_t, dn_link)); @@ -465,6 +500,7 @@ dmu_objset_open_impl(spa_t *spa, dsl_dat offsetof(dmu_buf_impl_t, db_link)); mutex_init(&os->os_lock, NULL, MUTEX_DEFAULT, NULL); + mutex_init(&os->os_userused_lock, NULL, MUTEX_DEFAULT, NULL); mutex_init(&os->os_obj_lock, NULL, MUTEX_DEFAULT, NULL); mutex_init(&os->os_user_ptr_lock, NULL, MUTEX_DEFAULT, NULL); @@ -748,8 +784,12 @@ dmu_objset_evict_done(objset_t *os) rw_exit(&os_lock); mutex_destroy(&os->os_lock); + mutex_destroy(&os->os_userused_lock); mutex_destroy(&os->os_obj_lock); mutex_destroy(&os->os_user_ptr_lock); + for (int i = 0; i < TXG_SIZE; i++) { + multilist_destroy(os->os_dirty_dnodes[i]); + } spa_evicting_os_deregister(os->os_spa, os); kmem_free(os, sizeof (objset_t)); } @@ -1027,11 +1067,11 @@ dmu_objset_snapshot_one(const char *fsna } static void -dmu_objset_sync_dnodes(list_t *list, list_t *newlist, dmu_tx_t *tx) +dmu_objset_sync_dnodes(multilist_sublist_t *list, dmu_tx_t *tx) { dnode_t *dn; - while (dn = list_head(list)) { + while ((dn = multilist_sublist_head(list)) != NULL) { ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); ASSERT(dn->dn_dbuf->db_data_pending); /* @@ -1042,11 +1082,12 @@ dmu_objset_sync_dnodes(list_t *list, lis ASSERT(dn->dn_zio); ASSERT3U(dn->dn_nlevels, <=, DN_MAX_LEVELS); - list_remove(list, dn); + multilist_sublist_remove(list, dn); - if (newlist) { + multilist_t *newlist = dn->dn_objset->os_synced_dnodes; + if (newlist != NULL) { (void) dnode_add_ref(dn, newlist); - list_insert_tail(newlist, dn); + multilist_insert(newlist, dn); } dnode_sync(dn, tx); @@ -1101,6 +1142,29 @@ dmu_objset_write_done(zio_t *zio, arc_bu kmem_free(bp, sizeof (*bp)); } +typedef struct sync_dnodes_arg { + multilist_t *sda_list; + int sda_sublist_idx; + multilist_t *sda_newlist; + dmu_tx_t *sda_tx; +} sync_dnodes_arg_t; + +static void +sync_dnodes_task(void *arg) +{ + sync_dnodes_arg_t *sda = arg; + + multilist_sublist_t *ms = + multilist_sublist_lock(sda->sda_list, sda->sda_sublist_idx); + + dmu_objset_sync_dnodes(ms, sda->sda_tx); + + multilist_sublist_unlock(ms); + + kmem_free(sda, sizeof (*sda)); +} + + /* called from dsl */ void dmu_objset_sync(objset_t *os, zio_t *pio, dmu_tx_t *tx) @@ -1110,7 +1174,6 @@ dmu_objset_sync(objset_t *os, zio_t *pio zio_prop_t zp; zio_t *zio; list_t *list; - list_t *newlist = NULL; dbuf_dirty_record_t *dr; blkptr_t *blkptr_copy = kmem_alloc(sizeof (*os->os_rootbp), KM_SLEEP); *blkptr_copy = *os->os_rootbp; @@ -1164,20 +1227,36 @@ dmu_objset_sync(objset_t *os, zio_t *pio txgoff = tx->tx_txg & TXG_MASK; if (dmu_objset_userused_enabled(os)) { - newlist = &os->os_synced_dnodes; /* * We must create the list here because it uses the - * dn_dirty_link[] of this txg. + * dn_dirty_link[] of this txg. But it may already + * exist because we call dsl_dataset_sync() twice per txg. */ - list_create(newlist, sizeof (dnode_t), - offsetof(dnode_t, dn_dirty_link[txgoff])); + if (os->os_synced_dnodes == NULL) { + os->os_synced_dnodes = + multilist_create(sizeof (dnode_t), + offsetof(dnode_t, dn_dirty_link[txgoff]), + dnode_multilist_index_func); + } else { + ASSERT3U(os->os_synced_dnodes->ml_offset, ==, + offsetof(dnode_t, dn_dirty_link[txgoff])); + } } - dmu_objset_sync_dnodes(&os->os_free_dnodes[txgoff], newlist, tx); - dmu_objset_sync_dnodes(&os->os_dirty_dnodes[txgoff], newlist, tx); + for (int i = 0; + i < multilist_get_num_sublists(os->os_dirty_dnodes[txgoff]); i++) { + sync_dnodes_arg_t *sda = kmem_alloc(sizeof (*sda), KM_SLEEP); + sda->sda_list = os->os_dirty_dnodes[txgoff]; + sda->sda_sublist_idx = i; + sda->sda_tx = tx; + (void) taskq_dispatch(dmu_objset_pool(os)->dp_sync_taskq, + sync_dnodes_task, sda, 0); + /* callback frees sda */ + } + taskq_wait(dmu_objset_pool(os)->dp_sync_taskq); list = &DMU_META_DNODE(os)->dn_dirty_records[txgoff]; - while (dr = list_head(list)) { + while ((dr = list_head(list)) != NULL) { ASSERT0(dr->dr_dbuf->db_level); list_remove(list, dr); if (dr->dr_zio) @@ -1201,8 +1280,7 @@ dmu_objset_sync(objset_t *os, zio_t *pio boolean_t dmu_objset_is_dirty(objset_t *os, uint64_t txg) { - return (!list_is_empty(&os->os_dirty_dnodes[txg & TXG_MASK]) || - !list_is_empty(&os->os_free_dnodes[txg & TXG_MASK])); + return (!multilist_is_empty(os->os_dirty_dnodes[txg & TXG_MASK])); } static objset_used_cb_t *used_cbs[DMU_OST_NUMTYPES]; @@ -1256,8 +1334,15 @@ do_userquota_cacheflush(objset_t *os, us cookie = NULL; while ((uqn = avl_destroy_nodes(&cache->uqc_user_deltas, &cookie)) != NULL) { + /* + * os_userused_lock protects against concurrent calls to + * zap_increment_int(). It's needed because zap_increment_int() + * is not thread-safe (i.e. not atomic). + */ + mutex_enter(&os->os_userused_lock); VERIFY0(zap_increment_int(os, DMU_USERUSED_OBJECT, uqn->uqn_id, uqn->uqn_delta, tx)); + mutex_exit(&os->os_userused_lock); kmem_free(uqn, sizeof (*uqn)); } avl_destroy(&cache->uqc_user_deltas); @@ -1265,8 +1350,10 @@ do_userquota_cacheflush(objset_t *os, us cookie = NULL; while ((uqn = avl_destroy_nodes(&cache->uqc_group_deltas, &cookie)) != NULL) { + mutex_enter(&os->os_userused_lock); VERIFY0(zap_increment_int(os, DMU_GROUPUSED_OBJECT, uqn->uqn_id, uqn->uqn_delta, tx)); + mutex_exit(&os->os_userused_lock); kmem_free(uqn, sizeof (*uqn)); } avl_destroy(&cache->uqc_group_deltas); @@ -1301,37 +1388,38 @@ do_userquota_update(userquota_cache_t *c } } -void -dmu_objset_do_userquota_updates(objset_t *os, dmu_tx_t *tx) +typedef struct userquota_updates_arg { + objset_t *uua_os; + int uua_sublist_idx; + dmu_tx_t *uua_tx; +} userquota_updates_arg_t; + +static void +userquota_updates_task(void *arg) { + userquota_updates_arg_t *uua = arg; + objset_t *os = uua->uua_os; + dmu_tx_t *tx = uua->uua_tx; dnode_t *dn; - list_t *list = &os->os_synced_dnodes; userquota_cache_t cache = { 0 }; - ASSERT(list_head(list) == NULL || dmu_objset_userused_enabled(os)); + multilist_sublist_t *list = + multilist_sublist_lock(os->os_synced_dnodes, uua->uua_sublist_idx); + ASSERT(multilist_sublist_head(list) == NULL || + dmu_objset_userused_enabled(os)); avl_create(&cache.uqc_user_deltas, userquota_compare, sizeof (userquota_node_t), offsetof(userquota_node_t, uqn_node)); avl_create(&cache.uqc_group_deltas, userquota_compare, sizeof (userquota_node_t), offsetof(userquota_node_t, uqn_node)); - while (dn = list_head(list)) { + while ((dn = multilist_sublist_head(list)) != NULL) { int flags; ASSERT(!DMU_OBJECT_IS_SPECIAL(dn->dn_object)); ASSERT(dn->dn_phys->dn_type == DMU_OT_NONE || dn->dn_phys->dn_flags & DNODE_FLAG_USERUSED_ACCOUNTED); - /* Allocate the user/groupused objects if necessary. */ - if (DMU_USERUSED_DNODE(os)->dn_type == DMU_OT_NONE) { - VERIFY0(zap_create_claim(os, - DMU_USERUSED_OBJECT, - DMU_OT_USERGROUP_USED, DMU_OT_NONE, 0, tx)); - VERIFY0(zap_create_claim(os, - DMU_GROUPUSED_OBJECT, - DMU_OT_USERGROUP_USED, DMU_OT_NONE, 0, tx)); - } - flags = dn->dn_id_flags; ASSERT(flags); if (flags & DN_ID_OLD_EXIST) { @@ -1361,10 +1449,42 @@ dmu_objset_do_userquota_updates(objset_t dn->dn_id_flags &= ~(DN_ID_NEW_EXIST); mutex_exit(&dn->dn_mtx); - list_remove(list, dn); - dnode_rele(dn, list); + multilist_sublist_remove(list, dn); + dnode_rele(dn, os->os_synced_dnodes); } do_userquota_cacheflush(os, &cache, tx); + multilist_sublist_unlock(list); + kmem_free(uua, sizeof (*uua)); +} + +void +dmu_objset_do_userquota_updates(objset_t *os, dmu_tx_t *tx) +{ + if (!dmu_objset_userused_enabled(os)) + return; + + /* Allocate the user/groupused objects if necessary. */ + if (DMU_USERUSED_DNODE(os)->dn_type == DMU_OT_NONE) { + VERIFY0(zap_create_claim(os, + DMU_USERUSED_OBJECT, + DMU_OT_USERGROUP_USED, DMU_OT_NONE, 0, tx)); + VERIFY0(zap_create_claim(os, + DMU_GROUPUSED_OBJECT, + DMU_OT_USERGROUP_USED, DMU_OT_NONE, 0, tx)); + } + + for (int i = 0; + i < multilist_get_num_sublists(os->os_synced_dnodes); i++) { + userquota_updates_arg_t *uua = + kmem_alloc(sizeof (*uua), KM_SLEEP); + uua->uua_os = os; + uua->uua_sublist_idx = i; + uua->uua_tx = tx; + /* note: caller does taskq_wait() */ + (void) taskq_dispatch(dmu_objset_pool(os)->dp_sync_taskq, + userquota_updates_task, uua, 0); + /* callback frees uua */ + } } /* Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dnode.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dnode.c Fri Apr 14 18:25:49 2017 (r316916) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dnode.c Fri Apr 14 18:26:24 2017 (r316917) @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2016 by Delphix. All rights reserved. + * Copyright (c) 2012, 2017 by Delphix. All rights reserved. * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. * Copyright (c) 2014 Integros [integros.com] */ @@ -1282,13 +1282,14 @@ dnode_setdirty(dnode_t *dn, dmu_tx_t *tx */ dmu_objset_userquota_get_ids(dn, B_TRUE, tx); - mutex_enter(&os->os_lock); + multilist_t *dirtylist = os->os_dirty_dnodes[txg & TXG_MASK]; + multilist_sublist_t *mls = multilist_sublist_lock_obj(dirtylist, dn); /* * If we are already marked dirty, we're done. */ if (list_link_active(&dn->dn_dirty_link[txg & TXG_MASK])) { - mutex_exit(&os->os_lock); + multilist_sublist_unlock(mls); return; } @@ -1302,13 +1303,9 @@ dnode_setdirty(dnode_t *dn, dmu_tx_t *tx dprintf_ds(os->os_dsl_dataset, "obj=%llu txg=%llu\n", dn->dn_object, txg); - if (dn->dn_free_txg > 0 && dn->dn_free_txg <= txg) { - list_insert_tail(&os->os_free_dnodes[txg&TXG_MASK], dn); - } else { - list_insert_tail(&os->os_dirty_dnodes[txg&TXG_MASK], dn); - } + multilist_sublist_insert_head(mls, dn); - mutex_exit(&os->os_lock); + multilist_sublist_unlock(mls); /* * The dnode maintains a hold on its containing dbuf as @@ -1329,13 +1326,6 @@ dnode_setdirty(dnode_t *dn, dmu_tx_t *tx void dnode_free(dnode_t *dn, dmu_tx_t *tx) { - int txgoff = tx->tx_txg & TXG_MASK; - - dprintf("dn=%p txg=%llu\n", dn, tx->tx_txg); - - /* we should be the only holder... hopefully */ - /* ASSERT3U(refcount_count(&dn->dn_holds), ==, 1); */ - mutex_enter(&dn->dn_mtx); if (dn->dn_type == DMU_OT_NONE || dn->dn_free_txg) { mutex_exit(&dn->dn_mtx); @@ -1344,19 +1334,7 @@ dnode_free(dnode_t *dn, dmu_tx_t *tx) dn->dn_free_txg = tx->tx_txg; mutex_exit(&dn->dn_mtx); - /* - * If the dnode is already dirty, it needs to be moved from - * the dirty list to the free list. - */ - mutex_enter(&dn->dn_objset->os_lock); - if (list_link_active(&dn->dn_dirty_link[txgoff])) { - list_remove(&dn->dn_objset->os_dirty_dnodes[txgoff], dn); - list_insert_tail(&dn->dn_objset->os_free_dnodes[txgoff], dn); - mutex_exit(&dn->dn_objset->os_lock); - } else { - mutex_exit(&dn->dn_objset->os_lock); - dnode_setdirty(dn, tx); - } + dnode_setdirty(dn, tx); } /* Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dnode_sync.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dnode_sync.c Fri Apr 14 18:25:49 2017 (r316916) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dnode_sync.c Fri Apr 14 18:26:24 2017 (r316917) @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2016 by Delphix. All rights reserved. + * Copyright (c) 2012, 2017 by Delphix. All rights reserved. * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. */ Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_dataset.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_dataset.c Fri Apr 14 18:25:49 2017 (r316916) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_dataset.c Fri Apr 14 18:26:24 2017 (r316917) @@ -1674,6 +1674,11 @@ dsl_dataset_sync_done(dsl_dataset_t *ds, bplist_iterate(&ds->ds_pending_deadlist, deadlist_enqueue_cb, &ds->ds_deadlist, tx); + if (os->os_synced_dnodes != NULL) { + multilist_destroy(os->os_synced_dnodes); + os->os_synced_dnodes = NULL; + } + ASSERT(!dmu_objset_is_dirty(os, dmu_tx_get_txg(tx))); dmu_buf_rele(ds->ds_dbuf, ds); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_pool.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_pool.c Fri Apr 14 18:25:49 2017 (r316916) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_pool.c Fri Apr 14 18:26:24 2017 (r316917) @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2016 by Delphix. All rights reserved. + * Copyright (c) 2011, 2017 by Delphix. All rights reserved. * Copyright (c) 2013 Steven Hartland. All rights reserved. * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. * Copyright (c) 2014 Integros [integros.com] @@ -127,6 +127,10 @@ int zfs_delay_min_dirty_percent = 60; */ uint64_t zfs_delay_scale = 1000 * 1000 * 1000 / 2000; +/* + * This determines the number of threads used by the dp_sync_taskq. + */ +int zfs_sync_taskq_batch_pct = 75; hrtime_t zfs_throttle_delay = MSEC2NSEC(10); hrtime_t zfs_throttle_resolution = MSEC2NSEC(10); @@ -167,6 +171,10 @@ dsl_pool_open_impl(spa_t *spa, uint64_t txg_list_create(&dp->dp_sync_tasks, offsetof(dsl_sync_task_t, dst_node)); + dp->dp_sync_taskq = taskq_create("dp_sync_taskq", + zfs_sync_taskq_batch_pct, minclsyspri, 1, INT_MAX, + TASKQ_THREADS_CPU_PCT); + mutex_init(&dp->dp_lock, NULL, MUTEX_DEFAULT, NULL); cv_init(&dp->dp_spaceavail_cv, NULL, CV_DEFAULT, NULL); @@ -317,6 +325,8 @@ dsl_pool_close(dsl_pool_t *dp) txg_list_destroy(&dp->dp_sync_tasks); txg_list_destroy(&dp->dp_dirty_dirs); + taskq_destroy(dp->dp_sync_taskq); + /* * We can't set retry to TRUE since we're explicitly specifying * a spa to flush. This is good enough; any missed buffers for @@ -505,12 +515,15 @@ dsl_pool_sync(dsl_pool_t *dp, uint64_t t /* * After the data blocks have been written (ensured by the zio_wait() - * above), update the user/group space accounting. + * above), update the user/group space accounting. This happens + * in tasks dispatched to dp_sync_taskq, so wait for them before + * continuing. */ for (ds = list_head(&synced_datasets); ds != NULL; ds = list_next(&synced_datasets, ds)) { dmu_objset_do_userquota_updates(ds->ds_objset, tx); } + taskq_wait(dp->dp_sync_taskq); /* * Sync the datasets again to push out the changes due to @@ -557,8 +570,7 @@ dsl_pool_sync(dsl_pool_t *dp, uint64_t t dp->dp_mos_uncompressed_delta = 0; } - if (list_head(&mos->os_dirty_dnodes[txg & TXG_MASK]) != NULL || - list_head(&mos->os_free_dnodes[txg & TXG_MASK]) != NULL) { + if (!multilist_is_empty(mos->os_dirty_dnodes[txg & TXG_MASK])) { dsl_pool_sync_mos(dp, tx); } @@ -616,7 +628,8 @@ int dsl_pool_sync_context(dsl_pool_t *dp) { return (curthread == dp->dp_tx.tx_sync_thread || - spa_is_initializing(dp->dp_spa)); + spa_is_initializing(dp->dp_spa) || + taskq_member(dp->dp_sync_taskq, curthread)); } uint64_t Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/multilist.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/multilist.c Fri Apr 14 18:25:49 2017 (r316916) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/multilist.c Fri Apr 14 18:26:24 2017 (r316917) @@ -65,16 +65,16 @@ multilist_d2l(multilist_t *ml, void *obj * requirement, but a general rule of thumb in order to garner the * best multi-threaded performance out of the data structure. */ -static void -multilist_create_impl(multilist_t *ml, size_t size, size_t offset, +static multilist_t * +multilist_create_impl(size_t size, size_t offset, unsigned int num, multilist_sublist_index_func_t *index_func) { - ASSERT3P(ml, !=, NULL); ASSERT3U(size, >, 0); ASSERT3U(size, >=, offset + sizeof (multilist_node_t)); ASSERT3U(num, >, 0); ASSERT3P(index_func, !=, NULL); + multilist_t *ml = kmem_alloc(sizeof (*ml), KM_SLEEP); ml->ml_offset = offset; ml->ml_num_sublists = num; ml->ml_index_func = index_func; @@ -89,15 +89,16 @@ multilist_create_impl(multilist_t *ml, s mutex_init(&mls->mls_lock, NULL, MUTEX_DEFAULT, NULL); list_create(&mls->mls_list, size, offset); } + return (ml); } /* - * Initialize a new sublist, using the default number of sublists + * Allocate a new multilist, using the default number of sublists * (the number of CPUs, or at least 4, or the tunable * zfs_multilist_num_sublists). */ -void -multilist_create(multilist_t *ml, size_t size, size_t offset, +multilist_t * +multilist_create(size_t size, size_t offset, multilist_sublist_index_func_t *index_func) { int num_sublists; @@ -108,7 +109,7 @@ multilist_create(multilist_t *ml, size_t num_sublists = MAX(boot_ncpus, 4); } - multilist_create_impl(ml, size, offset, num_sublists, index_func); + return (multilist_create_impl(size, offset, num_sublists, index_func)); } /* @@ -134,6 +135,7 @@ multilist_destroy(multilist_t *ml) ml->ml_num_sublists = 0; ml->ml_offset = 0; + kmem_free(ml, sizeof (multilist_t)); } /* @@ -285,6 +287,13 @@ multilist_sublist_lock(multilist_t *ml, return (mls); } +/* Lock and return the sublist that would be used to store the specified obj */ +multilist_sublist_t * +multilist_sublist_lock_obj(multilist_t *ml, void *obj) +{ + return (multilist_sublist_lock(ml, ml->ml_index_func(ml, obj))); +} + void multilist_sublist_unlock(multilist_sublist_t *mls) { Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/spa_misc.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/spa_misc.c Fri Apr 14 18:25:49 2017 (r316916) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/spa_misc.c Fri Apr 14 18:26:24 2017 (r316917) @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2015 by Delphix. All rights reserved. + * Copyright (c) 2011, 2017 by Delphix. All rights reserved. * Copyright 2015 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. * Copyright 2013 Saso Kiselkov. All rights reserved. @@ -620,7 +620,7 @@ spa_add(const char *name, nvlist_t *conf spa_active_count++; } - avl_create(&spa->spa_alloc_tree, zio_timestamp_compare, + avl_create(&spa->spa_alloc_tree, zio_bookmark_compare, sizeof (zio_t), offsetof(zio_t, io_alloc_node)); /* Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu_objset.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu_objset.h Fri Apr 14 18:25:49 2017 (r316916) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu_objset.h Fri Apr 14 18:26:24 2017 (r316917) @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2016 by Delphix. All rights reserved. + * Copyright (c) 2012, 2017 by Delphix. All rights reserved. * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. * Copyright (c) 2014 Integros [integros.com] @@ -110,7 +110,7 @@ struct objset { /* no lock needed: */ struct dmu_tx *os_synctx; /* XXX sketchy */ zil_header_t os_zil_header; - list_t os_synced_dnodes; + multilist_t *os_synced_dnodes; uint64_t os_flags; uint64_t os_freed_dnodes; boolean_t os_rescan_dnodes; @@ -121,11 +121,13 @@ struct objset { /* Protected by os_lock */ kmutex_t os_lock; - list_t os_dirty_dnodes[TXG_SIZE]; - list_t os_free_dnodes[TXG_SIZE]; + multilist_t *os_dirty_dnodes[TXG_SIZE]; list_t os_dnodes; list_t os_downgraded_dbufs; + /* Protects changes to DMU_{USER,GROUP}USED_OBJECT */ + kmutex_t os_userused_lock; + /* stuff we store for the user */ kmutex_t os_user_ptr_lock; void *os_user_ptr; Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dnode.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dnode.h Fri Apr 14 18:25:49 2017 (r316916) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dnode.h Fri Apr 14 18:26:24 2017 (r316917) @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2016 by Delphix. All rights reserved. + * Copyright (c) 2012, 2017 by Delphix. All rights reserved. * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. */ @@ -35,6 +35,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -203,7 +204,7 @@ struct dnode { uint32_t dn_dbufs_count; /* count of dn_dbufs */ /* protected by os_lock: */ - list_node_t dn_dirty_link[TXG_SIZE]; /* next on dataset's dirty */ + multilist_node_t dn_dirty_link[TXG_SIZE]; /* next on dataset's dirty */ /* protected by dn_mtx: */ kmutex_t dn_mtx; Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dsl_pool.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dsl_pool.h Fri Apr 14 18:25:49 2017 (r316916) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dsl_pool.h Fri Apr 14 18:26:24 2017 (r316917) @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2013 by Delphix. All rights reserved. + * Copyright (c) 2013, 2017 by Delphix. All rights reserved. * Copyright 2016 Nexenta Systems, Inc. All rights reserved. */ @@ -121,6 +121,7 @@ typedef struct dsl_pool { txg_list_t dp_dirty_zilogs; txg_list_t dp_dirty_dirs; txg_list_t dp_sync_tasks; + taskq_t *dp_sync_taskq; /* * Protects administrative changes (properties, namespace) Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/multilist.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/multilist.h Fri Apr 14 18:25:49 2017 (r316916) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/multilist.h Fri Apr 14 18:26:24 2017 (r316917) @@ -73,8 +73,7 @@ struct multilist { }; void multilist_destroy(multilist_t *); -void multilist_create(multilist_t *, size_t, size_t, - multilist_sublist_index_func_t *); +multilist_t *multilist_create(size_t, size_t, multilist_sublist_index_func_t *); void multilist_insert(multilist_t *, void *); void multilist_remove(multilist_t *, void *); @@ -84,6 +83,7 @@ unsigned int multilist_get_num_sublists( unsigned int multilist_get_random_index(multilist_t *); multilist_sublist_t *multilist_sublist_lock(multilist_t *, unsigned int); +multilist_sublist_t *multilist_sublist_lock_obj(multilist_t *, void *); void multilist_sublist_unlock(multilist_sublist_t *); void multilist_sublist_insert_head(multilist_sublist_t *, void *); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zio.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zio.h Fri Apr 14 18:25:49 2017 (r316916) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zio.h Fri Apr 14 18:26:24 2017 (r316917) @@ -22,7 +22,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2011 Nexenta Systems, Inc. All rights reserved. - * Copyright (c) 2012, 2016 by Delphix. All rights reserved. + * Copyright (c) 2012, 2017 by Delphix. All rights reserved. * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. * Copyright (c) 2013, Joyent, Inc. All rights reserved. * Copyright 2016 Toomas Soome @@ -455,7 +455,7 @@ struct zio { taskq_ent_t io_tqent; }; -extern int zio_timestamp_compare(const void *, const void *); +extern int zio_bookmark_compare(const void *, const void *); extern zio_t *zio_null(zio_t *pio, spa_t *spa, vdev_t *vd, zio_done_func_t *done, void *private, enum zio_flag flags); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zio.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zio.c Fri Apr 14 18:25:49 2017 (r316916) *** DIFF OUTPUT TRUNCATED AT 1000 LINES *** From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:27:13 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 9266FD3E333; Fri, 14 Apr 2017 18:27:13 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 61CD9382; Fri, 14 Apr 2017 18:27:13 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIRC3V070989; Fri, 14 Apr 2017 18:27:12 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIRCbD070988; Fri, 14 Apr 2017 18:27:12 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141827.v3EIRCbD070988@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:27:12 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316918 - vendor/illumos/dist/lib/libzfs/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:27:13 -0000 Author: avg Date: Fri Apr 14 18:27:12 2017 New Revision: 316918 URL: https://svnweb.freebsd.org/changeset/base/316918 Log: 7990 libzfs: snapspec_cb() does not need to call zfs_strdup() illumos/illumos-gate@d8584ba6fb7a5e46da1725845b99ae5fab5a4baf https://github.com/illumos/illumos-gate/commit/d8584ba6fb7a5e46da1725845b99ae5fab5a4baf https://www.illumos.org/issues/7990 The snapspec_cb() callback function in libzfs does not need to call zfs_strdup(). Reviewed by: Yuri Pankov Reviewed by: Toomas Soome Approved by: Matthew Ahrens Author: Marcel Telka Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_iter.c Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_iter.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_iter.c Fri Apr 14 18:26:24 2017 (r316917) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_iter.c Fri Apr 14 18:27:12 2017 (r316918) @@ -317,16 +317,17 @@ static int snapspec_cb(zfs_handle_t *zhp, void *arg) { snapspec_arg_t *ssa = arg; - char *shortsnapname; + const char *shortsnapname; int err = 0; if (ssa->ssa_seenlast) return (0); - shortsnapname = zfs_strdup(zhp->zfs_hdl, - strchr(zfs_get_name(zhp), '@') + 1); + shortsnapname = strchr(zfs_get_name(zhp), '@') + 1; if (!ssa->ssa_seenfirst && strcmp(shortsnapname, ssa->ssa_first) == 0) ssa->ssa_seenfirst = B_TRUE; + if (strcmp(shortsnapname, ssa->ssa_last) == 0) + ssa->ssa_seenlast = B_TRUE; if (ssa->ssa_seenfirst) { err = ssa->ssa_func(zhp, ssa->ssa_arg); @@ -334,10 +335,6 @@ snapspec_cb(zfs_handle_t *zhp, void *arg zfs_close(zhp); } - if (strcmp(shortsnapname, ssa->ssa_last) == 0) - ssa->ssa_seenlast = B_TRUE; - free(shortsnapname); - return (err); } From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:28:41 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id A4A86D3E3A5; Fri, 14 Apr 2017 18:28:41 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 664C26DB; Fri, 14 Apr 2017 18:28:41 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EISedQ071088; Fri, 14 Apr 2017 18:28:40 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EISePq071087; Fri, 14 Apr 2017 18:28:40 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141828.v3EISePq071087@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:28:40 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316919 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:28:41 -0000 Author: avg Date: Fri Apr 14 18:28:40 2017 New Revision: 316919 URL: https://svnweb.freebsd.org/changeset/base/316919 Log: 7885 zpool list can report 16.0e for expandsz illumos/illumos-gate@c040c10cdd1e4eab0fc88203758367dd81e057b7 https://github.com/illumos/illumos-gate/commit/c040c10cdd1e4eab0fc88203758367dd81e057b7 https://www.illumos.org/issues/7885 When a member of a RAIDZ has been replaced with a device smaller than the original, then the top level vdev can report its expand size as 16.0E. The reduced child asize causes the RAIDZ to have a vdev_asize lower than its vdev_max_asize which then results in an underflow during the calculation of the parents expand size. Also for RAIDZ vdevs the sum of their child vdev_min_asize could be smaller than the parents vdev_min_size. Fixed by: https://github.com/openzfs/openzfs/pull/296 Reviewed by: Matthew Ahrens Reviewed by: George Wilson Approved by: Gordon Ross Author: Steven Hartland Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/vdev.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/vdev.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/vdev.c Fri Apr 14 18:27:12 2017 (r316918) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/vdev.c Fri Apr 14 18:28:40 2017 (r316919) @@ -135,7 +135,8 @@ vdev_get_min_asize(vdev_t *vd) * so each child must provide at least 1/Nth of its asize. */ if (pvd->vdev_ops == &vdev_raidz_ops) - return (pvd->vdev_min_asize / pvd->vdev_children); + return ((pvd->vdev_min_asize + pvd->vdev_children - 1) / + pvd->vdev_children); return (pvd->vdev_min_asize); } @@ -1273,7 +1274,7 @@ vdev_open(vdev_t *vd) vd->vdev_psize = psize; /* - * Make sure the allocatable size hasn't shrunk. + * Make sure the allocatable size hasn't shrunk too much. */ if (asize < vd->vdev_min_asize) { vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN, @@ -1306,12 +1307,21 @@ vdev_open(vdev_t *vd) } /* - * If all children are healthy and the asize has increased, - * then we've experienced dynamic LUN growth. If automatic - * expansion is enabled then use the additional space. - */ - if (vd->vdev_state == VDEV_STATE_HEALTHY && asize > vd->vdev_asize && - (vd->vdev_expanding || spa->spa_autoexpand)) + * If all children are healthy we update asize if either: + * The asize has increased, due to a device expansion caused by dynamic + * LUN growth or vdev replacement, and automatic expansion is enabled; + * making the additional space available. + * + * The asize has decreased, due to a device shrink usually caused by a + * vdev replace with a smaller device. This ensures that calculations + * based of max_asize and asize e.g. esize are always valid. It's safe + * to do this as we've already validated that asize is greater than + * vdev_min_asize. + */ + if (vd->vdev_state == VDEV_STATE_HEALTHY && + ((asize > vd->vdev_asize && + (vd->vdev_expanding || spa->spa_autoexpand)) || + (asize < vd->vdev_asize))) vd->vdev_asize = asize; vdev_set_min_asize(vd); From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:29:15 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 3881BD3E3F9; Fri, 14 Apr 2017 18:29:15 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id EFAF7871; Fri, 14 Apr 2017 18:29:14 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EITE3d071163; Fri, 14 Apr 2017 18:29:14 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EITDmC071157; Fri, 14 Apr 2017 18:29:13 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141829.v3EITDmC071157@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:29:13 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316920 - in vendor-sys/illumos/dist/uts/common/fs/zfs: . sys X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:29:15 -0000 Author: avg Date: Fri Apr 14 18:29:13 2017 New Revision: 316920 URL: https://svnweb.freebsd.org/changeset/base/316920 Log: 8023 Panic destroying a metaslab deferred range tree illumos/illumos-gate@3991b535a8e990c0369be677746a87c259b13e9f https://github.com/illumos/illumos-gate/commit/3991b535a8e990c0369be677746a87c259b13e9f https://www.illumos.org/issues/8023 $C ffffff0011bc0970 vpanic() ffffff0011bc0a00 strlog() ffffff0011bc0a30 range_tree_destroy+0x72(ffffff043769ad00) ffffff0011bc0a70 metaslab_fini+0xd5(ffffff0449acf380) ffffff0011bc0ab0 vdev_metaslab_fini+0x56(ffffff0462bae800) ffffff0011bc0af0 spa_unload+0x9b(ffffff03e3dac000) ffffff0011bc0b70 spa_export_common+0x115(ffffff047f4b4000, 2, 0, 0, 0) ffffff0011bc0b90 spa_destroy+0x1d(ffffff047f4b4000) ffffff0011bc0bd0 zfs_ioc_pool_destroy+0x20(ffffff047f4b4000) ffffff0011bc0c80 zfsdev_ioctl+0x4d7(11400000000, 5a01, 8040190, 100003, ffffff03e1956b10, ffffff0011bc0e68) ffffff0011bc0cc0 cdev_ioctl+0x39(11400000000, 5a01, 8040190, 100003, ffffff03e1956b10, ffffff0011bc0e68) ffffff0011bc0d10 spec_ioctl+0x60(ffffff03d9153b00, 5a01, 8040190, 100003, ffffff03e1956b10, ffffff0011bc0e68, 0) ffffff0011bc0da0 fop_ioctl+0x55(ffffff03d9153b00, 5a01, 8040190, 100003, ffffff03e1956b10, ffffff0011bc0e68, 0) ffffff0011bc0ec0 ioctl+0x9b(3, 5a01, 8040190) ffffff0011bc0f10 _sys_sysenter_post_swapgs+0x149() Reviewed by: Brad Lewis Reviewed by: Matt Ahrens Reviewed by: Dan Kimmel Reviewed by: Saso Kiselkov Approved by: Dan McDonald Author: George Wilson Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c vendor-sys/illumos/dist/uts/common/fs/zfs/metaslab.c vendor-sys/illumos/dist/uts/common/fs/zfs/spa_misc.c vendor-sys/illumos/dist/uts/common/fs/zfs/space_map.c vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa.h Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c Fri Apr 14 18:28:40 2017 (r316919) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c Fri Apr 14 18:29:13 2017 (r316920) @@ -1552,6 +1552,7 @@ dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t * this assertion only if we're not already dirty. */ os = dn->dn_objset; + VERIFY3U(tx->tx_txg, <=, spa_final_dirty_txg(os->os_spa)); #ifdef DEBUG if (dn->dn_objset->os_dsl_dataset != NULL) rrw_enter(&os->os_dsl_dataset->ds_bp_rwlock, RW_READER, FTAG); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/metaslab.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/metaslab.c Fri Apr 14 18:28:40 2017 (r316919) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/metaslab.c Fri Apr 14 18:29:13 2017 (r316920) @@ -1563,11 +1563,19 @@ metaslab_set_fragmentation(metaslab_t *m uint64_t txg = spa_syncing_txg(spa); vdev_t *vd = msp->ms_group->mg_vd; - if (spa_writeable(spa)) { + /* + * If we've reached the final dirty txg, then we must + * be shutting down the pool. We don't want to dirty + * any data past this point so skip setting the condense + * flag. We can retry this action the next time the pool + * is imported. + */ + if (spa_writeable(spa) && txg < spa_final_dirty_txg(spa)) { msp->ms_condense_wanted = B_TRUE; vdev_dirty(vd, VDD_METASLAB, msp, txg + 1); spa_dbgmsg(spa, "txg %llu, requesting force condense: " - "msp %p, vd %p", txg, msp, vd); + "ms_id %llu, vdev_id %llu", txg, msp->ms_id, + vd->vdev_id); } msp->ms_fragmentation = ZFS_FRAG_INVALID; return; @@ -2189,13 +2197,17 @@ metaslab_sync(metaslab_t *msp, uint64_t /* * Normally, we don't want to process a metaslab if there * are no allocations or frees to perform. However, if the metaslab - * is being forced to condense we need to let it through. + * is being forced to condense and it's loaded, we need to let it + * through. */ if (range_tree_space(alloctree) == 0 && range_tree_space(msp->ms_freeingtree) == 0 && - !msp->ms_condense_wanted) + !(msp->ms_loaded && msp->ms_condense_wanted)) return; + + VERIFY(txg <= spa_final_dirty_txg(spa)); + /* * The only state that can actually be changing concurrently with * metaslab_sync() is the metaslab's ms_tree. No other thread can Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/spa_misc.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/spa_misc.c Fri Apr 14 18:28:40 2017 (r316919) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/spa_misc.c Fri Apr 14 18:29:13 2017 (r316920) @@ -1621,6 +1621,16 @@ spa_syncing_txg(spa_t *spa) return (spa->spa_syncing_txg); } +/* + * Return the last txg where data can be dirtied. The final txgs + * will be used to just clear out any deferred frees that remain. + */ +uint64_t +spa_final_dirty_txg(spa_t *spa) +{ + return (spa->spa_final_txg - TXG_DEFER_SIZE); +} + pool_state_t spa_state(spa_t *spa) { Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/space_map.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/space_map.c Fri Apr 14 18:28:40 2017 (r316919) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/space_map.c Fri Apr 14 18:29:13 2017 (r316920) @@ -23,7 +23,7 @@ * Use is subject to license terms. */ /* - * Copyright (c) 2012, 2014 by Delphix. All rights reserved. + * Copyright (c) 2012, 2016 by Delphix. All rights reserved. */ #include @@ -403,6 +403,7 @@ space_map_truncate(space_map_t *sm, dmu_ ASSERT(dsl_pool_sync_context(dmu_objset_pool(os))); ASSERT(dmu_tx_is_syncing(tx)); + VERIFY3U(dmu_tx_get_txg(tx), <=, spa_final_dirty_txg(spa)); dmu_object_info_from_db(sm->sm_dbuf, &doi); @@ -417,9 +418,10 @@ space_map_truncate(space_map_t *sm, dmu_ if ((spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_HISTOGRAM) && doi.doi_bonus_size != sizeof (space_map_phys_t)) || doi.doi_data_block_size != space_map_blksz) { - zfs_dbgmsg("txg %llu, spa %s, reallocating: " - "old bonus %u, old blocksz %u", dmu_tx_get_txg(tx), - spa_name(spa), doi.doi_bonus_size, doi.doi_data_block_size); + zfs_dbgmsg("txg %llu, spa %s, sm %p, reallocating " + "object[%llu]: old bonus %u, old blocksz %u", + dmu_tx_get_txg(tx), spa_name(spa), sm, sm->sm_object, + doi.doi_bonus_size, doi.doi_data_block_size); space_map_free(sm, tx); dmu_buf_rele(sm->sm_dbuf, sm); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa.h Fri Apr 14 18:28:40 2017 (r316919) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa.h Fri Apr 14 18:29:13 2017 (r316920) @@ -771,6 +771,7 @@ extern uint64_t spa_load_guid(spa_t *spa extern uint64_t spa_last_synced_txg(spa_t *spa); extern uint64_t spa_first_txg(spa_t *spa); extern uint64_t spa_syncing_txg(spa_t *spa); +extern uint64_t spa_final_dirty_txg(spa_t *spa); extern uint64_t spa_version(spa_t *spa); extern pool_state_t spa_state(spa_t *spa); extern spa_load_state_t spa_load_state(spa_t *spa); From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:29:37 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 3C9B9D3E445; Fri, 14 Apr 2017 18:29:37 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 0CB939BA; Fri, 14 Apr 2017 18:29:36 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EITags071219; Fri, 14 Apr 2017 18:29:36 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EITaZC071218; Fri, 14 Apr 2017 18:29:36 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141829.v3EITaZC071218@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:29:36 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316921 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:29:37 -0000 Author: avg Date: Fri Apr 14 18:29:35 2017 New Revision: 316921 URL: https://svnweb.freebsd.org/changeset/base/316921 Log: 8027 tighten up dsl_pool_dirty_delta illumos/illumos-gate@313ae1e182df6e6a04b56c4b73ded33e11b75666 https://github.com/illumos/illumos-gate/commit/313ae1e182df6e6a04b56c4b73ded33e11b75666 https://www.illumos.org/issues/8027 dsl_pool_dirty_delta() should not wake up waiters when dp->dp_dirty_total == zfs_dirty_data_max, because they wait for dp_dirty_total to fall strictly below the threshold. It's probably very rare for that condition to occur, but it's better to have more accurate code. Reviewed by: Matt Ahrens Reviewed by: Serapheim Dimitropoulos Reviewed by: Paul Dagnelie Approved by: Dan McDonald Author: Andriy Gapon Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_pool.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_pool.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_pool.c Fri Apr 14 18:29:13 2017 (r316920) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_pool.c Fri Apr 14 18:29:35 2017 (r316921) @@ -459,7 +459,7 @@ dsl_pool_dirty_delta(dsl_pool_t *dp, int * Note: we signal even when increasing dp_dirty_total. * This ensures forward progress -- each thread wakes the next waiter. */ - if (dp->dp_dirty_total <= zfs_dirty_data_max) + if (dp->dp_dirty_total < zfs_dirty_data_max) cv_signal(&dp->dp_spaceavail_cv); } From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:30:23 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 79712D3E4B8; Fri, 14 Apr 2017 18:30:23 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 2FDB0AF5; Fri, 14 Apr 2017 18:30:23 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIUMWT071313; Fri, 14 Apr 2017 18:30:22 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIUMxT071312; Fri, 14 Apr 2017 18:30:22 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141830.v3EIUMxT071312@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:30:22 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316922 - vendor/illumos/dist/lib/libzfs/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:30:23 -0000 Author: avg Date: Fri Apr 14 18:30:22 2017 New Revision: 316922 URL: https://svnweb.freebsd.org/changeset/base/316922 Log: 5380 receive of a send -p stream doesn't need to try renaming snapshots illumos/illumos-gate@471a88e499c660844f4590487ce7c4d5a7090294 https://github.com/illumos/illumos-gate/commit/471a88e499c660844f4590487ce7c4d5a7090294 https://www.illumos.org/issues/5380 A stream created with zfs send -p -I contains properties of all snapshots of a given dataset as opposed to only properties of snapshots in a given range. Not only this is suboptimal but the receive code also does not filter properties by the range. So, properties of earlier snapshots would be updated even though the snapshots themselves are not in the stream (just their properties). Given that modifying the snapshot properties requires a TXG sync and that the snapshots are updated one by one the described behavior may lead to a sever performance penalty. Reviewed by: Paul Dagnelie Reviewed by: Matt Ahrens Approved by: Dan McDonald Author: Andriy Gapon Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_sendrecv.c Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_sendrecv.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_sendrecv.c Fri Apr 14 18:29:35 2017 (r316921) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_sendrecv.c Fri Apr 14 18:30:22 2017 (r316922) @@ -2770,7 +2770,7 @@ zfs_receive_package(libzfs_handle_t *hdl goto out; } - if (fromsnap != NULL) { + if (fromsnap != NULL && recursive) { nvlist_t *renamed = NULL; nvpair_t *pair = NULL; @@ -2797,7 +2797,7 @@ zfs_receive_package(libzfs_handle_t *hdl *strchr(tofs, '@') = '\0'; } - if (recursive && !flags->dryrun && !flags->nomount) { + if (!flags->dryrun && !flags->nomount) { VERIFY(0 == nvlist_alloc(&renamed, NV_UNIQUE_NAME, 0)); } @@ -2866,7 +2866,7 @@ zfs_receive_package(libzfs_handle_t *hdl anyerr |= error; } while (error == 0); - if (drr->drr_payloadlen != 0 && fromsnap != NULL) { + if (drr->drr_payloadlen != 0 && recursive && fromsnap != NULL) { /* * Now that we have the fs's they sent us, try the * renames again. From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:32:13 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 83361D3E668; Fri, 14 Apr 2017 18:32:13 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 52FFAF3A; Fri, 14 Apr 2017 18:32:13 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIWCNf073510; Fri, 14 Apr 2017 18:32:12 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIWCAV073509; Fri, 14 Apr 2017 18:32:12 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141832.v3EIWCAV073509@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:32:12 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316923 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:32:13 -0000 Author: avg Date: Fri Apr 14 18:32:12 2017 New Revision: 316923 URL: https://svnweb.freebsd.org/changeset/base/316923 Log: 8026 retire zfs_throttle_delay and zfs_throttle_resolution illumos/illumos-gate@6b036259815954b7ad86d651af18efba672cb7a9 https://github.com/illumos/illumos-gate/commit/6b036259815954b7ad86d651af18efba672cb7a9 https://www.illumos.org/issues/8026 zfs_throttle_delay and zfs_throttle_resolution became disused since the new write throttling mechanism was introduced. Reviewed by: Matthew Ahrens Reviewed by: Serapheim Dimitropoulos Approved by: Richard Lowe Author: Andriy Gapon Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_pool.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_pool.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_pool.c Fri Apr 14 18:30:22 2017 (r316922) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_pool.c Fri Apr 14 18:32:12 2017 (r316923) @@ -132,9 +132,6 @@ uint64_t zfs_delay_scale = 1000 * 1000 * */ int zfs_sync_taskq_batch_pct = 75; -hrtime_t zfs_throttle_delay = MSEC2NSEC(10); -hrtime_t zfs_throttle_resolution = MSEC2NSEC(10); - int dsl_pool_open_special_dir(dsl_pool_t *dp, const char *name, dsl_dir_t **ddp) { From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:32:39 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id E83D1D3E709; Fri, 14 Apr 2017 18:32:39 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 9DCD9160; Fri, 14 Apr 2017 18:32:39 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIWcqC075267; Fri, 14 Apr 2017 18:32:38 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIWcAi075266; Fri, 14 Apr 2017 18:32:38 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141832.v3EIWcAi075266@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:32:38 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316924 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:32:40 -0000 Author: avg Date: Fri Apr 14 18:32:38 2017 New Revision: 316924 URL: https://svnweb.freebsd.org/changeset/base/316924 Log: 8061 sa_find_idx_tab can be declared more type-safely illumos/illumos-gate@7f0bdb4257bb4f1f76390b72665961e411da24c6 https://github.com/illumos/illumos-gate/commit/7f0bdb4257bb4f1f76390b72665961e411da24c6 https://www.illumos.org/issues/8061 sa_find_idx_tab() is declared as taking and returning "void *" parameters. These can be declared to be the specific types. Reviewed by: George Wilson Reviewed by: Chris Williamson Approved by: Dan McDonald Author: Matthew Ahrens Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sa.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sa.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sa.c Fri Apr 14 18:32:12 2017 (r316923) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sa.c Fri Apr 14 18:32:38 2017 (r316924) @@ -22,7 +22,7 @@ /* * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. * Portions Copyright 2011 iXsystems, Inc - * Copyright (c) 2013, 2016 by Delphix. All rights reserved. + * Copyright (c) 2013, 2017 by Delphix. All rights reserved. * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. * Copyright (c) 2014 Integros [integros.com] */ @@ -131,8 +131,8 @@ typedef void (sa_iterfunc_t)(void *hdr, static int sa_build_index(sa_handle_t *hdl, sa_buf_type_t buftype); static void sa_idx_tab_hold(objset_t *os, sa_idx_tab_t *idx_tab); -static void *sa_find_idx_tab(objset_t *os, dmu_object_type_t bonustype, - void *data); +static sa_idx_tab_t *sa_find_idx_tab(objset_t *os, dmu_object_type_t bonustype, + sa_hdr_phys_t *hdr); static void sa_idx_tab_rele(objset_t *os, void *arg); static void sa_copy_data(sa_data_locator_t *func, void *start, void *target, int buflen); @@ -1483,11 +1483,10 @@ sa_lookup_uio(sa_handle_t *hdl, sa_attr_ } #endif -void * -sa_find_idx_tab(objset_t *os, dmu_object_type_t bonustype, void *data) +static sa_idx_tab_t * +sa_find_idx_tab(objset_t *os, dmu_object_type_t bonustype, sa_hdr_phys_t *hdr) { sa_idx_tab_t *idx_tab; - sa_hdr_phys_t *hdr = (sa_hdr_phys_t *)data; sa_os_t *sa = os->os_sa; sa_lot_t *tb, search; avl_index_t loc; From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:33:22 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 5DF83D3E7B6; Fri, 14 Apr 2017 18:33:22 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 204942E3; Fri, 14 Apr 2017 18:33:22 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIXLF1075365; Fri, 14 Apr 2017 18:33:21 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIXLBi075363; Fri, 14 Apr 2017 18:33:21 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141833.v3EIXLBi075363@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:33:21 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316925 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:33:22 -0000 Author: avg Date: Fri Apr 14 18:33:20 2017 New Revision: 316925 URL: https://svnweb.freebsd.org/changeset/base/316925 Log: 6101 attempt to lzc_create() a filesystem under a volume results in a panic illumos/illumos-gate@b127fe3c059af7adf772735498680b4f2e1405ef https://github.com/illumos/illumos-gate/commit/b127fe3c059af7adf772735498680b4f2e1405ef https://www.illumos.org/issues/6101 lzc_create(), or more correctly, zfs_ioc_create() does not reject an attempt to create a filesystem as a child of a volume, instead it proceeds to a crash. A crash stack obtained on FreeBSD: page fault while in kernel mode zap_leaf_lookup() fzap_lookup() zap_lookup_norm() zap_lookup() zfs_get_zplprop() zfs_fill_zplprops_impl() zfs_ioc_create() zfsdev_ioctl() devfs_ioctl_f() kern_ioctl() sys_ioctl() This crash happened with a kernel without debugging assertions. The immediate cause of crash appears to an attempt to interpret a zvol object as a zap object. For filesystems: #define MASTER_NODE_OBJ 1 For zvols: #define ZVOL_OBJ 1ULL #define ZVOL_ZAP_OBJ 2ULL So, I see two problems here: 1. an attempt to create a filesystem under a zvol should be rejected as early as possible, maybe in zfs_fill_zplprops() 2. maybe zap_lookup / zap_lockdir should reject objects that are not of one of the zap object types Reviewed by: Matthew Ahrens Approved by: Dan McDonald Author: Andriy Gapon Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ioctl.c vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vfsops.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ioctl.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ioctl.c Fri Apr 14 18:32:38 2017 (r316924) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ioctl.c Fri Apr 14 18:33:20 2017 (r316925) @@ -3035,6 +3035,9 @@ zfs_fill_zplprops_impl(objset_t *os, uin ASSERT(zplprops != NULL); + if (os != NULL && os->os_phys->os_type != DMU_OST_ZFS) + return (SET_ERROR(EINVAL)); + /* * Pull out creator prop choices, if any. */ Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vfsops.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vfsops.c Fri Apr 14 18:32:38 2017 (r316924) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vfsops.c Fri Apr 14 18:33:20 2017 (r316925) @@ -2243,8 +2243,10 @@ zfs_get_zplprop(objset_t *os, zfs_prop_t else pname = zfs_prop_to_name(prop); - if (os != NULL) + if (os != NULL) { + ASSERT3U(os->os_phys->os_type, ==, DMU_OST_ZFS); error = zap_lookup(os, MASTER_NODE_OBJ, pname, 8, 1, value); + } if (error == ENOENT) { /* No value set, use the default value */ From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:34:05 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 3F9F2D3E810; Fri, 14 Apr 2017 18:34:05 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 014F263E; Fri, 14 Apr 2017 18:34:04 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIY4ck075447; Fri, 14 Apr 2017 18:34:04 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIY3Ya075442; Fri, 14 Apr 2017 18:34:03 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141834.v3EIY3Ya075442@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:34:03 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316926 - in vendor/illumos/dist: cmd/zfs lib/libzfs/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:34:05 -0000 Author: avg Date: Fri Apr 14 18:34:03 2017 New Revision: 316926 URL: https://svnweb.freebsd.org/changeset/base/316926 Log: 7955 libshare needs to initialize only those datasets being modified by the consumer illumos/illumos-gate@8a981c3356b194b3b5c0ae9276a9cc31cd2f93a3 https://github.com/illumos/illumos-gate/commit/8a981c3356b194b3b5c0ae9276a9cc31cd2f93a3 https://www.illumos.org/issues/7955 Libshare currently initializes all available filesystems when doing any libshare operation. This requires iterating through all the filesystem multiple times, which is a huge performance problem for sharing and unsharing operations. Reviewed by: Steve Gonczi Reviewed by: Sebastien Roy Reviewed by: Matthew Ahrens Reviewed by: George Wilson Reviewed by: Pavel Zakharov Reviewed by: Yuri Pankov Approved by: Gordon Ross Author: Daniel Hoffman Modified: vendor/illumos/dist/cmd/zfs/zfs_main.c vendor/illumos/dist/lib/libzfs/common/libzfs.h vendor/illumos/dist/lib/libzfs/common/libzfs_changelist.c vendor/illumos/dist/lib/libzfs/common/libzfs_impl.h vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c Modified: vendor/illumos/dist/cmd/zfs/zfs_main.c ============================================================================== --- vendor/illumos/dist/cmd/zfs/zfs_main.c Fri Apr 14 18:33:20 2017 (r316925) +++ vendor/illumos/dist/cmd/zfs/zfs_main.c Fri Apr 14 18:34:03 2017 (r316926) @@ -68,6 +68,7 @@ #include #include #include +#include #include "zfs_iter.h" #include "zfs_util.h" @@ -6111,6 +6112,15 @@ share_mount(int op, int argc, char **arg return (0); qsort(dslist, count, sizeof (void *), libzfs_dataset_cmp); + sa_init_selective_arg_t sharearg; + sharearg.zhandle_arr = dslist; + sharearg.zhandle_len = count; + if ((ret = zfs_init_libshare_arg(zfs_get_handle(dslist[0]), + SA_INIT_SHARE_API_SELECTIVE, &sharearg)) != SA_OK) { + (void) fprintf(stderr, + gettext("Could not initialize libshare, %d"), ret); + return (ret); + } for (i = 0; i < count; i++) { if (verbose) Modified: vendor/illumos/dist/lib/libzfs/common/libzfs.h ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs.h Fri Apr 14 18:33:20 2017 (r316925) +++ vendor/illumos/dist/lib/libzfs/common/libzfs.h Fri Apr 14 18:34:03 2017 (r316926) @@ -783,6 +783,17 @@ extern int zpool_fru_set(zpool_handle_t extern int zfs_get_hole_count(const char *, uint64_t *, uint64_t *); +/* Allow consumers to initialize libshare externally for optimal performance */ +extern int zfs_init_libshare_arg(libzfs_handle_t *, int, void *); +/* + * For most consumers, zfs_init_libshare_arg is sufficient on its own, and + * zfs_uninit_libshare is unnecessary. zfs_uninit_libshare should only be called + * if the caller has already initialized libshare for one set of zfs handles, + * and wishes to share or unshare filesystems outside of that set. In that case, + * the caller should uninitialize libshare, and then re-initialize it with the + * new handles being shared or unshared. + */ +extern void zfs_uninit_libshare(libzfs_handle_t *); #ifdef __cplusplus } #endif Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_changelist.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_changelist.c Fri Apr 14 18:33:20 2017 (r316925) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_changelist.c Fri Apr 14 18:34:03 2017 (r316926) @@ -24,7 +24,7 @@ * Use is subject to license terms. * * Portions Copyright 2007 Ramprakash Jelari - * Copyright (c) 2014, 2015 by Delphix. All rights reserved. + * Copyright (c) 2014, 2016 by Delphix. All rights reserved. * Copyright 2016 Igor Kozhukhov */ @@ -162,6 +162,9 @@ changelist_postfix(prop_changelist_t *cl char shareopts[ZFS_MAXPROPLEN]; int errors = 0; libzfs_handle_t *hdl; + size_t num_datasets = 0, i; + zfs_handle_t **zhandle_arr; + sa_init_selective_arg_t sharearg; /* * If we're changing the mountpoint, attempt to destroy the underlying @@ -186,8 +189,31 @@ changelist_postfix(prop_changelist_t *cl hdl = cn->cn_handle->zfs_hdl; assert(hdl != NULL); zfs_uninit_libshare(hdl); - } + /* + * For efficiencies sake, we initialize libshare for only a few + * shares (the ones affected here). Future initializations in + * this process should just use the cached initialization. + */ + for (cn = uu_list_last(clp->cl_list); cn != NULL; + cn = uu_list_prev(clp->cl_list, cn)) { + num_datasets++; + } + + zhandle_arr = zfs_alloc(hdl, + num_datasets * sizeof (zfs_handle_t *)); + for (i = 0, cn = uu_list_last(clp->cl_list); cn != NULL; + cn = uu_list_prev(clp->cl_list, cn)) { + zhandle_arr[i++] = cn->cn_handle; + zfs_refresh_properties(cn->cn_handle); + } + assert(i == num_datasets); + sharearg.zhandle_arr = zhandle_arr; + sharearg.zhandle_len = num_datasets; + errors = zfs_init_libshare_arg(hdl, SA_INIT_SHARE_API_SELECTIVE, + &sharearg); + free(zhandle_arr); + } /* * We walk the datasets in reverse, because we want to mount any parent * datasets before mounting the children. We walk all datasets even if @@ -212,8 +238,6 @@ changelist_postfix(prop_changelist_t *cl continue; cn->cn_needpost = B_FALSE; - zfs_refresh_properties(cn->cn_handle); - if (ZFS_IS_VOLUME(cn->cn_handle)) continue; Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_impl.h ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_impl.h Fri Apr 14 18:33:20 2017 (r316925) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_impl.h Fri Apr 14 18:34:03 2017 (r316926) @@ -203,7 +203,6 @@ void namespace_clear(libzfs_handle_t *); */ extern int zfs_init_libshare(libzfs_handle_t *, int); -extern void zfs_uninit_libshare(libzfs_handle_t *); extern int zfs_parse_options(char *, zfs_share_proto_t); extern int zfs_unshare_proto(zfs_handle_t *, Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c Fri Apr 14 18:33:20 2017 (r316925) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c Fri Apr 14 18:34:03 2017 (r316926) @@ -566,6 +566,7 @@ zfs_is_shared_smb(zfs_handle_t *zhp, cha */ static sa_handle_t (*_sa_init)(int); +static sa_handle_t (*_sa_init_arg)(int, void *); static void (*_sa_fini)(sa_handle_t); static sa_share_t (*_sa_find_share)(sa_handle_t, char *); static int (*_sa_enable_share)(sa_share_t, char *); @@ -605,6 +606,8 @@ _zfs_init_libshare(void) if ((libshare = dlopen(path, RTLD_LAZY | RTLD_GLOBAL)) != NULL) { _sa_init = (sa_handle_t (*)(int))dlsym(libshare, "sa_init"); + _sa_init_arg = (sa_handle_t (*)(int, void *))dlsym(libshare, + "sa_init_arg"); _sa_fini = (void (*)(sa_handle_t))dlsym(libshare, "sa_fini"); _sa_find_share = (sa_share_t (*)(sa_handle_t, char *)) dlsym(libshare, "sa_find_share"); @@ -624,14 +627,15 @@ _zfs_init_libshare(void) char *, char *))dlsym(libshare, "sa_zfs_process_share"); _sa_update_sharetab_ts = (void (*)(sa_handle_t)) dlsym(libshare, "sa_update_sharetab_ts"); - if (_sa_init == NULL || _sa_fini == NULL || - _sa_find_share == NULL || _sa_enable_share == NULL || - _sa_disable_share == NULL || _sa_errorstr == NULL || - _sa_parse_legacy_options == NULL || + if (_sa_init == NULL || _sa_init_arg == NULL || + _sa_fini == NULL || _sa_find_share == NULL || + _sa_enable_share == NULL || _sa_disable_share == NULL || + _sa_errorstr == NULL || _sa_parse_legacy_options == NULL || _sa_needs_refresh == NULL || _sa_get_zfs_handle == NULL || _sa_zfs_process_share == NULL || _sa_update_sharetab_ts == NULL) { _sa_init = NULL; + _sa_init_arg = NULL; _sa_fini = NULL; _sa_disable_share = NULL; _sa_enable_share = NULL; @@ -654,8 +658,8 @@ _zfs_init_libshare(void) * service value is which part(s) of the API to initialize and is a * direct map to the libshare sa_init(service) interface. */ -int -zfs_init_libshare(libzfs_handle_t *zhandle, int service) +static int +zfs_init_libshare_impl(libzfs_handle_t *zhandle, int service, void *arg) { if (_sa_init == NULL) return (SA_CONFIG_ERR); @@ -671,17 +675,29 @@ zfs_init_libshare(libzfs_handle_t *zhand if (_sa_needs_refresh != NULL && _sa_needs_refresh(zhandle->libzfs_sharehdl)) { zfs_uninit_libshare(zhandle); - zhandle->libzfs_sharehdl = _sa_init(service); + zhandle->libzfs_sharehdl = _sa_init_arg(service, arg); } if (zhandle && zhandle->libzfs_sharehdl == NULL) - zhandle->libzfs_sharehdl = _sa_init(service); + zhandle->libzfs_sharehdl = _sa_init_arg(service, arg); if (zhandle->libzfs_sharehdl == NULL) return (SA_NO_MEMORY); return (SA_OK); } +int +zfs_init_libshare(libzfs_handle_t *zhandle, int service) +{ + return (zfs_init_libshare_impl(zhandle, service, NULL)); +} + +int +zfs_init_libshare_arg(libzfs_handle_t *zhandle, int service, void *arg) +{ + return (zfs_init_libshare_impl(zhandle, service, arg)); +} + /* * zfs_uninit_libshare(zhandle) @@ -786,8 +802,8 @@ zfs_share_proto(zfs_handle_t *zhp, zfs_s ZFS_MAXPROPLEN, B_FALSE) != 0 || strcmp(shareopts, "off") == 0) continue; - - ret = zfs_init_libshare(hdl, SA_INIT_SHARE_API); + ret = zfs_init_libshare_arg(hdl, SA_INIT_ONE_SHARE_FROM_HANDLE, + zhp); if (ret != SA_OK) { (void) zfs_error_fmt(hdl, EZFS_SHARENFSFAILED, dgettext(TEXT_DOMAIN, "cannot share '%s': %s"), @@ -881,6 +897,7 @@ unshare_one(libzfs_handle_t *hdl, const sa_share_t share; int err; char *mntpt; + /* * Mountpoint could get trashed if libshare calls getmntany * which it does during API initialization, so strdup the @@ -888,8 +905,14 @@ unshare_one(libzfs_handle_t *hdl, const */ mntpt = zfs_strdup(hdl, mountpoint); - /* make sure libshare initialized */ - if ((err = zfs_init_libshare(hdl, SA_INIT_SHARE_API)) != SA_OK) { + /* + * make sure libshare initialized, initialize everything because we + * don't know what other unsharing may happen later. Functions up the + * stack are allowed to initialize instead a subset of shares at the + * time the set is known. + */ + if ((err = zfs_init_libshare_arg(hdl, SA_INIT_ONE_SHARE_FROM_NAME, + (void *)name)) != SA_OK) { free(mntpt); /* don't need the copy anymore */ return (zfs_error_fmt(hdl, EZFS_SHARENFSFAILED, dgettext(TEXT_DOMAIN, "cannot unshare '%s': %s"), @@ -1221,6 +1244,7 @@ zpool_disable_datasets(zpool_handle_t *z int i; int ret = -1; int flags = (force ? MS_FORCE : 0); + sa_init_selective_arg_t sharearg; namelen = strlen(zhp->zpool_name); @@ -1295,6 +1319,12 @@ zpool_disable_datasets(zpool_handle_t *z * At this point, we have the entire list of filesystems, so sort it by * mountpoint. */ + sharearg.zhandle_arr = datasets; + sharearg.zhandle_len = used; + ret = zfs_init_libshare_arg(hdl, SA_INIT_SHARE_API_SELECTIVE, + &sharearg); + if (ret != 0) + goto out; qsort(mountpoints, used, sizeof (char *), mountpoint_compare); /* From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:38:23 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 43A38D3E9E2; Fri, 14 Apr 2017 18:38:23 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id ED2BCAC7; Fri, 14 Apr 2017 18:38:22 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIcMYm075945; Fri, 14 Apr 2017 18:38:22 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIcM4G075944; Fri, 14 Apr 2017 18:38:22 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141838.v3EIcM4G075944@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:38:22 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316927 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:38:23 -0000 Author: avg Date: Fri Apr 14 18:38:21 2017 New Revision: 316927 URL: https://svnweb.freebsd.org/changeset/base/316927 Log: 5379 modifying a mmap()-ed file does not update its timestamps illumos/illumos-gate@80e10fd0d22bbf0d18bfdae035e06f44c68ae8e6 https://github.com/illumos/illumos-gate/commit/80e10fd0d22bbf0d18bfdae035e06f44c68ae8e6 https://www.illumos.org/issues/5379 The following is based on a review of the illumos code and on a similar problem reported for FreeBSD where the relevant code is different. Looking at this block of code http://src.illumos.org/source/xref/illumos-gate/ usr/src/uts/common/fs/zfs/zfs_vnops.c#4187 I see code to set up an sa_bulk_attr_t object, I see code to set up mtime and ctime values, but I do not see code to actually apply the attributes... I would expect there to be a call to sa_bulk_update(), there is such a call in zfs_write() for instance. mmap_write.c [Magnifier] - demo (1.42 KB) Andriy Gapon, 2015-11-11 01:53 PM Reviewed by: Matthew Ahrens Reviewed by: Prashanth Sreenivasa Reviewed by: Dan McDonald Approved by: Gordon Ross Author: Andriy Gapon Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vnops.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vnops.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vnops.c Fri Apr 14 18:34:03 2017 (r316926) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vnops.c Fri Apr 14 18:38:21 2017 (r316927) @@ -4266,6 +4266,8 @@ zfs_putapage(vnode_t *vp, page_t *pp, u_ &zp->z_pflags, 8); zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime, B_TRUE); + err = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); + ASSERT0(err); zfs_log_write(zfsvfs->z_log, tx, TX_WRITE, zp, off, len, 0); } dmu_tx_commit(tx); From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:38:55 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 46EBAD3EA3C; Fri, 14 Apr 2017 18:38:55 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 1397CC0F; Fri, 14 Apr 2017 18:38:55 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIcsFh076008; Fri, 14 Apr 2017 18:38:54 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIcs5V076007; Fri, 14 Apr 2017 18:38:54 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141838.v3EIcs5V076007@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:38:54 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316928 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:38:55 -0000 Author: avg Date: Fri Apr 14 18:38:53 2017 New Revision: 316928 URL: https://svnweb.freebsd.org/changeset/base/316928 Log: 7256 low probability race in zfs_get_data illumos/illumos-gate@0c94e1af6784c69a1dea25e0e35dd13b2b91e2e5 https://github.com/illumos/illumos-gate/commit/0c94e1af6784c69a1dea25e0e35dd13b2b91e2e5 https://www.illumos.org/issues/7256 error = dmu_sync(zio, lr->lr_common.lrc_txg, zfs_get_done, zgd); ASSERT(error || lr->lr_length <= zp->z_blksz); It's possible, although extremely rare, that the zfs_get_done() callback is executed before dmu_sync() returns. In that case the znode's range lock is dropped and the znode is unreferenced. Thus, the assertion can access some invalid or wrong data via the zp pointer. size variable caches the correct value of z_blksz and can be safely used here. Reviewed by: Matt Ahrens Reviewed by: Pavel Zakharov Approved by: Dan McDonald Author: Andriy Gapon Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vnops.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vnops.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vnops.c Fri Apr 14 18:38:21 2017 (r316927) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vnops.c Fri Apr 14 18:38:53 2017 (r316928) @@ -1142,7 +1142,7 @@ zfs_get_data(void *arg, lr_write_t *lr, error = dmu_sync(zio, lr->lr_common.lrc_txg, zfs_get_done, zgd); - ASSERT(error || lr->lr_length <= zp->z_blksz); + ASSERT(error || lr->lr_length <= size); /* * On success, we need to wait for the write I/O From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:41:38 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 4CBF0D3EB06; Fri, 14 Apr 2017 18:41:38 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 282EEEF5; Fri, 14 Apr 2017 18:41:38 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIfbGa078399; Fri, 14 Apr 2017 18:41:37 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIfb7g078398; Fri, 14 Apr 2017 18:41:37 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141841.v3EIfb7g078398@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:41:37 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316929 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:41:38 -0000 Author: avg Date: Fri Apr 14 18:41:37 2017 New Revision: 316929 URL: https://svnweb.freebsd.org/changeset/base/316929 Log: 6914 kernel virtual memory fragmentation leads to hang illumos/illumos-gate@af868f46a5b794687741d5424de9e3a2d684a84a https://github.com/illumos/illumos-gate/commit/af868f46a5b794687741d5424de9e3a2d684a84a https://www.illumos.org/issues/6914 This change allows the kernel to use more virtual address space. This will allow us to devote 1.5x physmem for the zio arena, and an additional 1.5x physmem for the kernel heap. We saw a hang when unable to find any 128K contiguous memory segments. Looking at the core file we see many threads in stacks similar to this: > ffffff68c9c87c00::findstack -v stack pointer for thread ffffff68c9c87c00: ffffff02cd63d8b0 [ ffffff02cd63d8b0 _resume_from_idle+0xf4() ] ffffff02cd63d8e0 swtch+0x141() ffffff02cd63d920 cv_wait+0x70(ffffff6009b1b01e, ffffff6009b1b020) ffffff02cd63da50 vmem_xalloc+0x640(ffffff6009b1b000, 20000, 1000, 0, 0, 0, 0, ffffff0200000004) ffffff02cd63dac0 vmem_alloc+0x135(ffffff6009b1b000, 20000, 4) ffffff02cd63db60 segkmem_xalloc+0x171(ffffff6009b1b000, 0, 20000, 4, 0, fffffffffb885fe0, fffffffffbcefa10) ffffff02cd63dbc0 segkmem_alloc_vn+0x4a(ffffff6009b1b000, 20000, 4, fffffffffbcefa10) ffffff02cd63dbf0 segkmem_zio_alloc+0x20(ffffff6009b1b000, 20000, 4) ffffff02cd63dd20 vmem_xalloc+0x5b1(ffffff6009b1c000, 20000, 1000, 0, 0, 0, 0, 4) ffffff02cd63dd90 vmem_alloc+0x135(ffffff6009b1c000, 20000, 4) ffffff02cd63de20 kmem_slab_create+0x8d(ffffff605fd37008, 4) ffffff02cd63de80 kmem_slab_alloc+0x11e(ffffff605fd37008, 4) ffffff02cd63dee0 kmem_cache_alloc+0x233(ffffff605fd37008, 4) ffffff02cd63df10 zio_data_buf_alloc+0x5b(20000) ffffff02cd63df70 arc_get_data_buf+0x92(ffffff6265a70588, 20000, ffffff901fd796f8) ffffff02cd63dfb0 arc_buf_alloc_impl+0x9c(ffffff6265a70588, ffffff6d233ab0b8) Reviewed by: George Wilson Reviewed by: Adam Leventhal Reviewed by: John Kennedy Reviewed by: Igor Kozhukhov Reviewed by: Josef 'Jeff' Sipek Approved by: Garrett D'Amore Author: Matthew Ahrens Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c Fri Apr 14 18:38:53 2017 (r316928) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c Fri Apr 14 18:41:37 2017 (r316929) @@ -5885,18 +5885,6 @@ arc_init(void) /* Convert seconds to clock ticks */ arc_min_prefetch_lifespan = 1 * hz; - /* Start out with 1/8 of all memory */ - arc_c = allmem / 8; - -#ifdef _KERNEL - /* - * On architectures where the physical memory can be larger - * than the addressable space (intel in 32-bit mode), we may - * need to limit the cache to 1/8 of VM size. - */ - arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8); -#endif - /* set min cache to 1/32 of all memory, or 64MB, whichever is more */ arc_c_min = MAX(allmem / 32, 64 << 20); /* set max to 3/4 of all memory, or all but 1GB, whichever is more */ @@ -5934,6 +5922,15 @@ arc_init(void) /* limit meta-data to 1/4 of the arc capacity */ arc_meta_limit = arc_c_max / 4; +#ifdef _KERNEL + /* + * Metadata is stored in the kernel's heap. Don't let us + * use more than half the heap for the ARC. + */ + arc_meta_limit = MIN(arc_meta_limit, + vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 2); +#endif + /* Allow the tunable to override if it is reasonable */ if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max) arc_meta_limit = zfs_arc_meta_limit; From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:43:12 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 2CC36D3ECB9; Fri, 14 Apr 2017 18:43:12 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id D73D0372; Fri, 14 Apr 2017 18:43:11 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIhACJ079998; Fri, 14 Apr 2017 18:43:10 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIhAA2079997; Fri, 14 Apr 2017 18:43:10 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141843.v3EIhAA2079997@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:43:10 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316930 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:43:12 -0000 Author: avg Date: Fri Apr 14 18:43:10 2017 New Revision: 316930 URL: https://svnweb.freebsd.org/changeset/base/316930 Log: 5814 bpobj_iterate_impl(): Close a refcount leak iterating on a sublist. illumos/illumos-gate@b67dde11a73a9455d641403cbbb65ec2add41b41 https://github.com/illumos/illumos-gate/commit/b67dde11a73a9455d641403cbbb65ec2add41b41 https://www.illumos.org/issues/5814 Lets pull in this patch from freebsd: http://svnweb.freebsd.org/base?view=revision&revision=271781 bpobj_iterate_impl(): Close a refcount leak iterating on a sublist. If bpobj_space() returned non-zero here, the sublist would have been left open, along with the bonus buffer hold it requires. This call does not invoke any calls to bpobj_close() itself. This bug doesn't have any known vector, but was found on inspection. MFC after: 1 week Sponsored by: Spectra Logic Affects: All ZFS versions starting 21 May 2010 (illumos cde58dbc) MFSpectraBSD: r1050998 on 2014/03/26 Fix bpobj_iterate_impl() to properly call bpobj_close() if bpobj_space() returns an error. Reviewed by: Prakash Surya Reviewed by: Matthew Ahrens Reviewed by: Paul Dagnelie Reviewed by: Simon Klinkert Approved by: Gordon Ross Author: Will Andrews Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c Fri Apr 14 18:41:37 2017 (r316929) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c Fri Apr 14 18:43:10 2017 (r316930) @@ -301,8 +301,10 @@ bpobj_iterate_impl(bpobj_t *bpo, bpobj_i if (free) { err = bpobj_space(&sublist, &used_before, &comp_before, &uncomp_before); - if (err) + if (err != 0) { + bpobj_close(&sublist); break; + } } err = bpobj_iterate_impl(&sublist, func, arg, tx, free); if (free) { From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:49:45 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id B6214D3EE02; Fri, 14 Apr 2017 18:49:45 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 74793A34; Fri, 14 Apr 2017 18:49:45 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EInigF080545; Fri, 14 Apr 2017 18:49:44 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIniuO080544; Fri, 14 Apr 2017 18:49:44 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141849.v3EIniuO080544@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:49:44 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316931 - vendor/illumos/dist/lib/libzfs/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:49:45 -0000 Author: avg Date: Fri Apr 14 18:49:44 2017 New Revision: 316931 URL: https://svnweb.freebsd.org/changeset/base/316931 Log: 6268 zfs diff confused by moving a file to another directory illumos/illumos-gate@aab04418a72c0a29040a5da7eec08efe19dbef04 https://github.com/illumos/illumos-gate/commit/aab04418a72c0a29040a5da7eec08efe19dbef04 https://www.illumos.org/issues/6268 The zfs diff command presents a description of the changes that have occurred to files within a filesystem between two snapshots. If a file is renamed, the tool is capable of reporting this, e.g.: cd /some/zfs/dataset/subdir mv file0 file1 Will result in a diff record like: R /some/zfs/dataset/subdir/file0 -> /some/zfs/dataset/subdir/file1 Unfortunately, it seems that rename detection only uses the base filename to determine if a file has been renamed or simply modified. This leads to misreporting only the original filename, omitting the more relevant destination filename entirely. For example: cd /some/zfs/dataset/subdir mv file0 ../otherdir/file0 Will result in a diff entry: M /some/zfs/dataset/subdir/file0 But it should really emit: R /some/zfs/dataset/subdir/file0 -> /some/zfs/dataset/otherdir/file0 Reviewed by: Matthew Ahrens Reviewed by: Justin Gibbs Approved by: Dan McDonald Author: Joshua M. Clulow Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_diff.c Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_diff.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_diff.c Fri Apr 14 18:43:10 2017 (r316930) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_diff.c Fri Apr 14 18:49:44 2017 (r316931) @@ -57,15 +57,6 @@ #define ZDIFF_REMOVED '-' #define ZDIFF_RENAMED 'R' -static boolean_t -do_name_cmp(const char *fpath, const char *tpath) -{ - char *fname, *tname; - fname = strrchr(fpath, '/') + 1; - tname = strrchr(tpath, '/') + 1; - return (strcmp(fname, tname) == 0); -} - typedef struct differ_info { zfs_handle_t *zhp; char *fromsnap; @@ -260,7 +251,6 @@ static int write_inuse_diffs_one(FILE *fp, differ_info_t *di, uint64_t dobj) { struct zfs_stat fsb, tsb; - boolean_t same_name; mode_t fmode, tmode; char fobjname[MAXPATHLEN], tobjname[MAXPATHLEN]; int fobjerr, tobjerr; @@ -321,7 +311,6 @@ write_inuse_diffs_one(FILE *fp, differ_i if (fmode != tmode && fsb.zs_gen == tsb.zs_gen) tsb.zs_gen++; /* Force a generational difference */ - same_name = do_name_cmp(fobjname, tobjname); /* Simple modification or no change */ if (fsb.zs_gen == tsb.zs_gen) { @@ -332,7 +321,7 @@ write_inuse_diffs_one(FILE *fp, differ_i if (change) { print_link_change(fp, di, change, change > 0 ? fobjname : tobjname, &tsb); - } else if (same_name) { + } else if (strcmp(fobjname, tobjname) == 0) { print_file(fp, di, ZDIFF_MODIFIED, fobjname, &tsb); } else { print_rename(fp, di, fobjname, tobjname, &tsb); From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:51:18 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 5CEF9D3EF05; Fri, 14 Apr 2017 18:51:18 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 13303DE1; Fri, 14 Apr 2017 18:51:18 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIpH74083600; Fri, 14 Apr 2017 18:51:17 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIpHEA083599; Fri, 14 Apr 2017 18:51:17 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141851.v3EIpHEA083599@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:51:17 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316932 - vendor/illumos/dist/lib/libzfs/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:51:18 -0000 Author: avg Date: Fri Apr 14 18:51:16 2017 New Revision: 316932 URL: https://svnweb.freebsd.org/changeset/base/316932 Log: 6280 libzfs: unshare_one() could fail with EZFS_SHARENFSFAILED illumos/illumos-gate@d1672efb6feac57c42788e27f739dfa3c4f3baf7 https://github.com/illumos/illumos-gate/commit/d1672efb6feac57c42788e27f739dfa3c4f3baf7 https://www.illumos.org/issues/6280 The unshare_one() in libzfs could fail with EZFS_SHARENFSFAILED at line 834 here: 831 /* make sure libshare initialized */ 832 if ((err = zfs_init_libshare(hdl, SA_INIT_SHARE_API)) != SA_OK) { 833 free(mntpt); /* don't need the copy anymore */ 834 return (zfs_error_fmt(hdl, EZFS_SHARENFSFAILED, 835 dgettext(TEXT_DOMAIN, "cannot unshare '%s': %s"), 836 name, _sa_errorstr(err))); 837 } The correct error should be EZFS_UNSHARENFSFAILED instead. Reviewed by: Toomas Soome Reviewed by: Dan McDonald Reviewed by: Matthew Ahrens Approved by: Gordon Ross Author: Marcel Telka Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c Fri Apr 14 18:49:44 2017 (r316931) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_mount.c Fri Apr 14 18:51:16 2017 (r316932) @@ -20,6 +20,7 @@ */ /* + * Copyright 2015 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, 2016 by Delphix. All rights reserved. * Copyright 2016 Igor Kozhukhov @@ -914,7 +915,7 @@ unshare_one(libzfs_handle_t *hdl, const if ((err = zfs_init_libshare_arg(hdl, SA_INIT_ONE_SHARE_FROM_NAME, (void *)name)) != SA_OK) { free(mntpt); /* don't need the copy anymore */ - return (zfs_error_fmt(hdl, EZFS_SHARENFSFAILED, + return (zfs_error_fmt(hdl, EZFS_UNSHARENFSFAILED, dgettext(TEXT_DOMAIN, "cannot unshare '%s': %s"), name, _sa_errorstr(err))); } From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:52:50 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 439FBD3E120; Fri, 14 Apr 2017 18:52:50 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 12FC1369; Fri, 14 Apr 2017 18:52:50 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIqnNN084647; Fri, 14 Apr 2017 18:52:49 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIqn87084646; Fri, 14 Apr 2017 18:52:49 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141852.v3EIqn87084646@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:52:49 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316933 - vendor/illumos/dist/lib/libzfs/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:52:50 -0000 Author: avg Date: Fri Apr 14 18:52:48 2017 New Revision: 316933 URL: https://svnweb.freebsd.org/changeset/base/316933 Log: 5142 libzfs support raidz root pool (loader project) illumos/illumos-gate@d5f26ad8122c3762fb16413a17bfb497db86a782 https://github.com/illumos/illumos-gate/commit/d5f26ad8122c3762fb16413a17bfb497db86a782 https://www.illumos.org/issues/5142 the current libzfs only allows simple disk and mirror setup for boot pool, as loader does support booting from raidz, this feature will remove raidz restriction from boot pool setup. Reviewed by: George Wilson Reviewed by: Yuri Pankov Reviewed by: Andrew Stormont Reviewed by: Albert Lee Approved by: Robert Mustacchi Author: Toomas Soome Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_pool.c Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_pool.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_pool.c Fri Apr 14 18:51:16 2017 (r316932) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_pool.c Fri Apr 14 18:52:48 2017 (r316933) @@ -2267,6 +2267,7 @@ vdev_get_physpaths(nvlist_t *nv, char *p return (ret); } } else if (strcmp(type, VDEV_TYPE_MIRROR) == 0 || + strcmp(type, VDEV_TYPE_RAIDZ) == 0 || strcmp(type, VDEV_TYPE_REPLACING) == 0 || (is_spare = (strcmp(type, VDEV_TYPE_SPARE) == 0))) { nvlist_t **child; From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:54:13 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 2E908D3E184; Fri, 14 Apr 2017 18:54:13 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id DC2736C6; Fri, 14 Apr 2017 18:54:12 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIsChq084742; Fri, 14 Apr 2017 18:54:12 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIsCWD084741; Fri, 14 Apr 2017 18:54:12 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141854.v3EIsCWD084741@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:54:12 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316934 - vendor/illumos/dist/lib/libzfs/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:54:13 -0000 Author: avg Date: Fri Apr 14 18:54:11 2017 New Revision: 316934 URL: https://svnweb.freebsd.org/changeset/base/316934 Log: 7340 receive manual origin should override automatic origin illumos/illumos-gate@ed4e7a6a5cbc5e8986dc649ad54435210487b102 https://github.com/illumos/illumos-gate/commit/ed4e7a6a5cbc5e8986dc649ad54435210487b102 https://www.illumos.org/issues/7340 When -o origin= is specified as part of a ZFS receive, that origin should override the automatic detection in libzfs. Reviewed by: George Wilson Reviewed by: Matthew Ahrens Approved by: Robert Mustacchi Author: Paul Dagnelie Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_sendrecv.c Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_sendrecv.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_sendrecv.c Fri Apr 14 18:52:48 2017 (r316933) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_sendrecv.c Fri Apr 14 18:54:11 2017 (r316934) @@ -3173,7 +3173,12 @@ zfs_receive_one(libzfs_handle_t *hdl, in /* * Determine the name of the origin snapshot, store in zc_string. */ - if (drrb->drr_flags & DRR_FLAG_CLONE) { + if (originsnap) { + (void) strncpy(zc.zc_string, originsnap, sizeof (zc.zc_string)); + if (flags->verbose) + (void) printf("using provided clone origin %s\n", + zc.zc_string); + } else if (drrb->drr_flags & DRR_FLAG_CLONE) { if (guid_to_name(hdl, zc.zc_value, drrb->drr_fromguid, B_FALSE, zc.zc_string) != 0) { zcmd_free_nvlists(&zc); @@ -3184,11 +3189,6 @@ zfs_receive_one(libzfs_handle_t *hdl, in } if (flags->verbose) (void) printf("found clone origin %s\n", zc.zc_string); - } else if (originsnap) { - (void) strncpy(zc.zc_string, originsnap, sizeof (zc.zc_string)); - if (flags->verbose) - (void) printf("using provided clone origin %s\n", - zc.zc_string); } boolean_t resuming = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) & From owner-svn-src-vendor@freebsd.org Fri Apr 14 18:56:01 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id E7E75D3E21C; Fri, 14 Apr 2017 18:56:01 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id B462A94F; Fri, 14 Apr 2017 18:56:01 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3EIu0pb085001; Fri, 14 Apr 2017 18:56:00 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3EIu0Du085000; Fri, 14 Apr 2017 18:56:00 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201704141856.v3EIu0Du085000@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Fri, 14 Apr 2017 18:56:00 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316935 - vendor/illumos/dist/lib/libzfs/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Apr 2017 18:56:02 -0000 Author: avg Date: Fri Apr 14 18:56:00 2017 New Revision: 316935 URL: https://svnweb.freebsd.org/changeset/base/316935 Log: 5704 libzfs can only handle 255 file descriptors illumos/illumos-gate@bde3d612a7c090234c60e6e4578821237a5db135 https://github.com/illumos/illumos-gate/commit/bde3d612a7c090234c60e6e4578821237a5db135 https://www.illumos.org/issues/5704 libzfs uses fopen(), at least in libzfs_init(). If there are more than 255 filedescriptors open, fopen() will fail unless you give 'F' as the last mode character. The fix would be to give 'rF' instead of 'r' as mode to fopen(). Reviewed by: Josef 'Jeff' Sipek Reviewed by: John Kennedy Approved by: Richard Lowe Author: Simon Klinkert Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_util.c Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_util.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_util.c Fri Apr 14 18:54:11 2017 (r316934) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_util.c Fri Apr 14 18:56:00 2017 (r316935) @@ -626,13 +626,13 @@ libzfs_init(void) return (NULL); } - if ((hdl->libzfs_mnttab = fopen(MNTTAB, "r")) == NULL) { + if ((hdl->libzfs_mnttab = fopen(MNTTAB, "rF")) == NULL) { (void) close(hdl->libzfs_fd); free(hdl); return (NULL); } - hdl->libzfs_sharetab = fopen("/etc/dfs/sharetab", "r"); + hdl->libzfs_sharetab = fopen("/etc/dfs/sharetab", "rF"); if (libzfs_core_init() != 0) { (void) close(hdl->libzfs_fd); From owner-svn-src-vendor@freebsd.org Sat Apr 15 00:51:21 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 791E1D3DF36; Sat, 15 Apr 2017 00:51:21 +0000 (UTC) (envelope-from sjg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 430EA8A; Sat, 15 Apr 2017 00:51:21 +0000 (UTC) (envelope-from sjg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3F0pKX0033592; Sat, 15 Apr 2017 00:51:20 GMT (envelope-from sjg@FreeBSD.org) Received: (from sjg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3F0pJJ1033578; Sat, 15 Apr 2017 00:51:19 GMT (envelope-from sjg@FreeBSD.org) Message-Id: <201704150051.v3F0pJJ1033578@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: sjg set sender to sjg@FreeBSD.org using -f From: "Simon J. Gerraty" Date: Sat, 15 Apr 2017 00:51:19 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316947 - in vendor/NetBSD/bmake/dist: . mk unit-tests X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 15 Apr 2017 00:51:21 -0000 Author: sjg Date: Sat Apr 15 00:51:18 2017 New Revision: 316947 URL: https://svnweb.freebsd.org/changeset/base/316947 Log: Import bmake-20170413 Modified: vendor/NetBSD/bmake/dist/ChangeLog vendor/NetBSD/bmake/dist/Makefile vendor/NetBSD/bmake/dist/job.c vendor/NetBSD/bmake/dist/main.c vendor/NetBSD/bmake/dist/mk/ChangeLog vendor/NetBSD/bmake/dist/mk/auto.obj.mk vendor/NetBSD/bmake/dist/mk/dirdeps.mk vendor/NetBSD/bmake/dist/mk/install-mk vendor/NetBSD/bmake/dist/mk/meta.stage.mk vendor/NetBSD/bmake/dist/mk/meta2deps.py vendor/NetBSD/bmake/dist/mk/mkopt.sh vendor/NetBSD/bmake/dist/str.c vendor/NetBSD/bmake/dist/unit-tests/modmatch.exp vendor/NetBSD/bmake/dist/unit-tests/modmatch.mk Modified: vendor/NetBSD/bmake/dist/ChangeLog ============================================================================== --- vendor/NetBSD/bmake/dist/ChangeLog Sat Apr 15 00:39:45 2017 (r316946) +++ vendor/NetBSD/bmake/dist/ChangeLog Sat Apr 15 00:51:18 2017 (r316947) @@ -1,8 +1,35 @@ +2017-04-13 Simon J. Gerraty + + * Makefile (_MAKE_VERSION): 20170413 + Merge with NetBSD make, pick up + o main.c: when setting .OBJDIR ignore '$' in paths. + + * job.c: use MALLOC_OPTIONS to set malloc_options. + +2017-04-11 Simon J. Gerraty + + * Makefile (_MAKE_VERSION): 20170411 + Merge with NetBSD make, pick up + o str.c: Str_Match: allow [^a-z] to behave as expected. + +2017-03-26 Simon J. Gerraty + + * Makefile (_MAKE_VERSION): 20170326 + Merge with NetBSD make, pick up + o main.c: purge relative paths from realpath cache when .OBJDIR + is changed. + +2017-03-11 Simon J. Gerraty + + * Makefile (_MAKE_VERSION): 20170311 + Merge with NetBSD make, pick up + o main.c: only use -C arg "as is" if it starts with '/'. + 2017-03-01 Simon J. Gerraty * Makefile (_MAKE_VERSION): 20170301 Merge with NetBSD make, pick up - o main.c: use -C arg as is rather than getcwd() + o main.c: use -C arg "as is" rather than getcwd() if they identify the same directory. o parse.c: ensure loadfile buffer is \n terminated in non-mmap case Modified: vendor/NetBSD/bmake/dist/Makefile ============================================================================== --- vendor/NetBSD/bmake/dist/Makefile Sat Apr 15 00:39:45 2017 (r316946) +++ vendor/NetBSD/bmake/dist/Makefile Sat Apr 15 00:51:18 2017 (r316947) @@ -1,7 +1,7 @@ -# $Id: Makefile,v 1.81 2017/03/01 17:01:23 sjg Exp $ +# $Id: Makefile,v 1.85 2017/04/13 16:29:40 sjg Exp $ # Base version on src date -_MAKE_VERSION= 20170301 +_MAKE_VERSION= 20170413 PROG= bmake Modified: vendor/NetBSD/bmake/dist/job.c ============================================================================== --- vendor/NetBSD/bmake/dist/job.c Sat Apr 15 00:39:45 2017 (r316946) +++ vendor/NetBSD/bmake/dist/job.c Sat Apr 15 00:51:18 2017 (r316947) @@ -373,7 +373,10 @@ static void JobSigLock(sigset_t *); static void JobSigUnlock(sigset_t *); static void JobSigReset(void); -const char *malloc_options="A"; +#if !defined(MALLOC_OPTIONS) +# define MALLOC_OPTIONS "A" +#endif +const char *malloc_options= MALLOC_OPTIONS; static void job_table_dump(const char *where) Modified: vendor/NetBSD/bmake/dist/main.c ============================================================================== --- vendor/NetBSD/bmake/dist/main.c Sat Apr 15 00:39:45 2017 (r316946) +++ vendor/NetBSD/bmake/dist/main.c Sat Apr 15 00:51:18 2017 (r316947) @@ -1,4 +1,4 @@ -/* $NetBSD: main.c,v 1.257 2017/02/08 17:47:36 christos Exp $ */ +/* $NetBSD: main.c,v 1.260 2017/04/13 13:55:23 christos Exp $ */ /* * Copyright (c) 1988, 1989, 1990, 1993 @@ -69,7 +69,7 @@ */ #ifndef MAKE_NATIVE -static char rcsid[] = "$NetBSD: main.c,v 1.257 2017/02/08 17:47:36 christos Exp $"; +static char rcsid[] = "$NetBSD: main.c,v 1.260 2017/04/13 13:55:23 christos Exp $"; #else #include #ifndef lint @@ -81,7 +81,7 @@ __COPYRIGHT("@(#) Copyright (c) 1988, 19 #if 0 static char sccsid[] = "@(#)main.c 8.3 (Berkeley) 3/19/94"; #else -__RCSID("$NetBSD: main.c,v 1.257 2017/02/08 17:47:36 christos Exp $"); +__RCSID("$NetBSD: main.c,v 1.260 2017/04/13 13:55:23 christos Exp $"); #endif #endif /* not lint */ #endif @@ -458,7 +458,8 @@ rearg: (void)fprintf(stderr, "%s: %s.\n", progname, strerror(errno)); exit(2); } - if (stat(argvalue, &sa) != -1 && + if (argvalue[0] == '/' && + stat(argvalue, &sa) != -1 && stat(curdir, &sb) != -1 && sa.st_ino == sb.st_ino && sa.st_dev == sb.st_dev) @@ -721,22 +722,15 @@ Boolean Main_SetObjdir(const char *fmt, ...) { struct stat sb; - char *p, *path; - char buf[MAXPATHLEN + 1], pbuf[MAXPATHLEN + 1]; + char *path; + char buf[MAXPATHLEN + 1]; Boolean rc = FALSE; va_list ap; va_start(ap, fmt); - vsnprintf(path = pbuf, MAXPATHLEN, fmt, ap); + vsnprintf(path = buf, MAXPATHLEN, fmt, ap); va_end(ap); - /* expand variable substitutions */ - if (strchr(path, '$') != 0) { - snprintf(buf, MAXPATHLEN, "%s", path); - path = p = Var_Subst(NULL, buf, VAR_GLOBAL, VARF_WANTRES); - } else - p = NULL; - if (path[0] != '/') { snprintf(buf, MAXPATHLEN, "%s/%s", curdir, path); path = buf; @@ -752,25 +746,35 @@ Main_SetObjdir(const char *fmt, ...) Var_Set(".OBJDIR", objdir, VAR_GLOBAL, 0); setenv("PWD", objdir, 1); Dir_InitDot(); + cached_realpath(".OBJDIR", NULL); /* purge */ rc = TRUE; if (enterFlag && strcmp(objdir, curdir) != 0) enterFlagObj = TRUE; } } - free(p); return rc; } static Boolean Main_SetVarObjdir(const char *var, const char *suffix) { - char *p1, *path; - if ((path = Var_Value(var, VAR_CMD, &p1)) == NULL) + char *p, *path, *xpath; + + if ((path = Var_Value(var, VAR_CMD, &p)) == NULL) return FALSE; - (void)Main_SetObjdir("%s%s", path, suffix); - free(p1); + /* expand variable substitutions */ + if (strchr(path, '$') != 0) + xpath = Var_Subst(NULL, path, VAR_GLOBAL, VARF_WANTRES); + else + xpath = path; + + (void)Main_SetObjdir("%s%s", xpath, suffix); + + if (xpath != path) + free(xpath); + free(p); return TRUE; } @@ -1922,7 +1926,23 @@ cached_realpath(const char *pathname, ch cache->flags = INTERNAL; #endif } - + if (resolved == NULL && strcmp(pathname, ".OBJDIR") == 0) { + /* purge any relative paths */ + Hash_Entry *he, *nhe; + Hash_Search hs; + + he = Hash_EnumFirst(&cache->context, &hs); + while (he) { + nhe = Hash_EnumNext(&hs); + if (he->name[0] != '/') { + if (DEBUG(DIR)) + fprintf(stderr, "cached_realpath: purging %s\n", he->name); + Hash_DeleteEntry(&cache->context, he); + } + he = nhe; + } + return NULL; + } if ((rp = Var_Value(pathname, cache, &cp)) != NULL) { /* a hit */ strlcpy(resolved, rp, MAXPATHLEN); Modified: vendor/NetBSD/bmake/dist/mk/ChangeLog ============================================================================== --- vendor/NetBSD/bmake/dist/mk/ChangeLog Sat Apr 15 00:39:45 2017 (r316946) +++ vendor/NetBSD/bmake/dist/mk/ChangeLog Sat Apr 15 00:51:18 2017 (r316947) @@ -1,3 +1,24 @@ +2017-04-01 Simon J. Gerraty + + * install-mk (MK_VERSION): 20170401 + + * meta2deps.py: add is_src so we can check if obj dependency + is also a src dependency. + +2017-03-26 Simon J. Gerraty + + * install-mk (MK_VERSION): 20170326 + + * meta.stage.mk: do nothing if NO_STAGING is defined. + +2017-03-24 Simon J. Gerraty + + * auto.obj.mk: handle the case of __objdir=obj or obj.${MACHINE} etc. + +2017-03-18 Simon J. Gerraty + + * mkopt.sh: treat WITH_*=NO like no; ie. WITHOUT_* + 2017-03-01 Simon J. Gerraty * install-mk (MK_VERSION): 20170301 Modified: vendor/NetBSD/bmake/dist/mk/auto.obj.mk ============================================================================== --- vendor/NetBSD/bmake/dist/mk/auto.obj.mk Sat Apr 15 00:39:45 2017 (r316946) +++ vendor/NetBSD/bmake/dist/mk/auto.obj.mk Sat Apr 15 00:51:18 2017 (r316947) @@ -1,4 +1,4 @@ -# $Id: auto.obj.mk,v 1.12 2015/12/16 01:57:06 sjg Exp $ +# $Id: auto.obj.mk,v 1.13 2017/03/24 20:53:22 sjg Exp $ # # @(#) Copyright (c) 2004, Simon J. Gerraty # @@ -57,7 +57,10 @@ __objdir_made != echo ${__objdir}/; umas # This causes make to use the specified directory as .OBJDIR .OBJDIR: ${__objdir} .if ${.OBJDIR:tA} != ${__objdir:tA} && ${__objdir_made:Uno:M${__objdir}/*} != "" +# watch out for __objdir being relative path +.if !(${__objdir:M/*} == "" && ${.OBJDIR:tA} == ${${.CURDIR}/${__objdir}:L:tA}) .error could not use ${__objdir}: .OBJDIR=${.OBJDIR} .endif .endif .endif +.endif Modified: vendor/NetBSD/bmake/dist/mk/dirdeps.mk ============================================================================== --- vendor/NetBSD/bmake/dist/mk/dirdeps.mk Sat Apr 15 00:39:45 2017 (r316946) +++ vendor/NetBSD/bmake/dist/mk/dirdeps.mk Sat Apr 15 00:51:18 2017 (r316947) @@ -1,4 +1,4 @@ -# $Id: dirdeps.mk,v 1.86 2017/03/01 20:26:51 sjg Exp $ +# $Id: dirdeps.mk,v 1.87 2017/03/07 01:49:03 sjg Exp $ # Copyright (c) 2010-2013, Juniper Networks, Inc. # All rights reserved. @@ -57,7 +57,7 @@ # distinguish them from others. # # Before each Makefile.depend file is read, we set -# DEP_RELDIR to be the the RELDIR (path relative to SRCTOP) for +# DEP_RELDIR to be the RELDIR (path relative to SRCTOP) for # its directory, and DEP_MACHINE etc according to the . # represented by the suffix of the corresponding target. # Modified: vendor/NetBSD/bmake/dist/mk/install-mk ============================================================================== --- vendor/NetBSD/bmake/dist/mk/install-mk Sat Apr 15 00:39:45 2017 (r316946) +++ vendor/NetBSD/bmake/dist/mk/install-mk Sat Apr 15 00:51:18 2017 (r316947) @@ -55,7 +55,7 @@ # Simon J. Gerraty # RCSid: -# $Id: install-mk,v 1.138 2017/03/01 20:26:51 sjg Exp $ +# $Id: install-mk,v 1.140 2017/04/03 21:04:09 sjg Exp $ # # @(#) Copyright (c) 1994 Simon J. Gerraty # @@ -70,7 +70,7 @@ # sjg@crufty.net # -MK_VERSION=20170301 +MK_VERSION=20170401 OWNER= GROUP= MODE=444 Modified: vendor/NetBSD/bmake/dist/mk/meta.stage.mk ============================================================================== --- vendor/NetBSD/bmake/dist/mk/meta.stage.mk Sat Apr 15 00:39:45 2017 (r316946) +++ vendor/NetBSD/bmake/dist/mk/meta.stage.mk Sat Apr 15 00:51:18 2017 (r316947) @@ -1,4 +1,4 @@ -# $Id: meta.stage.mk,v 1.48 2017/03/01 22:48:07 sjg Exp $ +# $Id: meta.stage.mk,v 1.49 2017/04/01 02:10:34 sjg Exp $ # # @(#) Copyright (c) 2011-2017, Simon J. Gerraty # @@ -13,6 +13,8 @@ # sjg@crufty.net # +.ifndef NO_STAGING + .if !target(__${.PARSEFILE}__) # the guard target is defined later @@ -324,3 +326,4 @@ stale_staged: staging .NOMETA .endif .endif .endif +.endif Modified: vendor/NetBSD/bmake/dist/mk/meta2deps.py ============================================================================== --- vendor/NetBSD/bmake/dist/mk/meta2deps.py Sat Apr 15 00:39:45 2017 (r316946) +++ vendor/NetBSD/bmake/dist/mk/meta2deps.py Sat Apr 15 00:51:18 2017 (r316947) @@ -37,7 +37,7 @@ We only pay attention to a subset of the """ RCSid: - $Id: meta2deps.py,v 1.24 2017/02/08 22:17:10 sjg Exp $ + $Id: meta2deps.py,v 1.25 2017/04/03 21:04:09 sjg Exp $ Copyright (c) 2011-2013, Juniper Networks, Inc. All rights reserved. @@ -491,6 +491,21 @@ class MetaFile: if not file: f.close() + def is_src(self, base, dir, rdir): + """is base in srctop""" + for dir in [dir,rdir]: + if not dir: + continue + path = '/'.join([dir,base]) + srctop = self.find_top(path, self.srctops) + if srctop: + if self.dpdeps: + self.add(self.file_deps, path.replace(srctop,''), 'file') + self.add(self.src_deps, dir.replace(srctop,''), 'src') + self.seenit(dir) + return True + return False + def parse_path(self, path, cwd, op=None, w=[]): """look at a path for the op specified""" @@ -519,10 +534,9 @@ class MetaFile: # to the src dir, we may need to add dependencies for each rdir = dir dir = abspath(dir, cwd, self.last_dir, self.debug, self.debug_out) - if rdir == dir or rdir.find('./') > 0: - rdir = None - if os.path.islink(dir): rdir = os.path.realpath(dir) + if rdir == dir: + rdir = None # now put path back together path = '/'.join([dir,base]) if self.debug > 1: @@ -543,17 +557,9 @@ class MetaFile: # finally, we get down to it if dir == self.cwd or dir == self.curdir: return - srctop = self.find_top(path, self.srctops) - if srctop: - if self.dpdeps: - self.add(self.file_deps, path.replace(srctop,''), 'file') - self.add(self.src_deps, dir.replace(srctop,''), 'src') + if self.is_src(base, dir, rdir): self.seenit(w[2]) - self.seenit(dir) - if rdir and not rdir.startswith(srctop): - dir = rdir # for below - rdir = None - else: + if not rdir: return objroot = None Modified: vendor/NetBSD/bmake/dist/mk/mkopt.sh ============================================================================== --- vendor/NetBSD/bmake/dist/mk/mkopt.sh Sat Apr 15 00:39:45 2017 (r316946) +++ vendor/NetBSD/bmake/dist/mk/mkopt.sh Sat Apr 15 00:51:18 2017 (r316947) @@ -1,5 +1,5 @@ : -# $Id: mkopt.sh,v 1.10 2015/06/07 17:29:08 sjg Exp $ +# $Id: mkopt.sh,v 1.11 2017/03/18 21:36:42 sjg Exp $ # # @(#) Copyright (c) 2014, Simon J. Gerraty # @@ -40,7 +40,7 @@ _mk_opt() { eval "_mov=\$$_mo _wov=\$$_wo _wiv=\$$_wi" case "$_wiv" in - no) _wov=no;; + [Nn][Oo]) _wov=no;; esac _v=${_mov:-${_wov:+no}} _v=${_v:-${_wiv:+yes}} Modified: vendor/NetBSD/bmake/dist/str.c ============================================================================== --- vendor/NetBSD/bmake/dist/str.c Sat Apr 15 00:39:45 2017 (r316946) +++ vendor/NetBSD/bmake/dist/str.c Sat Apr 15 00:51:18 2017 (r316947) @@ -1,4 +1,4 @@ -/* $NetBSD: str.c,v 1.36 2016/04/06 09:57:00 gson Exp $ */ +/* $NetBSD: str.c,v 1.37 2017/04/11 17:30:13 sjg Exp $ */ /*- * Copyright (c) 1988, 1989, 1990, 1993 @@ -69,14 +69,14 @@ */ #ifndef MAKE_NATIVE -static char rcsid[] = "$NetBSD: str.c,v 1.36 2016/04/06 09:57:00 gson Exp $"; +static char rcsid[] = "$NetBSD: str.c,v 1.37 2017/04/11 17:30:13 sjg Exp $"; #else #include #ifndef lint #if 0 static char sccsid[] = "@(#)str.c 5.8 (Berkeley) 6/1/90"; #else -__RCSID("$NetBSD: str.c,v 1.36 2016/04/06 09:57:00 gson Exp $"); +__RCSID("$NetBSD: str.c,v 1.37 2017/04/11 17:30:13 sjg Exp $"); #endif #endif /* not lint */ #endif @@ -373,16 +373,23 @@ Str_Match(const char *string, const char * by a range (two characters separated by "-"). */ if (*pattern == '[') { + int nomatch; + ++pattern; + if (*pattern == '^') { + ++pattern; + nomatch = 1; + } else + nomatch = 0; for (;;) { if ((*pattern == ']') || (*pattern == 0)) - return(0); + return(nomatch); if (*pattern == *string) break; if (pattern[1] == '-') { c2 = pattern[2]; if (c2 == 0) - return(0); + return(nomatch); if ((*pattern <= *string) && (c2 >= *string)) break; @@ -393,6 +400,8 @@ Str_Match(const char *string, const char } ++pattern; } + if (nomatch) + return 0; while ((*pattern != ']') && (*pattern != 0)) ++pattern; goto thisCharOK; Modified: vendor/NetBSD/bmake/dist/unit-tests/modmatch.exp ============================================================================== --- vendor/NetBSD/bmake/dist/unit-tests/modmatch.exp Sat Apr 15 00:39:45 2017 (r316946) +++ vendor/NetBSD/bmake/dist/unit-tests/modmatch.exp Sat Apr 15 00:51:18 2017 (r316947) @@ -14,4 +14,6 @@ LIB=e X_LIBS:M${LIB${LIB:tu}} is "/tmp/l LIB=e X_LIBS:M*/lib${LIB}.a is "/tmp/libe.a" LIB=e X_LIBS:M*/lib${LIB}.a:tu is "/TMP/LIBE.A" Mscanner=OK +Upper=One Two Three Four +Lower=five six seven exit status 0 Modified: vendor/NetBSD/bmake/dist/unit-tests/modmatch.mk ============================================================================== --- vendor/NetBSD/bmake/dist/unit-tests/modmatch.mk Sat Apr 15 00:39:45 2017 (r316946) +++ vendor/NetBSD/bmake/dist/unit-tests/modmatch.mk Sat Apr 15 00:51:18 2017 (r316947) @@ -15,7 +15,9 @@ res = no res = OK .endif -all: +all: show-libs check-cclass + +show-libs: @for x in $X; do ${.MAKE} -f ${MAKEFILE} show LIB=$$x; done @echo "Mscanner=${res}" @@ -23,3 +25,9 @@ show: @echo 'LIB=${LIB} X_LIBS:M$${LIB$${LIB:tu}} is "${X_LIBS:M${LIB${LIB:tu}}}"' @echo 'LIB=${LIB} X_LIBS:M*/lib$${LIB}.a is "${X_LIBS:M*/lib${LIB}.a}"' @echo 'LIB=${LIB} X_LIBS:M*/lib$${LIB}.a:tu is "${X_LIBS:M*/lib${LIB}.a:tu}"' + +LIST= One Two Three Four five six seven + +check-cclass: + @echo Upper=${LIST:M[A-Z]*} + @echo Lower=${LIST:M[^A-Z]*} From owner-svn-src-vendor@freebsd.org Sat Apr 15 00:51:37 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 67B8FD3DFFD; Sat, 15 Apr 2017 00:51:37 +0000 (UTC) (envelope-from sjg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 19EC118A; Sat, 15 Apr 2017 00:51:36 +0000 (UTC) (envelope-from sjg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3F0paUk033646; Sat, 15 Apr 2017 00:51:36 GMT (envelope-from sjg@FreeBSD.org) Received: (from sjg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3F0pawI033645; Sat, 15 Apr 2017 00:51:36 GMT (envelope-from sjg@FreeBSD.org) Message-Id: <201704150051.v3F0pawI033645@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: sjg set sender to sjg@FreeBSD.org using -f From: "Simon J. Gerraty" Date: Sat, 15 Apr 2017 00:51:36 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316948 - vendor/NetBSD/bmake/20170413 X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 15 Apr 2017 00:51:37 -0000 Author: sjg Date: Sat Apr 15 00:51:35 2017 New Revision: 316948 URL: https://svnweb.freebsd.org/changeset/base/316948 Log: tag bmake-20170413 Added: vendor/NetBSD/bmake/20170413/ - copied from r316947, vendor/NetBSD/bmake/dist/ From owner-svn-src-vendor@freebsd.org Sat Apr 15 17:03:17 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 3C3B3D3E56D; Sat, 15 Apr 2017 17:03:17 +0000 (UTC) (envelope-from julian@freebsd.org) Received: from vps1.elischer.org (vps1.elischer.org [204.109.63.16]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client CN "vps1.elischer.org", Issuer "CA Cert Signing Authority" (not verified)) by mx1.freebsd.org (Postfix) with ESMTPS id 1A79DD0B; Sat, 15 Apr 2017 17:03:16 +0000 (UTC) (envelope-from julian@freebsd.org) Received: from Julian-MBP3.local (58-7-95-3.dyn.iinet.net.au [58.7.95.3]) (authenticated bits=0) by vps1.elischer.org (8.15.2/8.15.2) with ESMTPSA id v3FH3AXw079539 (version=TLSv1.2 cipher=DHE-RSA-AES128-SHA bits=128 verify=NO); Sat, 15 Apr 2017 10:03:13 -0700 (PDT) (envelope-from julian@freebsd.org) Subject: Re: svn commit: r316930 - vendor-sys/illumos/dist/uts/common/fs/zfs To: Andriy Gapon , src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org References: <201704141843.v3EIhAA2079997@repo.freebsd.org> From: Julian Elischer Message-ID: Date: Sun, 16 Apr 2017 01:03:04 +0800 User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:45.0) Gecko/20100101 Thunderbird/45.8.0 MIME-Version: 1.0 In-Reply-To: <201704141843.v3EIhAA2079997@repo.freebsd.org> Content-Type: text/plain; charset=utf-8; format=flowed Content-Transfer-Encoding: 7bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 15 Apr 2017 17:03:17 -0000 MFCs to 11 and 10 for all this stuff you've done? On 15/4/17 2:43 am, Andriy Gapon wrote: > Author: avg > Date: Fri Apr 14 18:43:10 2017 > New Revision: 316930 > URL: https://svnweb.freebsd.org/changeset/base/316930 > > Log: > 5814 bpobj_iterate_impl(): Close a refcount leak iterating on a sublist. > > illumos/illumos-gate@b67dde11a73a9455d641403cbbb65ec2add41b41 > https://github.com/illumos/illumos-gate/commit/b67dde11a73a9455d641403cbbb65ec2add41b41 > > https://www.illumos.org/issues/5814 > Lets pull in this patch from freebsd: > http://svnweb.freebsd.org/base?view=revision&revision=271781 > bpobj_iterate_impl(): Close a refcount leak iterating on a sublist. > If bpobj_space() returned non-zero here, the sublist would have been > left open, along with the bonus buffer hold it requires. This call > does not invoke any calls to bpobj_close() itself. > This bug doesn't have any known vector, but was found on inspection. > MFC after: 1 week > Sponsored by: Spectra Logic > Affects: All ZFS versions starting 21 May 2010 (illumos cde58dbc) > MFSpectraBSD: r1050998 on 2014/03/26 > Fix bpobj_iterate_impl() to properly call bpobj_close() if bpobj_space() > returns an error. > > Reviewed by: Prakash Surya > Reviewed by: Matthew Ahrens > Reviewed by: Paul Dagnelie > Reviewed by: Simon Klinkert > Approved by: Gordon Ross > Author: Will Andrews > > Modified: > vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c > > Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c > ============================================================================== > --- vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c Fri Apr 14 18:41:37 2017 (r316929) > +++ vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c Fri Apr 14 18:43:10 2017 (r316930) > @@ -301,8 +301,10 @@ bpobj_iterate_impl(bpobj_t *bpo, bpobj_i > if (free) { > err = bpobj_space(&sublist, > &used_before, &comp_before, &uncomp_before); > - if (err) > + if (err != 0) { > + bpobj_close(&sublist); > break; > + } > } > err = bpobj_iterate_impl(&sublist, func, arg, tx, free); > if (free) { > > From owner-svn-src-vendor@freebsd.org Sat Apr 15 17:43:42 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 39AE9D3F31D; Sat, 15 Apr 2017 17:43:42 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from citapm.icyb.net.ua (citapm.icyb.net.ua [212.40.38.140]) by mx1.freebsd.org (Postfix) with ESMTP id D33586C7; Sat, 15 Apr 2017 17:43:37 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from porto.starpoint.kiev.ua (porto-e.starpoint.kiev.ua [212.40.38.100]) by citapm.icyb.net.ua (8.8.8p3/ICyb-2.3exp) with ESMTP id UAA13764; Sat, 15 Apr 2017 20:43:29 +0300 (EEST) (envelope-from avg@FreeBSD.org) Received: from localhost ([127.0.0.1]) by porto.starpoint.kiev.ua with esmtp (Exim 4.34 (FreeBSD)) id 1czRjN-000GqS-OH; Sat, 15 Apr 2017 20:43:29 +0300 Subject: Re: svn commit: r316930 - vendor-sys/illumos/dist/uts/common/fs/zfs To: Julian Elischer , src-committers@FreeBSD.org, svn-src-all@FreeBSD.org, svn-src-vendor@FreeBSD.org References: <201704141843.v3EIhAA2079997@repo.freebsd.org> From: Andriy Gapon Message-ID: Date: Sat, 15 Apr 2017 20:42:33 +0300 User-Agent: Mozilla/5.0 (X11; FreeBSD amd64; rv:52.0) Gecko/20100101 Thunderbird/52.0 MIME-Version: 1.0 In-Reply-To: Content-Type: text/plain; charset=utf-8 Content-Language: en-US Content-Transfer-Encoding: 7bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 15 Apr 2017 17:43:42 -0000 On 15/04/2017 20:03, Julian Elischer wrote: > MFCs to 11 and 10 for all this stuff you've done? Come on, it's not even in head yet! Seriously, any help with that would be greatly appreciated. > On 15/4/17 2:43 am, Andriy Gapon wrote: >> Author: avg >> Date: Fri Apr 14 18:43:10 2017 >> New Revision: 316930 >> URL: https://svnweb.freebsd.org/changeset/base/316930 >> >> Log: >> 5814 bpobj_iterate_impl(): Close a refcount leak iterating on a sublist. >> illumos/illumos-gate@b67dde11a73a9455d641403cbbb65ec2add41b41 >> >> https://github.com/illumos/illumos-gate/commit/b67dde11a73a9455d641403cbbb65ec2add41b41 >> >> https://www.illumos.org/issues/5814 >> Lets pull in this patch from freebsd: >> http://svnweb.freebsd.org/base?view=revision&revision=271781 >> bpobj_iterate_impl(): Close a refcount leak iterating on a sublist. >> If bpobj_space() returned non-zero here, the sublist would have been >> left open, along with the bonus buffer hold it requires. This call >> does not invoke any calls to bpobj_close() itself. >> This bug doesn't have any known vector, but was found on inspection. >> MFC after: 1 week >> Sponsored by: Spectra Logic >> Affects: All ZFS versions starting 21 May 2010 (illumos cde58dbc) >> MFSpectraBSD: r1050998 on 2014/03/26 >> Fix bpobj_iterate_impl() to properly call bpobj_close() if bpobj_space() >> returns an error. >> Reviewed by: Prakash Surya >> Reviewed by: Matthew Ahrens >> Reviewed by: Paul Dagnelie >> Reviewed by: Simon Klinkert >> Approved by: Gordon Ross >> Author: Will Andrews >> >> Modified: >> vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c >> >> Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c >> ============================================================================== >> --- vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c Fri Apr 14 18:41:37 >> 2017 (r316929) >> +++ vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c Fri Apr 14 18:43:10 >> 2017 (r316930) >> @@ -301,8 +301,10 @@ bpobj_iterate_impl(bpobj_t *bpo, bpobj_i >> if (free) { >> err = bpobj_space(&sublist, >> &used_before, &comp_before, &uncomp_before); >> - if (err) >> + if (err != 0) { >> + bpobj_close(&sublist); >> break; >> + } >> } >> err = bpobj_iterate_impl(&sublist, func, arg, tx, free); >> if (free) { >> >> > -- Andriy Gapon From owner-svn-src-vendor@freebsd.org Sat Apr 15 19:47:19 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 970F2D3F0A7; Sat, 15 Apr 2017 19:47:19 +0000 (UTC) (envelope-from bapt@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 318CAE5A; Sat, 15 Apr 2017 19:47:19 +0000 (UTC) (envelope-from bapt@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3FJlIZK094459; Sat, 15 Apr 2017 19:47:18 GMT (envelope-from bapt@FreeBSD.org) Received: (from bapt@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3FJlHl4094446; Sat, 15 Apr 2017 19:47:17 GMT (envelope-from bapt@FreeBSD.org) Message-Id: <201704151947.v3FJlHl4094446@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: bapt set sender to bapt@FreeBSD.org using -f From: Baptiste Daroussin Date: Sat, 15 Apr 2017 19:47:17 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316975 - in vendor/zstd: . dist dist/build dist/build/VS2005 dist/build/VS2005/fullbench dist/build/VS2005/fuzzer dist/build/VS2005/zstd dist/build/VS2005/zstdlib dist/build/VS2008 dis... X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 15 Apr 2017 19:47:19 -0000 Author: bapt Date: Sat Apr 15 19:47:16 2017 New Revision: 316975 URL: https://svnweb.freebsd.org/changeset/base/316975 Log: Import zstd 1.1.4 Added: vendor/zstd/ vendor/zstd/dist/ vendor/zstd/dist/.buckconfig vendor/zstd/dist/.buckversion vendor/zstd/dist/.gitattributes vendor/zstd/dist/.gitignore vendor/zstd/dist/.travis.yml vendor/zstd/dist/CONTRIBUTING.md vendor/zstd/dist/LICENSE vendor/zstd/dist/LICENSE-examples vendor/zstd/dist/Makefile (contents, props changed) vendor/zstd/dist/NEWS vendor/zstd/dist/PATENTS vendor/zstd/dist/README.md vendor/zstd/dist/TESTING.md vendor/zstd/dist/appveyor.yml vendor/zstd/dist/build/ vendor/zstd/dist/build/.gitignore vendor/zstd/dist/build/README.md vendor/zstd/dist/build/VS2005/ vendor/zstd/dist/build/VS2005/fullbench/ vendor/zstd/dist/build/VS2005/fullbench/fullbench.vcproj vendor/zstd/dist/build/VS2005/fuzzer/ vendor/zstd/dist/build/VS2005/fuzzer/fuzzer.vcproj vendor/zstd/dist/build/VS2005/zstd/ vendor/zstd/dist/build/VS2005/zstd.sln vendor/zstd/dist/build/VS2005/zstd/zstd.vcproj vendor/zstd/dist/build/VS2005/zstdlib/ vendor/zstd/dist/build/VS2005/zstdlib/zstdlib.vcproj vendor/zstd/dist/build/VS2008/ vendor/zstd/dist/build/VS2008/fullbench/ vendor/zstd/dist/build/VS2008/fullbench/fullbench.vcproj vendor/zstd/dist/build/VS2008/fuzzer/ vendor/zstd/dist/build/VS2008/fuzzer/fuzzer.vcproj vendor/zstd/dist/build/VS2008/zstd/ vendor/zstd/dist/build/VS2008/zstd.sln vendor/zstd/dist/build/VS2008/zstd/zstd.vcproj vendor/zstd/dist/build/VS2008/zstdlib/ vendor/zstd/dist/build/VS2008/zstdlib/zstdlib.vcproj vendor/zstd/dist/build/VS2010/ vendor/zstd/dist/build/VS2010/CompileAsCpp.props vendor/zstd/dist/build/VS2010/datagen/ vendor/zstd/dist/build/VS2010/datagen/datagen.vcxproj vendor/zstd/dist/build/VS2010/fullbench/ vendor/zstd/dist/build/VS2010/fullbench-dll/ vendor/zstd/dist/build/VS2010/fullbench-dll/fullbench-dll.vcxproj vendor/zstd/dist/build/VS2010/fullbench/fullbench.vcxproj vendor/zstd/dist/build/VS2010/fuzzer/ vendor/zstd/dist/build/VS2010/fuzzer/fuzzer.vcxproj vendor/zstd/dist/build/VS2010/libzstd/ vendor/zstd/dist/build/VS2010/libzstd-dll/ vendor/zstd/dist/build/VS2010/libzstd-dll/libzstd-dll.rc vendor/zstd/dist/build/VS2010/libzstd-dll/libzstd-dll.vcxproj vendor/zstd/dist/build/VS2010/libzstd/libzstd.vcxproj vendor/zstd/dist/build/VS2010/zstd/ vendor/zstd/dist/build/VS2010/zstd.sln vendor/zstd/dist/build/VS2010/zstd/zstd.rc vendor/zstd/dist/build/VS2010/zstd/zstd.vcxproj vendor/zstd/dist/build/VS_scripts/ vendor/zstd/dist/build/VS_scripts/README.md vendor/zstd/dist/build/VS_scripts/build.VS2010.cmd vendor/zstd/dist/build/VS_scripts/build.VS2012.cmd vendor/zstd/dist/build/VS_scripts/build.VS2013.cmd vendor/zstd/dist/build/VS_scripts/build.VS2015.cmd vendor/zstd/dist/build/VS_scripts/build.generic.cmd vendor/zstd/dist/build/cmake/ vendor/zstd/dist/build/cmake/.gitignore vendor/zstd/dist/build/cmake/CMakeLists.txt (contents, props changed) vendor/zstd/dist/build/cmake/CMakeModules/ vendor/zstd/dist/build/cmake/CMakeModules/AddExtraCompilationFlags.cmake vendor/zstd/dist/build/cmake/cmake_uninstall.cmake.in (contents, props changed) vendor/zstd/dist/build/cmake/contrib/ vendor/zstd/dist/build/cmake/contrib/CMakeLists.txt (contents, props changed) vendor/zstd/dist/build/cmake/contrib/pzstd/ vendor/zstd/dist/build/cmake/contrib/pzstd/CMakeLists.txt (contents, props changed) vendor/zstd/dist/build/cmake/lib/ vendor/zstd/dist/build/cmake/lib/CMakeLists.txt (contents, props changed) vendor/zstd/dist/build/cmake/programs/ vendor/zstd/dist/build/cmake/programs/.gitignore vendor/zstd/dist/build/cmake/programs/CMakeLists.txt (contents, props changed) vendor/zstd/dist/build/cmake/tests/ vendor/zstd/dist/build/cmake/tests/.gitignore vendor/zstd/dist/build/cmake/tests/CMakeLists.txt (contents, props changed) vendor/zstd/dist/circle.yml vendor/zstd/dist/contrib/ vendor/zstd/dist/contrib/gen_html/ vendor/zstd/dist/contrib/gen_html/.gitignore vendor/zstd/dist/contrib/gen_html/Makefile (contents, props changed) vendor/zstd/dist/contrib/gen_html/README.md vendor/zstd/dist/contrib/gen_html/gen-zstd-manual.sh (contents, props changed) vendor/zstd/dist/contrib/gen_html/gen_html.cpp (contents, props changed) vendor/zstd/dist/contrib/meson/ vendor/zstd/dist/contrib/meson/README vendor/zstd/dist/contrib/meson/meson.build vendor/zstd/dist/contrib/meson/meson_options.txt (contents, props changed) vendor/zstd/dist/contrib/pzstd/ vendor/zstd/dist/contrib/pzstd/.gitignore vendor/zstd/dist/contrib/pzstd/BUCK vendor/zstd/dist/contrib/pzstd/ErrorHolder.h (contents, props changed) vendor/zstd/dist/contrib/pzstd/Logging.h (contents, props changed) vendor/zstd/dist/contrib/pzstd/Makefile (contents, props changed) vendor/zstd/dist/contrib/pzstd/Options.cpp (contents, props changed) vendor/zstd/dist/contrib/pzstd/Options.h (contents, props changed) vendor/zstd/dist/contrib/pzstd/Pzstd.cpp (contents, props changed) vendor/zstd/dist/contrib/pzstd/Pzstd.h (contents, props changed) vendor/zstd/dist/contrib/pzstd/README.md vendor/zstd/dist/contrib/pzstd/SkippableFrame.cpp (contents, props changed) vendor/zstd/dist/contrib/pzstd/SkippableFrame.h (contents, props changed) vendor/zstd/dist/contrib/pzstd/images/ vendor/zstd/dist/contrib/pzstd/images/Cspeed.png (contents, props changed) vendor/zstd/dist/contrib/pzstd/images/Dspeed.png (contents, props changed) vendor/zstd/dist/contrib/pzstd/main.cpp (contents, props changed) vendor/zstd/dist/contrib/pzstd/test/ vendor/zstd/dist/contrib/pzstd/test/BUCK vendor/zstd/dist/contrib/pzstd/test/OptionsTest.cpp (contents, props changed) vendor/zstd/dist/contrib/pzstd/test/PzstdTest.cpp (contents, props changed) vendor/zstd/dist/contrib/pzstd/test/RoundTrip.h (contents, props changed) vendor/zstd/dist/contrib/pzstd/test/RoundTripTest.cpp (contents, props changed) vendor/zstd/dist/contrib/pzstd/utils/ vendor/zstd/dist/contrib/pzstd/utils/BUCK vendor/zstd/dist/contrib/pzstd/utils/Buffer.h (contents, props changed) vendor/zstd/dist/contrib/pzstd/utils/FileSystem.h (contents, props changed) vendor/zstd/dist/contrib/pzstd/utils/Likely.h (contents, props changed) vendor/zstd/dist/contrib/pzstd/utils/Range.h (contents, props changed) vendor/zstd/dist/contrib/pzstd/utils/ResourcePool.h (contents, props changed) vendor/zstd/dist/contrib/pzstd/utils/ScopeGuard.h (contents, props changed) vendor/zstd/dist/contrib/pzstd/utils/ThreadPool.h (contents, props changed) vendor/zstd/dist/contrib/pzstd/utils/WorkQueue.h (contents, props changed) vendor/zstd/dist/contrib/pzstd/utils/test/ vendor/zstd/dist/contrib/pzstd/utils/test/BUCK vendor/zstd/dist/contrib/pzstd/utils/test/BufferTest.cpp (contents, props changed) vendor/zstd/dist/contrib/pzstd/utils/test/RangeTest.cpp (contents, props changed) vendor/zstd/dist/contrib/pzstd/utils/test/ResourcePoolTest.cpp (contents, props changed) vendor/zstd/dist/contrib/pzstd/utils/test/ScopeGuardTest.cpp (contents, props changed) vendor/zstd/dist/contrib/pzstd/utils/test/ThreadPoolTest.cpp (contents, props changed) vendor/zstd/dist/contrib/pzstd/utils/test/WorkQueueTest.cpp (contents, props changed) vendor/zstd/dist/doc/ vendor/zstd/dist/doc/README.md vendor/zstd/dist/doc/educational_decoder/ vendor/zstd/dist/doc/educational_decoder/README.md vendor/zstd/dist/doc/educational_decoder/harness.c (contents, props changed) vendor/zstd/dist/doc/educational_decoder/zstd_decompress.c (contents, props changed) vendor/zstd/dist/doc/educational_decoder/zstd_decompress.h (contents, props changed) vendor/zstd/dist/doc/images/ vendor/zstd/dist/doc/images/Cspeed4.png (contents, props changed) vendor/zstd/dist/doc/images/DCspeed5.png (contents, props changed) vendor/zstd/dist/doc/images/Dspeed4.png (contents, props changed) vendor/zstd/dist/doc/images/dict-cr.png (contents, props changed) vendor/zstd/dist/doc/images/dict-cs.png (contents, props changed) vendor/zstd/dist/doc/images/dict-ds.png (contents, props changed) vendor/zstd/dist/doc/zstd_compression_format.md vendor/zstd/dist/doc/zstd_manual.html (contents, props changed) vendor/zstd/dist/examples/ vendor/zstd/dist/examples/.gitignore vendor/zstd/dist/examples/Makefile (contents, props changed) vendor/zstd/dist/examples/README.md vendor/zstd/dist/examples/dictionary_compression.c (contents, props changed) vendor/zstd/dist/examples/dictionary_decompression.c (contents, props changed) vendor/zstd/dist/examples/multiple_streaming_compression.c (contents, props changed) vendor/zstd/dist/examples/simple_compression.c (contents, props changed) vendor/zstd/dist/examples/simple_decompression.c (contents, props changed) vendor/zstd/dist/examples/streaming_compression.c (contents, props changed) vendor/zstd/dist/examples/streaming_decompression.c (contents, props changed) vendor/zstd/dist/lib/ vendor/zstd/dist/lib/.gitignore vendor/zstd/dist/lib/BUCK vendor/zstd/dist/lib/Makefile (contents, props changed) vendor/zstd/dist/lib/README.md vendor/zstd/dist/lib/common/ vendor/zstd/dist/lib/common/bitstream.h (contents, props changed) vendor/zstd/dist/lib/common/entropy_common.c (contents, props changed) vendor/zstd/dist/lib/common/error_private.c (contents, props changed) vendor/zstd/dist/lib/common/error_private.h (contents, props changed) vendor/zstd/dist/lib/common/fse.h (contents, props changed) vendor/zstd/dist/lib/common/fse_decompress.c (contents, props changed) vendor/zstd/dist/lib/common/huf.h (contents, props changed) vendor/zstd/dist/lib/common/mem.h (contents, props changed) vendor/zstd/dist/lib/common/pool.c (contents, props changed) vendor/zstd/dist/lib/common/pool.h (contents, props changed) vendor/zstd/dist/lib/common/threading.c (contents, props changed) vendor/zstd/dist/lib/common/threading.h (contents, props changed) vendor/zstd/dist/lib/common/xxhash.c (contents, props changed) vendor/zstd/dist/lib/common/xxhash.h (contents, props changed) vendor/zstd/dist/lib/common/zstd_common.c (contents, props changed) vendor/zstd/dist/lib/common/zstd_errors.h (contents, props changed) vendor/zstd/dist/lib/common/zstd_internal.h (contents, props changed) vendor/zstd/dist/lib/compress/ vendor/zstd/dist/lib/compress/fse_compress.c (contents, props changed) vendor/zstd/dist/lib/compress/huf_compress.c (contents, props changed) vendor/zstd/dist/lib/compress/zstd_compress.c (contents, props changed) vendor/zstd/dist/lib/compress/zstd_opt.h (contents, props changed) vendor/zstd/dist/lib/compress/zstdmt_compress.c (contents, props changed) vendor/zstd/dist/lib/compress/zstdmt_compress.h (contents, props changed) vendor/zstd/dist/lib/decompress/ vendor/zstd/dist/lib/decompress/huf_decompress.c (contents, props changed) vendor/zstd/dist/lib/decompress/zstd_decompress.c (contents, props changed) vendor/zstd/dist/lib/deprecated/ vendor/zstd/dist/lib/deprecated/zbuff.h (contents, props changed) vendor/zstd/dist/lib/deprecated/zbuff_common.c (contents, props changed) vendor/zstd/dist/lib/deprecated/zbuff_compress.c (contents, props changed) vendor/zstd/dist/lib/deprecated/zbuff_decompress.c (contents, props changed) vendor/zstd/dist/lib/dictBuilder/ vendor/zstd/dist/lib/dictBuilder/cover.c (contents, props changed) vendor/zstd/dist/lib/dictBuilder/divsufsort.c (contents, props changed) vendor/zstd/dist/lib/dictBuilder/divsufsort.h (contents, props changed) vendor/zstd/dist/lib/dictBuilder/zdict.c (contents, props changed) vendor/zstd/dist/lib/dictBuilder/zdict.h (contents, props changed) vendor/zstd/dist/lib/dll/ vendor/zstd/dist/lib/dll/example/ vendor/zstd/dist/lib/dll/example/Makefile (contents, props changed) vendor/zstd/dist/lib/dll/example/README.md vendor/zstd/dist/lib/dll/example/build_package.bat vendor/zstd/dist/lib/dll/example/fullbench-dll.sln vendor/zstd/dist/lib/dll/example/fullbench-dll.vcxproj vendor/zstd/dist/lib/dll/libzstd.def vendor/zstd/dist/lib/legacy/ vendor/zstd/dist/lib/legacy/zstd_legacy.h (contents, props changed) vendor/zstd/dist/lib/legacy/zstd_v01.c (contents, props changed) vendor/zstd/dist/lib/legacy/zstd_v01.h (contents, props changed) vendor/zstd/dist/lib/legacy/zstd_v02.c (contents, props changed) vendor/zstd/dist/lib/legacy/zstd_v02.h (contents, props changed) vendor/zstd/dist/lib/legacy/zstd_v03.c (contents, props changed) vendor/zstd/dist/lib/legacy/zstd_v03.h (contents, props changed) vendor/zstd/dist/lib/legacy/zstd_v04.c (contents, props changed) vendor/zstd/dist/lib/legacy/zstd_v04.h (contents, props changed) vendor/zstd/dist/lib/legacy/zstd_v05.c (contents, props changed) vendor/zstd/dist/lib/legacy/zstd_v05.h (contents, props changed) vendor/zstd/dist/lib/legacy/zstd_v06.c (contents, props changed) vendor/zstd/dist/lib/legacy/zstd_v06.h (contents, props changed) vendor/zstd/dist/lib/legacy/zstd_v07.c (contents, props changed) vendor/zstd/dist/lib/legacy/zstd_v07.h (contents, props changed) vendor/zstd/dist/lib/libzstd.pc.in (contents, props changed) vendor/zstd/dist/lib/zstd.h (contents, props changed) vendor/zstd/dist/programs/ vendor/zstd/dist/programs/.gitignore vendor/zstd/dist/programs/BUCK vendor/zstd/dist/programs/Makefile (contents, props changed) vendor/zstd/dist/programs/README.md vendor/zstd/dist/programs/bench.c (contents, props changed) vendor/zstd/dist/programs/bench.h (contents, props changed) vendor/zstd/dist/programs/datagen.c (contents, props changed) vendor/zstd/dist/programs/datagen.h (contents, props changed) vendor/zstd/dist/programs/dibio.c (contents, props changed) vendor/zstd/dist/programs/dibio.h (contents, props changed) vendor/zstd/dist/programs/fileio.c (contents, props changed) vendor/zstd/dist/programs/fileio.h (contents, props changed) vendor/zstd/dist/programs/platform.h (contents, props changed) vendor/zstd/dist/programs/util.h (contents, props changed) vendor/zstd/dist/programs/windres/ vendor/zstd/dist/programs/windres/generate_res.bat vendor/zstd/dist/programs/windres/verrsrc.h (contents, props changed) vendor/zstd/dist/programs/windres/zstd.rc vendor/zstd/dist/programs/windres/zstd32.res (contents, props changed) vendor/zstd/dist/programs/windres/zstd64.res (contents, props changed) vendor/zstd/dist/programs/zstd.1 (contents, props changed) vendor/zstd/dist/programs/zstdcli.c (contents, props changed) vendor/zstd/dist/programs/zstdgrep (contents, props changed) vendor/zstd/dist/programs/zstdless (contents, props changed) vendor/zstd/dist/tests/ vendor/zstd/dist/tests/.gitignore vendor/zstd/dist/tests/Makefile (contents, props changed) vendor/zstd/dist/tests/README.md vendor/zstd/dist/tests/datagencli.c (contents, props changed) vendor/zstd/dist/tests/decodecorpus.c (contents, props changed) vendor/zstd/dist/tests/fullbench.c (contents, props changed) vendor/zstd/dist/tests/fuzzer.c (contents, props changed) vendor/zstd/dist/tests/gzip/ vendor/zstd/dist/tests/gzip/Makefile (contents, props changed) vendor/zstd/dist/tests/gzip/gzip-env.sh (contents, props changed) vendor/zstd/dist/tests/gzip/helin-segv.sh (contents, props changed) vendor/zstd/dist/tests/gzip/help-version.sh (contents, props changed) vendor/zstd/dist/tests/gzip/hufts-segv.gz (contents, props changed) vendor/zstd/dist/tests/gzip/hufts.sh (contents, props changed) vendor/zstd/dist/tests/gzip/init.cfg vendor/zstd/dist/tests/gzip/init.sh (contents, props changed) vendor/zstd/dist/tests/gzip/keep.sh (contents, props changed) vendor/zstd/dist/tests/gzip/list.sh (contents, props changed) vendor/zstd/dist/tests/gzip/memcpy-abuse.sh (contents, props changed) vendor/zstd/dist/tests/gzip/mixed.sh (contents, props changed) vendor/zstd/dist/tests/gzip/null-suffix-clobber.sh (contents, props changed) vendor/zstd/dist/tests/gzip/stdin.sh (contents, props changed) vendor/zstd/dist/tests/gzip/test-driver.sh (contents, props changed) vendor/zstd/dist/tests/gzip/trailing-nul.sh (contents, props changed) vendor/zstd/dist/tests/gzip/unpack-invalid.sh (contents, props changed) vendor/zstd/dist/tests/gzip/z-suffix.sh (contents, props changed) vendor/zstd/dist/tests/gzip/zdiff.sh (contents, props changed) vendor/zstd/dist/tests/gzip/zgrep-context.sh (contents, props changed) vendor/zstd/dist/tests/gzip/zgrep-f.sh (contents, props changed) vendor/zstd/dist/tests/gzip/zgrep-signal.sh (contents, props changed) vendor/zstd/dist/tests/gzip/znew-k.sh (contents, props changed) vendor/zstd/dist/tests/invalidDictionaries.c (contents, props changed) vendor/zstd/dist/tests/legacy.c (contents, props changed) vendor/zstd/dist/tests/longmatch.c (contents, props changed) vendor/zstd/dist/tests/namespaceTest.c (contents, props changed) vendor/zstd/dist/tests/paramgrill.c (contents, props changed) vendor/zstd/dist/tests/playTests.sh (contents, props changed) vendor/zstd/dist/tests/pool.c (contents, props changed) vendor/zstd/dist/tests/roundTripCrash.c (contents, props changed) vendor/zstd/dist/tests/symbols.c (contents, props changed) vendor/zstd/dist/tests/test-zstd-speed.py (contents, props changed) vendor/zstd/dist/tests/test-zstd-versions.py (contents, props changed) vendor/zstd/dist/tests/zbufftest.c (contents, props changed) vendor/zstd/dist/tests/zstreamtest.c (contents, props changed) vendor/zstd/dist/zlibWrapper/ vendor/zstd/dist/zlibWrapper/.gitignore vendor/zstd/dist/zlibWrapper/BUCK vendor/zstd/dist/zlibWrapper/Makefile (contents, props changed) vendor/zstd/dist/zlibWrapper/README.md vendor/zstd/dist/zlibWrapper/examples/ vendor/zstd/dist/zlibWrapper/examples/example.c (contents, props changed) vendor/zstd/dist/zlibWrapper/examples/example_original.c (contents, props changed) vendor/zstd/dist/zlibWrapper/examples/fitblk.c (contents, props changed) vendor/zstd/dist/zlibWrapper/examples/fitblk_original.c (contents, props changed) vendor/zstd/dist/zlibWrapper/examples/minigzip.c (contents, props changed) vendor/zstd/dist/zlibWrapper/examples/zwrapbench.c (contents, props changed) vendor/zstd/dist/zlibWrapper/gzclose.c (contents, props changed) vendor/zstd/dist/zlibWrapper/gzcompatibility.h (contents, props changed) vendor/zstd/dist/zlibWrapper/gzguts.h (contents, props changed) vendor/zstd/dist/zlibWrapper/gzlib.c (contents, props changed) vendor/zstd/dist/zlibWrapper/gzread.c (contents, props changed) vendor/zstd/dist/zlibWrapper/gzwrite.c (contents, props changed) vendor/zstd/dist/zlibWrapper/zstd_zlibwrapper.c (contents, props changed) vendor/zstd/dist/zlibWrapper/zstd_zlibwrapper.h (contents, props changed) Added: vendor/zstd/dist/.buckconfig ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/zstd/dist/.buckconfig Sat Apr 15 19:47:16 2017 (r316975) @@ -0,0 +1,9 @@ +[cxx] + cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4 + cflags = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wpointer-arith + cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4 + cxxflags = -std=c++11 -Wno-deprecated-declarations + gtest_dep = //contrib/pzstd:gtest + +[httpserver] + port = 0 Added: vendor/zstd/dist/.buckversion ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/zstd/dist/.buckversion Sat Apr 15 19:47:16 2017 (r316975) @@ -0,0 +1 @@ +c8dec2e8da52d483f6dd7c6cd2ad694e8e6fed2b Added: vendor/zstd/dist/.gitattributes ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/zstd/dist/.gitattributes Sat Apr 15 19:47:16 2017 (r316975) @@ -0,0 +1,24 @@ +# Set the default behavior +* text eol=lf + +# Explicitly declare source files +*.c text eol=lf +*.h text eol=lf + +# Denote files that should not be modified. +*.odt binary +*.png binary + +# Visual Studio +*.sln text eol=crlf +*.vcxproj* text eol=crlf +*.vcproj* text eol=crlf +*.suo binary +*.rc text eol=crlf + +# Windows +*.bat text eol=crlf +*.cmd text eol=crlf + +# .travis.yml merging +.travis.yml merge=ours Added: vendor/zstd/dist/.gitignore ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/zstd/dist/.gitignore Sat Apr 15 19:47:16 2017 (r316975) @@ -0,0 +1,41 @@ +# Object files +*.o +*.ko +*.dSYM + +# Libraries +*.lib +*.a + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +zstd +zstdmt +*.exe +*.out +*.app + +# Test artefacts +tmp* +dictionary* + +# Other files +.directory +_codelite/ +_zstdbench/ +.clang_complete +*.idea +*.swp +.DS_Store +googletest/ +*.d + +# Directories +bin/ +.buckd/ +buck-out/ Added: vendor/zstd/dist/.travis.yml ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/zstd/dist/.travis.yml Sat Apr 15 19:47:16 2017 (r316975) @@ -0,0 +1,51 @@ +# Long tests: run on commits to master branch/cron builds + +language: c +sudo: required +dist: trusty +matrix: + include: + # Ubuntu 14.04 + - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-test' + - env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test32' + - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test' + - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream' + - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest' + + - env: Cmd='make arminstall && make armtest' + - env: Cmd='make arminstall && make aarch64test' + - env: Cmd='make ppcinstall && make ppctest' + - env: Cmd='make ppcinstall && make ppc64test' + + + - env: Cmd='make gpp6install valgrindinstall && make -C zlibWrapper test && make -C zlibWrapper valgrindTest' + - env: Cmd='make -C tests versionsTest' + - env: Cmd='make gpp6install && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' + install: + - export CXX="g++-6" CC="gcc-6" + - env: Cmd='make gcc6install && CC=gcc-6 make uasan-test-zstd-nolegacy' + - env: Cmd='make gcc6install && CC=gcc-6 make uasan-test-zbuff' + + # OS X Mavericks + - env: Cmd="make gnu90build && make clean && make test && make clean && make travis-install" + os: osx + +git: + depth: 1 + +branches: + only: + - dev + - master + +script: + - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') + - echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST + - export FUZZERTEST=-T5mn; + export ZSTREAM_TESTTIME=-T5mn; + export DECODECORPUS_TESTTIME=-T1mn; + if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then + git fetch origin dev; + git checkout -f FETCH_HEAD; + fi; + sh -c "$Cmd" || travis_terminate 1; Added: vendor/zstd/dist/CONTRIBUTING.md ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/zstd/dist/CONTRIBUTING.md Sat Apr 15 19:47:16 2017 (r316975) @@ -0,0 +1,42 @@ +# Contributing to Zstandard +We want to make contributing to this project as easy and transparent as +possible. + +## Our Development Process +New versions are being developed in the "dev" branch, +or in their own feature branch. +When they are deemed ready for a release, they are merged into "master". + +As a consequences, all contributions must stage first through "dev" +or their own feature branch. + +## Pull Requests +We actively welcome your pull requests. + +1. Fork the repo and create your branch from `dev`. +2. If you've added code that should be tested, add tests. +3. If you've changed APIs, update the documentation. +4. Ensure the test suite passes. +5. Make sure your code lints. +6. If you haven't already, complete the Contributor License Agreement ("CLA"). + +## Contributor License Agreement ("CLA") +In order to accept your pull request, we need you to submit a CLA. You only need +to do this once to work on any of Facebook's open source projects. + +Complete your CLA here: + +## Issues +We use GitHub issues to track public bugs. Please ensure your description is +clear and has sufficient instructions to be able to reproduce the issue. + +Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe +disclosure of security bugs. In those cases, please go through the process +outlined on that page and do not file a public issue. + +## Coding Style +* 4 spaces for indentation rather than tabs + +## License +By contributing to Zstandard, you agree that your contributions will be licensed +under the [LICENSE](LICENSE) file in the root directory of this source tree. Added: vendor/zstd/dist/LICENSE ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/zstd/dist/LICENSE Sat Apr 15 19:47:16 2017 (r316975) @@ -0,0 +1,30 @@ +BSD License + +For Zstandard software + +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Added: vendor/zstd/dist/LICENSE-examples ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/zstd/dist/LICENSE-examples Sat Apr 15 19:47:16 2017 (r316975) @@ -0,0 +1,11 @@ +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +The examples provided by Facebook are for non-commercial testing and evaluation +purposes only. Facebook reserves all rights not expressly granted. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +FACEBOOK BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Added: vendor/zstd/dist/Makefile ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/zstd/dist/Makefile Sat Apr 15 19:47:16 2017 (r316975) @@ -0,0 +1,315 @@ +# ################################################################ +# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. +# ################################################################ + +PRGDIR = programs +ZSTDDIR = lib +BUILDIR = build +ZWRAPDIR = zlibWrapper +TESTDIR = tests + +# Define nul output +VOID = /dev/null + +ifneq (,$(filter Windows%,$(OS))) +EXT =.exe +else +EXT = +endif + +.PHONY: default +default: lib-release zstd-release + +.PHONY: all +all: | allmost examples manual + +.PHONY: allmost +allmost: + $(MAKE) -C $(ZSTDDIR) all + $(MAKE) -C $(PRGDIR) all + $(MAKE) -C $(TESTDIR) all + $(MAKE) -C $(ZWRAPDIR) all + +#skip zwrapper, can't build that on alternate architectures without the proper zlib installed +.PHONY: allarch +allarch: + $(MAKE) -C $(ZSTDDIR) all + $(MAKE) -C $(PRGDIR) all + $(MAKE) -C $(TESTDIR) all + +.PHONY: all32 +all32: + $(MAKE) -C $(PRGDIR) zstd32 + $(MAKE) -C $(TESTDIR) all32 + +.PHONY: lib +lib: + @$(MAKE) -C $(ZSTDDIR) $@ + +.PHONY: lib-release +lib-release: + @$(MAKE) -C $(ZSTDDIR) + +.PHONY: zstd +zstd: + @$(MAKE) -C $(PRGDIR) $@ + cp $(PRGDIR)/zstd$(EXT) . + +.PHONY: zstd-release +zstd-release: + @$(MAKE) -C $(PRGDIR) + cp $(PRGDIR)/zstd$(EXT) . + +.PHONY: zstdmt +zstdmt: + @$(MAKE) -C $(PRGDIR) $@ + cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT) + +.PHONY: zlibwrapper +zlibwrapper: + $(MAKE) -C $(ZWRAPDIR) test + +.PHONY: shortest +shortest: + $(MAKE) -C $(TESTDIR) $@ + +.PHONY: test +test: + $(MAKE) -C $(TESTDIR) $@ + +.PHONY: examples +examples: + CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all + +.PHONY: manual +manual: + $(MAKE) -C contrib/gen_html $@ + +.PHONY: clean +clean: + @$(MAKE) -C $(ZSTDDIR) $@ > $(VOID) + @$(MAKE) -C $(PRGDIR) $@ > $(VOID) + @$(MAKE) -C $(TESTDIR) $@ > $(VOID) + @$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID) + @$(MAKE) -C examples/ $@ > $(VOID) + @$(MAKE) -C contrib/gen_html $@ > $(VOID) + @$(RM) zstd$(EXT) zstdmt$(EXT) tmp* + @echo Cleaning completed + +#------------------------------------------------------------------------------ +# make install is validated only for Linux, OSX, Hurd and some BSD targets +#------------------------------------------------------------------------------ +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD)) +HOST_OS = POSIX +.PHONY: install uninstall travis-install clangtest gpptest armtest usan asan uasan + +install: + @$(MAKE) -C $(ZSTDDIR) $@ + @$(MAKE) -C $(PRGDIR) $@ + +uninstall: + @$(MAKE) -C $(ZSTDDIR) $@ + @$(MAKE) -C $(PRGDIR) $@ + +travis-install: + $(MAKE) install PREFIX=~/install_test_dir + +gppbuild: clean + g++ -v + CC=g++ $(MAKE) -C programs all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" + +gcc5build: clean + gcc-5 -v + CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror" + +gcc6build: clean + gcc-6 -v + CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror" + +clangbuild: clean + clang -v + CXX=clang++ CC=clang $(MAKE) all MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" + +m32build: clean + gcc -v + $(MAKE) all32 + +armbuild: clean + CC=arm-linux-gnueabi-gcc CFLAGS="-Werror" $(MAKE) allarch + +aarch64build: clean + CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allarch + +ppcbuild: clean + CC=powerpc-linux-gnu-gcc CLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allarch + +ppc64build: clean + CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allarch + +armfuzz: clean + CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest + +aarch64fuzz: clean + CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest + +ppcfuzz: clean + CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest + +ppc64fuzz: clean + CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" $(MAKE) -C $(TESTDIR) fuzztest + +gpptest: clean + CC=g++ $(MAKE) -C $(PRGDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" + +gcc5test: clean + gcc-5 -v + $(MAKE) all CC=gcc-5 MOREFLAGS="-Werror" + +gcc6test: clean + gcc-6 -v + $(MAKE) all CC=gcc-6 MOREFLAGS="-Werror" + +clangtest: clean + clang -v + $(MAKE) all CXX=clang-++ CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" + +armtest: clean + $(MAKE) -C $(TESTDIR) datagen # use native, faster + $(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" + +aarch64test: + $(MAKE) -C $(TESTDIR) datagen # use native, faster + $(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static" + +ppctest: clean + $(MAKE) -C $(TESTDIR) datagen # use native, faster + $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" + +ppc64test: clean + $(MAKE) -C $(TESTDIR) datagen # use native, faster + $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" + +arm-ppc-compilation: + $(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" + $(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static" + $(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" + $(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" + +# run UBsan with -fsanitize-recover=signed-integer-overflow +# due to a bug in UBsan when doing pointer subtraction +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303 + +usan: clean + $(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=undefined" + +asan: clean + $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address" + +asan-%: clean + LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address" $(MAKE) -C $(TESTDIR) $* + +msan: clean + $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer" # datagen.c fails this test for no obvious reason + +msan-%: clean + LDFLAGS=-fuse-ld=gold MOREFLAGS="-fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer" $(MAKE) -C $(TESTDIR) $* + +asan32: clean + $(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address" + +uasan: clean + $(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined" + +uasan-%: clean + LDFLAGS=-fuse-ld=gold MOREFLAGS="-Og -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined" $(MAKE) -C $(TESTDIR) $* + +tsan-%: clean + LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread" $(MAKE) -C $(TESTDIR) $* +apt-install: + sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES) + +apt-add-repo: + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + sudo apt-get update -y -qq + +ppcinstall: + APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install + +arminstall: + APT_PACKAGES="qemu-system-arm qemu-user-static gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install + +valgrindinstall: + APT_PACKAGES="valgrind" $(MAKE) apt-install + +libc6install: + APT_PACKAGES="libc6-dev-i386 gcc-multilib" $(MAKE) apt-install + +gcc6install: apt-add-repo + APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-6 gcc-6-multilib" $(MAKE) apt-install + +gpp6install: apt-add-repo + APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install + +clang38install: + APT_PACKAGES="clang-3.8" $(MAKE) apt-install + +endif + + +ifneq (,$(filter MSYS%,$(shell uname))) +HOST_OS = MSYS +CMAKE_PARAMS = -G"MSYS Makefiles" +endif + + +#------------------------------------------------------------------------ +#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets +#------------------------------------------------------------------------ +ifneq (,$(filter $(HOST_OS),MSYS POSIX)) +cmakebuild: + cmake --version + $(RM) -r $(BUILDIR)/cmake/build + mkdir $(BUILDIR)/cmake/build + cd $(BUILDIR)/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall + +c90build: clean + gcc -v + CFLAGS="-std=c90" $(MAKE) allmost # will fail, due to missing support for `long long` + +gnu90build: clean + gcc -v + CFLAGS="-std=gnu90" $(MAKE) allmost + +c99build: clean + gcc -v + CFLAGS="-std=c99" $(MAKE) allmost + +gnu99build: clean + gcc -v + CFLAGS="-std=gnu99" $(MAKE) allmost + +c11build: clean + gcc -v + CFLAGS="-std=c11" $(MAKE) allmost + +bmix64build: clean + gcc -v + CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test + +bmix32build: clean + gcc -v + CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(TESTDIR) test + +bmi32build: clean + gcc -v + CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test + +staticAnalyze: clean + gcc -v + CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all +endif Added: vendor/zstd/dist/NEWS ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/zstd/dist/NEWS Sat Apr 15 19:47:16 2017 (r316975) @@ -0,0 +1,261 @@ +v1.1.4 +cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski +cli : new : advanced benchmark command --priority=rt +cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77 +cli : fix : --rm remains silent when input is stdin +cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski +speed : improved decompression speed in streaming mode for single shot scenarios (+5%) +memory : DDict (decompression dictionary) memory usage down from 150 KB to 20 KB +arch : 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell +API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize() +API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value) +build: new: meson build system in contrib/meson, by Dima Krasner +build: improved cmake script, by @Majlen +build: added -Wformat-security flag, as recommended by Padraig Brady +doc : new : educational decoder, by Sean Purcell + +v1.1.3 +cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`) +cli : new : experimental target `make zstdmt`, with multi-threading support +cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell, based on prior work by Giuseppe Ottaviano. +cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski +cli : fix zstdless on Mac OS-X, by Andrew Janke +cli : fix #232 "compress non-files" +dictBuilder : improved dictionary generation quality, thanks to Nick Terrell +API : new : lib/compress/ZSTDMT_compress.h multithreading API (experimental) +API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul +API : new : ZDICT_finalizeDictionary() +API : fix : ZSTD_initCStream_usingCDict() properly writes dictID into frame header, by Gregory Szorc (#511) +API : fix : all symbols properly exposed in libzstd, by Nick Terrell +build : support for Solaris target, by Przemyslaw Skibinski +doc : clarified specification, by Sean Purcell + +v1.1.2 +API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init +API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize() +API : zbuff : changed : prototypes now generate deprecation warnings +lib : improved : faster decompression speed at ultra compression settings and 32-bits mode +lib : changed : only public ZSTD_ symbols are now exposed +lib : changed : reduced usage of stack memory +lib : fixed : several corner case bugs, by Nick Terrell +cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski +cli : new : preserve file attributes +cli : new : added zstdless and zstdgrep tools +cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd) +cli : fixed : zstdcat +zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski +install : better compatibility with FreeBSD, by Dimitry Andric +source tree : changed : zbuff source files moved to lib/deprecated + +v1.1.1 +New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption +New : doc/zstd_manual.html, by Przemyslaw Skibinski +Improved : slightly better compression ratio at --ultra levels (>= 20) +Improved : better memory usage when using streaming compression API, thanks to @Rogier-5 report +Added : API : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section) +Added : example/multiple_streaming_compression.c +Changed : zstd_errors.h is now installed within /include (and replaces errors_public.h) +Updated man page +Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets + +v1.1.0 +New : contrib/pzstd, parallel version of zstd, by Nick Terrell +added : NetBSD install target (#338) +Improved : speed for batches of small files +Improved : speed of zlib wrapper, by Przemyslaw Skibinski +Changed : libzstd on Windows supports legacy formats, by Christophe Chevalier +Fixed : CLI -d output to stdout by default when input is stdin (#322) +Fixed : CLI correctly detects console on Mac OS-X +Fixed : CLI supports recursive mode `-r` on Mac OS-X +Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski +Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319) +Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365) +Fixed : zstd-pgo, reported by octoploid (#329) + +v1.0.0 +Change Licensing, all project is now BSD, Copyright Facebook +Small decompression speed improvement +API : Streaming API supports legacy format +API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParamter() +CLI supports legacy formats v0.4+ +Fixed : compression fails on certain huge files, reported by Jesse McGrew +Enhanced documentation, by Przemyslaw Skibinski + +v0.8.1 +New streaming API +Changed : --ultra now enables levels beyond 19 +Changed : -i# now selects benchmark time in second +Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell +Fixed : speed regression on specific patterns (#272) +Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291) +Fixed : ICC compilation, by Przemyslaw Skibinski + +v0.8.0 +Improved : better speed on clang and gcc -O2, thanks to Eric Biggers +New : Build on FreeBSD and DragonFly, thanks to JrMarino +Changed : modified API : ZSTD_compressEnd() +Fixed : legacy mode with ZSTD_HEAPMODE=0, by Christopher Bergqvist +Fixed : premature end of frame when zero-sized raw block, reported by Eric Biggers +Fixed : large dictionaries (> 384 KB), reported by Ilona Papava +Fixed : checksum correctly checked in single-pass mode +Fixed : combined --test amd --rm, reported by Andreas M. Nilsson +Modified : minor compression level adaptations +Updated : compression format specification to v0.2.0 +changed : zstd.h moved to /lib directory + +v0.7.5 +Transition version, supporting decoding of v0.8.x + +v0.7.4 +Added : homebrew for Mac, by Daniel Cade +Added : more examples +Fixed : segfault when using small dictionaries, reported by Felix Handte +Modified : default compression level for CLI is now 3 +Updated : specification, to v0.1.1 + +v0.7.3 +New : compression format specification +New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner. +New : `ZSTD_getDecompressedSize()` +New : OpenBSD target, by Juan Francisco Cantero Hurtado +New : `examples` directory +fixed : dictBuilder using HC levels, reported by Bartosz Taudul +fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte +fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by Greg Slazinski +modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section) +modified : legacy functions no longer need magic number + +v0.7.2 +fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski. +fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner. +fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner. + +v0.7.1 +fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier +fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв +fixed : corruption issue, reported by cj +modified : checksum enabled by default in command line mode + +v0.7.0 +New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski +New : Command `--rm`, to remove source file after successful de/compression +New : Visual build scripts, by Christophe Chevalier +New : Support for Sparse File-systems (do not use space for zero-filled sectors) +New : Frame checksum support +New : Support pass-through mode (when using `-df`) +API : more efficient Dictionary API : `ZSTD_compress_usingCDict()`, `ZSTD_decompress_usingDDict()` +API : create dictionary files from custom content, by Giuseppe Ottaviano +API : support for custom malloc/free functions +New : controllable Dictionary ID +New : Support for skippable frames + +v0.6.1 +New : zlib wrapper API, thanks to Przemyslaw Skibinski +New : Ability to compile compressor / decompressor separately +Changed : new lib directory structure +Fixed : Legacy codec v0.5 compatible with dictionary decompression +Fixed : Decoder corruption error (#173) +Fixed : null-string roundtrip (#176) +New : benchmark mode can select directory as input +Experimental : midipix support, VMS support + +v0.6.0 +Stronger high compression modes, thanks to Przemyslaw Skibinski +API : ZSTD_getFrameParams() provides size of decompressed content +New : highest compression modes require `--ultra` command to fully unleash their capacity +Fixed : zstd cli return error code > 0 and removes dst file artifact when decompression fails, thanks to Chip Turner + +v0.5.1 +New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski +Changed : Dictionary builder integrated into libzstd and zstd cli +Changed (!) : zstd cli now uses "multiple input files" as default mode. See `zstd -h`. +Fix : high compression modes for big-endian platforms +New : zstd cli : `-t` | `--test` command + +v0.5.0 +New : dictionary builder utility +Changed : streaming & dictionary API +Improved : better compression of small data + +v0.4.7 +Improved : small compression speed improvement in HC mode +Changed : `zstd_decompress.c` has ZSTD_LEGACY_SUPPORT to 0 by default +fix : bt search bug + +v0.4.6 +fix : fast compression mode on Windows +New : cmake configuration file, thanks to Artyom Dymchenko +Improved : high compression mode on repetitive data +New : block-level API +New : ZSTD_duplicateCCtx() + +v0.4.5 +new : -m/--multiple : compress/decompress multiple files + +v0.4.4 +Fixed : high compression modes for Windows 32 bits +new : external dictionary API extended to buffered mode and accessible through command line +new : windows DLL project, thanks to Christophe Chevalier + +v0.4.3 : +new : external dictionary API +new : zstd-frugal + +v0.4.2 : +Generic minor improvements for small blocks +Fixed : big-endian compatibility, by Peter Harris (#85) + +v0.4.1 +Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben) +removed `zstd.c` + +v0.4.0 +Command line utility compatible with high compression levels +Removed zstdhc => merged into zstd +Added : ZBUFF API (see zstd_buffered.h) +Rolling buffer support + +v0.3.6 +small blocks params + +v0.3.5 +minor generic compression improvements + +v0.3.4 +Faster fast cLevels + +v0.3.3 +Small compression ratio improvement + +v0.3.2 +Fixed Visual Studio + +v0.3.1 : +Small compression ratio improvement + +v0.3 +HC mode : compression levels 2-26 + +v0.2.2 +Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier + +v0.2.1 +Fix : Read errors, advanced fuzzer tests, by Hanno Böck + +v0.2.0 +**Breaking format change** +Faster decompression speed +Can still decode v0.1 format + +v0.1.3 +fix uninitialization warning, reported by Evan Nemerson + +v0.1.2 +frame concatenation support + +v0.1.1 +fix compression bug +detects write-flush errors + +v0.1.0 +first release Added: vendor/zstd/dist/PATENTS ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/zstd/dist/PATENTS Sat Apr 15 19:47:16 2017 (r316975) @@ -0,0 +1,33 @@ +Additional Grant of Patent Rights Version 2 + +"Software" means the Zstandard software distributed by Facebook, Inc. + +Facebook, Inc. ("Facebook") hereby grants to each recipient of the Software +("you") a perpetual, worldwide, royalty-free, non-exclusive, irrevocable +(subject to the termination provision below) license under any Necessary +Claims, to make, have made, use, sell, offer to sell, import, and otherwise +transfer the Software. For avoidance of doubt, no license is granted under +Facebook’s rights in any patent claims that are infringed by (i) modifications +to the Software made by you or any third party or (ii) the Software in +combination with any software or other technology. + +The license granted hereunder will terminate, automatically and without notice, +if you (or any of your subsidiaries, corporate affiliates or agents) initiate +directly or indirectly, or take a direct financial interest in, any Patent +Assertion: (i) against Facebook or any of its subsidiaries or corporate +affiliates, (ii) against any party if such Patent Assertion arises in whole or +in part from any software, technology, product or service of Facebook or any of +its subsidiaries or corporate affiliates, or (iii) against any party relating +to the Software. Notwithstanding the foregoing, if Facebook or any of its +subsidiaries or corporate affiliates files a lawsuit alleging patent +infringement against you in the first instance, and you respond by filing a +patent infringement counterclaim in that lawsuit against that party that is +unrelated to the Software, the license granted hereunder will not terminate +under section (i) of this paragraph due to such counterclaim. + +A "Necessary Claim" is a claim of a patent owned by Facebook that is +necessarily infringed by the Software standing alone. + +A "Patent Assertion" is any lawsuit or other action alleging direct, indirect, +or contributory infringement or inducement to infringe any patent, including a +cross-claim or counterclaim. Added: vendor/zstd/dist/README.md ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/zstd/dist/README.md Sat Apr 15 19:47:16 2017 (r316975) @@ -0,0 +1,141 @@ + __Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm, + targeting real-time compression scenarios at zlib-level and better compression ratios. + +It is provided as an open-source BSD-licensed **C** library, +and a command line utility producing and decoding `.zst` and `.gz` files. +For other programming languages, +you can consult a list of known ports on [Zstandard homepage](http://www.zstd.net/#other-languages). + +|Branch |Status | +|------------|---------| +|master | [![Build Status](https://travis-ci.org/facebook/zstd.svg?branch=master)](https://travis-ci.org/facebook/zstd) | +|dev | [![Build Status](https://travis-ci.org/facebook/zstd.svg?branch=dev)](https://travis-ci.org/facebook/zstd) | + +As a reference, several fast compression algorithms were tested and compared +on a server running Linux Mint Debian Edition (`Linux version 4.8.0-1-amd64`), +with a Core i7-6700K CPU @ 4.0GHz, +using [lzbench v1.6], an open-source in-memory benchmark by @inikep +compiled with GCC 6.3.0, +on the [Silesia compression corpus]. + +[lzbench v1.6]: https://github.com/inikep/lzbench +[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia + +| Compressor name | Ratio | Compression| Decompress.| +| --------------- | ------| -----------| ---------- | +| **zstd 1.1.3 -1** | 2.877 | 430 MB/s | 1110 MB/s | +| zlib 1.2.8 -1 | 2.743 | 110 MB/s | 400 MB/s | +| brotli 0.5.2 -0 | 2.708 | 400 MB/s | 430 MB/s | +| quicklz 1.5.0 -1 | 2.238 | 550 MB/s | 710 MB/s | +| lzo1x 2.09 -1 | 2.108 | 650 MB/s | 830 MB/s | +| lz4 1.7.5 | 2.101 | 720 MB/s | 3600 MB/s | +| snappy 1.1.3 | 2.091 | 500 MB/s | 1650 MB/s | +| lzf 3.6 -1 | 2.077 | 400 MB/s | 860 MB/s | + +[zlib]:http://www.zlib.net/ +[LZ4]: http://www.lz4.org/ + +Zstd can also offer stronger compression ratios at the cost of compression speed. +Speed vs Compression trade-off is configurable by small increments. Decompression speed is preserved and remains roughly the same at all settings, a property shared by most LZ compression algorithms, such as [zlib] or lzma. + +The following tests were run on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with GCC 5.2.1, on the [Silesia compression corpus]. + +Compression Speed vs Ratio | Decompression Speed +---------------------------|-------------------- +![Compression Speed vs Ratio](doc/images/Cspeed4.png "Compression Speed vs Ratio") | ![Decompression Speed](doc/images/Dspeed4.png "Decompression Speed") + +Several algorithms can produce higher compression ratios, but at slower speeds, falling outside of the graph. +For a larger picture including very slow modes, [click on this link](doc/images/DCspeed5.png) . + + +### The case for Small Data compression + +Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives. + +The smaller the amount of data to compress, the more difficult it is to compress. This problem is common to all compression algorithms, and reason is, compression algorithms learn from past data how to compress future data. But at the beginning of a new data set, there is no "past" to build upon. + +To solve this situation, Zstd offers a __training mode__, which can be used to tune the algorithm for a selected type of data. +Training Zstandard is achieved by provide it with a few samples (one file per sample). The result of this training is stored in a file called "dictionary", which must be loaded before compression and decompression. +Using this dictionary, the compression ratio achievable on small data improves dramatically. + +The following example uses the `github-users` [sample set](https://github.com/facebook/zstd/releases/tag/v1.1.3), created from [github public API](https://developer.github.com/v3/users/#get-all-users). +It consists of roughly 10K records weighting about 1KB each. + +Compression Ratio | Compression Speed | Decompression Speed +------------------|-------------------|-------------------- +![Compression Ratio](doc/images/dict-cr.png "Compression Ratio") | ![Compression Speed](doc/images/dict-cs.png "Compression Speed") | ![Decompression Speed](doc/images/dict-ds.png "Decompression Speed") + + +These compression gains are achieved while simultaneously providing _faster_ compression and decompression speeds. + +Training works if there is some correlation in a family of small data samples. The more data-specific a dictionary is, the more efficient it is (there is no _universal dictionary_). +Hence, deploying one dictionary per type of data will provide the greatest benefits. +Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file. + +#### Dictionary compression How To : + +1) Create the dictionary + +`zstd --train FullPathToTrainingSet/* -o dictionaryName` + +2) Compress with dictionary + +`zstd -D dictionaryName FILE` + +3) Decompress with dictionary + +`zstd -D dictionaryName --decompress FILE.zst` + + +### Build + +Once you have the repository cloned, there are multiple ways provided to build Zstandard. + +#### Makefile + +If your system is compatible with a standard `make` (or `gmake`) binary generator, +you can simply run it at the root directory. +It will generate `zstd` within root directory. + +Other available options include : +- `make install` : create and install zstd binary, library and man page +- `make test` : create and run `zstd` and test tools on local platform + +#### cmake + +A `cmake` project generator is provided within `build/cmake`. +It can generate Makefiles or other build scripts +to create `zstd` binary, and `libzstd` dynamic and static libraries. + +#### Meson + +A Meson project is provided within `contrib/meson`. + +#### Visual Studio (Windows) + +Going into `build` directory, you will find additional possibilities : +- Projects for Visual Studio 2005, 2008 and 2010 + + VS2010 project is compatible with VS2012, VS2013 and VS2015 +- Automated build scripts for Visual compiler by @KrzysFR , in `build/VS_scripts`, + which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution. + + +### Status + +Zstandard is currently deployed within Facebook. It is used daily to compress and decompress very large amounts of data in multiple formats and use cases. +Zstandard is considered safe for production environments. + +### License + +Zstandard is [BSD-licensed](LICENSE). We also provide an [additional patent grant](PATENTS). + +### Contributing + +The "dev" branch is the one where all contributions will be merged before reaching "master". +If you plan to propose a patch, please commit into the "dev" branch or its own feature branch. +Direct commit to "master" are not permitted. +For more information, please read [CONTRIBUTING](CONTRIBUTING.md). + +### Miscellaneous + +Zstd entropy stage is provided by [Huff0 and FSE, from Finite State Entropy library](https://github.com/Cyan4973/FiniteStateEntropy). Added: vendor/zstd/dist/TESTING.md ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/zstd/dist/TESTING.md Sat Apr 15 19:47:16 2017 (r316975) @@ -0,0 +1,44 @@ +Testing +======= *** DIFF OUTPUT TRUNCATED AT 1000 LINES *** From owner-svn-src-vendor@freebsd.org Sat Apr 15 19:47:51 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 76C82D3F136; Sat, 15 Apr 2017 19:47:51 +0000 (UTC) (envelope-from bapt@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 30CC8F93; Sat, 15 Apr 2017 19:47:51 +0000 (UTC) (envelope-from bapt@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3FJlo57094517; Sat, 15 Apr 2017 19:47:50 GMT (envelope-from bapt@FreeBSD.org) Received: (from bapt@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3FJlo7A094516; Sat, 15 Apr 2017 19:47:50 GMT (envelope-from bapt@FreeBSD.org) Message-Id: <201704151947.v3FJlo7A094516@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: bapt set sender to bapt@FreeBSD.org using -f From: Baptiste Daroussin Date: Sat, 15 Apr 2017 19:47:50 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316976 - vendor/zstd/1.1.4 X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 15 Apr 2017 19:47:51 -0000 Author: bapt Date: Sat Apr 15 19:47:50 2017 New Revision: 316976 URL: https://svnweb.freebsd.org/changeset/base/316976 Log: Tag import of zstd 1.1.4 Added: vendor/zstd/1.1.4/ - copied from r316975, vendor/zstd/dist/ From owner-svn-src-vendor@freebsd.org Sat Apr 15 21:25:40 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id A0E8ED3F8B2; Sat, 15 Apr 2017 21:25:40 +0000 (UTC) (envelope-from bapt@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 7284A8EF; Sat, 15 Apr 2017 21:25:40 +0000 (UTC) (envelope-from bapt@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3FLPdph035925; Sat, 15 Apr 2017 21:25:39 GMT (envelope-from bapt@FreeBSD.org) Received: (from bapt@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3FLPdOX035924; Sat, 15 Apr 2017 21:25:39 GMT (envelope-from bapt@FreeBSD.org) Message-Id: <201704152125.v3FLPdOX035924@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: bapt set sender to bapt@FreeBSD.org using -f From: Baptiste Daroussin Date: Sat, 15 Apr 2017 21:25:39 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316987 - in vendor/zstd/dist: build lib/dll programs programs/windres X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 15 Apr 2017 21:25:40 -0000 Author: bapt Date: Sat Apr 15 21:25:39 2017 New Revision: 316987 URL: https://svnweb.freebsd.org/changeset/base/316987 Log: Remove files confusing the git exporter Deleted: vendor/zstd/dist/build/ vendor/zstd/dist/lib/dll/ vendor/zstd/dist/programs/windres/ Modified: vendor/zstd/dist/programs/fileio.c Modified: vendor/zstd/dist/programs/fileio.c ============================================================================== --- vendor/zstd/dist/programs/fileio.c Sat Apr 15 21:19:49 2017 (r316986) +++ vendor/zstd/dist/programs/fileio.c Sat Apr 15 21:25:39 2017 (r316987) @@ -138,7 +138,7 @@ static U32 g_dictIDFlag = 1; void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; } static U32 g_checksumFlag = 1; void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; } -static U32 g_removeSrcFile = 0; +static U32 g_removeSrcFile = 1; void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } static U32 g_memLimit = 0; void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; } From owner-svn-src-vendor@freebsd.org Sat Apr 15 21:33:45 2017 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 8FC65D3FAC9; Sat, 15 Apr 2017 21:33:45 +0000 (UTC) (envelope-from bapt@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 57195CE0; Sat, 15 Apr 2017 21:33:45 +0000 (UTC) (envelope-from bapt@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v3FLXi8K039837; Sat, 15 Apr 2017 21:33:44 GMT (envelope-from bapt@FreeBSD.org) Received: (from bapt@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v3FLXiOU039836; Sat, 15 Apr 2017 21:33:44 GMT (envelope-from bapt@FreeBSD.org) Message-Id: <201704152133.v3FLXiOU039836@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: bapt set sender to bapt@FreeBSD.org using -f From: Baptiste Daroussin Date: Sat, 15 Apr 2017 21:33:44 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r316988 - vendor/zstd/dist/programs X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 15 Apr 2017 21:33:45 -0000 Author: bapt Date: Sat Apr 15 21:33:44 2017 New Revision: 316988 URL: https://svnweb.freebsd.org/changeset/base/316988 Log: revert a change committed by accident: it is freebsd specific and should remain in contrib, not vendor Modified: vendor/zstd/dist/programs/fileio.c Modified: vendor/zstd/dist/programs/fileio.c ============================================================================== --- vendor/zstd/dist/programs/fileio.c Sat Apr 15 21:25:39 2017 (r316987) +++ vendor/zstd/dist/programs/fileio.c Sat Apr 15 21:33:44 2017 (r316988) @@ -138,7 +138,7 @@ static U32 g_dictIDFlag = 1; void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; } static U32 g_checksumFlag = 1; void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; } -static U32 g_removeSrcFile = 1; +static U32 g_removeSrcFile = 0; void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } static U32 g_memLimit = 0; void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; }