Date: Tue, 17 Jun 2014 07:35:57 +0000 (UTC) From: Xin LI <delphij@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r267565 - vendor-sys/illumos/dist/common/zfs vendor-sys/illumos/dist/uts/common/fs/zfs vendor-sys/illumos/dist/uts/common/fs/zfs/sys vendor/illumos/dist/cmd/zdb vendor/illumos/dist/cmd/... Message-ID: <201406170735.s5H7Zvhh016469@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: delphij Date: Tue Jun 17 07:35:54 2014 New Revision: 267565 URL: http://svnweb.freebsd.org/changeset/base/267565 Log: 4757 ZFS embedded-data block pointers ("zero block compression") 4913 zfs release should not be subject to space checks Reviewed by: Adam Leventhal <ahl@delphix.com> Reviewed by: Max Grossman <max.grossman@delphix.com> Reviewed by: George Wilson <george.wilson@delphix.com> Reviewed by: Christopher Siden <christopher.siden@delphix.com> Reviewed by: Dan McDonald <danmcd@omniti.com> Approved by: Dan McDonald <danmcd@omniti.com> illumos/illumos-dist@5d7b4d438c4a51eccc95e77a83a437b4d48380eb Added: vendor-sys/illumos/dist/uts/common/fs/zfs/blkptr.c (contents, props changed) vendor-sys/illumos/dist/uts/common/fs/zfs/sys/blkptr.h (contents, props changed) Modified: vendor-sys/illumos/dist/common/zfs/zfeature_common.c vendor-sys/illumos/dist/common/zfs/zfeature_common.h vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c vendor-sys/illumos/dist/uts/common/fs/zfs/dmu.c vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_send.c vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_traverse.c vendor-sys/illumos/dist/uts/common/fs/zfs/dnode.c vendor-sys/illumos/dist/uts/common/fs/zfs/dnode_sync.c vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_dataset.c vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_destroy.c vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_scan.c vendor-sys/illumos/dist/uts/common/fs/zfs/dsl_userhold.c vendor-sys/illumos/dist/uts/common/fs/zfs/spa.c vendor-sys/illumos/dist/uts/common/fs/zfs/spa_misc.c vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dbuf.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu_impl.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/dmu_send.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa_impl.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zfs_ioctl.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zio.h vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ioctl.c vendor-sys/illumos/dist/uts/common/fs/zfs/zil.c vendor-sys/illumos/dist/uts/common/fs/zfs/zio.c vendor-sys/illumos/dist/uts/common/fs/zfs/zio_compress.c vendor-sys/illumos/dist/uts/common/fs/zfs/zvol.c Changes in other areas also in this revision: Modified: vendor/illumos/dist/cmd/zdb/zdb.c vendor/illumos/dist/cmd/zfs/zfs_main.c vendor/illumos/dist/cmd/zstreamdump/zstreamdump.c vendor/illumos/dist/cmd/ztest/ztest.c vendor/illumos/dist/lib/libzfs/common/libzfs.h vendor/illumos/dist/lib/libzfs/common/libzfs_sendrecv.c vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.c vendor/illumos/dist/lib/libzfs_core/common/libzfs_core.h vendor/illumos/dist/man/man1m/zfs.1m vendor/illumos/dist/man/man5/zpool-features.5 Modified: vendor-sys/illumos/dist/common/zfs/zfeature_common.c ============================================================================== --- vendor-sys/illumos/dist/common/zfs/zfeature_common.c Tue Jun 17 07:11:00 2014 (r267564) +++ vendor-sys/illumos/dist/common/zfs/zfeature_common.c Tue Jun 17 07:35:54 2014 (r267565) @@ -215,4 +215,9 @@ zpool_feature_init(void) "com.joyent:filesystem_limits", "filesystem_limits", "Filesystem and snapshot limits.", B_TRUE, B_FALSE, B_FALSE, filesystem_limits_deps); + + zfeature_register(SPA_FEATURE_EMBEDDED_DATA, + "com.delphix:embedded_data", "embedded_data", + "Blocks which compress very well use even less space.", + B_FALSE, B_TRUE, B_TRUE, NULL); } Modified: vendor-sys/illumos/dist/common/zfs/zfeature_common.h ============================================================================== --- vendor-sys/illumos/dist/common/zfs/zfeature_common.h Tue Jun 17 07:11:00 2014 (r267564) +++ vendor-sys/illumos/dist/common/zfs/zfeature_common.h Tue Jun 17 07:35:54 2014 (r267565) @@ -48,6 +48,7 @@ typedef enum spa_feature { SPA_FEATURE_ENABLED_TXG, SPA_FEATURE_HOLE_BIRTH, SPA_FEATURE_EXTENSIBLE_DATASET, + SPA_FEATURE_EMBEDDED_DATA, SPA_FEATURE_BOOKMARKS, SPA_FEATURE_FS_SS_LIMIT, SPA_FEATURES @@ -68,7 +69,7 @@ typedef struct zfeature_info { const spa_feature_t *fi_depends; } zfeature_info_t; -typedef int (zfeature_func_t)(zfeature_info_t *fi, void *arg); +typedef int (zfeature_func_t)(zfeature_info_t *, void *); #define ZFS_FEATURE_DEBUG @@ -77,8 +78,8 @@ extern zfeature_info_t spa_feature_table extern boolean_t zfeature_is_valid_guid(const char *); extern boolean_t zfeature_is_supported(const char *); -extern int zfeature_lookup_name(const char *name, spa_feature_t *res); -extern boolean_t zfeature_depends_on(spa_feature_t fid, spa_feature_t check); +extern int zfeature_lookup_name(const char *, spa_feature_t *); +extern boolean_t zfeature_depends_on(spa_feature_t, spa_feature_t); extern void zpool_feature_init(void); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c Tue Jun 17 07:11:00 2014 (r267564) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c Tue Jun 17 07:35:54 2014 (r267565) @@ -744,8 +744,10 @@ buf_discard_identity(arc_buf_hdr_t *hdr) } static arc_buf_hdr_t * -buf_hash_find(uint64_t spa, const dva_t *dva, uint64_t birth, kmutex_t **lockp) +buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp) { + const dva_t *dva = BP_IDENTITY(bp); + uint64_t birth = BP_PHYSICAL_BIRTH(bp); uint64_t idx = BUF_HASH_INDEX(spa, dva, birth); kmutex_t *hash_lock = BUF_HASH_LOCK(idx); arc_buf_hdr_t *buf; @@ -777,6 +779,8 @@ buf_hash_insert(arc_buf_hdr_t *buf, kmut arc_buf_hdr_t *fbuf; uint32_t i; + ASSERT(!DVA_IS_EMPTY(&buf->b_dva)); + ASSERT(buf->b_birth != 0); ASSERT(!HDR_IN_HASH_TABLE(buf)); *lockp = hash_lock; mutex_enter(hash_lock); @@ -2738,10 +2742,10 @@ arc_getbuf_func(zio_t *zio, arc_buf_t *b static void arc_read_done(zio_t *zio) { - arc_buf_hdr_t *hdr, *found; + arc_buf_hdr_t *hdr; arc_buf_t *buf; arc_buf_t *abuf; /* buffer we're assigning to callback */ - kmutex_t *hash_lock; + kmutex_t *hash_lock = NULL; arc_callback_t *callback_list, *acb; int freeable = FALSE; @@ -2756,12 +2760,22 @@ arc_read_done(zio_t *zio) * reason for it not to be found is if we were freed during the * read. */ - found = buf_hash_find(hdr->b_spa, &hdr->b_dva, hdr->b_birth, - &hash_lock); - - ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) || - (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) || - (found == hdr && HDR_L2_READING(hdr))); + if (HDR_IN_HASH_TABLE(hdr)) { + ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp)); + ASSERT3U(hdr->b_dva.dva_word[0], ==, + BP_IDENTITY(zio->io_bp)->dva_word[0]); + ASSERT3U(hdr->b_dva.dva_word[1], ==, + BP_IDENTITY(zio->io_bp)->dva_word[1]); + + arc_buf_hdr_t *found = buf_hash_find(hdr->b_spa, zio->io_bp, + &hash_lock); + + ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && + hash_lock == NULL) || + (found == hdr && + DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) || + (found == hdr && HDR_L2_READING(hdr))); + } hdr->b_flags &= ~ARC_L2_EVICTED; if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH)) @@ -2885,16 +2899,25 @@ arc_read(zio_t *pio, spa_t *spa, const b void *private, zio_priority_t priority, int zio_flags, uint32_t *arc_flags, const zbookmark_t *zb) { - arc_buf_hdr_t *hdr; + arc_buf_hdr_t *hdr = NULL; arc_buf_t *buf = NULL; - kmutex_t *hash_lock; + kmutex_t *hash_lock = NULL; zio_t *rzio; uint64_t guid = spa_load_guid(spa); + ASSERT(!BP_IS_EMBEDDED(bp) || + BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA); + top: - hdr = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp), - &hash_lock); - if (hdr && hdr->b_datacnt > 0) { + if (!BP_IS_EMBEDDED(bp)) { + /* + * Embedded BP's have no DVA and require no I/O to "read". + * Create an anonymous arc buf to back it. + */ + hdr = buf_hash_find(guid, bp, &hash_lock); + } + + if (hdr != NULL && hdr->b_datacnt > 0) { *arc_flags |= ARC_CACHED; @@ -2968,7 +2991,7 @@ top: done(NULL, buf, private); } else { uint64_t size = BP_GET_LSIZE(bp); - arc_callback_t *acb; + arc_callback_t *acb; vdev_t *vd = NULL; uint64_t addr = 0; boolean_t devw = B_FALSE; @@ -2977,15 +3000,17 @@ top: if (hdr == NULL) { /* this block is not in the cache */ - arc_buf_hdr_t *exists; + arc_buf_hdr_t *exists = NULL; arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp); buf = arc_buf_alloc(spa, size, private, type); hdr = buf->b_hdr; - hdr->b_dva = *BP_IDENTITY(bp); - hdr->b_birth = BP_PHYSICAL_BIRTH(bp); - hdr->b_cksum0 = bp->blk_cksum.zc_word[0]; - exists = buf_hash_insert(hdr, &hash_lock); - if (exists) { + if (!BP_IS_EMBEDDED(bp)) { + hdr->b_dva = *BP_IDENTITY(bp); + hdr->b_birth = BP_PHYSICAL_BIRTH(bp); + hdr->b_cksum0 = bp->blk_cksum.zc_word[0]; + exists = buf_hash_insert(hdr, &hash_lock); + } + if (exists != NULL) { /* somebody beat us to the hash insert */ mutex_exit(hash_lock); buf_discard_identity(hdr); @@ -3057,7 +3082,8 @@ top: vd = NULL; } - mutex_exit(hash_lock); + if (hash_lock != NULL) + mutex_exit(hash_lock); /* * At this point, we have a level 1 cache miss. Try again in @@ -3192,8 +3218,9 @@ arc_freed(spa_t *spa, const blkptr_t *bp kmutex_t *hash_lock; uint64_t guid = spa_load_guid(spa); - hdr = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp), - &hash_lock); + ASSERT(!BP_IS_EMBEDDED(bp)); + + hdr = buf_hash_find(guid, bp, &hash_lock); if (hdr == NULL) return; if (HDR_BUF_AVAILABLE(hdr)) { @@ -3509,7 +3536,7 @@ arc_write_done(zio_t *zio) ASSERT(hdr->b_acb == NULL); if (zio->io_error == 0) { - if (BP_IS_HOLE(zio->io_bp)) { + if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) { buf_discard_identity(hdr); } else { hdr->b_dva = *BP_IDENTITY(zio->io_bp); @@ -3521,10 +3548,10 @@ arc_write_done(zio_t *zio) } /* - * If the block to be written was all-zero, we may have - * compressed it away. In this case no write was performed - * so there will be no dva/birth/checksum. The buffer must - * therefore remain anonymous (and uncached). + * If the block to be written was all-zero or compressed enough to be + * embedded in the BP, no write was performed so there will be no + * dva/birth/checksum. The buffer must therefore remain anonymous + * (and uncached). */ if (!BUF_EMPTY(hdr)) { arc_buf_hdr_t *exists; @@ -4818,7 +4845,7 @@ static boolean_t l2arc_compress_buf(l2arc_buf_hdr_t *l2hdr) { void *cdata; - size_t csize, len; + size_t csize, len, rounded; ASSERT(l2hdr->b_compress == ZIO_COMPRESS_OFF); ASSERT(l2hdr->b_tmp_cdata != NULL); @@ -4828,6 +4855,12 @@ l2arc_compress_buf(l2arc_buf_hdr_t *l2hd csize = zio_compress_data(ZIO_COMPRESS_LZ4, l2hdr->b_tmp_cdata, cdata, l2hdr->b_asize); + rounded = P2ROUNDUP(csize, (size_t)SPA_MINBLOCKSIZE); + if (rounded > csize) { + bzero((char *)cdata + csize, rounded - csize); + csize = rounded; + } + if (csize == 0) { /* zero block, indicate that there's nothing to write */ zio_data_buf_free(cdata, len); Added: vendor-sys/illumos/dist/uts/common/fs/zfs/blkptr.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/blkptr.c Tue Jun 17 07:35:54 2014 (r267565) @@ -0,0 +1,119 @@ +/* + * CDDL HEADER START + * + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2013 by Delphix. All rights reserved. + */ + +#include <sys/zfs_context.h> +#include <sys/zio.h> +#include <sys/zio_compress.h> + +/* + * Embedded-data Block Pointers + * + * Normally, block pointers point (via their DVAs) to a block which holds data. + * If the data that we need to store is very small, this is an inefficient + * use of space, because a block must be at minimum 1 sector (typically 512 + * bytes or 4KB). Additionally, reading these small blocks tends to generate + * more random reads. + * + * Embedded-data Block Pointers allow small pieces of data (the "payload", + * up to 112 bytes) to be stored in the block pointer itself, instead of + * being pointed to. The "Pointer" part of this name is a bit of a + * misnomer, as nothing is pointed to. + * + * BP_EMBEDDED_TYPE_DATA block pointers allow highly-compressible data to + * be embedded in the block pointer. The logic for this is handled in + * the SPA, by the zio pipeline. Therefore most code outside the zio + * pipeline doesn't need special-cases to handle these block pointers. + * + * See spa.h for details on the exact layout of embedded block pointers. + */ + +void +encode_embedded_bp_compressed(blkptr_t *bp, void *data, + enum zio_compress comp, int uncompressed_size, int compressed_size) +{ + uint64_t *bp64 = (uint64_t *)bp; + uint64_t w = 0; + uint8_t *data8 = data; + + ASSERT3U(compressed_size, <=, BPE_PAYLOAD_SIZE); + ASSERT(uncompressed_size == compressed_size || + comp != ZIO_COMPRESS_OFF); + ASSERT3U(comp, >=, ZIO_COMPRESS_OFF); + ASSERT3U(comp, <, ZIO_COMPRESS_FUNCTIONS); + + bzero(bp, sizeof (*bp)); + BP_SET_EMBEDDED(bp, B_TRUE); + BP_SET_COMPRESS(bp, comp); + BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); + BPE_SET_LSIZE(bp, uncompressed_size); + BPE_SET_PSIZE(bp, compressed_size); + + /* + * Encode the byte array into the words of the block pointer. + * First byte goes into low bits of first word (little endian). + */ + for (int i = 0; i < compressed_size; i++) { + BF64_SET(w, (i % sizeof (w)) * NBBY, NBBY, data8[i]); + if (i % sizeof (w) == sizeof (w) - 1) { + /* we've reached the end of a word */ + ASSERT3P(bp64, <, bp + 1); + *bp64 = w; + bp64++; + if (!BPE_IS_PAYLOADWORD(bp, bp64)) + bp64++; + w = 0; + } + } + /* write last partial word */ + if (bp64 < (uint64_t *)(bp + 1)) + *bp64 = w; +} + +/* + * buf must be at least BPE_GET_PSIZE(bp) bytes long (which will never be + * more than BPE_PAYLOAD_SIZE bytes). + */ +void +decode_embedded_bp_compressed(const blkptr_t *bp, void *buf) +{ + int psize; + uint8_t *buf8 = buf; + uint64_t w = 0; + const uint64_t *bp64 = (const uint64_t *)bp; + + ASSERT(BP_IS_EMBEDDED(bp)); + + psize = BPE_GET_PSIZE(bp); + + /* + * Decode the words of the block pointer into the byte array. + * Low bits of first word are the first byte (little endian). + */ + for (int i = 0; i < psize; i++) { + if (i % sizeof (w) == 0) { + /* beginning of a word */ + ASSERT3P(bp64, <, bp + 1); + w = *bp64; + bp64++; + if (!BPE_IS_PAYLOADWORD(bp, bp64)) + bp64++; + } + buf8[i] = BF64_GET(w, (i % sizeof (w)) * NBBY, NBBY); + } +} Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c Tue Jun 17 07:11:00 2014 (r267564) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/bpobj.c Tue Jun 17 07:35:54 2014 (r267565) @@ -192,6 +192,13 @@ bpobj_close(bpobj_t *bpo) mutex_destroy(&bpo->bpo_lock); } +static boolean_t +bpobj_hasentries(bpobj_t *bpo) +{ + return (bpo->bpo_phys->bpo_num_blkptrs != 0 || + (bpo->bpo_havesubobj && bpo->bpo_phys->bpo_num_subobjs != 0)); +} + static int bpobj_iterate_impl(bpobj_t *bpo, bpobj_itor_t func, void *arg, dmu_tx_t *tx, boolean_t free) @@ -332,9 +339,11 @@ bpobj_iterate_impl(bpobj_t *bpo, bpobj_i out: /* If there are no entries, there should be no bytes. */ - ASSERT(bpo->bpo_phys->bpo_num_blkptrs > 0 || - (bpo->bpo_havesubobj && bpo->bpo_phys->bpo_num_subobjs > 0) || - bpo->bpo_phys->bpo_bytes == 0); + if (!bpobj_hasentries(bpo)) { + ASSERT0(bpo->bpo_phys->bpo_bytes); + ASSERT0(bpo->bpo_phys->bpo_comp); + ASSERT0(bpo->bpo_phys->bpo_uncomp); + } mutex_exit(&bpo->bpo_lock); return (err); @@ -377,7 +386,7 @@ bpobj_enqueue_subobj(bpobj_t *bpo, uint6 VERIFY3U(0, ==, bpobj_open(&subbpo, bpo->bpo_os, subobj)); VERIFY3U(0, ==, bpobj_space(&subbpo, &used, &comp, &uncomp)); - if (used == 0) { + if (!bpobj_hasentries(&subbpo)) { /* No point in having an empty subobj. */ bpobj_close(&subbpo); bpobj_free(bpo->bpo_os, subobj, tx); @@ -453,13 +462,29 @@ bpobj_enqueue(bpobj_t *bpo, const blkptr ASSERT(!BP_IS_HOLE(bp)); ASSERT(bpo->bpo_object != dmu_objset_pool(bpo->bpo_os)->dp_empty_bpobj); + if (BP_IS_EMBEDDED(bp)) { + /* + * The bpobj will compress better without the payload. + * + * Note that we store EMBEDDED bp's because they have an + * uncompressed size, which must be accounted for. An + * alternative would be to add their size to bpo_uncomp + * without storing the bp, but that would create additional + * complications: bpo_uncomp would be inconsistent with the + * set of BP's stored, and bpobj_iterate() wouldn't visit + * all the space accounted for in the bpobj. + */ + bzero(&stored_bp, sizeof (stored_bp)); + stored_bp.blk_prop = bp->blk_prop; + stored_bp.blk_birth = bp->blk_birth; + } else if (!BP_GET_DEDUP(bp)) { + /* The bpobj will compress better without the checksum */ + bzero(&stored_bp.blk_cksum, sizeof (stored_bp.blk_cksum)); + } + /* We never need the fill count. */ stored_bp.blk_fill = 0; - /* The bpobj will compress better if we can leave off the checksum */ - if (!BP_GET_DEDUP(bp)) - bzero(&stored_bp.blk_cksum, sizeof (stored_bp.blk_cksum)); - mutex_enter(&bpo->bpo_lock); offset = bpo->bpo_phys->bpo_num_blkptrs * sizeof (stored_bp); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c Tue Jun 17 07:11:00 2014 (r267564) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dbuf.c Tue Jun 17 07:35:54 2014 (r267565) @@ -40,6 +40,8 @@ #include <sys/dmu_zfetch.h> #include <sys/sa.h> #include <sys/sa_impl.h> +#include <sys/zfeature.h> +#include <sys/blkptr.h> #include <sys/range_tree.h> /* @@ -1435,6 +1437,38 @@ dbuf_fill_done(dmu_buf_impl_t *db, dmu_t mutex_exit(&db->db_mtx); } +void +dmu_buf_write_embedded(dmu_buf_t *dbuf, void *data, + bp_embedded_type_t etype, enum zio_compress comp, + int uncompressed_size, int compressed_size, int byteorder, + dmu_tx_t *tx) +{ + dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; + struct dirty_leaf *dl; + dmu_object_type_t type; + + DB_DNODE_ENTER(db); + type = DB_DNODE(db)->dn_type; + DB_DNODE_EXIT(db); + + ASSERT0(db->db_level); + ASSERT(db->db_blkid != DMU_BONUS_BLKID); + + dmu_buf_will_not_fill(dbuf, tx); + + ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); + dl = &db->db_last_dirty->dt.dl; + encode_embedded_bp_compressed(&dl->dr_overridden_by, + data, comp, uncompressed_size, compressed_size); + BPE_SET_ETYPE(&dl->dr_overridden_by, etype); + BP_SET_TYPE(&dl->dr_overridden_by, type); + BP_SET_LEVEL(&dl->dr_overridden_by, 0); + BP_SET_BYTEORDER(&dl->dr_overridden_by, byteorder); + + dl->dr_override_state = DR_OVERRIDDEN; + dl->dr_overridden_by.blk_birth = db->db_last_dirty->dr_txg; +} + /* * Directly assign a provided arc buf to a given dbuf if it's not referenced * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf. @@ -1819,7 +1853,7 @@ dbuf_prefetch(dnode_t *dn, uint64_t blki } if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) { - if (bp && !BP_IS_HOLE(bp)) { + if (bp && !BP_IS_HOLE(bp) && !BP_IS_EMBEDDED(bp)) { dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH; zbookmark_t zb; @@ -2451,7 +2485,7 @@ dbuf_write_ready(zio_t *zio, arc_buf_t * uint64_t fill = 0; int i; - ASSERT(db->db_blkptr == bp); + ASSERT3P(db->db_blkptr, ==, bp); DB_DNODE_ENTER(db); dn = DB_DNODE(db); @@ -2463,7 +2497,8 @@ dbuf_write_ready(zio_t *zio, arc_buf_t * ASSERT((db->db_blkid != DMU_SPILL_BLKID && BP_GET_TYPE(bp) == dn->dn_type) || (db->db_blkid == DMU_SPILL_BLKID && - BP_GET_TYPE(bp) == dn->dn_bonustype)); + BP_GET_TYPE(bp) == dn->dn_bonustype) || + BP_IS_EMBEDDED(bp)); ASSERT(BP_GET_LEVEL(bp) == db->db_level); } @@ -2504,12 +2539,13 @@ dbuf_write_ready(zio_t *zio, arc_buf_t * for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { if (BP_IS_HOLE(ibp)) continue; - fill += ibp->blk_fill; + fill += BP_GET_FILL(ibp); } } DB_DNODE_EXIT(db); - bp->blk_fill = fill; + if (!BP_IS_EMBEDDED(bp)) + bp->blk_fill = fill; mutex_exit(&db->db_mtx); } @@ -2621,7 +2657,8 @@ dbuf_write_done(zio_t *zio, arc_buf_t *b dn->dn_phys->dn_maxblkid >> (db->db_level * epbs)); ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, db->db.db_size); - arc_set_callback(db->db_buf, dbuf_do_evict, db); + if (!arc_released(db->db_buf)) + arc_set_callback(db->db_buf, dbuf_do_evict, db); } DB_DNODE_EXIT(db); mutex_destroy(&dr->dt.di.dr_mtx); @@ -2747,10 +2784,16 @@ dbuf_write(dbuf_dirty_record_t *dr, arc_ dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); DB_DNODE_EXIT(db); - if (db->db_level == 0 && dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { - ASSERT(db->db_state != DB_NOFILL); + if (db->db_level == 0 && + dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { + /* + * The BP for this block has been provided by open context + * (by dmu_sync() or dmu_buf_write_embedded()). + */ + void *contents = (data != NULL) ? data->b_data : NULL; + dr->dr_zio = zio_write(zio, os->os_spa, txg, - db->db_blkptr, data->b_data, arc_buf_size(data), &zp, + db->db_blkptr, contents, db->db.db_size, &zp, dbuf_write_override_ready, NULL, dbuf_write_override_done, dr, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); mutex_enter(&db->db_mtx); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dmu.c Tue Jun 17 07:11:00 2014 (r267564) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dmu.c Tue Jun 17 07:35:54 2014 (r267565) @@ -125,17 +125,13 @@ const dmu_object_byteswap_info_t dmu_ot_ }; int -dmu_buf_hold(objset_t *os, uint64_t object, uint64_t offset, - void *tag, dmu_buf_t **dbp, int flags) +dmu_buf_hold_noread(objset_t *os, uint64_t object, uint64_t offset, + void *tag, dmu_buf_t **dbp) { dnode_t *dn; uint64_t blkid; dmu_buf_impl_t *db; int err; - int db_flags = DB_RF_CANFAIL; - - if (flags & DMU_READ_NO_PREFETCH) - db_flags |= DB_RF_NOPREFETCH; err = dnode_hold(os, object, FTAG, &dn); if (err) @@ -144,18 +140,37 @@ dmu_buf_hold(objset_t *os, uint64_t obje rw_enter(&dn->dn_struct_rwlock, RW_READER); db = dbuf_hold(dn, blkid, tag); rw_exit(&dn->dn_struct_rwlock); + dnode_rele(dn, FTAG); + if (db == NULL) { - err = SET_ERROR(EIO); - } else { + *dbp = NULL; + return (SET_ERROR(EIO)); + } + + *dbp = &db->db; + return (err); +} + +int +dmu_buf_hold(objset_t *os, uint64_t object, uint64_t offset, + void *tag, dmu_buf_t **dbp, int flags) +{ + int err; + int db_flags = DB_RF_CANFAIL; + + if (flags & DMU_READ_NO_PREFETCH) + db_flags |= DB_RF_NOPREFETCH; + + err = dmu_buf_hold_noread(os, object, offset, tag, dbp); + if (err == 0) { + dmu_buf_impl_t *db = (dmu_buf_impl_t *)(*dbp); err = dbuf_read(db, NULL, db_flags); - if (err) { + if (err != 0) { dbuf_rele(db, tag); - db = NULL; + *dbp = NULL; } } - dnode_rele(dn, FTAG); - *dbp = &db->db; /* NULL db plus first field offset is NULL */ return (err); } @@ -848,6 +863,25 @@ dmu_prealloc(objset_t *os, uint64_t obje dmu_buf_rele_array(dbp, numbufs, FTAG); } +void +dmu_write_embedded(objset_t *os, uint64_t object, uint64_t offset, + void *data, uint8_t etype, uint8_t comp, int uncompressed_size, + int compressed_size, int byteorder, dmu_tx_t *tx) +{ + dmu_buf_t *db; + + ASSERT3U(etype, <, NUM_BP_EMBEDDED_TYPES); + ASSERT3U(comp, <, ZIO_COMPRESS_FUNCTIONS); + VERIFY0(dmu_buf_hold_noread(os, object, offset, + FTAG, &db)); + + dmu_buf_write_embedded(db, + data, (bp_embedded_type_t)etype, (enum zio_compress)comp, + uncompressed_size, compressed_size, byteorder, tx); + + dmu_buf_rele(db, FTAG); +} + /* * DMU support for xuio */ @@ -1263,7 +1297,7 @@ dmu_sync_ready(zio_t *zio, arc_buf_t *bu * block size still needs to be known for replay. */ BP_SET_LSIZE(bp, db->db_size); - } else { + } else if (!BP_IS_EMBEDDED(bp)) { ASSERT(BP_GET_LEVEL(bp) == 0); bp->blk_fill = 1; } @@ -1534,9 +1568,15 @@ dmu_object_set_checksum(objset_t *os, ui { dnode_t *dn; - /* XXX assumes dnode_hold will not get an i/o error */ - (void) dnode_hold(os, object, FTAG, &dn); - ASSERT(checksum < ZIO_CHECKSUM_FUNCTIONS); + /* + * Send streams include each object's checksum function. This + * check ensures that the receiving system can understand the + * checksum function transmitted. + */ + ASSERT3U(checksum, <, ZIO_CHECKSUM_LEGACY_FUNCTIONS); + + VERIFY0(dnode_hold(os, object, FTAG, &dn)); + ASSERT3U(checksum, <, ZIO_CHECKSUM_FUNCTIONS); dn->dn_checksum = checksum; dnode_setdirty(dn, tx); dnode_rele(dn, FTAG); @@ -1548,9 +1588,14 @@ dmu_object_set_compress(objset_t *os, ui { dnode_t *dn; - /* XXX assumes dnode_hold will not get an i/o error */ - (void) dnode_hold(os, object, FTAG, &dn); - ASSERT(compress < ZIO_COMPRESS_FUNCTIONS); + /* + * Send streams include each object's compression function. This + * check ensures that the receiving system can understand the + * compression function transmitted. + */ + ASSERT3U(compress, <, ZIO_COMPRESS_LEGACY_FUNCTIONS); + + VERIFY0(dnode_hold(os, object, FTAG, &dn)); dn->dn_compress = compress; dnode_setdirty(dn, tx); dnode_rele(dn, FTAG); @@ -1717,7 +1762,7 @@ dmu_object_info_from_dnode(dnode_t *dn, doi->doi_max_offset = (dn->dn_maxblkid + 1) * dn->dn_datablksz; doi->doi_fill_count = 0; for (int i = 0; i < dnp->dn_nblkptr; i++) - doi->doi_fill_count += dnp->dn_blkptr[i].blk_fill; + doi->doi_fill_count += BP_GET_FILL(&dnp->dn_blkptr[i]); mutex_exit(&dn->dn_mtx); rw_exit(&dn->dn_struct_rwlock); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c Tue Jun 17 07:11:00 2014 (r267564) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c Tue Jun 17 07:35:54 2014 (r267565) @@ -338,7 +338,7 @@ dmu_objset_open_impl(spa_t *spa, dsl_dat * default (fletcher2/off). Snapshots don't need to know about * checksum/compression/copies. */ - if (ds) { + if (ds != NULL) { err = dsl_prop_register(ds, zfs_prop_to_name(ZFS_PROP_PRIMARYCACHE), primary_cache_changed_cb, os); @@ -391,7 +391,7 @@ dmu_objset_open_impl(spa_t *spa, dsl_dat kmem_free(os, sizeof (objset_t)); return (err); } - } else if (ds == NULL) { + } else { /* It's the meta-objset. */ os->os_checksum = ZIO_CHECKSUM_FLETCHER_4; os->os_compress = ZIO_COMPRESS_LZJB; @@ -435,17 +435,6 @@ dmu_objset_open_impl(spa_t *spa, dsl_dat &os->os_groupused_dnode); } - /* - * We should be the only thread trying to do this because we - * have ds_opening_lock - */ - if (ds) { - mutex_enter(&ds->ds_lock); - ASSERT(ds->ds_objset == NULL); - ds->ds_objset = os; - mutex_exit(&ds->ds_lock); - } - *osp = os; return (0); } @@ -456,11 +445,19 @@ dmu_objset_from_ds(dsl_dataset_t *ds, ob int err = 0; mutex_enter(&ds->ds_opening_lock); - *osp = ds->ds_objset; - if (*osp == NULL) { + if (ds->ds_objset == NULL) { + objset_t *os; err = dmu_objset_open_impl(dsl_dataset_get_spa(ds), - ds, dsl_dataset_get_blkptr(ds), osp); + ds, dsl_dataset_get_blkptr(ds), &os); + + if (err == 0) { + mutex_enter(&ds->ds_lock); + ASSERT(ds->ds_objset == NULL); + ds->ds_objset = os; + mutex_exit(&ds->ds_lock); + } } + *osp = ds->ds_objset; mutex_exit(&ds->ds_opening_lock); return (err); } @@ -986,6 +983,7 @@ dmu_objset_write_ready(zio_t *zio, arc_b objset_t *os = arg; dnode_phys_t *dnp = &os->os_phys->os_meta_dnode; + ASSERT(!BP_IS_EMBEDDED(bp)); ASSERT3P(bp, ==, os->os_rootbp); ASSERT3U(BP_GET_TYPE(bp), ==, DMU_OT_OBJSET); ASSERT0(BP_GET_LEVEL(bp)); @@ -998,7 +996,7 @@ dmu_objset_write_ready(zio_t *zio, arc_b */ bp->blk_fill = 0; for (int i = 0; i < dnp->dn_nblkptr; i++) - bp->blk_fill += dnp->dn_blkptr[i].blk_fill; + bp->blk_fill += BP_GET_FILL(&dnp->dn_blkptr[i]); } /* ARGSUSED */ Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_send.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_send.c Tue Jun 17 07:11:00 2014 (r267564) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_send.c Tue Jun 17 07:35:54 2014 (r267565) @@ -48,7 +48,9 @@ #include <sys/zfs_onexit.h> #include <sys/dmu_send.h> #include <sys/dsl_destroy.h> +#include <sys/blkptr.h> #include <sys/dsl_bookmark.h> +#include <sys/zfeature.h> /* Set this tunable to TRUE to replace corrupt data with 0x2f5baddb10c */ int zfs_send_corrupt_data = B_FALSE; @@ -168,7 +170,7 @@ dump_free(dmu_sendarg_t *dsp, uint64_t o } static int -dump_data(dmu_sendarg_t *dsp, dmu_object_type_t type, +dump_write(dmu_sendarg_t *dsp, dmu_object_type_t type, uint64_t object, uint64_t offset, int blksz, const blkptr_t *bp, void *data) { struct drr_write *drrw = &(dsp->dsa_drr->drr_u.drr_write); @@ -203,13 +205,22 @@ dump_data(dmu_sendarg_t *dsp, dmu_object drrw->drr_offset = offset; drrw->drr_length = blksz; drrw->drr_toguid = dsp->dsa_toguid; - drrw->drr_checksumtype = BP_GET_CHECKSUM(bp); - if (zio_checksum_table[drrw->drr_checksumtype].ci_dedup) - drrw->drr_checksumflags |= DRR_CHECKSUM_DEDUP; - DDK_SET_LSIZE(&drrw->drr_key, BP_GET_LSIZE(bp)); - DDK_SET_PSIZE(&drrw->drr_key, BP_GET_PSIZE(bp)); - DDK_SET_COMPRESS(&drrw->drr_key, BP_GET_COMPRESS(bp)); - drrw->drr_key.ddk_cksum = bp->blk_cksum; + if (BP_IS_EMBEDDED(bp)) { + /* + * There's no pre-computed checksum of embedded BP's, so + * (like fletcher4-checkummed blocks) userland will have + * to compute a dedup-capable checksum itself. + */ + drrw->drr_checksumtype = ZIO_CHECKSUM_OFF; + } else { + drrw->drr_checksumtype = BP_GET_CHECKSUM(bp); + if (zio_checksum_table[drrw->drr_checksumtype].ci_dedup) + drrw->drr_checksumflags |= DRR_CHECKSUM_DEDUP; + DDK_SET_LSIZE(&drrw->drr_key, BP_GET_LSIZE(bp)); + DDK_SET_PSIZE(&drrw->drr_key, BP_GET_PSIZE(bp)); + DDK_SET_COMPRESS(&drrw->drr_key, BP_GET_COMPRESS(bp)); + drrw->drr_key.ddk_cksum = bp->blk_cksum; + } if (dump_bytes(dsp, dsp->dsa_drr, sizeof (dmu_replay_record_t)) != 0) return (SET_ERROR(EINTR)); @@ -219,6 +230,43 @@ dump_data(dmu_sendarg_t *dsp, dmu_object } static int +dump_write_embedded(dmu_sendarg_t *dsp, uint64_t object, uint64_t offset, + int blksz, const blkptr_t *bp) +{ + char buf[BPE_PAYLOAD_SIZE]; + struct drr_write_embedded *drrw = + &(dsp->dsa_drr->drr_u.drr_write_embedded); + + if (dsp->dsa_pending_op != PENDING_NONE) { + if (dump_bytes(dsp, dsp->dsa_drr, + sizeof (dmu_replay_record_t)) != 0) + return (EINTR); + dsp->dsa_pending_op = PENDING_NONE; + } + + ASSERT(BP_IS_EMBEDDED(bp)); + + bzero(dsp->dsa_drr, sizeof (dmu_replay_record_t)); + dsp->dsa_drr->drr_type = DRR_WRITE_EMBEDDED; + drrw->drr_object = object; + drrw->drr_offset = offset; + drrw->drr_length = blksz; + drrw->drr_toguid = dsp->dsa_toguid; + drrw->drr_compression = BP_GET_COMPRESS(bp); + drrw->drr_etype = BPE_GET_ETYPE(bp); + drrw->drr_lsize = BPE_GET_LSIZE(bp); + drrw->drr_psize = BPE_GET_PSIZE(bp); + + decode_embedded_bp_compressed(bp, buf); + + if (dump_bytes(dsp, dsp->dsa_drr, sizeof (dmu_replay_record_t)) != 0) + return (EINTR); + if (dump_bytes(dsp, buf, P2ROUNDUP(drrw->drr_psize, 8)) != 0) + return (EINTR); + return (0); +} + +static int dump_spill(dmu_sendarg_t *dsp, uint64_t object, int blksz, void *data) { struct drr_spill *drrs = &(dsp->dsa_drr->drr_u.drr_spill); @@ -338,6 +386,33 @@ dump_dnode(dmu_sendarg_t *dsp, uint64_t return (0); } +static boolean_t +backup_do_embed(dmu_sendarg_t *dsp, const blkptr_t *bp) +{ + if (!BP_IS_EMBEDDED(bp)) + return (B_FALSE); + + /* + * Compression function must be legacy, or explicitly enabled. + */ + if ((BP_GET_COMPRESS(bp) >= ZIO_COMPRESS_LEGACY_FUNCTIONS && + !(dsp->dsa_featureflags & DMU_BACKUP_FEATURE_EMBED_DATA_LZ4))) + return (B_FALSE); + + /* + * Embed type must be explicitly enabled. + */ + switch (BPE_GET_ETYPE(bp)) { + case BP_EMBEDDED_TYPE_DATA: + if (dsp->dsa_featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) + return (B_TRUE); + break; + default: + return (B_FALSE); + } + return (B_FALSE); +} + #define BP_SPAN(dnp, level) \ (((uint64_t)dnp->dn_datablkszsec) << (SPA_MINBLOCKSHIFT + \ (level) * (dnp->dn_indblkshift - SPA_BLKPTRSHIFT))) @@ -406,11 +481,17 @@ backup_cb(spa_t *spa, zilog_t *zilog, co err = dump_spill(dsp, zb->zb_object, blksz, abuf->b_data); (void) arc_buf_remove_ref(abuf, &abuf); + } else if (backup_do_embed(dsp, bp)) { + /* it's an embedded level-0 block of a regular object */ + int blksz = dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT; + err = dump_write_embedded(dsp, zb->zb_object, + zb->zb_blkid * blksz, blksz, bp); } else { /* it's a level-0 block of a regular object */ uint32_t aflags = ARC_WAIT; arc_buf_t *abuf; int blksz = BP_GET_LSIZE(bp); + ASSERT3U(blksz, ==, dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT); ASSERT0(zb->zb_level); if (arc_read(NULL, spa, bp, arc_getbuf_func, &abuf, ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL, @@ -429,7 +510,7 @@ backup_cb(spa_t *spa, zilog_t *zilog, co } } - err = dump_data(dsp, type, zb->zb_object, zb->zb_blkid * blksz, + err = dump_write(dsp, type, zb->zb_object, zb->zb_blkid * blksz, blksz, bp, abuf->b_data); (void) arc_buf_remove_ref(abuf, &abuf); } @@ -443,14 +524,15 @@ backup_cb(spa_t *spa, zilog_t *zilog, co */ static int dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *ds, - zfs_bookmark_phys_t *fromzb, boolean_t is_clone, int outfd, - vnode_t *vp, offset_t *off) + zfs_bookmark_phys_t *fromzb, boolean_t is_clone, boolean_t embedok, + int outfd, vnode_t *vp, offset_t *off) { objset_t *os; dmu_replay_record_t *drr; dmu_sendarg_t *dsp; int err; uint64_t fromtxg = 0; + uint64_t featureflags = 0; err = dmu_objset_from_ds(ds, &os); if (err != 0) { @@ -473,13 +555,23 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, return (SET_ERROR(EINVAL)); } if (version >= ZPL_VERSION_SA) { - DMU_SET_FEATUREFLAGS( - drr->drr_u.drr_begin.drr_versioninfo, - DMU_BACKUP_FEATURE_SA_SPILL); + featureflags |= DMU_BACKUP_FEATURE_SA_SPILL; } } #endif + if (embedok && + spa_feature_is_active(dp->dp_spa, SPA_FEATURE_EMBEDDED_DATA)) { + featureflags |= DMU_BACKUP_FEATURE_EMBED_DATA; + if (spa_feature_is_active(dp->dp_spa, SPA_FEATURE_LZ4_COMPRESS)) + featureflags |= DMU_BACKUP_FEATURE_EMBED_DATA_LZ4; + } else { + embedok = B_FALSE; + } + + DMU_SET_FEATUREFLAGS(drr->drr_u.drr_begin.drr_versioninfo, + featureflags); + drr->drr_u.drr_begin.drr_creation_time = ds->ds_phys->ds_creation_time; drr->drr_u.drr_begin.drr_type = dmu_objset_type(os); @@ -511,6 +603,7 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, ZIO_SET_CHECKSUM(&dsp->dsa_zc, 0, 0, 0, 0); dsp->dsa_pending_op = PENDING_NONE; dsp->dsa_incremental = (fromzb != NULL); + dsp->dsa_featureflags = featureflags; mutex_enter(&ds->ds_sendstream_lock); list_insert_head(&ds->ds_sendstreams, dsp); @@ -562,7 +655,7 @@ out: int dmu_send_obj(const char *pool, uint64_t tosnap, uint64_t fromsnap, - int outfd, vnode_t *vp, offset_t *off) + boolean_t embedok, int outfd, vnode_t *vp, offset_t *off) { *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201406170735.s5H7Zvhh016469>