Date: Wed, 10 Mar 2021 09:36:13 GMT From: Martin Matuska <mm@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org Subject: git: 21b12f3e451b - stable/13 - zfs: merge OpenZFS master-436ab35a5 Message-ID: <202103100936.12A9aDPl013647@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch stable/13 has been updated by mm: URL: https://cgit.FreeBSD.org/src/commit/?id=21b12f3e451b0d21ff4a3776cb5d11750c7cc822 commit 21b12f3e451b0d21ff4a3776cb5d11750c7cc822 Author: Martin Matuska <mm@FreeBSD.org> AuthorDate: 2021-02-16 00:39:34 +0000 Commit: Martin Matuska <mm@FreeBSD.org> CommitDate: 2021-03-10 01:24:58 +0000 zfs: merge OpenZFS master-436ab35a5 - speed up writing to ZFS pools without ZIL devices (aa755b3) - speed up importing ZFS pools (2d8f72d, a0e0199, cf0977a) ... Reviewed by: mjg (partial) Tested by: pho Differential Revision: https://reviews.freebsd.org/D28677 (cherry picked from commit 184c1b943937986c81e1996d999d21626ec7a4ff) --- cddl/lib/libzfs/Makefile | 1 - cddl/lib/libzpool/Makefile | 1 - .../openzfs/.github/PULL_REQUEST_TEMPLATE.md | 42 +++ .../.github/workflows/zfs-tests-functional.yml | 64 ++++ .../openzfs/.github/workflows/zfs-tests-sanity.yml | 60 ++++ sys/contrib/openzfs/META | 2 +- sys/contrib/openzfs/Makefile.am | 13 +- sys/contrib/openzfs/cmd/Makefile.am | 10 + sys/contrib/openzfs/cmd/mount_zfs/Makefile.am | 2 + sys/contrib/openzfs/cmd/mount_zfs/mount_zfs.c | 15 + sys/contrib/openzfs/cmd/raidz_test/Makefile.am | 2 + sys/contrib/openzfs/cmd/raidz_test/raidz_test.c | 5 +- sys/contrib/openzfs/cmd/vdev_id/vdev_id | 399 +++++++++++++++------ sys/contrib/openzfs/cmd/zdb/Makefile.am | 2 + sys/contrib/openzfs/cmd/zdb/zdb.c | 138 ++++++- sys/contrib/openzfs/cmd/zed/Makefile.am | 2 + sys/contrib/openzfs/cmd/zed/agents/zfs_agents.c | 19 +- sys/contrib/openzfs/cmd/zed/agents/zfs_retire.c | 2 +- sys/contrib/openzfs/cmd/zed/zed.d/zed-functions.sh | 0 sys/contrib/openzfs/cmd/zfs/Makefile.am | 2 + .../openzfs/cmd/zfs_ids_to_path/Makefile.am | 2 + sys/contrib/openzfs/cmd/zgenhostid/Makefile.am | 4 +- sys/contrib/openzfs/cmd/zhack/Makefile.am | 2 + sys/contrib/openzfs/cmd/zinject/Makefile.am | 2 + sys/contrib/openzfs/cmd/zpool/Makefile.am | 2 + sys/contrib/openzfs/cmd/zpool/zpool_main.c | 15 +- sys/contrib/openzfs/cmd/zpool_influxdb/Makefile.am | 2 + sys/contrib/openzfs/cmd/zstream/Makefile.am | 2 + sys/contrib/openzfs/cmd/ztest/Makefile.am | 2 + sys/contrib/openzfs/cmd/ztest/ztest.c | 388 ++++++++++---------- sys/contrib/openzfs/cmd/zvol_id/Makefile.am | 2 + sys/contrib/openzfs/config/CppCheck.am | 11 + sys/contrib/openzfs/config/Rules.am | 1 + sys/contrib/openzfs/config/always-cppcheck.m4 | 6 + sys/contrib/openzfs/config/ax_count_cpus.m4 | 101 ++++++ .../openzfs/config/kernel-get-disk-and-module.m4 | 0 sys/contrib/openzfs/config/kernel-vfs-iov_iter.m4 | 44 --- sys/contrib/openzfs/config/zfs-build.m4 | 10 +- .../contrib/dracut/90zfs/module-setup.sh.in | 4 +- .../contrib/dracut/90zfs/zfs-env-bootfs.service.in | 2 +- .../contrib/dracut/90zfs/zfs-generator.sh.in | 8 + .../contrib/dracut/90zfs/zfs-load-key.sh.in | 2 +- .../pyzfs/libzfs_core/test/test_libzfs_core.py | 4 +- sys/contrib/openzfs/cppcheck-suppressions.txt | 8 - .../etc/systemd/system/zfs-import-cache.service.in | 2 +- .../etc/systemd/system/zfs-import-scan.service.in | 2 +- .../openzfs/include/os/freebsd/spl/sys/Makefile.am | 1 + .../openzfs/include/os/freebsd/spl/sys/ccompile.h | 3 - .../openzfs/include/os/freebsd/spl/sys/fcntl.h | 38 ++ .../openzfs/include/os/freebsd/spl/sys/uio.h | 79 ++-- .../include/os/freebsd/zfs/sys/freebsd_crypto.h | 2 +- .../include/os/freebsd/zfs/sys/zfs_znode_impl.h | 5 +- sys/contrib/openzfs/include/os/linux/spl/sys/uio.h | 73 ++-- .../include/os/linux/zfs/sys/zfs_vnops_os.h | 2 +- .../include/os/linux/zfs/sys/zfs_znode_impl.h | 5 +- sys/contrib/openzfs/include/sys/abd.h | 75 +++- sys/contrib/openzfs/include/sys/abd_impl.h | 48 +-- sys/contrib/openzfs/include/sys/crypto/common.h | 2 +- sys/contrib/openzfs/include/sys/dmu.h | 15 +- sys/contrib/openzfs/include/sys/fs/zfs.h | 1 + sys/contrib/openzfs/include/sys/sa.h | 2 +- sys/contrib/openzfs/include/sys/spa.h | 1 + sys/contrib/openzfs/include/sys/spa_impl.h | 1 + sys/contrib/openzfs/include/sys/uio_impl.h | 8 +- sys/contrib/openzfs/include/sys/vdev.h | 4 + sys/contrib/openzfs/include/sys/vdev_impl.h | 5 + sys/contrib/openzfs/include/sys/vdev_raidz_impl.h | 1 + sys/contrib/openzfs/include/sys/zfs_debug.h | 1 + sys/contrib/openzfs/include/sys/zfs_sa.h | 2 +- sys/contrib/openzfs/include/sys/zfs_vnops.h | 8 +- sys/contrib/openzfs/include/sys/zfs_znode.h | 2 + sys/contrib/openzfs/lib/Makefile.am | 10 +- sys/contrib/openzfs/lib/libavl/Makefile.am | 2 + sys/contrib/openzfs/lib/libefi/Makefile.am | 2 + sys/contrib/openzfs/lib/libefi/rdwr_efi.c | 1 + sys/contrib/openzfs/lib/libicp/Makefile.am | 2 + sys/contrib/openzfs/lib/libnvpair/Makefile.am | 3 +- sys/contrib/openzfs/lib/libshare/Makefile.am | 2 + sys/contrib/openzfs/lib/libspl/Makefile.am | 9 + .../lib/libspl/include/os/freebsd/Makefile.am | 4 + .../openzfs/lib/libspl/include/os/freebsd/fcntl.h | 33 ++ .../lib/libspl/include/os/freebsd/sys/Makefile.am | 1 + .../lib/libspl/include/os/freebsd/sys/fcntl.h | 38 ++ sys/contrib/openzfs/lib/libspl/include/sys/uio.h | 44 +-- sys/contrib/openzfs/lib/libtpool/Makefile.am | 2 + sys/contrib/openzfs/lib/libunicode/Makefile.am | 2 + sys/contrib/openzfs/lib/libuutil/Makefile.am | 3 +- sys/contrib/openzfs/lib/libuutil/uu_avl.c | 1 + sys/contrib/openzfs/lib/libzfs/Makefile.am | 3 +- sys/contrib/openzfs/lib/libzfs/libzfs_import.c | 2 +- sys/contrib/openzfs/lib/libzfs/libzfs_pool.c | 51 ++- sys/contrib/openzfs/lib/libzfs/libzfs_sendrecv.c | 205 +++++------ sys/contrib/openzfs/lib/libzfs_core/Makefile.am | 3 +- sys/contrib/openzfs/lib/libzfsbootenv/Makefile.am | 3 +- sys/contrib/openzfs/lib/libzpool/Makefile.am | 2 + sys/contrib/openzfs/lib/libzstd/Makefile.am | 2 + sys/contrib/openzfs/lib/libzutil/Makefile.am | 5 +- .../lib/libzutil/os/freebsd/zutil_import_os.c | 20 +- sys/contrib/openzfs/lib/libzutil/zutil_import.c | 52 ++- .../openzfs/man/man5/zfs-module-parameters.5 | 16 + sys/contrib/openzfs/man/man8/zdb.8 | 18 +- sys/contrib/openzfs/man/man8/zfs-list.8 | 11 +- sys/contrib/openzfs/man/man8/zfs-program.8 | 4 +- sys/contrib/openzfs/man/man8/zfsprops.8 | 2 +- sys/contrib/openzfs/module/Makefile.in | 22 +- sys/contrib/openzfs/module/avl/avl.c | 3 - sys/contrib/openzfs/module/icp/algs/modes/modes.c | 12 +- sys/contrib/openzfs/module/icp/core/kcf_prov_lib.c | 32 +- sys/contrib/openzfs/module/icp/io/sha1_mod.c | 48 +-- sys/contrib/openzfs/module/icp/io/sha2_mod.c | 48 +-- sys/contrib/openzfs/module/icp/io/skein_mod.c | 40 +-- sys/contrib/openzfs/module/lua/ldebug.c | 1 - sys/contrib/openzfs/module/lua/ldo.c | 1 - .../openzfs/module/os/freebsd/spl/spl_uio.c | 38 +- .../openzfs/module/os/freebsd/spl/spl_vfs.c | 2 + sys/contrib/openzfs/module/os/freebsd/zfs/abd_os.c | 31 +- .../openzfs/module/os/freebsd/zfs/crypto_os.c | 26 +- .../openzfs/module/os/freebsd/zfs/zfs_ctldir.c | 40 ++- .../openzfs/module/os/freebsd/zfs/zfs_file_os.c | 2 +- .../openzfs/module/os/freebsd/zfs/zfs_vnops_os.c | 89 ++--- .../openzfs/module/os/freebsd/zfs/zfs_znode.c | 6 +- .../openzfs/module/os/freebsd/zfs/zio_crypt.c | 77 ++-- .../openzfs/module/os/freebsd/zfs/zvol_os.c | 44 ++- .../openzfs/module/os/linux/spl/spl-generic.c | 2 - .../openzfs/module/os/linux/spl/spl-kstat.c | 2 +- .../openzfs/module/os/linux/spl/spl-taskq.c | 3 +- sys/contrib/openzfs/module/os/linux/zfs/abd_os.c | 49 ++- .../openzfs/module/os/linux/zfs/vdev_disk.c | 49 ++- .../module/{zcommon => os/linux/zfs}/zfs_uio.c | 95 +++-- .../openzfs/module/os/linux/zfs/zfs_vfsops.c | 2 +- .../openzfs/module/os/linux/zfs/zfs_vnops_os.c | 90 ++--- .../openzfs/module/os/linux/zfs/zfs_znode.c | 18 +- .../openzfs/module/os/linux/zfs/zio_crypt.c | 30 +- sys/contrib/openzfs/module/os/linux/zfs/zpl_file.c | 18 +- .../openzfs/module/os/linux/zfs/zpl_inode.c | 4 +- .../openzfs/module/os/linux/zfs/zpl_xattr.c | 6 +- sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c | 8 +- sys/contrib/openzfs/module/zfs/abd.c | 293 ++++++++------- sys/contrib/openzfs/module/zfs/arc.c | 42 ++- sys/contrib/openzfs/module/zfs/dbuf.c | 2 +- sys/contrib/openzfs/module/zfs/dmu.c | 54 ++- sys/contrib/openzfs/module/zfs/dmu_objset.c | 2 +- sys/contrib/openzfs/module/zfs/dmu_tx.c | 16 + sys/contrib/openzfs/module/zfs/dsl_dataset.c | 13 +- sys/contrib/openzfs/module/zfs/dsl_destroy.c | 15 +- sys/contrib/openzfs/module/zfs/metaslab.c | 100 ++++-- sys/contrib/openzfs/module/zfs/sa.c | 6 +- sys/contrib/openzfs/module/zfs/spa.c | 51 ++- sys/contrib/openzfs/module/zfs/spa_history.c | 8 +- sys/contrib/openzfs/module/zfs/spa_misc.c | 67 ++-- sys/contrib/openzfs/module/zfs/txg.c | 24 +- sys/contrib/openzfs/module/zfs/vdev.c | 225 ++++++++++-- sys/contrib/openzfs/module/zfs/vdev_draid.c | 34 +- sys/contrib/openzfs/module/zfs/vdev_indirect.c | 2 +- sys/contrib/openzfs/module/zfs/vdev_label.c | 38 +- sys/contrib/openzfs/module/zfs/vdev_queue.c | 2 +- sys/contrib/openzfs/module/zfs/vdev_raidz.c | 49 +-- sys/contrib/openzfs/module/zfs/vdev_removal.c | 13 + sys/contrib/openzfs/module/zfs/zfs_ioctl.c | 3 + sys/contrib/openzfs/module/zfs/zfs_sa.c | 11 +- sys/contrib/openzfs/module/zfs/zfs_vnops.c | 69 ++-- sys/contrib/openzfs/module/zfs/zil.c | 2 +- sys/contrib/openzfs/module/zfs/zio.c | 75 ++-- sys/contrib/openzfs/rpm/generic/zfs.spec.in | 2 +- sys/contrib/openzfs/tests/runfiles/common.run | 2 +- sys/contrib/openzfs/tests/runfiles/linux.run | 2 +- .../openzfs/tests/test-runner/bin/zts-report.py.in | 35 ++ .../openzfs/tests/test-runner/include/logapi.shlib | 6 +- .../tests/zfs-tests/cmd/mmapwrite/mmapwrite.c | 4 +- .../tests/functional/acl/posix-sa/cleanup.ksh | 0 .../functional/acl/posix-sa/posix_001_pos.ksh | 0 .../functional/acl/posix-sa/posix_002_pos.ksh | 0 .../functional/acl/posix-sa/posix_003_pos.ksh | 0 .../functional/acl/posix-sa/posix_004_pos.ksh | 0 .../tests/functional/acl/posix-sa/setup.ksh | 0 .../tests/functional/acl/posix/posix_004_pos.ksh | 0 .../tests/functional/cli_root/zdb/Makefile.am | 4 +- .../tests/functional/cli_root/zdb/zdb_args_neg.ksh | 2 +- .../tests/functional/cli_root/zdb/zdb_recover.ksh | 55 +++ .../functional/cli_root/zdb/zdb_recover_2.ksh | 57 +++ .../cli_root/zfs_mount/zfs_mount_013_pos.ksh | 20 +- .../cli_root/zfs_receive/zfs_receive_005_neg.ksh | 4 +- .../cli_root/zfs_receive/zfs_receive_014_pos.ksh | 25 -- .../cli_root/zfs_receive/zfs_receive_new_props.ksh | 0 .../cli_root/zfs_rename/zfs_rename_nounmount.ksh | 0 .../cli_root/zfs_rollback/zfs_rollback_001_pos.ksh | 1 + .../functional/cli_root/zfs_send/zfs_send-b.ksh | 10 +- .../cli_root/zfs_send/zfs_send_003_pos.ksh | 2 +- .../cli_root/zfs_send/zfs_send_004_neg.ksh | 2 +- .../cli_root/zfs_send/zfs_send_005_pos.ksh | 2 +- .../cli_root/zfs_send/zfs_send_encrypted.ksh | 10 +- .../zfs_send/zfs_send_encrypted_unloaded.ksh | 2 +- .../functional/cli_root/zfs_send/zfs_send_raw.ksh | 14 +- .../cli_root/zfs_set/zfs_set_common.kshlib | 14 +- .../functional/cli_root/zpool_create/draidcfg.gz | Bin 0 -> 21672412 bytes .../zpool_events/zpool_events_duplicates.ksh | 0 .../functional/cli_root/zpool_export/Makefile.am | 3 +- .../functional/cli_root/zpool_export/setup.ksh | 4 - .../cli_root/zpool_export/zpool_export.cfg | 37 +- .../cli_root/zpool_export/zpool_export.kshlib | 32 ++ .../cli_root/zpool_export/zpool_export_001_pos.ksh | 17 +- .../cli_root/zpool_export/zpool_export_002_pos.ksh | 13 +- .../cli_root/zpool_export/zpool_export_003_neg.ksh | 15 +- .../cli_root/zpool_export/zpool_export_004_pos.ksh | 21 +- .../zfs-tests/tests/functional/io/Makefile.am | 1 + .../zfs-tests/tests/functional/io/io_uring.ksh | 72 ++++ .../{persist_l2arc => l2arc}/Makefile.am | 0 .../{persist_l2arc => l2arc}/cleanup.ksh | 0 .../persist_l2arc.cfg => l2arc/l2arc.cfg} | 0 .../persist_l2arc_001_pos.ksh | 0 .../persist_l2arc_002_pos.ksh | 0 .../persist_l2arc_003_neg.ksh | 0 .../persist_l2arc_004_pos.ksh | 0 .../persist_l2arc_005_pos.ksh | 0 .../persist_l2arc_006_pos.ksh | 0 .../persist_l2arc_007_pos.ksh | 0 .../persist_l2arc_008_pos.ksh | 0 .../functional/{persist_l2arc => l2arc}/setup.ksh | 0 .../tests/functional/procfs/pool_state.ksh | 18 +- .../functional/redacted_send/redacted_negative.ksh | 8 +- .../functional/redacted_send/redacted_resume.ksh | 2 +- ...edundancy_003_pos.ksh => redundancy_mirror.ksh} | 0 ...edundancy_001_pos.ksh => redundancy_raidz1.ksh} | 0 ...edundancy_002_pos.ksh => redundancy_raidz2.ksh} | 0 ...edundancy_004_neg.ksh => redundancy_stripe.ksh} | 0 .../tests/functional/removal/removal_with_send.ksh | 2 +- .../tests/functional/rsend/send_invalid.ksh | 2 +- .../functional/rsend/send_partial_dataset.ksh | 2 +- .../tests/functional/userquota/Makefile.am | 3 +- .../tests/functional/zvol/zvol_swap/zvol_swap.cfg | 2 + sys/modules/zfs/zfs_config.h | 6 +- 231 files changed, 3260 insertions(+), 1780 deletions(-) diff --git a/cddl/lib/libzfs/Makefile b/cddl/lib/libzfs/Makefile index 611eb91d76e7..d0b3458eda64 100644 --- a/cddl/lib/libzfs/Makefile +++ b/cddl/lib/libzfs/Makefile @@ -67,7 +67,6 @@ KERNEL_C = \ zfs_fletcher_superscalar4.c \ zfs_namecheck.c \ zfs_prop.c \ - zfs_uio.c \ zpool_prop.c \ zprop_common.c diff --git a/cddl/lib/libzpool/Makefile b/cddl/lib/libzpool/Makefile index 2cb1b0446ade..819f67ceec5a 100644 --- a/cddl/lib/libzpool/Makefile +++ b/cddl/lib/libzpool/Makefile @@ -46,7 +46,6 @@ KERNEL_C = \ zfs_fletcher_superscalar4.c \ zfs_namecheck.c \ zfs_prop.c \ - zfs_uio.c \ zfs_zstd.c \ zpool_prop.c \ zprop_common.c \ diff --git a/sys/contrib/openzfs/.github/PULL_REQUEST_TEMPLATE.md b/sys/contrib/openzfs/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000000..465ee182c497 --- /dev/null +++ b/sys/contrib/openzfs/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,42 @@ +<!--- Please fill out the following template, which will help other contributors review your Pull Request. --> + +<!--- Provide a general summary of your changes in the Title above --> + +<!--- +Documentation on ZFS Buildbot options can be found at +https://openzfs.github.io/openzfs-docs/Developer%20Resources/Buildbot%20Options.html +--> + +### Motivation and Context +<!--- Why is this change required? What problem does it solve? --> +<!--- If it fixes an open issue, please link to the issue here. --> + +### Description +<!--- Describe your changes in detail --> + +### How Has This Been Tested? +<!--- Please describe in detail how you tested your changes. --> +<!--- Include details of your testing environment, and the tests you ran to --> +<!--- see how your change affects other areas of the code, etc. --> +<!--- If your change is a performance enhancement, please provide benchmarks here. --> +<!--- Please think about using the draft PR feature if appropriate --> + +### Types of changes +<!--- What types of changes does your code introduce? Put an `x` in all the boxes that apply: --> +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Performance enhancement (non-breaking change which improves efficiency) +- [ ] Code cleanup (non-breaking change which makes code smaller or more readable) +- [ ] Breaking change (fix or feature that would cause existing functionality to change) +- [ ] Library ABI change (libzfs, libzfs\_core, libnvpair, libuutil and libzfsbootenv) +- [ ] Documentation (a change to man pages or other documentation) + +### Checklist: +<!--- Go over all the following points, and put an `x` in all the boxes that apply. --> +<!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! --> +- [ ] My code follows the OpenZFS [code style requirements](https://github.com/openzfs/zfs/blob/master/.github/CONTRIBUTING.md#coding-conventions). +- [ ] I have updated the documentation accordingly. +- [ ] I have read the [**contributing** document](https://github.com/openzfs/zfs/blob/master/.github/CONTRIBUTING.md). +- [ ] I have added [tests](https://github.com/openzfs/zfs/tree/master/tests) to cover my changes. +- [ ] I have run the ZFS Test Suite with this change applied. +- [ ] All commit messages are properly formatted and contain [`Signed-off-by`](https://github.com/openzfs/zfs/blob/master/.github/CONTRIBUTING.md#signed-off-by). diff --git a/sys/contrib/openzfs/.github/workflows/zfs-tests-functional.yml b/sys/contrib/openzfs/.github/workflows/zfs-tests-functional.yml new file mode 100644 index 000000000000..631f174b74fd --- /dev/null +++ b/sys/contrib/openzfs/.github/workflows/zfs-tests-functional.yml @@ -0,0 +1,64 @@ +name: zfs-tests-functional + +on: + push: + pull_request: + +jobs: + tests-functional-ubuntu: + strategy: + fail-fast: false + matrix: + os: [18.04, 20.04] + runs-on: ubuntu-${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install --yes -qq build-essential autoconf libtool gdb lcov \ + git alien fakeroot wget curl bc fio acl \ + sysstat mdadm lsscsi parted gdebi attr dbench watchdog ksh \ + nfs-kernel-server samba rng-tools xz-utils \ + zlib1g-dev uuid-dev libblkid-dev libselinux-dev \ + xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \ + libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \ + libpam0g-dev pamtester python-dev python-setuptools python-cffi \ + python3 python3-dev python3-setuptools python3-cffi + - name: Autogen.sh + run: | + sh autogen.sh + - name: Configure + run: | + ./configure --enable-debug --enable-debuginfo + - name: Make + run: | + make --no-print-directory -s pkg-utils pkg-kmod + - name: Install + run: | + sudo dpkg -i *.deb + # Update order of directories to search for modules, otherwise + # Ubuntu will load kernel-shipped ones. + sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf + sudo depmod + sudo modprobe zfs + - name: Tests + run: | + /usr/share/zfs/zfs-tests.sh -v -s 3G + - name: Prepare artifacts + if: failure() + run: | + RESULTS_PATH=$(readlink -f /var/tmp/test_results/current) + sudo dmesg > $RESULTS_PATH/dmesg + sudo cp /var/log/syslog $RESULTS_PATH/ + sudo chmod +r $RESULTS_PATH/* + # Replace ':' in dir names, actions/upload-artifact doesn't support it + for f in $(find $RESULTS_PATH -name '*:*'); do mv "$f" "${f//:/__}"; done + - uses: actions/upload-artifact@v2 + if: failure() + with: + name: Test logs Ubuntu-${{ matrix.os }} + path: /var/tmp/test_results/20*/ + if-no-files-found: ignore diff --git a/sys/contrib/openzfs/.github/workflows/zfs-tests-sanity.yml b/sys/contrib/openzfs/.github/workflows/zfs-tests-sanity.yml new file mode 100644 index 000000000000..e03399757575 --- /dev/null +++ b/sys/contrib/openzfs/.github/workflows/zfs-tests-sanity.yml @@ -0,0 +1,60 @@ +name: zfs-tests-sanity + +on: + push: + pull_request: + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install --yes -qq build-essential autoconf libtool gdb lcov \ + git alien fakeroot wget curl bc fio acl \ + sysstat mdadm lsscsi parted gdebi attr dbench watchdog ksh \ + nfs-kernel-server samba rng-tools xz-utils \ + zlib1g-dev uuid-dev libblkid-dev libselinux-dev \ + xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \ + libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \ + libpam0g-dev pamtester python-dev python-setuptools python-cffi \ + python3 python3-dev python3-setuptools python3-cffi + - name: Autogen.sh + run: | + sh autogen.sh + - name: Configure + run: | + ./configure --enable-debug --enable-debuginfo + - name: Make + run: | + make --no-print-directory -s pkg-utils pkg-kmod + - name: Install + run: | + sudo dpkg -i *.deb + # Update order of directories to search for modules, otherwise + # Ubuntu will load kernel-shipped ones. + sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf + sudo depmod + sudo modprobe zfs + - name: Tests + run: | + /usr/share/zfs/zfs-tests.sh -v -s 3G -r sanity + - name: Prepare artifacts + if: failure() + run: | + RESULTS_PATH=$(readlink -f /var/tmp/test_results/current) + sudo dmesg > $RESULTS_PATH/dmesg + sudo cp /var/log/syslog $RESULTS_PATH/ + sudo chmod +r $RESULTS_PATH/* + # Replace ':' in dir names, actions/upload-artifact doesn't support it + for f in $(find $RESULTS_PATH -name '*:*'); do mv "$f" "${f//:/__}"; done + - uses: actions/upload-artifact@v2 + if: failure() + with: + name: Test logs + path: /var/tmp/test_results/20*/ + if-no-files-found: ignore diff --git a/sys/contrib/openzfs/META b/sys/contrib/openzfs/META index 886da443357d..abced52178a7 100644 --- a/sys/contrib/openzfs/META +++ b/sys/contrib/openzfs/META @@ -6,5 +6,5 @@ Release: rc1 Release-Tags: relext License: CDDL Author: OpenZFS -Linux-Maximum: 5.10 +Linux-Maximum: 5.11 Linux-Minimum: 3.10 diff --git a/sys/contrib/openzfs/Makefile.am b/sys/contrib/openzfs/Makefile.am index 436b78d76282..b7cc4ce85655 100644 --- a/sys/contrib/openzfs/Makefile.am +++ b/sys/contrib/openzfs/Makefile.am @@ -25,7 +25,6 @@ endif AUTOMAKE_OPTIONS = foreign EXTRA_DIST = autogen.sh copy-builtin -EXTRA_DIST += cppcheck-suppressions.txt EXTRA_DIST += config/config.awk config/rpm.am config/deb.am config/tgz.am EXTRA_DIST += META AUTHORS COPYRIGHT LICENSE NEWS NOTICE README.md EXTRA_DIST += CODE_OF_CONDUCT.md @@ -204,13 +203,13 @@ vcscheck: PHONY += lint lint: cppcheck paxcheck +CPPCHECKDIRS = cmd lib module PHONY += cppcheck -cppcheck: - @if type cppcheck > /dev/null 2>&1; then \ - cppcheck --quiet --force --error-exitcode=2 --inline-suppr \ - --suppressions-list=${top_srcdir}/cppcheck-suppressions.txt \ - -UHAVE_SSE2 -UHAVE_AVX512F -UHAVE_UIO_ZEROCOPY \ - ${top_srcdir}; \ +cppcheck: $(CPPCHECKDIRS) + @if test -n "$(CPPCHECK)"; then \ + set -e ; for dir in $(CPPCHECKDIRS) ; do \ + $(MAKE) -C $$dir cppcheck ; \ + done \ else \ echo "skipping cppcheck because cppcheck is not installed"; \ fi diff --git a/sys/contrib/openzfs/cmd/Makefile.am b/sys/contrib/openzfs/cmd/Makefile.am index d99d1dc382cc..473fcb0e07a1 100644 --- a/sys/contrib/openzfs/cmd/Makefile.am +++ b/sys/contrib/openzfs/cmd/Makefile.am @@ -2,10 +2,20 @@ SUBDIRS = zfs zpool zdb zhack zinject zstream zstreamdump ztest SUBDIRS += fsck_zfs vdev_id raidz_test zfs_ids_to_path SUBDIRS += zpool_influxdb +CPPCHECKDIRS = zfs zpool zdb zhack zinject zstream ztest +CPPCHECKDIRS += raidz_test zfs_ids_to_path zpool_influxdb + if USING_PYTHON SUBDIRS += arcstat arc_summary dbufstat endif if BUILD_LINUX SUBDIRS += mount_zfs zed zgenhostid zvol_id zvol_wait +CPPCHECKDIRS += mount_zfs zed zgenhostid zvol_id endif + +PHONY = cppcheck +cppcheck: $(CPPCHECKDIRS) + set -e ; for dir in $(CPPCHECKDIRS) ; do \ + $(MAKE) -C $$dir cppcheck ; \ + done diff --git a/sys/contrib/openzfs/cmd/mount_zfs/Makefile.am b/sys/contrib/openzfs/cmd/mount_zfs/Makefile.am index 6c4d6ff79f16..3957602d27ad 100644 --- a/sys/contrib/openzfs/cmd/mount_zfs/Makefile.am +++ b/sys/contrib/openzfs/cmd/mount_zfs/Makefile.am @@ -18,3 +18,5 @@ mount_zfs_LDADD = \ $(abs_top_builddir)/lib/libnvpair/libnvpair.la mount_zfs_LDADD += $(LTLIBINTL) + +include $(top_srcdir)/config/CppCheck.am diff --git a/sys/contrib/openzfs/cmd/mount_zfs/mount_zfs.c b/sys/contrib/openzfs/cmd/mount_zfs/mount_zfs.c index ca39d228479e..5196c3e5cb5f 100644 --- a/sys/contrib/openzfs/cmd/mount_zfs/mount_zfs.c +++ b/sys/contrib/openzfs/cmd/mount_zfs/mount_zfs.c @@ -50,6 +50,21 @@ libzfs_handle_t *g_zfs; static void parse_dataset(const char *target, char **dataset) { + /* + * Prior to util-linux 2.36.2, if a file or directory in the + * current working directory was named 'dataset' then mount(8) + * would prepend the current working directory to the dataset. + * Check for it and strip the prepended path when it is added. + */ + char cwd[PATH_MAX]; + if (getcwd(cwd, PATH_MAX) == NULL) { + perror("getcwd"); + return; + } + int len = strlen(cwd); + if (strncmp(cwd, target, len) == 0) + target += len; + /* Assume pool/dataset is more likely */ strlcpy(*dataset, target, PATH_MAX); diff --git a/sys/contrib/openzfs/cmd/raidz_test/Makefile.am b/sys/contrib/openzfs/cmd/raidz_test/Makefile.am index 72c914e641e4..983ff25dc92a 100644 --- a/sys/contrib/openzfs/cmd/raidz_test/Makefile.am +++ b/sys/contrib/openzfs/cmd/raidz_test/Makefile.am @@ -18,3 +18,5 @@ raidz_test_LDADD = \ $(abs_top_builddir)/lib/libzfs_core/libzfs_core.la raidz_test_LDADD += -lm + +include $(top_srcdir)/config/CppCheck.am diff --git a/sys/contrib/openzfs/cmd/raidz_test/raidz_test.c b/sys/contrib/openzfs/cmd/raidz_test/raidz_test.c index 4e2639f3676d..e3eb4f4ce44a 100644 --- a/sys/contrib/openzfs/cmd/raidz_test/raidz_test.c +++ b/sys/contrib/openzfs/cmd/raidz_test/raidz_test.c @@ -492,8 +492,9 @@ vdev_raidz_map_alloc_expanded(abd_t *abd, uint64_t size, uint64_t offset, (dc - r) * (rows - 1) + row; } rr->rr_col[c].rc_size = 1ULL << ashift; - rr->rr_col[c].rc_abd = - abd_get_offset(abd, off << ashift); + rr->rr_col[c].rc_abd = abd_get_offset_struct( + &rr->rr_col[c].rc_abdstruct, + abd, off << ashift, 1 << ashift); } asize += rr->rr_col[c].rc_size; diff --git a/sys/contrib/openzfs/cmd/vdev_id/vdev_id b/sys/contrib/openzfs/cmd/vdev_id/vdev_id index 8a75e638b67e..95a4e483b876 100755 --- a/sys/contrib/openzfs/cmd/vdev_id/vdev_id +++ b/sys/contrib/openzfs/cmd/vdev_id/vdev_id @@ -79,6 +79,34 @@ # channel 86:00.0 1 A # channel 86:00.0 0 B +# # +# # Example vdev_id.conf - multipath / multijbod-daisychaining +# # +# +# multipath yes +# multijbod yes +# +# # PCI_ID HBA PORT CHANNEL NAME +# channel 85:00.0 1 A +# channel 85:00.0 0 B +# channel 86:00.0 1 A +# channel 86:00.0 0 B + +# # +# # Example vdev_id.conf - multipath / mixed +# # +# +# multipath yes +# slot mix +# +# # PCI_ID HBA PORT CHANNEL NAME +# channel 85:00.0 3 A +# channel 85:00.0 2 B +# channel 86:00.0 3 A +# channel 86:00.0 2 B +# channel af:00.0 0 C +# channel af:00.0 1 C + # # # # Example vdev_id.conf - alias # # @@ -92,9 +120,10 @@ PATH=/bin:/sbin:/usr/bin:/usr/sbin CONFIG=/etc/zfs/vdev_id.conf PHYS_PER_PORT= DEV= -MULTIPATH= TOPOLOGY= BAY= +ENCL_ID="" +UNIQ_ENCL_ID="" usage() { cat << EOF @@ -107,6 +136,7 @@ Usage: vdev_id [-h] -e Create enclose device symlinks only (/dev/by-enclosure) -g Storage network topology [default="$TOPOLOGY"] -m Run in multipath mode + -j Run in multijbod mode -p number of phy's per switch port [default=$PHYS_PER_PORT] -h show this summary EOF @@ -117,12 +147,12 @@ map_slot() { LINUX_SLOT=$1 CHANNEL=$2 - MAPPED_SLOT=`awk "\\$1 == \"slot\" && \\$2 == ${LINUX_SLOT} && \ - \\$4 ~ /^${CHANNEL}$|^$/ { print \\$3; exit }" $CONFIG` + MAPPED_SLOT=$(awk '$1 == "slot" && $2 == "${LINUX_SLOT}" && \ + $4 ~ /^${CHANNEL}$|^$/ { print $3; exit}' $CONFIG) if [ -z "$MAPPED_SLOT" ] ; then MAPPED_SLOT=$LINUX_SLOT fi - printf "%d" ${MAPPED_SLOT} + printf "%d" "${MAPPED_SLOT}" } map_channel() { @@ -132,40 +162,120 @@ map_channel() { case $TOPOLOGY in "sas_switch") - MAPPED_CHAN=`awk "\\$1 == \"channel\" && \\$2 == ${PORT} \ - { print \\$3; exit }" $CONFIG` + MAPPED_CHAN=$(awk -v port="$PORT" \ + '$1 == "channel" && $2 == ${PORT} \ + { print $3; exit }' $CONFIG) ;; "sas_direct"|"scsi") - MAPPED_CHAN=`awk "\\$1 == \"channel\" && \ - \\$2 == \"${PCI_ID}\" && \\$3 == ${PORT} \ - { print \\$4; exit }" $CONFIG` + MAPPED_CHAN=$(awk -v pciID="$PCI_ID" -v port="$PORT" \ + '$1 == "channel" && $2 == pciID && $3 == port \ + {print $4}' $CONFIG) ;; esac - printf "%s" ${MAPPED_CHAN} + printf "%s" "${MAPPED_CHAN}" +} + +get_encl_id() { + set -- $(echo $1) + count=$# + + i=1 + while [ $i -le $count ] ; do + d=$(eval echo '$'{$i}) + id=$(cat "/sys/class/enclosure/${d}/id") + ENCL_ID="${ENCL_ID} $id" + i=$((i + 1)) + done +} + +get_uniq_encl_id() { + for uuid in ${ENCL_ID}; do + found=0 + + for count in ${UNIQ_ENCL_ID}; do + if [ $count = $uuid ]; then + found=1 + break + fi + done + + if [ $found -eq 0 ]; then + UNIQ_ENCL_ID="${UNIQ_ENCL_ID} $uuid" + fi + done +} + +# map_jbod explainer: The bsg driver knows the difference between a SAS +# expander and fanout expander. Use hostX instance along with top-level +# (whole enclosure) expander instances in /sys/class/enclosure and +# matching a field in an array of expanders, using the index of the +# matched array field as the enclosure instance, thereby making jbod IDs +# dynamic. Avoids reliance on high overhead userspace commands like +# multipath and lsscsi and instead uses existing sysfs data. $HOSTCHAN +# variable derived from devpath gymnastics in sas_handler() function. +map_jbod() { + DEVEXP=$(ls -l "/sys/block/$DEV/device/" | grep enclos | awk -F/ '{print $(NF-1) }') + DEV=$1 + + # Use "set --" to create index values (Arrays) + set -- $(ls -l /sys/class/enclosure | grep -v "^total" | awk '{print $9}') + # Get count of total elements + JBOD_COUNT=$# + JBOD_ITEM=$* + + # Build JBODs (enclosure) id from sys/class/enclosure/<dev>/id + get_encl_id "$JBOD_ITEM" + # Different expander instances for each paths. + # Filter out and keep only unique id. + get_uniq_encl_id + + # Identify final 'mapped jbod' + j=0 + for count in ${UNIQ_ENCL_ID}; do + i=1 + j=$((j + 1)) + while [ $i -le $JBOD_COUNT ] ; do + d=$(eval echo '$'{$i}) + id=$(cat "/sys/class/enclosure/${d}/id") + if [ "$d" = "$DEVEXP" ] && [ $id = $count ] ; then + MAPPED_JBOD=$j + break + fi + i=$((i + 1)) + done + done + + printf "%d" "${MAPPED_JBOD}" } sas_handler() { if [ -z "$PHYS_PER_PORT" ] ; then - PHYS_PER_PORT=`awk "\\$1 == \"phys_per_port\" \ - {print \\$2; exit}" $CONFIG` + PHYS_PER_PORT=$(awk '$1 == "phys_per_port" \ + {print $2; exit}' $CONFIG) fi PHYS_PER_PORT=${PHYS_PER_PORT:-4} - if ! echo $PHYS_PER_PORT | grep -q -E '^[0-9]+$' ; then + + if ! echo "$PHYS_PER_PORT" | grep -q -E '^[0-9]+$' ; then echo "Error: phys_per_port value $PHYS_PER_PORT is non-numeric" exit 1 fi if [ -z "$MULTIPATH_MODE" ] ; then - MULTIPATH_MODE=`awk "\\$1 == \"multipath\" \ - {print \\$2; exit}" $CONFIG` + MULTIPATH_MODE=$(awk '$1 == "multipath" \ + {print $2; exit}' $CONFIG) + fi + + if [ -z "$MULTIJBOD_MODE" ] ; then + MULTIJBOD_MODE=$(awk '$1 == "multijbod" \ + {print $2; exit}' $CONFIG) fi # Use first running component device if we're handling a dm-mpath device if [ "$MULTIPATH_MODE" = "yes" ] ; then # If udev didn't tell us the UUID via DM_NAME, check /dev/mapper if [ -z "$DM_NAME" ] ; then - DM_NAME=`ls -l --full-time /dev/mapper | - awk "/\/$DEV$/{print \\$9}"` + DM_NAME=$(ls -l --full-time /dev/mapper | + grep "$DEV"$ | awk '{print $9}') fi # For raw disks udev exports DEVTYPE=partition when @@ -175,28 +285,41 @@ sas_handler() { # we have to append the -part suffix directly in the # helper. if [ "$DEVTYPE" != "partition" ] ; then - PART=`echo $DM_NAME | awk -Fp '/p/{print "-part"$2}'` + PART=$(echo "$DM_NAME" | awk -Fp '/p/{print "-part"$2}') fi # Strip off partition information. - DM_NAME=`echo $DM_NAME | sed 's/p[0-9][0-9]*$//'` + DM_NAME=$(echo "$DM_NAME" | sed 's/p[0-9][0-9]*$//') if [ -z "$DM_NAME" ] ; then return fi - # Get the raw scsi device name from multipath -ll. Strip off - # leading pipe symbols to make field numbering consistent. - DEV=`multipath -ll $DM_NAME | - awk '/running/{gsub("^[|]"," "); print $3 ; exit}'` + # Utilize DM device name to gather subordinate block devices + # using sysfs to avoid userspace utilities + DMDEV=$(ls -l --full-time /dev/mapper | grep $DM_NAME | + awk '{gsub("../", " "); print $NF}') + + # Use sysfs pointers in /sys/block/dm-X/slaves because using + # userspace tools creates lots of overhead and should be avoided + # whenever possible. Use awk to isolate lowest instance of + # sd device member in dm device group regardless of string + # length. + DEV=$(ls "/sys/block/$DMDEV/slaves" | awk ' + { len=sprintf ("%20s",length($0)); gsub(/ /,0,str); a[NR]=len "_" $0; } + END { + asort(a) + print substr(a[1],22) + }') + if [ -z "$DEV" ] ; then return fi fi - if echo $DEV | grep -q ^/devices/ ; then + if echo "$DEV" | grep -q ^/devices/ ; then sys_path=$DEV else - sys_path=`udevadm info -q path -p /sys/block/$DEV 2>/dev/null` + sys_path=$(udevadm info -q path -p "/sys/block/$DEV" 2>/dev/null) fi # Use positional parameters as an ad-hoc array @@ -206,84 +329,104 @@ sas_handler() { # Get path up to /sys/.../hostX i=1 - while [ $i -le $num_dirs ] ; do - d=$(eval echo \${$i}) + + while [ $i -le "$num_dirs" ] ; do + d=$(eval echo '$'{$i}) scsi_host_dir="$scsi_host_dir/$d" - echo $d | grep -q -E '^host[0-9]+$' && break - i=$(($i + 1)) + echo "$d" | grep -q -E '^host[0-9]+$' && break + i=$((i + 1)) done - if [ $i = $num_dirs ] ; then + # Lets grab the SAS host channel number and save it for JBOD sorting later + HOSTCHAN=$(echo "$d" | awk -F/ '{ gsub("host","",$NF); print $NF}') + + if [ $i = "$num_dirs" ] ; then return fi - PCI_ID=$(eval echo \${$(($i -1))} | awk -F: '{print $2":"$3}') + PCI_ID=$(eval echo '$'{$((i -1))} | awk -F: '{print $2":"$3}') # In sas_switch mode, the directory four levels beneath # /sys/.../hostX contains symlinks to phy devices that reveal # the switch port number. In sas_direct mode, the phy links one # directory down reveal the HBA port. port_dir=$scsi_host_dir + case $TOPOLOGY in - "sas_switch") j=$(($i + 4)) ;; - "sas_direct") j=$(($i + 1)) ;; + "sas_switch") j=$((i + 4)) ;; + "sas_direct") j=$((i + 1)) ;; esac - i=$(($i + 1)) + i=$((i + 1)) + while [ $i -le $j ] ; do - port_dir="$port_dir/$(eval echo \${$i})" - i=$(($i + 1)) + port_dir="$port_dir/$(eval echo '$'{$i})" + i=$((i + 1)) done - PHY=`ls -d $port_dir/phy* 2>/dev/null | head -1 | awk -F: '{print $NF}'` + PHY=$(ls -d "$port_dir"/phy* 2>/dev/null | head -1 | awk -F: '{print $NF}') if [ -z "$PHY" ] ; then PHY=0 fi - PORT=$(( $PHY / $PHYS_PER_PORT )) + PORT=$((PHY / PHYS_PER_PORT)) # Look in /sys/.../sas_device/end_device-X for the bay_identifier # attribute. end_device_dir=$port_dir - while [ $i -lt $num_dirs ] ; do - d=$(eval echo \${$i}) + + while [ $i -lt "$num_dirs" ] ; do + d=$(eval echo '$'{$i}) end_device_dir="$end_device_dir/$d" - if echo $d | grep -q '^end_device' ; then + if echo "$d" | grep -q '^end_device' ; then end_device_dir="$end_device_dir/sas_device/$d" break fi - i=$(($i + 1)) + i=$((i + 1)) done + # Add 'mix' slot type for environments where dm-multipath devices + # include end-devices connected via SAS expanders or direct connection + # to SAS HBA. A mixed connectivity environment such as pool devices + # contained in a SAS JBOD and spare drives or log devices directly + # connected in a server backplane without expanders in the I/O path. SLOT= + case $BAY in "bay") - SLOT=`cat $end_device_dir/bay_identifier 2>/dev/null` + SLOT=$(cat "$end_device_dir/bay_identifier" 2>/dev/null) + ;; + "mix") + if [ $(cat "$end_device_dir/bay_identifier" 2>/dev/null) ] ; then + SLOT=$(cat "$end_device_dir/bay_identifier" 2>/dev/null) + else + SLOT=$(cat "$end_device_dir/phy_identifier" 2>/dev/null) + fi ;; "phy") - SLOT=`cat $end_device_dir/phy_identifier 2>/dev/null` + SLOT=$(cat "$end_device_dir/phy_identifier" 2>/dev/null) ;; "port") - d=$(eval echo \${$i}) - SLOT=`echo $d | sed -e 's/^.*://'` + d=$(eval echo '$'{$i}) + SLOT=$(echo "$d" | sed -e 's/^.*://') ;; "id") - i=$(($i + 1)) - d=$(eval echo \${$i}) - SLOT=`echo $d | sed -e 's/^.*://'` + i=$((i + 1)) + d=$(eval echo '$'{$i}) + SLOT=$(echo "$d" | sed -e 's/^.*://') ;; "lun") - i=$(($i + 2)) - d=$(eval echo \${$i}) - SLOT=`echo $d | sed -e 's/^.*://'` + i=$((i + 2)) + d=$(eval echo '$'{$i}) + SLOT=$(echo "$d" | sed -e 's/^.*://') ;; "ses") # look for this SAS path in all SCSI Enclosure Services # (SES) enclosures - sas_address=`cat $end_device_dir/sas_address 2>/dev/null` - enclosures=`lsscsi -g | \ - sed -n -e '/enclosu/s/^.* \([^ ][^ ]*\) *$/\1/p'` + sas_address=$(cat "$end_device_dir/sas_address" 2>/dev/null) + enclosures=$(lsscsi -g | \ + sed -n -e '/enclosu/s/^.* \([^ ][^ ]*\) *$/\1/p') for enclosure in $enclosures; do - set -- $(sg_ses -p aes $enclosure | \ + set -- $(sg_ses -p aes "$enclosure" | \ awk "/device slot number:/{slot=\$12} \ /SAS address: $sas_address/\ *** 11320 LINES SKIPPED ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202103100936.12A9aDPl013647>