Date: Tue, 22 Aug 2017 11:03:50 +0000 (UTC) From: Baptiste Daroussin <bapt@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r322780 - in vendor/zstd/1.3.1: . doc doc/educational_decoder examples lib lib/common lib/compress lib/decompress lib/deprecated lib/dictBuilder lib/legacy programs tests tests/files te... Message-ID: <201708221103.v7MB3oB4051158@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: bapt Date: Tue Aug 22 11:03:50 2017 New Revision: 322780 URL: https://svnweb.freebsd.org/changeset/base/322780 Log: Tag import of zstandard 1.3.1 Added: vendor/zstd/1.3.1/ - copied from r322652, vendor/zstd/dist/ vendor/zstd/1.3.1/tests/files/ - copied from r322779, vendor/zstd/dist/tests/files/ vendor/zstd/1.3.1/tests/fuzz/ - copied from r322779, vendor/zstd/dist/tests/fuzz/ vendor/zstd/1.3.1/tests/poolTests.c - copied unchanged from r322779, vendor/zstd/dist/tests/poolTests.c Replaced: vendor/zstd/1.3.1/CONTRIBUTING.md - copied unchanged from r322779, vendor/zstd/dist/CONTRIBUTING.md vendor/zstd/1.3.1/Makefile - copied unchanged from r322779, vendor/zstd/dist/Makefile vendor/zstd/1.3.1/NEWS - copied unchanged from r322779, vendor/zstd/dist/NEWS vendor/zstd/1.3.1/README.md - copied unchanged from r322779, vendor/zstd/dist/README.md vendor/zstd/1.3.1/circle.yml - copied unchanged from r322779, vendor/zstd/dist/circle.yml vendor/zstd/1.3.1/doc/educational_decoder/harness.c - copied unchanged from r322779, vendor/zstd/dist/doc/educational_decoder/harness.c vendor/zstd/1.3.1/doc/educational_decoder/zstd_decompress.c - copied unchanged from r322779, vendor/zstd/dist/doc/educational_decoder/zstd_decompress.c vendor/zstd/1.3.1/doc/educational_decoder/zstd_decompress.h - copied unchanged from r322779, vendor/zstd/dist/doc/educational_decoder/zstd_decompress.h vendor/zstd/1.3.1/doc/zstd_compression_format.md - copied unchanged from r322779, vendor/zstd/dist/doc/zstd_compression_format.md vendor/zstd/1.3.1/doc/zstd_manual.html - copied unchanged from r322779, vendor/zstd/dist/doc/zstd_manual.html vendor/zstd/1.3.1/lib/common/bitstream.h - copied unchanged from r322779, vendor/zstd/dist/lib/common/bitstream.h vendor/zstd/1.3.1/lib/common/error_private.c - copied unchanged from r322779, vendor/zstd/dist/lib/common/error_private.c vendor/zstd/1.3.1/lib/common/error_private.h - copied unchanged from r322779, vendor/zstd/dist/lib/common/error_private.h vendor/zstd/1.3.1/lib/common/fse.h - copied unchanged from r322779, vendor/zstd/dist/lib/common/fse.h vendor/zstd/1.3.1/lib/common/fse_decompress.c - copied unchanged from r322779, vendor/zstd/dist/lib/common/fse_decompress.c vendor/zstd/1.3.1/lib/common/huf.h - copied unchanged from r322779, vendor/zstd/dist/lib/common/huf.h vendor/zstd/1.3.1/lib/common/mem.h - copied unchanged from r322779, vendor/zstd/dist/lib/common/mem.h vendor/zstd/1.3.1/lib/common/pool.c - copied unchanged from r322779, vendor/zstd/dist/lib/common/pool.c vendor/zstd/1.3.1/lib/common/pool.h - copied unchanged from r322779, vendor/zstd/dist/lib/common/pool.h vendor/zstd/1.3.1/lib/common/threading.h - copied unchanged from r322779, vendor/zstd/dist/lib/common/threading.h vendor/zstd/1.3.1/lib/common/xxhash.c - copied unchanged from r322779, vendor/zstd/dist/lib/common/xxhash.c vendor/zstd/1.3.1/lib/common/zstd_common.c - copied unchanged from r322779, vendor/zstd/dist/lib/common/zstd_common.c vendor/zstd/1.3.1/lib/common/zstd_errors.h - copied unchanged from r322779, vendor/zstd/dist/lib/common/zstd_errors.h vendor/zstd/1.3.1/lib/common/zstd_internal.h - copied unchanged from r322779, vendor/zstd/dist/lib/common/zstd_internal.h vendor/zstd/1.3.1/lib/compress/fse_compress.c - copied unchanged from r322779, vendor/zstd/dist/lib/compress/fse_compress.c vendor/zstd/1.3.1/lib/compress/huf_compress.c - copied unchanged from r322779, vendor/zstd/dist/lib/compress/huf_compress.c vendor/zstd/1.3.1/lib/compress/zstd_compress.c - copied unchanged from r322779, vendor/zstd/dist/lib/compress/zstd_compress.c vendor/zstd/1.3.1/lib/compress/zstd_opt.h - copied unchanged from r322779, vendor/zstd/dist/lib/compress/zstd_opt.h vendor/zstd/1.3.1/lib/compress/zstdmt_compress.c - copied unchanged from r322779, vendor/zstd/dist/lib/compress/zstdmt_compress.c vendor/zstd/1.3.1/lib/compress/zstdmt_compress.h - copied unchanged from r322779, vendor/zstd/dist/lib/compress/zstdmt_compress.h vendor/zstd/1.3.1/lib/decompress/huf_decompress.c - copied unchanged from r322779, vendor/zstd/dist/lib/decompress/huf_decompress.c vendor/zstd/1.3.1/lib/decompress/zstd_decompress.c - copied unchanged from r322779, vendor/zstd/dist/lib/decompress/zstd_decompress.c vendor/zstd/1.3.1/lib/deprecated/zbuff.h - copied unchanged from r322779, vendor/zstd/dist/lib/deprecated/zbuff.h vendor/zstd/1.3.1/lib/deprecated/zbuff_common.c - copied unchanged from r322779, vendor/zstd/dist/lib/deprecated/zbuff_common.c vendor/zstd/1.3.1/lib/deprecated/zbuff_compress.c - copied unchanged from r322779, vendor/zstd/dist/lib/deprecated/zbuff_compress.c vendor/zstd/1.3.1/lib/deprecated/zbuff_decompress.c - copied unchanged from r322779, vendor/zstd/dist/lib/deprecated/zbuff_decompress.c vendor/zstd/1.3.1/lib/dictBuilder/cover.c - copied unchanged from r322779, vendor/zstd/dist/lib/dictBuilder/cover.c vendor/zstd/1.3.1/lib/dictBuilder/zdict.c - copied unchanged from r322779, vendor/zstd/dist/lib/dictBuilder/zdict.c vendor/zstd/1.3.1/lib/dictBuilder/zdict.h - copied unchanged from r322779, vendor/zstd/dist/lib/dictBuilder/zdict.h vendor/zstd/1.3.1/lib/legacy/zstd_legacy.h - copied unchanged from r322779, vendor/zstd/dist/lib/legacy/zstd_legacy.h vendor/zstd/1.3.1/lib/legacy/zstd_v01.c - copied unchanged from r322779, vendor/zstd/dist/lib/legacy/zstd_v01.c vendor/zstd/1.3.1/lib/legacy/zstd_v01.h - copied unchanged from r322779, vendor/zstd/dist/lib/legacy/zstd_v01.h vendor/zstd/1.3.1/lib/legacy/zstd_v02.c - copied unchanged from r322779, vendor/zstd/dist/lib/legacy/zstd_v02.c vendor/zstd/1.3.1/lib/legacy/zstd_v02.h - copied unchanged from r322779, vendor/zstd/dist/lib/legacy/zstd_v02.h vendor/zstd/1.3.1/lib/legacy/zstd_v03.c - copied unchanged from r322779, vendor/zstd/dist/lib/legacy/zstd_v03.c vendor/zstd/1.3.1/lib/legacy/zstd_v03.h - copied unchanged from r322779, vendor/zstd/dist/lib/legacy/zstd_v03.h vendor/zstd/1.3.1/lib/legacy/zstd_v04.c - copied unchanged from r322779, vendor/zstd/dist/lib/legacy/zstd_v04.c vendor/zstd/1.3.1/lib/legacy/zstd_v04.h - copied unchanged from r322779, vendor/zstd/dist/lib/legacy/zstd_v04.h vendor/zstd/1.3.1/lib/legacy/zstd_v05.c - copied unchanged from r322779, vendor/zstd/dist/lib/legacy/zstd_v05.c vendor/zstd/1.3.1/lib/legacy/zstd_v05.h - copied unchanged from r322779, vendor/zstd/dist/lib/legacy/zstd_v05.h vendor/zstd/1.3.1/lib/legacy/zstd_v06.c - copied unchanged from r322779, vendor/zstd/dist/lib/legacy/zstd_v06.c vendor/zstd/1.3.1/lib/legacy/zstd_v06.h - copied unchanged from r322779, vendor/zstd/dist/lib/legacy/zstd_v06.h vendor/zstd/1.3.1/lib/legacy/zstd_v07.c - copied unchanged from r322779, vendor/zstd/dist/lib/legacy/zstd_v07.c vendor/zstd/1.3.1/lib/legacy/zstd_v07.h - copied unchanged from r322779, vendor/zstd/dist/lib/legacy/zstd_v07.h vendor/zstd/1.3.1/lib/zstd.h - copied unchanged from r322779, vendor/zstd/dist/lib/zstd.h vendor/zstd/1.3.1/programs/.gitignore - copied unchanged from r322779, vendor/zstd/dist/programs/.gitignore vendor/zstd/1.3.1/programs/Makefile - copied unchanged from r322779, vendor/zstd/dist/programs/Makefile vendor/zstd/1.3.1/programs/README.md - copied unchanged from r322779, vendor/zstd/dist/programs/README.md vendor/zstd/1.3.1/programs/bench.c - copied unchanged from r322779, vendor/zstd/dist/programs/bench.c vendor/zstd/1.3.1/programs/bench.h - copied unchanged from r322779, vendor/zstd/dist/programs/bench.h vendor/zstd/1.3.1/programs/datagen.c - copied unchanged from r322779, vendor/zstd/dist/programs/datagen.c vendor/zstd/1.3.1/programs/datagen.h - copied unchanged from r322779, vendor/zstd/dist/programs/datagen.h vendor/zstd/1.3.1/programs/dibio.c - copied unchanged from r322779, vendor/zstd/dist/programs/dibio.c vendor/zstd/1.3.1/programs/dibio.h - copied unchanged from r322779, vendor/zstd/dist/programs/dibio.h vendor/zstd/1.3.1/programs/fileio.c - copied unchanged from r322779, vendor/zstd/dist/programs/fileio.c vendor/zstd/1.3.1/programs/fileio.h - copied unchanged from r322779, vendor/zstd/dist/programs/fileio.h vendor/zstd/1.3.1/programs/platform.h - copied unchanged from r322779, vendor/zstd/dist/programs/platform.h vendor/zstd/1.3.1/programs/util.h - copied unchanged from r322779, vendor/zstd/dist/programs/util.h vendor/zstd/1.3.1/programs/zstd.1 - copied unchanged from r322779, vendor/zstd/dist/programs/zstd.1 vendor/zstd/1.3.1/programs/zstd.1.md - copied unchanged from r322779, vendor/zstd/dist/programs/zstd.1.md vendor/zstd/1.3.1/programs/zstdcli.c - copied unchanged from r322779, vendor/zstd/dist/programs/zstdcli.c vendor/zstd/1.3.1/tests/Makefile - copied unchanged from r322779, vendor/zstd/dist/tests/Makefile vendor/zstd/1.3.1/tests/datagencli.c - copied unchanged from r322779, vendor/zstd/dist/tests/datagencli.c vendor/zstd/1.3.1/tests/decodecorpus.c - copied unchanged from r322779, vendor/zstd/dist/tests/decodecorpus.c vendor/zstd/1.3.1/tests/fullbench.c - copied unchanged from r322779, vendor/zstd/dist/tests/fullbench.c vendor/zstd/1.3.1/tests/fuzzer.c - copied unchanged from r322779, vendor/zstd/dist/tests/fuzzer.c vendor/zstd/1.3.1/tests/invalidDictionaries.c - copied unchanged from r322779, vendor/zstd/dist/tests/invalidDictionaries.c vendor/zstd/1.3.1/tests/legacy.c - copied unchanged from r322779, vendor/zstd/dist/tests/legacy.c vendor/zstd/1.3.1/tests/longmatch.c - copied unchanged from r322779, vendor/zstd/dist/tests/longmatch.c vendor/zstd/1.3.1/tests/namespaceTest.c - copied unchanged from r322779, vendor/zstd/dist/tests/namespaceTest.c vendor/zstd/1.3.1/tests/paramgrill.c - copied unchanged from r322779, vendor/zstd/dist/tests/paramgrill.c vendor/zstd/1.3.1/tests/playTests.sh - copied unchanged from r322779, vendor/zstd/dist/tests/playTests.sh vendor/zstd/1.3.1/tests/roundTripCrash.c - copied unchanged from r322779, vendor/zstd/dist/tests/roundTripCrash.c vendor/zstd/1.3.1/tests/symbols.c - copied unchanged from r322779, vendor/zstd/dist/tests/symbols.c vendor/zstd/1.3.1/tests/zbufftest.c - copied unchanged from r322779, vendor/zstd/dist/tests/zbufftest.c vendor/zstd/1.3.1/tests/zstreamtest.c - copied unchanged from r322779, vendor/zstd/dist/tests/zstreamtest.c vendor/zstd/1.3.1/zlibWrapper/examples/zwrapbench.c - copied unchanged from r322779, vendor/zstd/dist/zlibWrapper/examples/zwrapbench.c vendor/zstd/1.3.1/zlibWrapper/gzcompatibility.h - copied unchanged from r322779, vendor/zstd/dist/zlibWrapper/gzcompatibility.h vendor/zstd/1.3.1/zlibWrapper/gzlib.c - copied unchanged from r322779, vendor/zstd/dist/zlibWrapper/gzlib.c vendor/zstd/1.3.1/zlibWrapper/gzread.c - copied unchanged from r322779, vendor/zstd/dist/zlibWrapper/gzread.c vendor/zstd/1.3.1/zlibWrapper/gzwrite.c - copied unchanged from r322779, vendor/zstd/dist/zlibWrapper/gzwrite.c vendor/zstd/1.3.1/zlibWrapper/zstd_zlibwrapper.c - copied unchanged from r322779, vendor/zstd/dist/zlibWrapper/zstd_zlibwrapper.c vendor/zstd/1.3.1/zlibWrapper/zstd_zlibwrapper.h - copied unchanged from r322779, vendor/zstd/dist/zlibWrapper/zstd_zlibwrapper.h Deleted: vendor/zstd/1.3.1/PATENTS vendor/zstd/1.3.1/examples/ vendor/zstd/1.3.1/tests/pool.c Copied: vendor/zstd/1.3.1/CONTRIBUTING.md (from r322779, vendor/zstd/dist/CONTRIBUTING.md) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/zstd/1.3.1/CONTRIBUTING.md Tue Aug 22 11:03:50 2017 (r322780, copy of r322779, vendor/zstd/dist/CONTRIBUTING.md) @@ -0,0 +1,42 @@ +# Contributing to Zstandard +We want to make contributing to this project as easy and transparent as +possible. + +## Our Development Process +New versions are being developed in the "dev" branch, +or in their own feature branch. +When they are deemed ready for a release, they are merged into "master". + +As a consequences, all contributions must stage first through "dev" +or their own feature branch. + +## Pull Requests +We actively welcome your pull requests. + +1. Fork the repo and create your branch from `dev`. +2. If you've added code that should be tested, add tests. +3. If you've changed APIs, update the documentation. +4. Ensure the test suite passes. +5. Make sure your code lints. +6. If you haven't already, complete the Contributor License Agreement ("CLA"). + +## Contributor License Agreement ("CLA") +In order to accept your pull request, we need you to submit a CLA. You only need +to do this once to work on any of Facebook's open source projects. + +Complete your CLA here: <https://code.facebook.com/cla>; + +## Issues +We use GitHub issues to track public bugs. Please ensure your description is +clear and has sufficient instructions to be able to reproduce the issue. + +Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe +disclosure of security bugs. In those cases, please go through the process +outlined on that page and do not file a public issue. + +## Coding Style +* 4 spaces for indentation rather than tabs + +## License +By contributing to Zstandard, you agree that your contributions will be licensed +under both the [LICENSE](LICENSE) file and the [COPYING](COPYING) file in the root directory of this source tree. Copied: vendor/zstd/1.3.1/Makefile (from r322779, vendor/zstd/dist/Makefile) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/zstd/1.3.1/Makefile Tue Aug 22 11:03:50 2017 (r322780, copy of r322779, vendor/zstd/dist/Makefile) @@ -0,0 +1,334 @@ +# ################################################################ +# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. +# ################################################################ + +PRGDIR = programs +ZSTDDIR = lib +BUILDIR = build +ZWRAPDIR = zlibWrapper +TESTDIR = tests + +# Define nul output +VOID = /dev/null + +ifneq (,$(filter Windows%,$(OS))) +EXT =.exe +else +EXT = +endif + +.PHONY: default +default: lib-release zstd-release + +.PHONY: all +all: | allmost examples manual + +.PHONY: allmost +allmost: + $(MAKE) -C $(ZSTDDIR) all + $(MAKE) -C $(PRGDIR) all + $(MAKE) -C $(TESTDIR) all + $(MAKE) -C $(ZWRAPDIR) all + +#skip zwrapper, can't build that on alternate architectures without the proper zlib installed +.PHONY: allarch +allarch: + $(MAKE) -C $(ZSTDDIR) all + $(MAKE) -C $(PRGDIR) all + $(MAKE) -C $(TESTDIR) all + +.PHONY: all32 +all32: + $(MAKE) -C $(PRGDIR) zstd32 + $(MAKE) -C $(TESTDIR) all32 + +.PHONY: lib +lib: + @$(MAKE) -C $(ZSTDDIR) $@ + +.PHONY: lib-release +lib-release: + @$(MAKE) -C $(ZSTDDIR) + +.PHONY: zstd +zstd: + @$(MAKE) -C $(PRGDIR) $@ + cp $(PRGDIR)/zstd$(EXT) . + +.PHONY: zstd-release +zstd-release: + @$(MAKE) -C $(PRGDIR) + cp $(PRGDIR)/zstd$(EXT) . + +.PHONY: zstdmt +zstdmt: + @$(MAKE) -C $(PRGDIR) $@ + cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT) + +.PHONY: zlibwrapper +zlibwrapper: + $(MAKE) -C $(ZWRAPDIR) test + +.PHONY: test shortest +test shortest: + $(MAKE) -C $(PRGDIR) allVariants + $(MAKE) -C $(TESTDIR) $@ + +.PHONY: examples +examples: + CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all + +.PHONY: manual +manual: + $(MAKE) -C contrib/gen_html $@ + +.PHONY: cleanTabs +cleanTabs: + cd contrib; ./cleanTabs + +.PHONY: clean +clean: + @$(MAKE) -C $(ZSTDDIR) $@ > $(VOID) + @$(MAKE) -C $(PRGDIR) $@ > $(VOID) + @$(MAKE) -C $(TESTDIR) $@ > $(VOID) + @$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID) + @$(MAKE) -C examples/ $@ > $(VOID) + @$(MAKE) -C contrib/gen_html $@ > $(VOID) + @$(RM) zstd$(EXT) zstdmt$(EXT) tmp* + @echo Cleaning completed + +#------------------------------------------------------------------------------ +# make install is validated only for Linux, OSX, Hurd and some BSD targets +#------------------------------------------------------------------------------ +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD MSYS_NT)) + +HOST_OS = POSIX +CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON + +.PHONY: list +list: + @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs + +.PHONY: install clangtest gpptest armtest usan asan uasan +install: + @$(MAKE) -C $(ZSTDDIR) $@ + @$(MAKE) -C $(PRGDIR) $@ + +.PHONY: uninstall +uninstall: + @$(MAKE) -C $(ZSTDDIR) $@ + @$(MAKE) -C $(PRGDIR) $@ + +.PHONY: travis-install +travis-install: + $(MAKE) install PREFIX=~/install_test_dir + +.PHONY: gppbuild +gppbuild: clean + g++ -v + CC=g++ $(MAKE) -C programs all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" + +.PHONY: gcc5build +gcc5build: clean + gcc-5 -v + CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror" + +.PHONY: gcc6build +gcc6build: clean + gcc-6 -v + CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror" + +.PHONY: gcc7build +gcc7build: clean + gcc-7 -v + CC=gcc-7 $(MAKE) all MOREFLAGS="-Werror" + +.PHONY: clangbuild +clangbuild: clean + clang -v + CXX=clang++ CC=clang $(MAKE) all MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" + +m32build: clean + gcc -v + $(MAKE) all32 + +armbuild: clean + CC=arm-linux-gnueabi-gcc CFLAGS="-Werror" $(MAKE) allarch + +aarch64build: clean + CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allarch + +ppcbuild: clean + CC=powerpc-linux-gnu-gcc CLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allarch + +ppc64build: clean + CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allarch + +armfuzz: clean + CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest + +aarch64fuzz: clean + CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest + +ppcfuzz: clean + CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest + +ppc64fuzz: clean + CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest + +gpptest: clean + CC=$(CXX) $(MAKE) -C $(PRGDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" + +gcc5test: clean + gcc-5 -v + $(MAKE) all CC=gcc-5 MOREFLAGS="-Werror" + +gcc6test: clean + gcc-6 -v + $(MAKE) all CC=gcc-6 MOREFLAGS="-Werror" + +clangtest: clean + clang -v + $(MAKE) all CXX=clang-++ CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" + +armtest: clean + $(MAKE) -C $(TESTDIR) datagen # use native, faster + $(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests + +aarch64test: + $(MAKE) -C $(TESTDIR) datagen # use native, faster + $(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests + +ppctest: clean + $(MAKE) -C $(TESTDIR) datagen # use native, faster + $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" FUZZER_FLAGS=--no-big-tests + +ppc64test: clean + $(MAKE) -C $(TESTDIR) datagen # use native, faster + $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests + +arm-ppc-compilation: + $(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" + $(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static" + $(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" + $(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" + +# run UBsan with -fsanitize-recover=signed-integer-overflow +# due to a bug in UBsan when doing pointer subtraction +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303 + +usan: clean + $(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=undefined" + +asan: clean + $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address" + +asan-%: clean + LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address" $(MAKE) -C $(TESTDIR) $* + +msan: clean + $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer" HAVE_LZMA=0 # datagen.c fails this test for no obvious reason + +msan-%: clean + LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) HAVE_LZMA=0 $* + +asan32: clean + $(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address" + +uasan: clean + $(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined" + +uasan-%: clean + LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined" $(MAKE) -C $(TESTDIR) $* + +tsan-%: clean + LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests + +apt-install: + sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES) + +apt-add-repo: + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + sudo apt-get update -y -qq + +ppcinstall: + APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install + +arminstall: + APT_PACKAGES="qemu-system-arm qemu-user-static gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install + +valgrindinstall: + APT_PACKAGES="valgrind" $(MAKE) apt-install + +libc6install: + APT_PACKAGES="libc6-dev-i386 gcc-multilib" $(MAKE) apt-install + +gcc6install: apt-add-repo + APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-6 gcc-6-multilib" $(MAKE) apt-install + +gpp6install: apt-add-repo + APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install + +clang38install: + APT_PACKAGES="clang-3.8" $(MAKE) apt-install + +endif + + +ifneq (,$(filter MSYS%,$(shell uname))) +HOST_OS = MSYS +CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON +endif + + +#------------------------------------------------------------------------ +#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets +#------------------------------------------------------------------------ +ifneq (,$(filter $(HOST_OS),MSYS POSIX)) +cmakebuild: + cmake --version + $(RM) -r $(BUILDIR)/cmake/build + mkdir $(BUILDIR)/cmake/build + cd $(BUILDIR)/cmake/build ; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall + +c90build: clean + gcc -v + CFLAGS="-std=c90" $(MAKE) allmost # will fail, due to missing support for `long long` + +gnu90build: clean + gcc -v + CFLAGS="-std=gnu90" $(MAKE) allmost + +c99build: clean + gcc -v + CFLAGS="-std=c99" $(MAKE) allmost + +gnu99build: clean + gcc -v + CFLAGS="-std=gnu99" $(MAKE) allmost + +c11build: clean + gcc -v + CFLAGS="-std=c11" $(MAKE) allmost + +bmix64build: clean + gcc -v + CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test + +bmix32build: clean + gcc -v + CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(TESTDIR) test + +bmi32build: clean + gcc -v + CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test + +staticAnalyze: clean + gcc -v + CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all +endif Copied: vendor/zstd/1.3.1/NEWS (from r322779, vendor/zstd/dist/NEWS) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/zstd/1.3.1/NEWS Tue Aug 22 11:03:50 2017 (r322780, copy of r322779, vendor/zstd/dist/NEWS) @@ -0,0 +1,313 @@ +v1.3.1 +New license : BSD + GPLv2 +perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt (@mcmilk) +perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760) +cli : improved and fixed --list command, by @ib (#772) +cli : command -vV to list supported formats, by @ib (#771) +build : fixed binary variants, reported by @svenha (#788) +build : fix Visual compilation for non x86/x64 targets, reported by Greg Slazinski (@GregSlazinski) (#718) +API exp : breaking change : ZSTD_getframeHeader() provides more information +API exp : breaking change : pinned down values of error codes +doc : fixed huffman example, by Ulrich Kunitz (@ulikunitz) +new : contrib/adaptive-compression, I/O driven compression strength, by Paul Cruz (@paulcruz74) +new : contrib/long_distance_matching, statistics by Stella Lau (@stellamplau) +updated : contrib/linux-kernel, by Nick Terrell (@terrelln) + +v1.3.0 +cli : new : `--list` command, by Paul Cruz +cli : changed : xz/lzma support enabled by default +cli : changed : `-t *` continue processing list after a decompression error +API : added : ZSTD_versionString() +API : promoted to stable status : ZSTD_getFrameContentSize(), by Sean Purcell +API exp : new advanced API : ZSTD_compress_generic(), ZSTD_CCtx_setParameter() +API exp : new : API for static or external allocation : ZSTD_initStatic?Ctx() +API exp : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700) +API exp : clarified memory estimation / measurement functions. +API exp : changed : strongest strategy renamed ZSTD_btultra, fastest strategy ZSTD_fast set to 1 +tools : decodecorpus can generate random dictionary-compressed samples, by Paul Cruz +new : contrib/seekable_format, demo and API, by Sean Purcell +changed : contrib/linux-kernel, updated version and license, by Nick Terrell + +v1.2.0 +cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable) +cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell +cli : new : zstdmt symlink hardwired to `zstd -T0` +cli : new : command --threads=# (#671) +cli : changed : cover dictionary builder by default, for improved quality, by Nick Terrell +cli : new : commands --train-cover and --train-legacy, to select dictionary algorithm and parameters +cli : experimental targets `zstd4` and `xzstd4`, with support for lz4 format, by Sean Purcell +cli : fix : does not output compressed data on console +cli : fix : ignore symbolic links unless --force specified, +API : breaking change : ZSTD_createCDict_advanced(), only use compressionParameters as argument +API : added : prototypes ZSTD_*_usingCDict_advanced(), for direct control over frameParameters. +API : improved: ZSTDMT_compressCCtx() reduced memory usage +API : fix : ZSTDMT_compressCCtx() now provides srcSize in header (#634) +API : fix : src size stored in frame header is controlled at end of frame +API : fix : enforced consistent rules for pledgedSrcSize==0 (#641) +API : fix : error code "GENERIC" replaced by "dstSizeTooSmall" when appropriate +build: improved cmake script, by @Majlen +build: enabled Multi-threading support for *BSD, by Baptiste Daroussin +tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target. +new : contrib/linux-kernel version, by Nick Terrell + +v1.1.4 +cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski +cli : new : advanced benchmark command --priority=rt +cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77 +cli : fix : --rm remains silent when input is stdin +cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski +speed : improved decompression speed in streaming mode for single shot scenarios (+5%) +memory: DDict (decompression dictionary) memory usage down from 150 KB to 20 KB +arch: 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell +API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize() +API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value) +build : new: meson build system in contrib/meson, by Dima Krasner +build : improved cmake script, by @Majlen +build : added -Wformat-security flag, as recommended by Padraig Brady +doc : new : educational decoder, by Sean Purcell + +v1.1.3 +cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`) +cli : new : experimental target `make zstdmt`, with multi-threading support +cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell, based on prior work by Giuseppe Ottaviano. +cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski +cli : fix zstdless on Mac OS-X, by Andrew Janke +cli : fix #232 "compress non-files" +dictBuilder : improved dictionary generation quality, thanks to Nick Terrell +API : new : lib/compress/ZSTDMT_compress.h multithreading API (experimental) +API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul +API : new : ZDICT_finalizeDictionary() +API : fix : ZSTD_initCStream_usingCDict() properly writes dictID into frame header, by Gregory Szorc (#511) +API : fix : all symbols properly exposed in libzstd, by Nick Terrell +build : support for Solaris target, by Przemyslaw Skibinski +doc : clarified specification, by Sean Purcell + +v1.1.2 +API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init +API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize() +API : zbuff : changed : prototypes now generate deprecation warnings +lib : improved : faster decompression speed at ultra compression settings and 32-bits mode +lib : changed : only public ZSTD_ symbols are now exposed +lib : changed : reduced usage of stack memory +lib : fixed : several corner case bugs, by Nick Terrell +cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski +cli : new : preserve file attributes +cli : new : added zstdless and zstdgrep tools +cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd) +cli : fixed : zstdcat +zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski +install : better compatibility with FreeBSD, by Dimitry Andric +source tree : changed : zbuff source files moved to lib/deprecated + +v1.1.1 +New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption +New : doc/zstd_manual.html, by Przemyslaw Skibinski +Improved : slightly better compression ratio at --ultra levels (>= 20) +Improved : better memory usage when using streaming compression API, thanks to @Rogier-5 report +Added : API : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section) +Added : example/multiple_streaming_compression.c +Changed : zstd_errors.h is now installed within /include (and replaces errors_public.h) +Updated man page +Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets + +v1.1.0 +New : contrib/pzstd, parallel version of zstd, by Nick Terrell +added : NetBSD install target (#338) +Improved : speed for batches of small files +Improved : speed of zlib wrapper, by Przemyslaw Skibinski +Changed : libzstd on Windows supports legacy formats, by Christophe Chevalier +Fixed : CLI -d output to stdout by default when input is stdin (#322) +Fixed : CLI correctly detects console on Mac OS-X +Fixed : CLI supports recursive mode `-r` on Mac OS-X +Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski +Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319) +Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365) +Fixed : zstd-pgo, reported by octoploid (#329) + +v1.0.0 +Change Licensing, all project is now BSD, Copyright Facebook +Small decompression speed improvement +API : Streaming API supports legacy format +API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParamter() +CLI supports legacy formats v0.4+ +Fixed : compression fails on certain huge files, reported by Jesse McGrew +Enhanced documentation, by Przemyslaw Skibinski + +v0.8.1 +New streaming API +Changed : --ultra now enables levels beyond 19 +Changed : -i# now selects benchmark time in second +Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell +Fixed : speed regression on specific patterns (#272) +Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291) +Fixed : ICC compilation, by Przemyslaw Skibinski + +v0.8.0 +Improved : better speed on clang and gcc -O2, thanks to Eric Biggers +New : Build on FreeBSD and DragonFly, thanks to JrMarino +Changed : modified API : ZSTD_compressEnd() +Fixed : legacy mode with ZSTD_HEAPMODE=0, by Christopher Bergqvist +Fixed : premature end of frame when zero-sized raw block, reported by Eric Biggers +Fixed : large dictionaries (> 384 KB), reported by Ilona Papava +Fixed : checksum correctly checked in single-pass mode +Fixed : combined --test amd --rm, reported by Andreas M. Nilsson +Modified : minor compression level adaptations +Updated : compression format specification to v0.2.0 +changed : zstd.h moved to /lib directory + +v0.7.5 +Transition version, supporting decoding of v0.8.x + +v0.7.4 +Added : homebrew for Mac, by Daniel Cade +Added : more examples +Fixed : segfault when using small dictionaries, reported by Felix Handte +Modified : default compression level for CLI is now 3 +Updated : specification, to v0.1.1 + +v0.7.3 +New : compression format specification +New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner. +New : `ZSTD_getDecompressedSize()` +New : OpenBSD target, by Juan Francisco Cantero Hurtado +New : `examples` directory +fixed : dictBuilder using HC levels, reported by Bartosz Taudul +fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte +fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by Greg Slazinski +modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section) +modified : legacy functions no longer need magic number + +v0.7.2 +fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski. +fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner. +fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner. + +v0.7.1 +fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier +fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв +fixed : corruption issue, reported by cj +modified : checksum enabled by default in command line mode + +v0.7.0 +New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski +New : Command `--rm`, to remove source file after successful de/compression +New : Visual build scripts, by Christophe Chevalier +New : Support for Sparse File-systems (do not use space for zero-filled sectors) +New : Frame checksum support +New : Support pass-through mode (when using `-df`) +API : more efficient Dictionary API : `ZSTD_compress_usingCDict()`, `ZSTD_decompress_usingDDict()` +API : create dictionary files from custom content, by Giuseppe Ottaviano +API : support for custom malloc/free functions +New : controllable Dictionary ID +New : Support for skippable frames + +v0.6.1 +New : zlib wrapper API, thanks to Przemyslaw Skibinski +New : Ability to compile compressor / decompressor separately +Changed : new lib directory structure +Fixed : Legacy codec v0.5 compatible with dictionary decompression +Fixed : Decoder corruption error (#173) +Fixed : null-string roundtrip (#176) +New : benchmark mode can select directory as input +Experimental : midipix support, VMS support + +v0.6.0 +Stronger high compression modes, thanks to Przemyslaw Skibinski +API : ZSTD_getFrameParams() provides size of decompressed content +New : highest compression modes require `--ultra` command to fully unleash their capacity +Fixed : zstd cli return error code > 0 and removes dst file artifact when decompression fails, thanks to Chip Turner + +v0.5.1 +New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski +Changed : Dictionary builder integrated into libzstd and zstd cli +Changed (!) : zstd cli now uses "multiple input files" as default mode. See `zstd -h`. +Fix : high compression modes for big-endian platforms +New : zstd cli : `-t` | `--test` command + +v0.5.0 +New : dictionary builder utility +Changed : streaming & dictionary API +Improved : better compression of small data + +v0.4.7 +Improved : small compression speed improvement in HC mode +Changed : `zstd_decompress.c` has ZSTD_LEGACY_SUPPORT to 0 by default +fix : bt search bug + +v0.4.6 +fix : fast compression mode on Windows +New : cmake configuration file, thanks to Artyom Dymchenko +Improved : high compression mode on repetitive data +New : block-level API +New : ZSTD_duplicateCCtx() + +v0.4.5 +new : -m/--multiple : compress/decompress multiple files + +v0.4.4 +Fixed : high compression modes for Windows 32 bits +new : external dictionary API extended to buffered mode and accessible through command line +new : windows DLL project, thanks to Christophe Chevalier + +v0.4.3 : +new : external dictionary API +new : zstd-frugal + +v0.4.2 : +Generic minor improvements for small blocks +Fixed : big-endian compatibility, by Peter Harris (#85) + +v0.4.1 +Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben) +removed `zstd.c` + +v0.4.0 +Command line utility compatible with high compression levels +Removed zstdhc => merged into zstd +Added : ZBUFF API (see zstd_buffered.h) +Rolling buffer support + +v0.3.6 +small blocks params + +v0.3.5 +minor generic compression improvements + +v0.3.4 +Faster fast cLevels + +v0.3.3 +Small compression ratio improvement + +v0.3.2 +Fixed Visual Studio + +v0.3.1 : +Small compression ratio improvement + +v0.3 +HC mode : compression levels 2-26 + +v0.2.2 +Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier + +v0.2.1 +Fix : Read errors, advanced fuzzer tests, by Hanno Böck + +v0.2.0 +**Breaking format change** +Faster decompression speed +Can still decode v0.1 format + +v0.1.3 +fix uninitialization warning, reported by Evan Nemerson + +v0.1.2 +frame concatenation support + +v0.1.1 +fix compression bug +detects write-flush errors + +v0.1.0 +first release Copied: vendor/zstd/1.3.1/README.md (from r322779, vendor/zstd/dist/README.md) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/zstd/1.3.1/README.md Tue Aug 22 11:03:50 2017 (r322780, copy of r322779, vendor/zstd/dist/README.md) @@ -0,0 +1,153 @@ + __Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm, + targeting real-time compression scenarios at zlib-level and better compression ratios. + +It is provided as an open-source BSD-licensed **C** library, +and a command line utility producing and decoding `.zst` and `.gz` files. +For other programming languages, +you can consult a list of known ports on [Zstandard homepage](http://www.zstd.net/#other-languages). + +| dev branch status | +|-------------------| +| [![Build Status][travisDevBadge]][travisLink] [![Build status][AppveyorDevBadge]][AppveyorLink] [![Build status][CircleDevBadge]][CircleLink] + +[travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite" +[travisLink]: https://travis-ci.org/facebook/zstd +[AppveyorDevBadge]: https://ci.appveyor.com/api/projects/status/xt38wbdxjk5mrbem/branch/dev?svg=true "Windows test suite" +[AppveyorLink]: https://ci.appveyor.com/project/YannCollet/zstd-p0yf0 +[CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite" +[CircleLink]: https://circleci.com/gh/facebook/zstd + + +As a reference, several fast compression algorithms were tested and compared +on a server running Linux Debian (`Linux version 4.8.0-1-amd64`), +with a Core i7-6700K CPU @ 4.0GHz, +using [lzbench], an open-source in-memory benchmark by @inikep +compiled with GCC 6.3.0, +on the [Silesia compression corpus]. + +[lzbench]: https://github.com/inikep/lzbench +[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia + +| Compressor name | Ratio | Compression| Decompress.| +| --------------- | ------| -----------| ---------- | +| **zstd 1.1.3 -1** | 2.877 | 430 MB/s | 1110 MB/s | +| zlib 1.2.8 -1 | 2.743 | 110 MB/s | 400 MB/s | +| brotli 0.5.2 -0 | 2.708 | 400 MB/s | 430 MB/s | +| quicklz 1.5.0 -1 | 2.238 | 550 MB/s | 710 MB/s | +| lzo1x 2.09 -1 | 2.108 | 650 MB/s | 830 MB/s | +| lz4 1.7.5 | 2.101 | 720 MB/s | 3600 MB/s | +| snappy 1.1.3 | 2.091 | 500 MB/s | 1650 MB/s | +| lzf 3.6 -1 | 2.077 | 400 MB/s | 860 MB/s | + +[zlib]:http://www.zlib.net/ +[LZ4]: http://www.lz4.org/ + +Zstd can also offer stronger compression ratios at the cost of compression speed. +Speed vs Compression trade-off is configurable by small increments. Decompression speed is preserved and remains roughly the same at all settings, a property shared by most LZ compression algorithms, such as [zlib] or lzma. + +The following tests were run +on a server running Linux Debian (`Linux version 4.8.0-1-amd64`) +with a Core i7-6700K CPU @ 4.0GHz, +using [lzbench], an open-source in-memory benchmark by @inikep +compiled with GCC 6.3.0, +on the [Silesia compression corpus]. + +Compression Speed vs Ratio | Decompression Speed +---------------------------|-------------------- + |  + +Several algorithms can produce higher compression ratios, but at slower speeds, falling outside of the graph. +For a larger picture including very slow modes, [click on this link](doc/images/DCspeed5.png) . + + +### The case for Small Data compression + +Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives. + +The smaller the amount of data to compress, the more difficult it is to compress. This problem is common to all compression algorithms, and reason is, compression algorithms learn from past data how to compress future data. But at the beginning of a new data set, there is no "past" to build upon. + +To solve this situation, Zstd offers a __training mode__, which can be used to tune the algorithm for a selected type of data. +Training Zstandard is achieved by providing it with a few samples (one file per sample). The result of this training is stored in a file called "dictionary", which must be loaded before compression and decompression. +Using this dictionary, the compression ratio achievable on small data improves dramatically. + +The following example uses the `github-users` [sample set](https://github.com/facebook/zstd/releases/tag/v1.1.3), created from [github public API](https://developer.github.com/v3/users/#get-all-users). +It consists of roughly 10K records weighing about 1KB each. + +Compression Ratio | Compression Speed | Decompression Speed +------------------|-------------------|-------------------- + |  |  + + +These compression gains are achieved while simultaneously providing _faster_ compression and decompression speeds. + +Training works if there is some correlation in a family of small data samples. The more data-specific a dictionary is, the more efficient it is (there is no _universal dictionary_). +Hence, deploying one dictionary per type of data will provide the greatest benefits. +Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file. + +#### Dictionary compression How To : + +1) Create the dictionary + +`zstd --train FullPathToTrainingSet/* -o dictionaryName` + +2) Compress with dictionary + +`zstd -D dictionaryName FILE` + +3) Decompress with dictionary + +`zstd -D dictionaryName --decompress FILE.zst` + + +### Build + +Once you have the repository cloned, there are multiple ways provided to build Zstandard. + +#### Makefile + +If your system is compatible with a standard `make` (or `gmake`) binary generator, +you can simply run it at the root directory. +It will generate `zstd` within root directory. + +Other available options include : +- `make install` : create and install zstd binary, library and man page +- `make test` : create and run `zstd` and test tools on local platform + +#### cmake + +A `cmake` project generator is provided within `build/cmake`. +It can generate Makefiles or other build scripts +to create `zstd` binary, and `libzstd` dynamic and static libraries. + +#### Meson + +A Meson project is provided within `contrib/meson`. + +#### Visual Studio (Windows) + +Going into `build` directory, you will find additional possibilities : +- Projects for Visual Studio 2005, 2008 and 2010 + + VS2010 project is compatible with VS2012, VS2013 and VS2015 +- Automated build scripts for Visual compiler by @KrzysFR , in `build/VS_scripts`, + which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution. + + +### Status + +Zstandard is currently deployed within Facebook. It is used continuously to compress large amounts of data in multiple formats and use cases. +Zstandard is considered safe for production environments. + +### License + +Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING). + +### Contributing + +The "dev" branch is the one where all contributions will be merged before reaching "master". +If you plan to propose a patch, please commit into the "dev" branch or its own feature branch. +Direct commit to "master" are not permitted. +For more information, please read [CONTRIBUTING](CONTRIBUTING.md). + +### Miscellaneous + +Zstd entropy stage is provided by [Huff0 and FSE, from Finite State Entropy library](https://github.com/Cyan4973/FiniteStateEntropy). Copied: vendor/zstd/1.3.1/circle.yml (from r322779, vendor/zstd/dist/circle.yml) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/zstd/1.3.1/circle.yml Tue Aug 22 11:03:50 2017 (r322780, copy of r322779, vendor/zstd/dist/circle.yml) @@ -0,0 +1,75 @@ +dependencies: + override: + - sudo dpkg --add-architecture i386 + - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update + - sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross + - sudo apt-get -y install libstdc++-7-dev clang gcc g++ gcc-5 gcc-6 gcc-7 zlib1g-dev liblzma-dev + - sudo apt-get -y install linux-libc-dev:i386 libc6-dev-i386 + +test: + override: + - ? | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then cc -v; make all && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build && make clean; fi + : + parallel: true + - ? | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c99build && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu99build && make clean; fi + : + parallel: true + - ? | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c11build && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make cmakebuild && make clean; fi + : + parallel: true + - ? | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make gppbuild && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc5build && make clean; fi + : + parallel: true + - ? | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make gcc6build && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make clangbuild && make clean; fi + : + parallel: true + - ? | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make m32build && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild && make clean; fi + : + parallel: true + - ? | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make aarch64build && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppcbuild && make clean; fi + : + parallel: true + - ? | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make ppc64build && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc7build && make clean; fi #could add another test here + : + parallel: true + - ? | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make shortest && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi + : + parallel: true + + post: + - echo Circle CI tests finished + + # Longer tests + #- make -C tests test-zstd-nolegacy && make clean + #- pyenv global 3.4.4; make -C tests versionsTest && make clean + #- make zlibwrapper && make clean + #- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean + #- make uasan && make clean + #- make asan32 && make clean + #- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu" + # Valgrind tests + #- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean + #- make -C tests valgrindTest && make clean + # ARM, AArch64, PowerPC, PowerPC64 tests + #- make ppctest && make clean + #- make ppc64test && make clean + #- make armtest && make clean + #- make aarch64test && make clean Copied: vendor/zstd/1.3.1/doc/educational_decoder/harness.c (from r322779, vendor/zstd/dist/doc/educational_decoder/harness.c) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/zstd/1.3.1/doc/educational_decoder/harness.c Tue Aug 22 11:03:50 2017 (r322780, copy of r322779, vendor/zstd/dist/doc/educational_decoder/harness.c) @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2017-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#include <stdio.h> +#include <stdlib.h> + +#include "zstd_decompress.h" + +typedef unsigned char u8; + +// If the data doesn't have decompressed size with it, fallback on assuming the +// compression ratio is at most 16 +#define MAX_COMPRESSION_RATIO (16) + +// Protect against allocating too much memory for output +#define MAX_OUTPUT_SIZE ((size_t)1024 * 1024 * 1024) + +u8 *input; +u8 *output; +u8 *dict; + +size_t read_file(const char *path, u8 **ptr) { + FILE *f = fopen(path, "rb"); + if (!f) { + fprintf(stderr, "failed to open file %s\n", path); + exit(1); + } + + fseek(f, 0L, SEEK_END); + size_t size = ftell(f); + rewind(f); + + *ptr = malloc(size); + if (!ptr) { + fprintf(stderr, "failed to allocate memory to hold %s\n", path); + exit(1); + } + + size_t pos = 0; + while (!feof(f)) { + size_t read = fread(&(*ptr)[pos], 1, size, f); + if (ferror(f)) { + fprintf(stderr, "error while reading file %s\n", path); + exit(1); + } + pos += read; + } + + fclose(f); + + return pos; +} + +void write_file(const char *path, const u8 *ptr, size_t size) { + FILE *f = fopen(path, "wb"); + + size_t written = 0; + while (written < size) { + written += fwrite(&ptr[written], 1, size, f); *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201708221103.v7MB3oB4051158>