Skip site navigation (1)Skip section navigation (2)
Date:      Sat, 6 May 2017 10:26:40 +0000 (UTC)
From:      Baptiste Daroussin <bapt@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r317877 - in head: contrib/zstd contrib/zstd/contrib contrib/zstd/contrib/pzstd contrib/zstd/contrib/pzstd/utils/test contrib/zstd/doc contrib/zstd/doc/educational_decoder contrib/zstd/...
Message-ID:  <201705061026.v46AQesA089962@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: bapt
Date: Sat May  6 10:26:40 2017
New Revision: 317877
URL: https://svnweb.freebsd.org/changeset/base/317877

Log:
  Import Zstandard 1.2.0
  
  Among new things it is now threaded by default, use zstd -T# to chose the
  number of threads not that -T0 will automatically determine the number of
  threads based on the number of CPU online.

Added:
  head/contrib/zstd/contrib/cleanTabs
     - copied unchanged from r317876, vendor/zstd/dist/contrib/cleanTabs
  head/contrib/zstd/programs/zstd.1.md
     - copied unchanged from r317876, vendor/zstd/dist/programs/zstd.1.md
Deleted:
  head/contrib/zstd/.gitignore
  head/contrib/zstd/.travis.yml
Modified:
  head/contrib/zstd/Makefile
  head/contrib/zstd/NEWS
  head/contrib/zstd/README.md
  head/contrib/zstd/appveyor.yml
  head/contrib/zstd/circle.yml
  head/contrib/zstd/contrib/pzstd/Options.cpp
  head/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp
  head/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp
  head/contrib/zstd/doc/educational_decoder/zstd_decompress.c
  head/contrib/zstd/doc/images/Cspeed4.png
  head/contrib/zstd/doc/images/Dspeed4.png
  head/contrib/zstd/doc/images/dict-cr.png
  head/contrib/zstd/doc/images/dict-cs.png
  head/contrib/zstd/doc/images/dict-ds.png
  head/contrib/zstd/doc/zstd_compression_format.md
  head/contrib/zstd/doc/zstd_manual.html
  head/contrib/zstd/examples/simple_compression.c
  head/contrib/zstd/examples/streaming_compression.c
  head/contrib/zstd/examples/streaming_decompression.c
  head/contrib/zstd/lib/Makefile
  head/contrib/zstd/lib/README.md
  head/contrib/zstd/lib/common/bitstream.h
  head/contrib/zstd/lib/common/error_private.c
  head/contrib/zstd/lib/common/fse.h
  head/contrib/zstd/lib/common/huf.h
  head/contrib/zstd/lib/common/mem.h
  head/contrib/zstd/lib/common/zstd_errors.h
  head/contrib/zstd/lib/common/zstd_internal.h
  head/contrib/zstd/lib/compress/fse_compress.c
  head/contrib/zstd/lib/compress/zstd_compress.c
  head/contrib/zstd/lib/compress/zstd_opt.h
  head/contrib/zstd/lib/compress/zstdmt_compress.c
  head/contrib/zstd/lib/decompress/zstd_decompress.c
  head/contrib/zstd/lib/dictBuilder/cover.c
  head/contrib/zstd/lib/dictBuilder/zdict.c
  head/contrib/zstd/lib/dictBuilder/zdict.h
  head/contrib/zstd/lib/legacy/zstd_v01.c
  head/contrib/zstd/lib/legacy/zstd_v02.c
  head/contrib/zstd/lib/legacy/zstd_v03.c
  head/contrib/zstd/lib/legacy/zstd_v04.c
  head/contrib/zstd/lib/legacy/zstd_v05.c
  head/contrib/zstd/lib/legacy/zstd_v06.c
  head/contrib/zstd/lib/zstd.h
  head/contrib/zstd/programs/Makefile
  head/contrib/zstd/programs/README.md
  head/contrib/zstd/programs/bench.c
  head/contrib/zstd/programs/dibio.c
  head/contrib/zstd/programs/fileio.c
  head/contrib/zstd/programs/fileio.h
  head/contrib/zstd/programs/platform.h
  head/contrib/zstd/programs/util.h
  head/contrib/zstd/programs/zstd.1
  head/contrib/zstd/programs/zstdcli.c
  head/contrib/zstd/tests/Makefile
  head/contrib/zstd/tests/decodecorpus.c
  head/contrib/zstd/tests/fullbench.c
  head/contrib/zstd/tests/fuzzer.c
  head/contrib/zstd/tests/paramgrill.c
  head/contrib/zstd/tests/playTests.sh
  head/contrib/zstd/tests/test-zstd-speed.py
  head/contrib/zstd/tests/zbufftest.c
  head/contrib/zstd/tests/zstreamtest.c
  head/contrib/zstd/zlibWrapper/examples/zwrapbench.c
  head/usr.bin/zstd/Makefile
Directory Properties:
  head/contrib/zstd/   (props changed)

Modified: head/contrib/zstd/Makefile
==============================================================================
--- head/contrib/zstd/Makefile	Sat May  6 10:18:45 2017	(r317876)
+++ head/contrib/zstd/Makefile	Sat May  6 10:26:40 2017	(r317877)
@@ -90,6 +90,10 @@ examples:
 manual:
 	$(MAKE) -C contrib/gen_html $@
 
+.PHONY: cleanTabs
+cleanTabs:
+	cd contrib; ./cleanTabs
+
 .PHONY: clean
 clean:
 	@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
@@ -105,9 +109,15 @@ clean:
 # make install is validated only for Linux, OSX, Hurd and some BSD targets
 #------------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
+
 HOST_OS = POSIX
-.PHONY: install uninstall travis-install clangtest gpptest armtest usan asan uasan
+CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON
 
+.PHONY: list
+list:
+	@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
+
+.PHONY: install uninstall travis-install clangtest gpptest armtest usan asan uasan
 install:
 	@$(MAKE) -C $(ZSTDDIR) $@
 	@$(MAKE) -C $(PRGDIR) $@
@@ -152,16 +162,16 @@ ppc64build: clean
 	CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allarch
 
 armfuzz: clean
-	CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest
+	CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
 
 aarch64fuzz: clean
-	CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest
+	CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
 
 ppcfuzz: clean
-	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest
+	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
 
 ppc64fuzz: clean
-	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" $(MAKE) -C $(TESTDIR) fuzztest
+	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
 
 gpptest: clean
 	CC=g++ $(MAKE) -C $(PRGDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
@@ -180,19 +190,19 @@ clangtest: clean
 
 armtest: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
-	$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
+	$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests
 
 aarch64test:
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
-	$(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
+	$(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests
 
 ppctest: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
-	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
+	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" FUZZER_FLAGS=--no-big-tests
 
 ppc64test: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
-	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
+	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests
 
 arm-ppc-compilation:
 	$(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
@@ -263,7 +273,7 @@ endif
 
 ifneq (,$(filter MSYS%,$(shell uname)))
 HOST_OS = MSYS
-CMAKE_PARAMS = -G"MSYS Makefiles"
+CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
 endif
 
 
@@ -275,7 +285,7 @@ cmakebuild:
 	cmake --version
 	$(RM) -r $(BUILDIR)/cmake/build
 	mkdir $(BUILDIR)/cmake/build
-	cd $(BUILDIR)/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
+	cd $(BUILDIR)/cmake/build ; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
 
 c90build: clean
 	gcc -v

Modified: head/contrib/zstd/NEWS
==============================================================================
--- head/contrib/zstd/NEWS	Sat May  6 10:18:45 2017	(r317876)
+++ head/contrib/zstd/NEWS	Sat May  6 10:26:40 2017	(r317877)
@@ -1,3 +1,25 @@
+v1.2.0
+cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable)
+cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell
+cli : new : zstdmt symlink hardwired to `zstd -T0`
+cli : new : command --threads=# (#671)
+cli : changed : cover dictionary builder by default, for improved quality, by Nick Terrell
+cli : new : commands --train-cover and --train-legacy, to select dictionary algorithm and parameters
+cli : experimental targets `zstd4` and `xzstd4`, with support for lz4 format, by Sean Purcell
+cli : fix : does not output compressed data on console
+cli : fix : ignore symbolic links unless --force specified,
+API : breaking change : ZSTD_createCDict_advanced(), only use compressionParameters as argument
+API : added : prototypes ZSTD_*_usingCDict_advanced(), for direct control over frameParameters.
+API : improved: ZSTDMT_compressCCtx() reduced memory usage
+API : fix : ZSTDMT_compressCCtx() now provides srcSize in header (#634)
+API : fix : src size stored in frame header is controlled at end of frame
+API : fix : enforced consistent rules for pledgedSrcSize==0 (#641)
+API : fix : error code "GENERIC" replaced by "dstSizeTooSmall" when appropriate
+build: improved cmake script, by @Majlen
+build: enabled Multi-threading support for *BSD, by Baptiste Daroussin
+tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target.
+new : contrib/linux-kernel version, by Nick Terrell
+
 v1.1.4
 cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski
 cli : new : advanced benchmark command --priority=rt
@@ -5,13 +27,13 @@ cli : fix : write on sparse-enabled file
 cli : fix : --rm remains silent when input is stdin
 cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski
 speed : improved decompression speed in streaming mode for single shot scenarios (+5%)
-memory : DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
-arch : 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell
+memory: DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
+arch: 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell
 API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize()
 API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value)
-build: new: meson build system in contrib/meson, by Dima Krasner
-build: improved cmake script, by @Majlen
-build: added -Wformat-security flag, as recommended by Padraig Brady
+build : new: meson build system in contrib/meson, by Dima Krasner
+build : improved cmake script, by @Majlen
+build : added -Wformat-security flag, as recommended by Padraig Brady
 doc : new : educational decoder, by Sean Purcell
 
 v1.1.3

Modified: head/contrib/zstd/README.md
==============================================================================
--- head/contrib/zstd/README.md	Sat May  6 10:18:45 2017	(r317876)
+++ head/contrib/zstd/README.md	Sat May  6 10:26:40 2017	(r317877)
@@ -12,13 +12,13 @@ you can consult a list of known ports on
 |dev         | [![Build Status](https://travis-ci.org/facebook/zstd.svg?branch=dev)](https://travis-ci.org/facebook/zstd) |
 
 As a reference, several fast compression algorithms were tested and compared
-on a server running Linux Mint Debian Edition (`Linux version 4.8.0-1-amd64`),
+on a server running Linux Debian (`Linux version 4.8.0-1-amd64`),
 with a Core i7-6700K CPU @ 4.0GHz,
-using [lzbench v1.6], an open-source in-memory benchmark by @inikep
+using [lzbench], an open-source in-memory benchmark by @inikep
 compiled with GCC 6.3.0,
 on the [Silesia compression corpus].
 
-[lzbench v1.6]: https://github.com/inikep/lzbench
+[lzbench]: https://github.com/inikep/lzbench
 [Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
 
 | Compressor name         | Ratio | Compression| Decompress.|
@@ -38,7 +38,12 @@ on the [Silesia compression corpus].
 Zstd can also offer stronger compression ratios at the cost of compression speed.
 Speed vs Compression trade-off is configurable by small increments. Decompression speed is preserved and remains roughly the same at all settings, a property shared by most LZ compression algorithms, such as [zlib] or lzma.
 
-The following tests were run on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with GCC 5.2.1, on the [Silesia compression corpus].
+The following tests were run
+on a server running Linux Debian (`Linux version 4.8.0-1-amd64`)
+with a Core i7-6700K CPU @ 4.0GHz,
+using [lzbench], an open-source in-memory benchmark by @inikep
+compiled with GCC 6.3.0,
+on the [Silesia compression corpus].
 
 Compression Speed vs Ratio | Decompression Speed
 ---------------------------|--------------------

Modified: head/contrib/zstd/appveyor.yml
==============================================================================
--- head/contrib/zstd/appveyor.yml	Sat May  6 10:18:45 2017	(r317876)
+++ head/contrib/zstd/appveyor.yml	Sat May  6 10:26:40 2017	(r317877)
@@ -1,66 +1,103 @@
-version: 1.0.{build}
-environment:
-  matrix:
-  - COMPILER: "gcc"
-    PLATFORM: "mingw64"
-    MAKE_PARAMS: '"make test && make lib && make -C tests test-symbols fullbench-dll fullbench-lib"'
-  - COMPILER: "gcc"
-    PLATFORM: "mingw32"
-    MAKE_PARAMS: '"make -C tests test-zstd test-fullbench test-fuzzer test-invalidDictionaries"'
-  - COMPILER: "gcc"
-    PLATFORM: "clang"
-    MAKE_PARAMS: '"make -C tests zstd fullbench fuzzer paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion""'
-  - COMPILER: "visual"
-    CONFIGURATION: "Debug"
-    PLATFORM: "x64"
-  - COMPILER: "visual"
-    CONFIGURATION: "Debug"
-    PLATFORM: "Win32"
-  - COMPILER: "visual"
-    CONFIGURATION: "Release"
-    PLATFORM: "x64"
-  - COMPILER: "visual"
-    CONFIGURATION: "Release"
-    PLATFORM: "Win32"
+-
+  version: 1.0.{build}
+  branches:
+    only:
+    - dev
+    - master
+  environment:
+    matrix:
+    - COMPILER: "gcc"
+      HOST:     "mingw"
+      PLATFORM: "x64"
+      SCRIPT:   "make allarch && make -C tests test-symbols fullbench-dll fullbench-lib"
+      ARTIFACT: "true"
+      BUILD:    "true"
+    - COMPILER: "gcc"
+      HOST:     "mingw"
+      PLATFORM: "x86"
+      SCRIPT:   "make allarch"
+      ARTIFACT: "true"
+      BUILD:    "true"
+    - COMPILER: "clang"
+      HOST:     "mingw"
+      PLATFORM: "x64"
+      SCRIPT:   "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allarch"
+      BUILD:    "true"
+
+    - COMPILER: "gcc"
+      HOST:     "mingw"
+      PLATFORM: "x64"
+      SCRIPT:   ""
+      TEST:     "cmake"
+
+    - COMPILER: "gcc"
+      HOST:     "mingw"
+      PLATFORM: "x64"
+      SCRIPT:   ""
+      TEST:     "pzstd"
+
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "x64"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "x64"
+      CONFIGURATION: "Release"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Release"
 
-install:
+  install:
   - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
-  - MKDIR bin
-  - if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH%
-  - if [%COMPILER%]==[gcc] (
-      SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" &&
-      SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" &&
-      COPY C:\msys64\usr\bin\make.exe C:\MinGW\bin\make.exe &&
-      COPY C:\MinGW\bin\gcc.exe C:\MinGW\bin\cc.exe
-    ) else (
-      IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;")
+  - SET PATH_ORIGINAL=%PATH%
+  - if [%HOST%]==[mingw] (
+      SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" &&
+      SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" &&
+      COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe &&
+      COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe
+    )
+  - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] (
+      SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;"
+    )
+
+  build_script:
+  - if [%HOST%]==[mingw] (
+      ( if [%PLATFORM%]==[x64] (
+        SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%"
+      ) else if [%PLATFORM%]==[x86] (
+        SET "PATH=%PATH_MINGW32%;%PATH_ORIGINAL%"
+      ) )
     )
-
-build_script:
-  - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION%
-  - if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL%
-  - if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL%
-  - if [%PLATFORM%]==[clang] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL%
-  - if [%COMPILER%]==[gcc] (
-      ECHO *** &&
-      ECHO *** Building %PLATFORM% &&
-      ECHO *** &&
+  - if [%HOST%]==[mingw] if [%BUILD%]==[true] (
       make -v &&
-      cc -v &&
-      ECHO %MAKE_PARAMS% &&
-      sh -c %MAKE_PARAMS%
-    )
-  - if [%PLATFORM%]==[clang] COPY tests\fuzzer.exe tests\fuzzer_clang.exe
-  - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw64] (
-      COPY programs\zstd.exe bin\zstd.exe &&
-      appveyor PushArtifact bin\zstd.exe
-    )
-  - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw32] (
-      COPY programs\zstd.exe bin\zstd32.exe &&
-      appveyor PushArtifact bin\zstd32.exe
+      sh -c "%COMPILER% -v" &&
+      ECHO Building zlib to static link &&
+      SET "CC=%COMPILER%" &&
+      sh -c "cd .. && git clone --depth 1 --branch v1.2.11 https://github.com/madler/zlib" &&
+      sh -c "cd ../zlib && make -f win32/Makefile.gcc libz.a"
+      ECHO Building zstd &&
+      SET "CPPFLAGS=-I../../zlib" &&
+      SET "LDFLAGS=../../zlib/libz.a" &&
+      sh -c "%SCRIPT%" &&
+      ( if [%COMPILER%]==[gcc] if [%ARTIFACT%]==[true]
+          lib\dll\example\build_package.bat &&
+          make -C programs DEBUGFLAGS= clean zstd &&
+          cp programs\zstd.exe zstd_%PLATFORM%.exe &&
+          appveyor PushArtifact zstd_%PLATFORM%.exe &&
+          cp programs\zstd.exe bin\zstd.exe &&
+          make -C programs DEBUGFLAGS= clean zstdmt &&
+          cp programs\zstd.exe bin\zstdmt.exe &&
+          cd bin\ && 7z a -tzip zstd-win-release-%PLATFORM%.zip * &&
+          appveyor PushArtifact zstd-win-release-%PLATFORM%.zip
+      )
     )
-  - if [%COMPILER%]==[gcc] make clean
-  - if [%COMPILER%]==[visual] (
+  - if [%HOST%]==[visual] (
       ECHO *** &&
       ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% in %APPVEYOR_BUILD_FOLDER% &&
       ECHO *** &&
@@ -111,29 +148,26 @@ build_script:
       COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
     )
 
-test_script:
+  test_script:
   - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
-  - SET FUZZERTEST=-T1mn
-  - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[clang] (
-      tests\fuzzer_clang.exe %FUZZERTEST% &&
-      ECHO *** &&
-      ECHO *** Building cmake for %PLATFORM% &&
-      ECHO *** &&
+  - SET "CC=gcc"
+  - SET "CXX=g++"
+  - if [%TEST%]==[cmake] (
       mkdir build\cmake\build &&
       cd build\cmake\build &&
       cmake -G "Visual Studio 14 2015 Win64" .. &&
       cd ..\..\.. &&
-      make clean &&
-      ECHO *** &&
-      ECHO *** Building pzstd for %PLATFORM% &&
-      ECHO *** &&
+      make clean
+    )
+  - if [%TEST%]==[pzstd] (
       make -C contrib\pzstd googletest-mingw64 &&
       make -C contrib\pzstd pzstd.exe &&
       make -C contrib\pzstd tests &&
       make -C contrib\pzstd check &&
       make -C contrib\pzstd clean
     )
-  - if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] (
+  - SET "FUZZERTEST=-T30s"
+  - if [%HOST%]==[visual] if [%CONFIGURATION%]==[Release] (
       CD tests &&
       SET ZSTD=./zstd.exe &&
       sh -e playTests.sh --test-large-data &&
@@ -146,33 +180,76 @@ test_script:
       fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
     )
 
-branches:
-  only:
-  - dev
-  - master
-
-artifacts:
-  - path: bin\zstd.exe
-  - path: bin\zstd32.exe
-
-deploy:
-- provider: GitHub
-  auth_token:
-    secure: LgJo8emYc3sFnlNWkGl4/VYK3nk/8+RagcsqDlAi3xeqNGNutnKjcftjg84uJoT4
-  artifact: bin\zstd.exe
-  force_update: true
-  on:
-    branch: autobuild
-    COMPILER: gcc
-    PLATFORM: "mingw64"
-    appveyor_repo_tag: true
-- provider: GitHub
-  auth_token:
-    secure: LgJo8emYc3sFnlNWkGl4/VYK3nk/8+RagcsqDlAi3xeqNGNutnKjcftjg84uJoT4
-  artifact: bin\zstd32.exe
-  force_update: true
-  on:
-    branch: autobuild
-    COMPILER: gcc
-    PLATFORM: "mingw32"
-    appveyor_repo_tag: true
+-
+  version: 1.0.{build}
+  environment:
+    matrix:
+    - COMPILER: "gcc"
+      HOST:     "mingw"
+      PLATFORM: "x64"
+      SCRIPT:   "make allarch"
+    - COMPILER: "gcc"
+      HOST:     "mingw"
+      PLATFORM: "x86"
+      SCRIPT:   "make allarch"
+    - COMPILER: "clang"
+      HOST:     "mingw"
+      PLATFORM: "x64"
+      SCRIPT:   "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allarch"
+
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "x64"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "x64"
+      CONFIGURATION: "Release"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Release"
+
+  install:
+  - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
+  - SET PATH_ORIGINAL=%PATH%
+  - if [%HOST%]==[mingw] (
+      SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" &&
+      SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" &&
+      COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe &&
+      COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe
+    )
+  - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] (
+      SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;"
+    )
+
+  build_script:
+  - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION%
+  - if [%HOST%]==[mingw] (
+      ( if [%PLATFORM%]==[x64] (
+        SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%"
+      ) else if [%PLATFORM%]==[x86] (
+        SET "PATH=%PATH_MINGW32%;%PATH_ORIGINAL%"
+      ) ) &&
+      make -v &&
+      sh -c "%COMPILER% -v" &&
+      set "CC=%COMPILER%" &&
+      sh -c "%SCRIPT%"
+    )
+  - if [%HOST%]==[visual] (
+      ECHO *** &&
+      ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
+      ECHO *** &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
+    )

Modified: head/contrib/zstd/circle.yml
==============================================================================
--- head/contrib/zstd/circle.yml	Sat May  6 10:18:45 2017	(r317876)
+++ head/contrib/zstd/circle.yml	Sat May  6 10:26:40 2017	(r317877)
@@ -3,7 +3,7 @@ dependencies:
     - sudo dpkg --add-architecture i386
     - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
     - sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
-    - sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6
+    - sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6 zlib1g-dev liblzma-dev
     - sudo apt-get -y install linux-libc-dev:i386 libc6-dev-i386
 
 test:

Copied: head/contrib/zstd/contrib/cleanTabs (from r317876, vendor/zstd/dist/contrib/cleanTabs)
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ head/contrib/zstd/contrib/cleanTabs	Sat May  6 10:26:40 2017	(r317877, copy of r317876, vendor/zstd/dist/contrib/cleanTabs)
@@ -0,0 +1,2 @@
+#!/bin/sh
+sed -i '' $'s/\t/    /g' ../lib/**/*.{h,c} ../programs/*.{h,c} ../tests/*.c ./**/*.{h,cpp} ../examples/*.c ../zlibWrapper/*.{h,c}

Modified: head/contrib/zstd/contrib/pzstd/Options.cpp
==============================================================================
--- head/contrib/zstd/contrib/pzstd/Options.cpp	Sat May  6 10:18:45 2017	(r317876)
+++ head/contrib/zstd/contrib/pzstd/Options.cpp	Sat May  6 10:26:40 2017	(r317877)
@@ -91,7 +91,7 @@ void usage() {
   std::fprintf(stderr, "  -#                     : # compression level (1-%d, default:%d)\n", kMaxNonUltraCompressionLevel, kDefaultCompressionLevel);
   std::fprintf(stderr, "  -d, --decompress       : decompression\n");
   std::fprintf(stderr, "  -o                file : result stored into `file` (only if 1 input file)\n");
-  std::fprintf(stderr, "  -f, --force            : overwrite output without prompting\n");
+  std::fprintf(stderr, "  -f, --force            : overwrite output without prompting, (de)compress links\n");
   std::fprintf(stderr, "      --rm               : remove source file(s) after successful (de)compression\n");
   std::fprintf(stderr, "  -k, --keep             : preserve source file(s) (default)\n");
   std::fprintf(stderr, "  -h, --help             : display help and exit\n");
@@ -121,6 +121,7 @@ Options::Status Options::parse(int argc,
   bool recursive = false;
   bool ultra = false;
   bool forceStdout = false;
+  bool followLinks = false;
   // Local copy of input files, which are pointers into argv.
   std::vector<const char *> localInputFiles;
   for (int i = 1; i < argc; ++i) {
@@ -255,6 +256,7 @@ Options::Status Options::parse(int argc,
       case 'f':
         overwrite = true;
         forceStdout = true;
+        followLinks = true;
         break;
       case 't':
         test = true;
@@ -328,13 +330,29 @@ Options::Status Options::parse(int argc,
     }
   }
 
+  g_utilDisplayLevel = verbosity;
+  // Remove local input files that are symbolic links
+  if (!followLinks) {
+      std::remove_if(localInputFiles.begin(), localInputFiles.end(),
+                     [&](const char *path) {
+                        bool isLink = UTIL_isLink(path);
+                        if (isLink && verbosity >= 2) {
+                            std::fprintf(
+                                    stderr,
+                                    "Warning : %s is symbolic link, ignoring\n",
+                                    path);
+                        }
+                        return isLink;
+                    });
+  }
+
   // Translate input files/directories into files to (de)compress
   if (recursive) {
     char *scratchBuffer = nullptr;
     unsigned numFiles = 0;
     const char **files =
         UTIL_createFileList(localInputFiles.data(), localInputFiles.size(),
-                            &scratchBuffer, &numFiles);
+                            &scratchBuffer, &numFiles, followLinks);
     if (files == nullptr) {
       std::fprintf(stderr, "Error traversing directories\n");
       return Status::Failure;

Modified: head/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp
==============================================================================
--- head/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp	Sat May  6 10:18:45 2017	(r317876)
+++ head/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp	Sat May  6 10:26:40 2017	(r317877)
@@ -10,6 +10,7 @@
 
 #include <gtest/gtest.h>
 #include <atomic>
+#include <iostream>
 #include <thread>
 #include <vector>
 
@@ -34,16 +35,19 @@ TEST(ThreadPool, AllJobsFinished) {
   std::atomic<unsigned> numFinished{0};
   std::atomic<bool> start{false};
   {
+    std::cerr << "Creating executor" << std::endl;
     ThreadPool executor(5);
     for (int i = 0; i < 10; ++i) {
       executor.add([ &numFinished, &start ] {
         while (!start.load()) {
-          // spin
+          std::this_thread::yield();
         }
         ++numFinished;
       });
     }
+    std::cerr << "Starting" << std::endl;
     start.store(true);
+    std::cerr << "Finishing" << std::endl;
   }
   EXPECT_EQ(10, numFinished.load());
 }

Modified: head/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp
==============================================================================
--- head/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp	Sat May  6 10:18:45 2017	(r317876)
+++ head/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp	Sat May  6 10:26:40 2017	(r317877)
@@ -10,6 +10,7 @@
 #include "utils/WorkQueue.h"
 
 #include <gtest/gtest.h>
+#include <iostream>
 #include <memory>
 #include <mutex>
 #include <thread>
@@ -201,11 +202,13 @@ TEST(WorkQueue, BoundedSizeMPMC) {
   WorkQueue<int> queue(10);
   std::vector<int> results(200, -1);
   std::mutex mutex;
+  std::cerr << "Creating popperThreads" << std::endl;
   std::vector<std::thread> popperThreads;
   for (int i = 0; i < 4; ++i) {
     popperThreads.emplace_back(Popper{&queue, results.data(), &mutex});
   }
 
+  std::cerr << "Creating pusherThreads" << std::endl;
   std::vector<std::thread> pusherThreads;
   for (int i = 0; i < 2; ++i) {
     auto min = i * 100;
@@ -218,15 +221,19 @@ TEST(WorkQueue, BoundedSizeMPMC) {
         });
   }
 
+  std::cerr << "Joining pusherThreads" << std::endl;
   for (auto& thread : pusherThreads) {
     thread.join();
   }
+  std::cerr << "Finishing queue" << std::endl;
   queue.finish();
 
+  std::cerr << "Joining popperThreads" << std::endl;
   for (auto& thread : popperThreads) {
     thread.join();
   }
 
+  std::cerr << "Inspecting results" << std::endl;
   for (int i = 0; i < 200; ++i) {
     EXPECT_EQ(i, results[i]);
   }

Modified: head/contrib/zstd/doc/educational_decoder/zstd_decompress.c
==============================================================================
--- head/contrib/zstd/doc/educational_decoder/zstd_decompress.c	Sat May  6 10:18:45 2017	(r317876)
+++ head/contrib/zstd/doc/educational_decoder/zstd_decompress.c	Sat May  6 10:26:40 2017	(r317877)
@@ -27,16 +27,19 @@ size_t ZSTD_decompress_with_dict(void *c
 
 /// Get the decompressed size of an input stream so memory can be allocated in
 /// advance
+/// Returns -1 if the size can't be determined
 size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len);
 
 /******* UTILITY MACROS AND TYPES *********************************************/
-// Max block size decompressed size is 128 KB and literal blocks must be smaller
-// than that
+// Max block size decompressed size is 128 KB and literal blocks can't be
+// larger than their block
 #define MAX_LITERALS_SIZE ((size_t)128 * 1024)
 
 #define MAX(a, b) ((a) > (b) ? (a) : (b))
 #define MIN(a, b) ((a) < (b) ? (a) : (b))
 
+/// This decoder calls exit(1) when it encounters an error, however a production
+/// library should propagate error codes
 #define ERROR(s)                                                               \
     do {                                                                       \
         fprintf(stderr, "Error: %s\n", s);                                     \
@@ -67,29 +70,31 @@ typedef int64_t i64;
 /// decompression functions.
 
 /*** IO STREAM OPERATIONS *************/
-/// These structs are the interface for IO, and do bounds checking on all
-/// operations.  They should be used opaquely to ensure safety.
 
-/// Output is always done byte-by-byte
+/// ostream_t/istream_t are used to wrap the pointers/length data passed into
+/// ZSTD_decompress, so that all IO operations are safely bounds checked
+/// They are written/read forward, and reads are treated as little-endian
+/// They should be used opaquely to ensure safety
 typedef struct {
     u8 *ptr;
     size_t len;
 } ostream_t;
 
-/// Input often reads a few bits at a time, so maintain an internal offset
 typedef struct {
     const u8 *ptr;
-    int bit_offset;
     size_t len;
+
+    // Input often reads a few bits at a time, so maintain an internal offset
+    int bit_offset;
 } istream_t;
 
 /// The following two functions are the only ones that allow the istream to be
 /// non-byte aligned
 
 /// Reads `num` bits from a bitstream, and updates the internal offset
-static inline u64 IO_read_bits(istream_t *const in, const int num);
-/// Rewinds the stream by `num` bits
-static inline void IO_rewind_bits(istream_t *const in, const int num);
+static inline u64 IO_read_bits(istream_t *const in, const int num_bits);
+/// Backs-up the stream by `num` bits so they can be read again
+static inline void IO_rewind_bits(istream_t *const in, const int num_bits);
 /// If the remaining bits in a byte will be unused, advance to the end of the
 /// byte
 static inline void IO_align_stream(istream_t *const in);
@@ -101,30 +106,31 @@ static inline void IO_write_byte(ostream
 /// be byte aligned.
 static inline size_t IO_istream_len(const istream_t *const in);
 
-/// Returns a pointer where `len` bytes can be read, and advances the internal
-/// state.  The stream must be byte aligned.
+/// Advances the stream by `len` bytes, and returns a pointer to the chunk that
+/// was skipped.  The stream must be byte aligned.
 static inline const u8 *IO_read_bytes(istream_t *const in, size_t len);
-/// Returns a pointer where `len` bytes can be written, and advances the internal
-/// state.  The stream must be byte aligned.
+/// Advances the stream by `len` bytes, and returns a pointer to the chunk that
+/// was skipped so it can be written to.
 static inline u8 *IO_write_bytes(ostream_t *const out, size_t len);
 
 /// Advance the inner state by `len` bytes.  The stream must be byte aligned.
 static inline void IO_advance_input(istream_t *const in, size_t len);
 
-/// Returns an `ostream_t` constructed from the given pointer and length
+/// Returns an `ostream_t` constructed from the given pointer and length.
 static inline ostream_t IO_make_ostream(u8 *out, size_t len);
-/// Returns an `istream_t` constructed from the given pointer and length
+/// Returns an `istream_t` constructed from the given pointer and length.
 static inline istream_t IO_make_istream(const u8 *in, size_t len);
 
-/// Returns an `istream_t` with the same base as `in`, and length `len`
-/// Then, advance `in` to account for the consumed bytes
-/// `in` must be byte aligned
+/// Returns an `istream_t` with the same base as `in`, and length `len`.
+/// Then, advance `in` to account for the consumed bytes.
+/// `in` must be byte aligned.
 static inline istream_t IO_make_sub_istream(istream_t *const in, size_t len);
 /*** END IO STREAM OPERATIONS *********/
 
 /*** BITSTREAM OPERATIONS *************/
-/// Read `num` bits (up to 64) from `src + offset`, where `offset` is in bits
-static inline u64 read_bits_LE(const u8 *src, const int num,
+/// Read `num` bits (up to 64) from `src + offset`, where `offset` is in bits,
+/// and return them interpreted as a little-endian unsigned integer.
+static inline u64 read_bits_LE(const u8 *src, const int num_bits,
                                const size_t offset);
 
 /// Read bits from the end of a HUF or FSE bitstream.  `offset` is in bits, so
@@ -136,9 +142,8 @@ static inline u64 STREAM_read_bits(const
 /*** END BITSTREAM OPERATIONS *********/
 
 /*** BIT COUNTING OPERATIONS **********/
-/// Returns `x`, where `2^x` is the largest power of 2 less than or equal to
-/// `num`, or `-1` if `num == 0`.
-static inline int log2inf(const u64 num);
+/// Returns the index of the highest set bit in `num`, or `-1` if `num == 0`
+static inline int highest_set_bit(const u64 num);
 /*** END BIT COUNTING OPERATIONS ******/
 
 /*** HUFFMAN PRIMITIVES ***************/
@@ -384,8 +389,8 @@ size_t ZSTD_decompress_with_dict(void *c
         parse_dictionary(&parsed_dict, (const u8 *)dict, dict_len);
     }
 
-    istream_t in = {(const u8 *)src, 0, src_len};
-    ostream_t out = {(u8 *)dst, dst_len};
+    istream_t in = IO_make_istream(src, src_len);
+    ostream_t out = IO_make_ostream(dst, dst_len);
 
     // "A content compressed by Zstandard is transformed into a Zstandard frame.
     // Multiple frames can be appended into a single file or stream. A frame is
@@ -633,6 +638,7 @@ static void frame_context_apply_dict(fra
         FSE_copy_dtable(&ctx->of_dtable, &dict->of_dtable);
         FSE_copy_dtable(&ctx->ml_dtable, &dict->ml_dtable);
 
+        // Copy the repeated offsets
         memcpy(ctx->previous_offsets, dict->previous_offsets,
                sizeof(ctx->previous_offsets));
     }
@@ -668,7 +674,7 @@ static void decompress_data(frame_contex
             // number of bytes to read and copy."
             const u8 *const read_ptr = IO_read_bytes(in, block_len);
             u8 *const write_ptr = IO_write_bytes(out, block_len);
-            //
+
             // Copy the raw data into the output
             memcpy(write_ptr, read_ptr, block_len);
 
@@ -682,7 +688,7 @@ static void decompress_data(frame_contex
             const u8 *const read_ptr = IO_read_bytes(in, 1);
             u8 *const write_ptr = IO_write_bytes(out, block_len);
 
-            // Copy `block_len` copies of `streams->src[0]` to the output
+            // Copy `block_len` copies of `read_ptr[0]` to the output
             memset(write_ptr, read_ptr[0], block_len);
 
             ctx->current_total_output += block_len;
@@ -751,7 +757,7 @@ static size_t decode_literals_compressed
                                          u8 **const literals,
                                          const int block_type,
                                          const int size_format);
-static void decode_huf_table(istream_t *const in, HUF_dtable *const dtable);
+static void decode_huf_table(HUF_dtable *const dtable, istream_t *const in);
 static void fse_decode_hufweights(ostream_t *weights, istream_t *const in,
                                     int *const num_symbs);
 
@@ -894,12 +900,12 @@ static size_t decode_literals_compressed
     istream_t huf_stream = IO_make_sub_istream(in, compressed_size);
 
     if (block_type == 2) {
-        // Decode provided Huffman table
+        // Decode the provided Huffman table
         // "This section is only present when Literals_Block_Type type is
         // Compressed_Literals_Block (2)."
 
         HUF_free_dtable(&ctx->literals_dtable);
-        decode_huf_table(&huf_stream, &ctx->literals_dtable);
+        decode_huf_table(&ctx->literals_dtable, &huf_stream);
     } else {
         // If the previous Huffman table is being repeated, ensure it exists
         if (!ctx->literals_dtable.symbols) {
@@ -922,13 +928,13 @@ static size_t decode_literals_compressed
 }
 
 // Decode the Huffman table description
-static void decode_huf_table(istream_t *const in, HUF_dtable *const dtable) {
-    const u8 header = IO_read_bits(in, 8);
-
+static void decode_huf_table(HUF_dtable *const dtable, istream_t *const in) {
     // "All literal values from zero (included) to last present one (excluded)
     // are represented by Weight with values from 0 to Max_Number_of_Bits."
 
     // "This is a single byte value (0-255), which describes how to decode the list of weights."
+    const u8 header = IO_read_bits(in, 8);
+
     u8 weights[HUF_MAX_SYMBS];
     memset(weights, 0, sizeof(weights));
 
@@ -997,7 +1003,7 @@ typedef struct {
     u16 ll_state;
     u16 of_state;
     u16 ml_state;
-} sequence_state_t;
+} sequence_states_t;
 
 /// Different modes to signal to decode_seq_tables what to do
 typedef enum {
@@ -1052,10 +1058,10 @@ static void decompress_sequences(frame_c
                                  istream_t *const in,
                                  sequence_command_t *const sequences,
                                  const size_t num_sequences);
-static sequence_command_t decode_sequence(sequence_state_t *const state,
+static sequence_command_t decode_sequence(sequence_states_t *const state,
                                           const u8 *const src,
                                           i64 *const offset);
-static void decode_seq_table(istream_t *const in, FSE_dtable *const table,
+static void decode_seq_table(FSE_dtable *const table, istream_t *const in,
                                const seq_part_t type, const seq_mode_t mode);
 
 static size_t decode_sequences(frame_context_t *const ctx, istream_t *in,
@@ -1131,34 +1137,33 @@ static void decompress_sequences(frame_c
     // Offsets
     // Match Lengths"
     // Update the tables we have stored in the context
-    decode_seq_table(in, &ctx->ll_dtable, seq_literal_length,
+    decode_seq_table(&ctx->ll_dtable, in, seq_literal_length,
                      (compression_modes >> 6) & 3);
 
-    decode_seq_table(in, &ctx->of_dtable, seq_offset,
+    decode_seq_table(&ctx->of_dtable, in, seq_offset,
                      (compression_modes >> 4) & 3);
 
-    decode_seq_table(in, &ctx->ml_dtable, seq_match_length,
+    decode_seq_table(&ctx->ml_dtable, in, seq_match_length,
                      (compression_modes >> 2) & 3);
 
-    // Check to make sure none of the tables are uninitialized
-    if (!ctx->ll_dtable.symbols || !ctx->of_dtable.symbols ||
-        !ctx->ml_dtable.symbols) {
-        CORRUPTION();
-    }
 
-    sequence_state_t state;
-    // Copy the context's tables into the local state
-    memcpy(&state.ll_table, &ctx->ll_dtable, sizeof(FSE_dtable));
-    memcpy(&state.of_table, &ctx->of_dtable, sizeof(FSE_dtable));
-    memcpy(&state.ml_table, &ctx->ml_dtable, sizeof(FSE_dtable));
+    sequence_states_t states;
+
+    // Initialize the decoding tables
+    {
+        states.ll_table = ctx->ll_dtable;
+        states.of_table = ctx->of_dtable;
+        states.ml_table = ctx->ml_dtable;
+    }
 
-    size_t len = IO_istream_len(in);
+    const size_t len = IO_istream_len(in);
     const u8 *const src = IO_read_bytes(in, len);
 
     // "After writing the last bit containing information, the compressor writes
     // a single 1-bit and then fills the byte with 0-7 0 bits of padding."
-    const int padding = 8 - log2inf(src[len - 1]);
-    i64 offset = len * 8 - padding;
+    const int padding = 8 - highest_set_bit(src[len - 1]);
+    // The offset starts at the end because FSE streams are read backwards
+    i64 bit_offset = len * 8 - padding;
 
     // "The bitstream starts with initial state values, each using the required
     // number of bits in their respective accuracy, decoded previously from
@@ -1166,24 +1171,22 @@ static void decompress_sequences(frame_c
     //
     // It starts by Literals_Length_State, followed by Offset_State, and finally
     // Match_Length_State."
-    FSE_init_state(&state.ll_table, &state.ll_state, src, &offset);
-    FSE_init_state(&state.of_table, &state.of_state, src, &offset);
-    FSE_init_state(&state.ml_table, &state.ml_state, src, &offset);
+    FSE_init_state(&states.ll_table, &states.ll_state, src, &bit_offset);
+    FSE_init_state(&states.of_table, &states.of_state, src, &bit_offset);
+    FSE_init_state(&states.ml_table, &states.ml_state, src, &bit_offset);
 
     for (size_t i = 0; i < num_sequences; i++) {
         // Decode sequences one by one
-        sequences[i] = decode_sequence(&state, src, &offset);
+        sequences[i] = decode_sequence(&states, src, &bit_offset);
     }
 
-    if (offset != 0) {
+    if (bit_offset != 0) {
         CORRUPTION();
     }
-
-    // Don't free tables so they can be used in the next block
 }
 
 // Decode a single sequence and update the state
-static sequence_command_t decode_sequence(sequence_state_t *const state,
+static sequence_command_t decode_sequence(sequence_states_t *const states,
                                           const u8 *const src,
                                           i64 *const offset) {
     // "Each symbol is a code in its own context, which specifies Baseline and
@@ -1191,9 +1194,9 @@ static sequence_command_t decode_sequenc
     // additional bits in the same bitstream."
 
     // Decode symbols, but don't update states
-    const u8 of_code = FSE_peek_symbol(&state->of_table, state->of_state);
-    const u8 ll_code = FSE_peek_symbol(&state->ll_table, state->ll_state);
-    const u8 ml_code = FSE_peek_symbol(&state->ml_table, state->ml_state);
+    const u8 of_code = FSE_peek_symbol(&states->of_table, states->of_state);
+    const u8 ll_code = FSE_peek_symbol(&states->ll_table, states->ll_state);
+    const u8 ml_code = FSE_peek_symbol(&states->ml_table, states->ml_state);
 
     // Offset doesn't need a max value as it's not decoded using a table
     if (ll_code > SEQ_MAX_CODES[seq_literal_length] ||
@@ -1221,17 +1224,18 @@ static sequence_command_t decode_sequenc
     // then Offset_State."
     // If the stream is complete don't read bits to update state
     if (*offset != 0) {
-        FSE_update_state(&state->ll_table, &state->ll_state, src, offset);
-        FSE_update_state(&state->ml_table, &state->ml_state, src, offset);
-        FSE_update_state(&state->of_table, &state->of_state, src, offset);
+        FSE_update_state(&states->ll_table, &states->ll_state, src, offset);
+        FSE_update_state(&states->ml_table, &states->ml_state, src, offset);
+        FSE_update_state(&states->of_table, &states->of_state, src, offset);
     }
 
     return seq;
 }
 
 /// Given a sequence part and table mode, decode the FSE distribution
-static void decode_seq_table(istream_t *const in, FSE_dtable *const table,
-                               const seq_part_t type, const seq_mode_t mode) {
+/// Errors if the mode is `seq_repeat` without a pre-existing table in `table`
+static void decode_seq_table(FSE_dtable *const table, istream_t *const in,
+                             const seq_part_t type, const seq_mode_t mode) {
     // Constant arrays indexed by seq_part_t
     const i16 *const default_distributions[] = {SEQ_LITERAL_LENGTH_DEFAULT_DIST,
                                                 SEQ_OFFSET_DEFAULT_DIST,
@@ -1272,12 +1276,17 @@ static void decode_seq_table(istream_t *
         // "Repeat_Mode : re-use distribution table from previous compressed
         // block."
         // Nothing to do here, table will be unchanged
+        if (!table->symbols) {
+            // This mode is invalid if we don't already have a table
+            CORRUPTION();
+        }
         break;
     default:
         // Impossible, as mode is from 0-3
         IMPOSSIBLE();
         break;
     }
+
 }
 /******* END SEQUENCE DECODING ************************************************/
 
@@ -1296,6 +1305,8 @@ static void execute_sequences(frame_cont
         const sequence_command_t seq = sequences[i];
 
         {
+            // If the sequence asks for more literals than are left, the
+            // sequence must be corrupted
             if (seq.literal_length > IO_istream_len(&litstream)) {
                 CORRUPTION();
             }
@@ -1336,7 +1347,8 @@ static void execute_sequences(frame_cont
                 // as per the exception listed above
                 offset = idx < 3 ? offset_hist[idx] : offset_hist[0] - 1;
 

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201705061026.v46AQesA089962>