Date: Wed, 20 Jul 2016 16:36:17 +0000 (UTC) From: Maxim Sobolev <sobomax@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org Subject: svn commit: r303095 - stable/10/usr.bin/mkuzip Message-ID: <201607201636.u6KGaHbC003138@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: sobomax Date: Wed Jul 20 16:36:17 2016 New Revision: 303095 URL: https://svnweb.freebsd.org/changeset/base/303095 Log: MFC: merge in all new features and improvements into mkuzip(8) from current, which includes: o LZMA compression; o block de-duplication; o performance improvements; o multi-thread support. This includes the following revisions: r295943,r296626,r296628,r296810,r298504,r298505,r298577 Suggested by: emaste Added: stable/10/usr.bin/mkuzip/mkuz_blk.c (contents, props changed) stable/10/usr.bin/mkuzip/mkuz_blk.h (contents, props changed) stable/10/usr.bin/mkuzip/mkuz_blk_chain.h (contents, props changed) stable/10/usr.bin/mkuzip/mkuz_blockcache.c (contents, props changed) stable/10/usr.bin/mkuzip/mkuz_blockcache.h (contents, props changed) stable/10/usr.bin/mkuzip/mkuz_cfg.h (contents, props changed) stable/10/usr.bin/mkuzip/mkuz_cloop.h (contents, props changed) stable/10/usr.bin/mkuzip/mkuz_conveyor.c (contents, props changed) stable/10/usr.bin/mkuzip/mkuz_conveyor.h (contents, props changed) stable/10/usr.bin/mkuzip/mkuz_format.h (contents, props changed) stable/10/usr.bin/mkuzip/mkuz_fqueue.c (contents, props changed) stable/10/usr.bin/mkuzip/mkuz_fqueue.h (contents, props changed) stable/10/usr.bin/mkuzip/mkuz_lzma.c (contents, props changed) stable/10/usr.bin/mkuzip/mkuz_lzma.h (contents, props changed) stable/10/usr.bin/mkuzip/mkuz_time.c (contents, props changed) stable/10/usr.bin/mkuzip/mkuz_time.h (contents, props changed) stable/10/usr.bin/mkuzip/mkuz_zlib.c (contents, props changed) stable/10/usr.bin/mkuzip/mkuz_zlib.h (contents, props changed) stable/10/usr.bin/mkuzip/mkuzip.h (contents, props changed) Modified: stable/10/usr.bin/mkuzip/Makefile stable/10/usr.bin/mkuzip/mkuzip.8 stable/10/usr.bin/mkuzip/mkuzip.c Modified: stable/10/usr.bin/mkuzip/Makefile ============================================================================== --- stable/10/usr.bin/mkuzip/Makefile Wed Jul 20 15:59:37 2016 (r303094) +++ stable/10/usr.bin/mkuzip/Makefile Wed Jul 20 16:36:17 2016 (r303095) @@ -2,9 +2,10 @@ PROG= mkuzip MAN= mkuzip.8 +SRCS= mkuzip.c mkuz_blockcache.c mkuz_lzma.c mkuz_zlib.c mkuz_conveyor.c \ + mkuz_blk.c mkuz_fqueue.c mkuz_time.c -DPADD= ${LIBZ} -LDADD= -lz - +DPADD= ${LIBZ} ${LIBMD} ${LIBLZMA} ${LIBPTHREAD} +LDADD= -lz -lmd -llzma -lpthread .include <bsd.prog.mk> Added: stable/10/usr.bin/mkuzip/mkuz_blk.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ stable/10/usr.bin/mkuzip/mkuz_blk.c Wed Jul 20 16:36:17 2016 (r303095) @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2004-2016 Maxim Sobolev <sobomax@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <stdint.h> +#include <stdlib.h> + +#include "mkuzip.h" +#include "mkuz_blk.h" + +struct mkuz_blk * +mkuz_blk_ctor(size_t blen) +{ + struct mkuz_blk *rval; + + rval = mkuz_safe_zmalloc(sizeof(struct mkuz_blk) + blen); + rval->alen = blen; + rval->br_offset = OFFSET_UNDEF; + return (rval); +} Added: stable/10/usr.bin/mkuzip/mkuz_blk.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ stable/10/usr.bin/mkuzip/mkuz_blk.h Wed Jul 20 16:36:17 2016 (r303095) @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2004-2016 Maxim Sobolev <sobomax@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#define OFFSET_UNDEF UINT64_MAX + +struct mkuz_blk_info { + uint64_t offset; + size_t len; + uint32_t blkno; + unsigned char digest[16]; +}; + +#define MKUZ_BLK_EOF (void *)0x1 +#define MKUZ_BLK_MORE (void *)0x2 + +struct mkuz_blk { + struct mkuz_blk_info info; + size_t alen; + uint64_t br_offset; + unsigned char data[]; +}; + +struct mkuz_blk *mkuz_blk_ctor(size_t); Added: stable/10/usr.bin/mkuzip/mkuz_blk_chain.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ stable/10/usr.bin/mkuzip/mkuz_blk_chain.h Wed Jul 20 16:36:17 2016 (r303095) @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2016 Maxim Sobolev <sobomax@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +struct mkuz_blk; +struct mkuz_bchain_link; + +struct mkuz_bchain_link { + struct mkuz_blk *this; + struct mkuz_bchain_link *prev; +}; Added: stable/10/usr.bin/mkuzip/mkuz_blockcache.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ stable/10/usr.bin/mkuzip/mkuz_blockcache.c Wed Jul 20 16:36:17 2016 (r303095) @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2016 Maxim Sobolev <sobomax@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <err.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#if defined(MKUZ_DEBUG) +# include <assert.h> +# include <stdio.h> +#endif + +#include "mkuz_blockcache.h" +#include "mkuz_blk.h" + +struct mkuz_blkcache_itm { + struct mkuz_blk_info hit; + struct mkuz_blkcache_itm *next; +}; + +static struct mkuz_blkcache { + struct mkuz_blkcache_itm first[256]; +} blkcache; + +static int +verify_match(int fd, const struct mkuz_blk *cbp, struct mkuz_blkcache_itm *bcep) +{ + void *vbuf; + ssize_t rlen; + int rval; + + rval = -1; + vbuf = malloc(cbp->info.len); + if (vbuf == NULL) { + goto e0; + } + if (lseek(fd, bcep->hit.offset, SEEK_SET) < 0) { + goto e1; + } + rlen = read(fd, vbuf, cbp->info.len); + if (rlen < 0 || (unsigned)rlen != cbp->info.len) { + goto e2; + } + rval = (memcmp(cbp->data, vbuf, cbp->info.len) == 0) ? 1 : 0; +e2: + lseek(fd, cbp->info.offset, SEEK_SET); +e1: + free(vbuf); +e0: + return (rval); +} + +#define I2J(x) ((intmax_t)(x)) +#define U2J(x) ((uintmax_t)(x)) + +static unsigned char +digest_fold(const unsigned char *mdigest) +{ + int i; + unsigned char rval; + + rval = mdigest[0]; + for (i = 1; i < 16; i++) { + rval = rval ^ mdigest[i]; + } + return (rval); +} + +struct mkuz_blk_info * +mkuz_blkcache_regblock(int fd, const struct mkuz_blk *bp) +{ + struct mkuz_blkcache_itm *bcep; + int rval; + unsigned char h; + +#if defined(MKUZ_DEBUG) + assert((unsigned)lseek(fd, 0, SEEK_CUR) == bp->info.offset); +#endif + h = digest_fold(bp->info.digest); + if (blkcache.first[h].hit.len == 0) { + bcep = &blkcache.first[h]; + } else { + for (bcep = &blkcache.first[h]; bcep != NULL; bcep = bcep->next) { + if (bcep->hit.len != bp->info.len) + continue; + if (memcmp(bp->info.digest, bcep->hit.digest, + sizeof(bp->info.digest)) == 0) { + break; + } + } + if (bcep != NULL) { + rval = verify_match(fd, bp, bcep); + if (rval == 1) { +#if defined(MKUZ_DEBUG) + fprintf(stderr, "cache hit %jd, %jd, %jd, %jd\n", + I2J(bcep->hit.blkno), I2J(bcep->hit.offset), + I2J(bp->info.offset), I2J(bp->info.len)); +#endif + return (&bcep->hit); + } + if (rval == 0) { +#if defined(MKUZ_DEBUG) + fprintf(stderr, "block MD5 collision, you should try lottery, " + "man!\n"); +#endif + return (NULL); + } + warn("verify_match"); + return (NULL); + } + bcep = malloc(sizeof(struct mkuz_blkcache_itm)); + if (bcep == NULL) + return (NULL); + memset(bcep, '\0', sizeof(struct mkuz_blkcache_itm)); + bcep->next = blkcache.first[h].next; + blkcache.first[h].next = bcep; + } + bcep->hit = bp->info; + return (NULL); +} Added: stable/10/usr.bin/mkuzip/mkuz_blockcache.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ stable/10/usr.bin/mkuzip/mkuz_blockcache.h Wed Jul 20 16:36:17 2016 (r303095) @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2016 Maxim Sobolev <sobomax@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +struct mkuz_blk; + +struct mkuz_blk_info *mkuz_blkcache_regblock(int, const struct mkuz_blk *); Added: stable/10/usr.bin/mkuzip/mkuz_cfg.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ stable/10/usr.bin/mkuzip/mkuz_cfg.h Wed Jul 20 16:36:17 2016 (r303095) @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2016 Maxim Sobolev <sobomax@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +struct mkuz_conveyor; + +struct mkuz_cfg { + int fdr; + int fdw; + int verbose; + int no_zcomp; + int en_dedup; + int nworkers; + int blksz; + const struct mkuz_format *handler; +}; Added: stable/10/usr.bin/mkuzip/mkuz_cloop.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ stable/10/usr.bin/mkuzip/mkuz_cloop.h Wed Jul 20 16:36:17 2016 (r303095) @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2004-2016 Maxim Sobolev <sobomax@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +/* CLOOP format and related constants */ + +/* + * Integer values (block size, number of blocks, offsets) + * are stored in big-endian (network) order on disk. + */ + +#define CLOOP_MAGIC_LEN 128 +#define CLOOP_OFS_COMPR 0x0b +#define CLOOP_OFS_VERSN (CLOOP_OFS_COMPR + 1) + +#define CLOOP_MAJVER_2 '2' +#define CLOOP_MAJVER_3 '3' + +#define CLOOP_COMP_LIBZ 'V' +#define CLOOP_COMP_LZMA 'L' + +struct cloop_header { + char magic[CLOOP_MAGIC_LEN]; /* cloop magic */ + uint32_t blksz; /* block size */ + uint32_t nblocks; /* number of blocks */ +}; Added: stable/10/usr.bin/mkuzip/mkuz_conveyor.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ stable/10/usr.bin/mkuzip/mkuz_conveyor.c Wed Jul 20 16:36:17 2016 (r303095) @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2004-2016 Maxim Sobolev <sobomax@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/types.h> +#include <err.h> +#include <inttypes.h> +#include <md5.h> +#include <pthread.h> +#include <stdlib.h> +#include <strings.h> + +#if defined(MKUZ_DEBUG) +# include <stdio.h> +#endif + +#include "mkuz_conveyor.h" +#include "mkuz_cfg.h" +#include "mkuzip.h" +#include "mkuz_format.h" +#include "mkuz_blk.h" +#include "mkuz_fqueue.h" +#include "mkuz_blk_chain.h" + +static void compute_digest(struct mkuz_blk *); + +struct cw_args { + struct mkuz_conveyor *cvp; + struct mkuz_cfg *cfp; +}; + +static void * +cworker(void *p) +{ + struct cw_args *cwp; + struct mkuz_cfg *cfp; + struct mkuz_blk *oblk, *iblk; + struct mkuz_conveyor *cvp; + void *c_ctx; + + cwp = (struct cw_args *)p; + cfp = cwp->cfp; + cvp = cwp->cvp; + free(cwp); + c_ctx = cfp->handler->f_init(cfp->blksz); + for (;;) { + iblk = mkuz_fqueue_deq(cvp->wrk_queue); + if (iblk == MKUZ_BLK_EOF) { + /* Let other threads to see the EOF block */ + mkuz_fqueue_enq(cvp->wrk_queue, iblk); + break; + } + if (cfp->no_zcomp == 0 && + mkuz_memvcmp(iblk->data, '\0', iblk->info.len) != 0) { + /* All zeroes block */ + oblk = mkuz_blk_ctor(0); + } else { + oblk = cfp->handler->f_compress(c_ctx, iblk); + if (cfp->en_dedup != 0) { + compute_digest(oblk); + } + } + oblk->info.blkno = iblk->info.blkno; + mkuz_fqueue_enq(cvp->results, oblk); + free(iblk); + } + return (NULL); +} + +static void +compute_digest(struct mkuz_blk *bp) +{ + MD5_CTX mcontext; + + MD5Init(&mcontext); + MD5Update(&mcontext, bp->data, bp->info.len); + MD5Final(bp->info.digest, &mcontext); +} + +struct mkuz_conveyor * +mkuz_conveyor_ctor(struct mkuz_cfg *cfp) +{ + struct mkuz_conveyor *cp; + struct cw_args *cwp; + int i, r; + + cp = mkuz_safe_zmalloc(sizeof(struct mkuz_conveyor) + + (sizeof(pthread_t) * cfp->nworkers)); + + cp->wrk_queue = mkuz_fqueue_ctor(1); + cp->results = mkuz_fqueue_ctor(1); + + for (i = 0; i < cfp->nworkers; i++) { + cwp = mkuz_safe_zmalloc(sizeof(struct cw_args)); + cwp->cfp = cfp; + cwp->cvp = cp; + r = pthread_create(&cp->wthreads[i], NULL, cworker, (void *)cwp); + if (r != 0) { + errx(1, "mkuz_conveyor_ctor: pthread_create() failed"); + /* Not reached */ + } + } + return (cp); +} Added: stable/10/usr.bin/mkuzip/mkuz_conveyor.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ stable/10/usr.bin/mkuzip/mkuz_conveyor.h Wed Jul 20 16:36:17 2016 (r303095) @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2016 Maxim Sobolev <sobomax@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +struct mkuz_fifo_queue; + +#define ITEMS_PER_WORKER 4 + +#define MAX_WORKERS_AUTO 24 + +struct mkuz_conveyor { + /* + * Work items are places in here, and picked up by workers in a FIFO + * fashion. + */ + struct mkuz_fifo_queue *wrk_queue; + /* + * Results are dropped into this FIFO and consumer is buzzed to pick them + * up + */ + struct mkuz_fifo_queue *results; + + pthread_t wthreads[]; +}; + +struct mkuz_cfg; + +struct mkuz_conveyor *mkuz_conveyor_ctor(struct mkuz_cfg *); Added: stable/10/usr.bin/mkuzip/mkuz_format.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ stable/10/usr.bin/mkuzip/mkuz_format.h Wed Jul 20 16:36:17 2016 (r303095) @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2016 Maxim Sobolev <sobomax@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +DEFINE_RAW_METHOD(f_init, void *, uint32_t); +DEFINE_RAW_METHOD(f_compress, struct mkuz_blk *, void *, const struct mkuz_blk *); + +struct mkuz_format { + const char *magic; + const char *default_sufx; + f_init_t f_init; + f_compress_t f_compress; +}; Added: stable/10/usr.bin/mkuzip/mkuz_fqueue.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ stable/10/usr.bin/mkuzip/mkuz_fqueue.c Wed Jul 20 16:36:17 2016 (r303095) @@ -0,0 +1,214 @@ +/* + * Copyright (c) 2004-2016 Maxim Sobolev <sobomax@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <err.h> +#include <pthread.h> +#include <stdint.h> +#include <stdlib.h> + +#if defined(MKUZ_DEBUG) +# include <assert.h> +#endif + +#include "mkuzip.h" +#include "mkuz_fqueue.h" +#include "mkuz_conveyor.h" +#include "mkuz_blk.h" +#include "mkuz_blk_chain.h" + +struct mkuz_fifo_queue * +mkuz_fqueue_ctor(int wakeup_len) +{ + struct mkuz_fifo_queue *fqp; + + fqp = mkuz_safe_zmalloc(sizeof(struct mkuz_fifo_queue)); + fqp->wakeup_len = wakeup_len; + if (pthread_mutex_init(&fqp->mtx, NULL) != 0) { + errx(1, "pthread_mutex_init() failed"); + } + if (pthread_cond_init(&fqp->cvar, NULL) != 0) { + errx(1, "pthread_cond_init() failed"); + } + return (fqp); +} + +void +mkuz_fqueue_enq(struct mkuz_fifo_queue *fqp, struct mkuz_blk *bp) +{ + struct mkuz_bchain_link *ip; + + ip = mkuz_safe_zmalloc(sizeof(struct mkuz_bchain_link)); + ip->this = bp; + + pthread_mutex_lock(&fqp->mtx); + if (fqp->first != NULL) { + fqp->first->prev = ip; + } else { + fqp->last = ip; + } + fqp->first = ip; + fqp->length += 1; + if (fqp->length >= fqp->wakeup_len) { + pthread_cond_signal(&fqp->cvar); + } + pthread_mutex_unlock(&fqp->mtx); +} + +#if defined(NOTYET) +int +mkuz_fqueue_enq_all(struct mkuz_fifo_queue *fqp, struct mkuz_bchain_link *cip_f, + struct mkuz_bchain_link *cip_l, int clen) +{ + int rval; + + pthread_mutex_lock(&fqp->mtx); + if (fqp->first != NULL) { + fqp->first->prev = cip_l; + } else { + fqp->last = cip_l; + } + fqp->first = cip_f; + fqp->length += clen; + rval = fqp->length; + if (fqp->length >= fqp->wakeup_len) { + pthread_cond_signal(&fqp->cvar); + } + pthread_mutex_unlock(&fqp->mtx); + return (rval); +} +#endif + +static int +mkuz_fqueue_check(struct mkuz_fifo_queue *fqp, cmp_cb_t cmp_cb, void *cap) +{ + struct mkuz_bchain_link *ip; + + for (ip = fqp->last; ip != NULL; ip = ip->prev) { + if (cmp_cb(ip->this, cap)) { + return (1); + } + } + return (0); +} + +struct mkuz_blk * +mkuz_fqueue_deq_when(struct mkuz_fifo_queue *fqp, cmp_cb_t cmp_cb, void *cap) +{ + struct mkuz_bchain_link *ip, *newlast, *newfirst, *mip; + struct mkuz_blk *bp; + + pthread_mutex_lock(&fqp->mtx); + while (fqp->last == NULL || !mkuz_fqueue_check(fqp, cmp_cb, cap)) { + pthread_cond_wait(&fqp->cvar, &fqp->mtx); + } + if (cmp_cb(fqp->last->this, cap)) { + mip = fqp->last; + fqp->last = mip->prev; + if (fqp->last == NULL) { +#if defined(MKUZ_DEBUG) + assert(fqp->length == 1); +#endif + fqp->first = NULL; + } + } else { +#if defined(MKUZ_DEBUG) + assert(fqp->length > 1); +#endif + newfirst = newlast = fqp->last; + mip = NULL; + for (ip = fqp->last->prev; ip != NULL; ip = ip->prev) { + if (cmp_cb(ip->this, cap)) { + mip = ip; + continue; + } + newfirst->prev = ip; + newfirst = ip; + } + newfirst->prev = NULL; + fqp->first = newfirst; + fqp->last = newlast; + } + fqp->length -= 1; + pthread_mutex_unlock(&fqp->mtx); + bp = mip->this; + free(mip); + + return bp; +} + +struct mkuz_blk * +mkuz_fqueue_deq(struct mkuz_fifo_queue *fqp) +{ + struct mkuz_bchain_link *ip; + struct mkuz_blk *bp; + + pthread_mutex_lock(&fqp->mtx); + while (fqp->last == NULL) { + pthread_cond_wait(&fqp->cvar, &fqp->mtx); + } +#if defined(MKUZ_DEBUG) + assert(fqp->length > 0); +#endif + ip = fqp->last; + fqp->last = ip->prev; + if (fqp->last == NULL) { +#if defined(MKUZ_DEBUG) + assert(fqp->length == 1); +#endif + fqp->first = NULL; + } + fqp->length -= 1; + pthread_mutex_unlock(&fqp->mtx); + bp = ip->this; + free(ip); + + return bp; +} + +#if defined(NOTYET) +struct mkuz_bchain_link * +mkuz_fqueue_deq_all(struct mkuz_fifo_queue *fqp, int *rclen) +{ + struct mkuz_bchain_link *rchain; + + pthread_mutex_lock(&fqp->mtx); + while (fqp->last == NULL) { + pthread_cond_wait(&fqp->cvar, &fqp->mtx); + } +#if defined(MKUZ_DEBUG) + assert(fqp->length > 0); +#endif + rchain = fqp->last; + fqp->first = fqp->last = NULL; + *rclen = fqp->length; + fqp->length = 0; + pthread_mutex_unlock(&fqp->mtx); + return (rchain); +} +#endif Added: stable/10/usr.bin/mkuzip/mkuz_fqueue.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ stable/10/usr.bin/mkuzip/mkuz_fqueue.h Wed Jul 20 16:36:17 2016 (r303095) @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2004-2016 Maxim Sobolev <sobomax@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +struct mkuz_fifo_queue { + pthread_mutex_t mtx; + pthread_cond_t cvar; + struct mkuz_bchain_link *first; + struct mkuz_bchain_link *last; + int length; + int wakeup_len; +}; + +struct mkuz_blk; +struct mkuz_bchain_link; + +DEFINE_RAW_METHOD(cmp_cb, int, const struct mkuz_blk *, void *); + +struct mkuz_fifo_queue *mkuz_fqueue_ctor(int); +void mkuz_fqueue_enq(struct mkuz_fifo_queue *, struct mkuz_blk *); +struct mkuz_blk *mkuz_fqueue_deq(struct mkuz_fifo_queue *); +struct mkuz_blk *mkuz_fqueue_deq_when(struct mkuz_fifo_queue *, cmp_cb_t, void *); +#if defined(NOTYET) +struct mkuz_bchain_link *mkuz_fqueue_deq_all(struct mkuz_fifo_queue *, int *); +int mkuz_fqueue_enq_all(struct mkuz_fifo_queue *, struct mkuz_bchain_link *, + struct mkuz_bchain_link *, int); +#endif Added: stable/10/usr.bin/mkuzip/mkuz_lzma.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ stable/10/usr.bin/mkuzip/mkuz_lzma.c Wed Jul 20 16:36:17 2016 (r303095) @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2004-2016 Maxim Sobolev <sobomax@FreeBSD.org> + * Copyright (c) 2011 Aleksandr Rybalko <ray@ddteam.net> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <err.h> +#include <stdint.h> + +#include <lzma.h> + +#include "mkuzip.h" +#include "mkuz_lzma.h" +#include "mkuz_blk.h" + +#define USED_BLOCKSIZE DEV_BSIZE + +struct mkuz_lzma { + lzma_filter filters[2]; + lzma_options_lzma opt_lzma; + lzma_stream strm; + uint32_t blksz; +}; + +static const lzma_stream lzma_stream_init = LZMA_STREAM_INIT; + +void * +mkuz_lzma_init(uint32_t blksz) +{ + struct mkuz_lzma *ulp; + + if (blksz % USED_BLOCKSIZE != 0) { + errx(1, "cluster size should be multiple of %d", + USED_BLOCKSIZE); + /* Not reached */ + } + if (blksz > MAXPHYS) { + errx(1, "cluster size is too large"); + /* Not reached */ + } *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201607201636.u6KGaHbC003138>