From owner-svn-src-projects@FreeBSD.ORG Sat Oct 25 10:54:28 2008 Return-Path: Delivered-To: svn-src-projects@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id A82D91065675; Sat, 25 Oct 2008 10:54:28 +0000 (UTC) (envelope-from lulf@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 959B38FC08; Sat, 25 Oct 2008 10:54:28 +0000 (UTC) (envelope-from lulf@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id m9PAsS4u030584; Sat, 25 Oct 2008 10:54:28 GMT (envelope-from lulf@svn.freebsd.org) Received: (from lulf@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id m9PAsSmS030578; Sat, 25 Oct 2008 10:54:28 GMT (envelope-from lulf@svn.freebsd.org) Message-Id: <200810251054.m9PAsSmS030578@svn.freebsd.org> From: Ulf Lilleengen Date: Sat, 25 Oct 2008 10:54:28 +0000 (UTC) To: src-committers@freebsd.org, svn-src-projects@freebsd.org X-SVN-Group: projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r184257 - projects/csup_cvsmode/contrib/csup X-BeenThere: svn-src-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the src " projects" tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 25 Oct 2008 10:54:28 -0000 Author: lulf Date: Sat Oct 25 10:54:28 2008 New Revision: 184257 URL: http://svn.freebsd.org/changeset/base/184257 Log: - Implement rsync support in csup, which is chosen as a protocol for regular files if the client supports it. The support is implemented with an API to operate on files, calculating the rolling checksum and md5 checksum for the blocks etc. - Remove unneeded stream_filter_stop and stream_flush before stream_close. Added: projects/csup_cvsmode/contrib/csup/rsyncfile.c (contents, props changed) projects/csup_cvsmode/contrib/csup/rsyncfile.h (contents, props changed) Modified: projects/csup_cvsmode/contrib/csup/Makefile projects/csup_cvsmode/contrib/csup/config.c projects/csup_cvsmode/contrib/csup/detailer.c projects/csup_cvsmode/contrib/csup/updater.c Modified: projects/csup_cvsmode/contrib/csup/Makefile ============================================================================== --- projects/csup_cvsmode/contrib/csup/Makefile Sat Oct 25 10:52:22 2008 (r184256) +++ projects/csup_cvsmode/contrib/csup/Makefile Sat Oct 25 10:54:28 2008 (r184257) @@ -10,7 +10,7 @@ PROG= csup SRCS= attrstack.c config.c detailer.c diff.c fattr.c fixups.c fnmatch.c \ globtree.c idcache.c keyword.c lister.c main.c misc.c mux.c parse.y \ pathcomp.c proto.c status.c stream.c threads.c token.l updater.c \ - rcsfile.c rcsparse.c lex.rcs.c + rcsfile.c rcsparse.c lex.rcs.c rsyncfile.c CFLAGS+= -I. -I${.CURDIR} -g -pthread -DHAVE_FFLAGS -DNDEBUG WARNS?= 1 Modified: projects/csup_cvsmode/contrib/csup/config.c ============================================================================== --- projects/csup_cvsmode/contrib/csup/config.c Sat Oct 25 10:52:22 2008 (r184256) +++ projects/csup_cvsmode/contrib/csup/config.c Sat Oct 25 10:54:28 2008 (r184257) @@ -133,8 +133,6 @@ config_init(const char *file, struct col coll->co_options &= ~CO_CHECKRCS; /* In recent versions, we always try to set the file modes. */ coll->co_options |= CO_SETMODE; - /* XXX We don't support the rsync updating algorithm yet. */ - coll->co_options |= CO_NORSYNC; error = config_parse_refusefiles(coll); if (error) goto bad; Modified: projects/csup_cvsmode/contrib/csup/detailer.c ============================================================================== --- projects/csup_cvsmode/contrib/csup/detailer.c Sat Oct 25 10:52:22 2008 (r184256) +++ projects/csup_cvsmode/contrib/csup/detailer.c Sat Oct 25 10:54:28 2008 (r184257) @@ -43,6 +43,7 @@ #include "mux.h" #include "proto.h" #include "rcsfile.h" +#include "rsyncfile.h" #include "status.h" #include "stream.h" @@ -67,6 +68,7 @@ static int detailer_dofile_co(struct det static int detailer_dofile_rcs(struct detailer *, struct coll *, char *, char *); static int detailer_dofile_regular(struct detailer *, char *, char *); +static int detailer_dofile_rsync(struct detailer *, char *, char *); static int detailer_checkrcsattr(struct detailer *, struct coll *, char *, struct fattr *, int); int detailer_send_details(struct detailer *, struct coll *, char *, @@ -106,7 +108,6 @@ detailer(void *arg) xasprintf(&args->errmsg, "Detailer failed: " "Network read failure: %s", strerror(errno)); } - lprintf(-1, "Error is '%s'\n", args->errmsg); args->status = STATUS_TRANSIENTFAILURE; break; case DETAILER_ERR_WRITE: @@ -343,7 +344,6 @@ detailer_coll(struct detailer *d, struct lprintf(-1, "Server warning: %s\n", msg); break; default: - lprintf(-1, "Line: %s, cmd %s\n", line, cmd); return (DETAILER_ERR_PROTO); } stream_flush(wr); @@ -390,6 +390,33 @@ detailer_dofile_regular(struct detailer } /* + * Tell the server to update a file with the rsync algorithm. + */ +static int +detailer_dofile_rsync(struct detailer *d, char *name, char *path) +{ + struct stream *wr; + struct rsyncfile *rf; + + rf = rsync_open(path, 0, 1); + if (rf == NULL) { + /* Fallback if we fail in opening it. */ + proto_printf(wr, "A %s\n", name); + return (0); + } + wr = d->wr; + proto_printf(wr, "r %s %z %z\n", name, rsync_filesize(rf), + rsync_blocksize(rf)); + /* Detail the blocks. */ + while (rsync_nextblock(rf) != 0) { + proto_printf(wr, "%s %s\n", rsync_rsum(rf), rsync_blockmd5(rf)); + lprintf(-1, "%s %s\n", rsync_rsum(rf), rsync_blockmd5(rf)); + } + proto_printf(wr, ".\n"); + rsync_close(rf); +} + +/* * Tell the server to update an RCS file that we have, or send it if we don't. */ static int @@ -572,6 +599,9 @@ detailer_send_details(struct detailer *d } else if (fattr_type(fa) == FT_FILE) { if (isrcs(name, &len) && !(coll->co_options & CO_NORCS)) { detailer_dofile_rcs(d, coll, name, path); + } else if (!(coll->co_options & CO_NORSYNC) || + !globtree_test(coll->co_norsync, name)) { + detailer_dofile_rsync(d, name, path); } else { detailer_dofile_regular(d, name, path); } Added: projects/csup_cvsmode/contrib/csup/rsyncfile.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/csup_cvsmode/contrib/csup/rsyncfile.c Sat Oct 25 10:54:28 2008 (r184257) @@ -0,0 +1,228 @@ +/*- + * Copyright (c) 2008, Ulf Lilleengen + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "misc.h" +#include "fattr.h" +#include "rsyncfile.h" + +#define MINBLOCKSIZE 1024 +#define MAXBLOCKSIZE (16 * 1024) +#define RECEIVEBUFFERSIZE (15 * 1024) +#define BLOCKINFOSIZE 26 +#define SEARCHREGION 10 +#define MAXBLOCKS (RECEIVEBUFFERSIZE / BLOCKINFOSIZE) + +#define CHAR_OFFSET 3 +#define RSUM_SIZE 9 + +struct rsyncfile { + char *start; + char *buf; + char *end; + size_t blocksize; + size_t fsize; + struct fattr *fa; + int fd; + + char *blockptr; + int blocknum; + char blockmd5[MD5_DIGEST_SIZE]; + char rsumstr[RSUM_SIZE]; + uint32_t rsum; +}; + +static size_t rsync_chooseblocksize(size_t); +static uint32_t rsync_rollsum(uint8_t *, size_t); + +/* Open a file and initialize variable for rsync operation. */ +struct rsyncfile * +rsync_open(char *path, size_t blocksize, int read) +{ + struct rsyncfile *rf; + struct stat st; + int error; + + rf = malloc(sizeof(*rf)); + if (rf == NULL) + return (NULL); + error = stat(path, &st); + if (error) { + free(rf); + return (NULL); + } + rf->fsize = st.st_size; + rf->fa = fattr_fromstat(&st); + + rf->fd = open(path, read ? O_RDONLY : O_RDWR); + if (rf->fd < 0) { + free(rf); + return (NULL); + } + rf->buf = mmap(0, rf->fsize, PROT_READ, MAP_SHARED, rf->fd, 0); + if (rf->buf == MAP_FAILED) { + free(rf); + return (NULL); + } + rf->start = rf->buf; + rf->end = rf->buf + rf->fsize; + rf->blocksize = (blocksize == 0 ? rsync_chooseblocksize(rf->fsize) : + blocksize); + rf->blockptr = rf->buf; + rf->blocknum = 0; + return (rf); +} + +/* Close and free all resources related to an rsync file transfer. */ +int +rsync_close(struct rsyncfile *rf) +{ + int error; + + error = munmap(rf->buf, rf->fsize); + if (error) + return (error); + close(rf->fd); + free(rf); +} + +/* + * Choose the most appropriate block size for an rsync transfer. Modeled + * algorithm after cvsup. + */ +static size_t +rsync_chooseblocksize(size_t fsize) +{ + size_t bestrem, blocksize, bs, hisearch, losearch, rem; + + blocksize = fsize / MAXBLOCKS; + losearch = blocksize - SEARCHREGION; + hisearch = blocksize + SEARCHREGION; + + if (losearch < MINBLOCKSIZE) { + losearch = MINBLOCKSIZE; + hisearch = losearch + (2 * SEARCHREGION); + } else if (hisearch > MAXBLOCKSIZE) { + hisearch = MAXBLOCKSIZE; + losearch = hisearch - (2 * SEARCHREGION); + } + + bestrem = MAXBLOCKSIZE; + for (bs = losearch; bs <= hisearch;) { + rem = fsize % bs; + if (rem < bestrem) { + bestrem = rem; + blocksize = bs; + } + } + return (bestrem); +} + +/* Get the next rsync block of a file. */ +int +rsync_nextblock(struct rsyncfile *rf) +{ + uint32_t rolling; + char *ptr; + MD5_CTX ctx; + size_t blocksize, i; + + if (rf->blockptr >= rf->end) + return (0); + blocksize = min((rf->end - rf->blockptr), rf->blocksize); + /* Calculate MD5 of the block. */ + MD5_Init(&ctx); + MD5_Update(&ctx, rf->blockptr, blocksize); + MD5_End(rf->blockmd5, &ctx); + + rf->rsum = rsync_rollsum(rf->blockptr, blocksize); + snprintf(rf->rsumstr, RSUM_SIZE, "%x", rf->rsum); + rf->blocknum++; + rf->blockptr += blocksize; + return (1); +} + +/* Get the rolling checksum of a file. */ +static uint32_t +rsync_rollsum(uint8_t *buf, size_t len) +{ + uint32_t a, b; + uint8_t *ptr, *limit; + + a = b = 0; + ptr = buf; + limit = buf + len; + + while (ptr < limit) { + a += *ptr + CHAR_OFFSET; + b += a; + ptr++; + } + return ((b << 16) | a); +} + +/* Get running sum so far. */ +char * +rsync_rsum(struct rsyncfile *rf) +{ + + return (rf->rsumstr); +} + +/* Get MD5 of current block. */ +char * +rsync_blockmd5(struct rsyncfile *rf) +{ + + return (rf->blockmd5); +} + +/* Accessor for blocksize. */ +size_t +rsync_blocksize(struct rsyncfile *rf) +{ + + return (rf->blocksize); +} + +/* Accessor for filesize. */ +size_t +rsync_filesize(struct rsyncfile *rf) +{ + + return (rf->fsize); +} Added: projects/csup_cvsmode/contrib/csup/rsyncfile.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/csup_cvsmode/contrib/csup/rsyncfile.h Sat Oct 25 10:54:28 2008 (r184257) @@ -0,0 +1,42 @@ +/*- + * Copyright (c) 2008, Ulf Lilleengen + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _RSYNCFILE_H_ +#define _RSYNCFILE_H_ + +struct rsyncfile; + +struct rsyncfile *rsync_open(char *, size_t, int); +int rsync_nextblock(struct rsyncfile *); +char *rsync_rsum(struct rsyncfile *); +char *rsync_blockmd5(struct rsyncfile *); +int rsync_close(struct rsyncfile *); +size_t rsync_blocksize(struct rsyncfile *); +size_t rsync_filesize(struct rsyncfile *); + +#endif /* !_RSYNCFILE_H_ */ Modified: projects/csup_cvsmode/contrib/csup/updater.c ============================================================================== --- projects/csup_cvsmode/contrib/csup/updater.c Sat Oct 25 10:52:22 2008 (r184256) +++ projects/csup_cvsmode/contrib/csup/updater.c Sat Oct 25 10:54:28 2008 (r184257) @@ -115,6 +115,8 @@ static int updater_rcsedit(struct updat char *); int updater_append_file(struct updater *, struct file_update *, off_t); +static int updater_rsync(struct updater *, struct file_update *, size_t); +static int updater_read_checkout(struct stream *, struct stream *); static struct file_update * fup_new(struct coll *coll, struct status *st) @@ -345,7 +347,7 @@ updater_docoll(struct updater *up, struc struct coll *coll; struct statusrec srbuf, *sr; struct fattr *rcsattr, *tmp; - char *cmd, *line, *msg, *attr; + char *cmd, *blocksize, *line, *msg, *attr; char *name, *tag, *date, *revdate; char *expand, *wantmd5, *revnum; char *optstr, *rcsopt, *pos; @@ -601,7 +603,30 @@ updater_docoll(struct updater *up, struc if (error) return (error); break; - case 'r': /* XXX: rsync support. */ + case 'r': + name = proto_get_ascii(&line); + attr = proto_get_ascii(&line); + blocksize = proto_get_ascii(&line); + wantmd5 = proto_get_ascii(&line); + if (name == NULL || attr == NULL || blocksize == NULL || + wantmd5 == NULL) { + return (UPDATER_ERR_PROTO); + } + error = fup_prepare(fup, name, 0); + if (error) + return (UPDATER_ERR_PROTO); + sr->sr_type = SR_FILELIVE; + fup->wantmd5 = xstrdup(wantmd5); + fup->temppath = tempname(fup->destpath); + sr = &fup->srbuf; + sr->sr_file = xstrdup(name); + sr->sr_serverattr = fattr_decode(attr); + if (sr->sr_serverattr == NULL) + return (UPDATER_ERR_PROTO); + error = updater_rsync(up, fup, strtol(blocksize, NULL, + 10)); + if (error) + return (error); break; case 'I': @@ -1457,8 +1482,6 @@ updater_addfile(struct updater *up, stru remains -= nread; stream_write(to, buf, nread); } while (remains > 0); - stream_flush(to); - stream_filter_stop(to); stream_close(to); line = stream_getln(up->rd, NULL); if (line == NULL) @@ -1749,8 +1772,6 @@ updater_rcsedit(struct updater *up, stru } stream_filter_start(dest, STREAM_FILTER_MD5RCS, md5); error = rcsfile_write(rf, dest); - stream_flush(dest); - stream_filter_stop(dest); stream_close(dest); rcsfile_free(rf); if (error) @@ -1927,8 +1948,6 @@ updater_append_file(struct updater *up, bytes -= nread; stream_write(to, buf, nread); } while (bytes > 0); - stream_flush(to); - stream_filter_stop(to); stream_close(to); line = stream_getln(up->rd, NULL); @@ -1959,3 +1978,142 @@ updater_append_file(struct updater *up, return (error); return (0); } + +/* + * Read file data from stream of checkout commands, and write it to the + * destination. + */ +static int +updater_read_checkout(struct stream *src, struct stream *dest) +{ + char *line; + size_t size; + ssize_t nbytes; + int error, first; + + first = 1; + line = stream_getln(src, &size); + while (line != NULL) { + if (line[size - 1] == '\n') + size--; + if ((size == 1 && *line == '.') || + (size == 2 && strncmp(line, ".+", 2) == 0)) + break; + if (size >= 2 && strncmp(line, "..", 2) == 0) { + size--; + line++; + } + if (!first) { + nbytes = stream_write(dest, "\n", 1); + if (nbytes == -1) + return (UPDATER_ERR_MSG); + } + nbytes = stream_write(dest, line, size); + if (nbytes == -1) + return (UPDATER_ERR_MSG); + line = stream_getln(src, &size); + first = 0; + } + if (line == NULL) + return (UPDATER_ERR_READ); + if (size == 1 && *line == '.') { + nbytes = stream_write(dest, "\n", 1); + if (nbytes == -1) + return (UPDATER_ERR_MSG); + } + return (0); +} + +/* Update file using the rsync protocol. */ +static int +updater_rsync(struct updater *up, struct file_update *fup, size_t blocksize) +{ + struct statusrec *sr; + struct coll *coll; + struct stream *to; + char md5[MD5_DIGEST_SIZE]; + char *buf, *line; + int error, orig; + size_t size, blocknum, blockstart, blockcount; + ssize_t nbytes; + + sr = &fup->srbuf; + + lprintf(1, " Rsync %s\n", fup->coname); + /* First open all files that we are going to work on. */ + to = stream_open_file(fup->temppath, O_WRONLY | O_CREAT | O_TRUNC, + 0600); + if (to == NULL) { + xasprintf(&up->errmsg, "%s: Cannot create: %s", + fup->temppath, strerror(errno)); + return (UPDATER_ERR_MSG); + } + orig = open(fup->destpath, O_RDONLY); + if (orig < 0) { + xasprintf(&up->errmsg, "%s: Cannot open: %s", + fup->destpath, strerror(errno)); + return (UPDATER_ERR_MSG); + } + stream_filter_start(to, STREAM_FILTER_MD5, md5); + + error = updater_read_checkout(up->rd, to); + if (error) { + xasprintf(&up->errmsg, "%s: Cannot write: %s", fup->temppath, + strerror(errno)); + return (error); + } + + /* Buffer must contain blocksize bytes. */ + buf = xmalloc(blocksize); + /* Done with the initial text, read and write chunks. */ + line = stream_getln(up->rd, NULL); + while (line != NULL) { + if (strcmp(line, ".") == 0) + break; + error = UPDATER_ERR_PROTO; + if (proto_get_sizet(&line, &blockstart, 10) != 0) + goto bad; + if (proto_get_sizet(&line, &blockcount, 10) != 0) + goto bad; + /* Read blocks from original file. */ + lseek(orig, SEEK_SET, (blocksize * blockstart)); + blocknum = 0; + error = UPDATER_ERR_MSG; + for (blocknum = 0; blocknum < blockcount; blocknum++) { + nbytes = read(orig, buf, blocksize); + if (nbytes < 0) { + xasprintf(&up->errmsg, "%s: Cannot read: %s", + fup->destpath, strerror(errno)); + goto bad; + } + nbytes = stream_write(to, buf, nbytes); + if (nbytes == -1) { + xasprintf(&up->errmsg, "%s: Cannot write: %s", + fup->temppath, strerror(errno)); + goto bad; + } + } + /* Get the remaining text from the server. */ + error = updater_read_checkout(up->rd, to); + if (error) { + xasprintf(&up->errmsg, "%s: Cannot write: %s", + fup->temppath, strerror(errno)); + goto bad; + } + line = stream_getln(up->rd, NULL); + } + stream_close(to); + close(orig); + + sr->sr_clientattr = fattr_frompath(fup->destpath, FATTR_NOFOLLOW); + if (sr->sr_clientattr == NULL) + return (UPDATER_ERR_PROTO); + fattr_override(sr->sr_clientattr, sr->sr_serverattr, + FA_MODTIME | FA_MASK); + + error = updater_updatefile(up, fup, md5, 0); + fup->wantmd5 = NULL; /* So that it doesn't get freed. */ +bad: + free(buf); + return (error); +}