Date: Mon, 28 Jun 2010 00:12:29 GMT From: Ivan Voras <ivoras@FreeBSD.org> To: Perforce Change Reviews <perforce@FreeBSD.org> Subject: PERFORCE change 180278 for review Message-ID: <201006280012.o5S0CTXu008122@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://p4web.freebsd.org/@@180278?ac=10 Change 180278 by ivoras@betelgeuse on 2010/06/28 00:12:23 Finish file patching, start work on finishing steps of the operation. Affected files ... .. //depot/projects/soc2010/pkg_patch/src/patch/Makefile#17 edit .. //depot/projects/soc2010/pkg_patch/src/patch/applypatch.c#7 edit .. //depot/projects/soc2010/pkg_patch/src/patch/applypatch.h#7 edit .. //depot/projects/soc2010/pkg_patch/src/patch/hashjob.c#16 edit .. //depot/projects/soc2010/pkg_patch/src/patch/hashjob.h#16 edit .. //depot/projects/soc2010/pkg_patch/src/patch/main.c#17 edit .. //depot/projects/soc2010/pkg_patch/src/patch/mkpatch.c#15 edit .. //depot/projects/soc2010/pkg_patch/src/patch/mkpatch.h#15 edit .. //depot/projects/soc2010/pkg_patch/src/patch/pkg_patch.h#15 edit .. //depot/projects/soc2010/pkg_patch/src/patch/support.c#14 edit Differences ... ==== //depot/projects/soc2010/pkg_patch/src/patch/Makefile#17 (text+ko) ==== ==== //depot/projects/soc2010/pkg_patch/src/patch/applypatch.c#7 (text+ko) ==== @@ -29,6 +29,7 @@ #include <string.h> #include <assert.h> #include <err.h> +#include <md5.h> #include <pkg.h> #include "pkg_patch.h" @@ -100,15 +101,148 @@ } +/* + * Convert a package-relative filename to the absolute (live) filename, using + * the "cwd" information from the Package plist information. + */ +static int +pkg_to_live_filename(char *afilename, char *pfilename, Package *pkg, + const char *msg) +{ + char *last_cwd = NULL; + PackingList pli; + Boolean found = FALSE; + + pli = pkg->head; + while (pli != NULL) { + if (pli->type == PLIST_CWD) + last_cwd = pli->name; + if (pli->type == PLIST_FILE && + strncmp(pfilename, pli->name, PATH_MAX) == 0) { + found = TRUE; + break; + } + pli = pli->next; + } + if (!found) { + warnx("File not found in package metadata: %s [%s]", + pfilename, msg); + return (-1); + } + if (last_cwd == NULL) { + warnx("Failure to detect package cwd info [%s]", msg); + return (-1); + } + snprintf(afilename, PATH_MAX, "%s/%s", last_cwd, pli->name); + return (0); +} + + +/* + * Convert a package-relative directory name to the absolute (live) directory + * name. This is actually somewhat evil and heuristic because the package + * metadata doesn't reference directories and we try to find any file whose + * path begins with the given directory. This should work for all official + * packages but as the metadata format is badly designed, there can be lots of + * edge cases. Solving those is left as an excercise for the reader. + */ +static int +pkg_to_live_dirname(char *adirname, char *pdirname, Package *pkg, + const char *msg) +{ + PackingList pli; + char *last_cwd = NULL; + int pdirlen; + Boolean found = FALSE; + + pdirlen = strlen(pdirname); + pli = pkg->head; + while (pli != NULL) { + if (pli->type == PLIST_CWD) + last_cwd = pli->name; + if (pli->type == PLIST_FILE && + strncmp(pdirname, pli->name, pdirlen) == 0) { + /* Just find anything which begins with the pdirname */ + found = TRUE; + break; + } + pli = pli->next; + } + if (!found) { + warnx("Directory cannot be divined from package metadata: " + "%s [%s]", pdirname, msg); + return (-1); + } + if (last_cwd == NULL) { + warnx("Failure to detect package cwd info [%s]", msg); + return (-1); + } + snprintf(adirname, PATH_MAX, "%s/%s", last_cwd, pli->name); + return (0); +} + + +/* + * Finds a MD5 comment for the given file within the given package metadata. + * As the package metadata is informally specified, nothing guarantees this + * will succeed :) + * The MD5 comment is expected to be after the file node and before any + * following file node. + */ +static int +pkg_find_md5(char *filename, Package *pkg, char *md5) +{ + PackingList pli; + Boolean found = FALSE; + char *p; + + pli = pkg->head; + *md5 = '\0'; + while (pli != NULL) { + if (pli->type == PLIST_FILE) { + if (strncmp(filename, pli->name, PATH_MAX) == 0) { + found = TRUE; + break; + } + } + pli = pli->next; + } + if (!found) + return (-1); + found = FALSE; + while (pli != NULL) { + if (pli->type == PLIST_COMMENT) { + p = strchr(pli->name, ':'); + if (p != NULL && strncmp(pli->name, "MD5:", 4) == 0) { + strlcpy(md5, p+1, 33); + found = TRUE; + break; + } + } + pli = pli->next; + if (pli != NULL) + if (pli->type == PLIST_FILE) + return (-1); + } + if (!found) + return (-1); + return (0); +} + + +/* + * Apply patch command + */ void perform_applypatch() { - char fpatch[PATH_MAX], dpatch[PATH_MAX], tmp[PATH_MAX]; + unsigned int err_count, n_patched_files, i; + char fpatch[PATH_MAX], dpatch[PATH_MAX], tmp[PATH_MAX], pext[10]; struct pkgxjob xpatch; struct pkg_patch pp; Package pkg_live, pkg_new; struct pplist *pl; - unsigned int err_count = 0; + FILE **fpvect; if (argc < 1) errx(1, "Expecting argument: patch filename"); @@ -129,6 +263,7 @@ dpatch); snprintf(tmp, PATH_MAX, "%s/%s", dpatch, PKGPATCH_FNAME); + err_count = 0; /* Step 1 - read the patch metadata */ read_pkgpatch_file(tmp, &pp); @@ -166,16 +301,162 @@ printf("Verifying live system and patch data consistency...\n"); /* Check that files to be added don't exist already. */ STAILQ_FOREACH(pl, &pp.pp_add, linkage) { - snprintf(tmp, PATH_MAX, "%s/%s", PREFIX, pl->filename); + if (pkg_to_live_filename(tmp, pl->filename, &pkg_new, "pp_add") + != 0) { + err_count++; + continue; + } if (access(tmp, F_OK) == 0) { warnx("File exists but shouldn't: %s", tmp); err_count++; } } + /* Check that files to be removed actually exist. */ + STAILQ_FOREACH(pl, &pp.pp_remove, linkage) { + if (pkg_to_live_filename(tmp, pl->filename, &pkg_live, + "pp_remove") != 0) { + err_count++; + continue; + } + if (access(tmp, F_OK) != 0) { + warnx("File should exist but doesn't: %s", tmp); + err_count++; + } + } + /* Check that directories to be removed actually exist. */ + STAILQ_FOREACH(pl, &pp.pp_rmdir, linkage) { + if (pkg_to_live_dirname(tmp, pl->filename, &pkg_live, + "pp_rmdir") != 0) { + err_count++; + continue; + } + if (access(tmp, F_OK) != 0) { + warnx("Directory should exist but doesn't: %s", tmp); + err_count++; + } + } + /* Check that files to be patched actually exist. */ + STAILQ_FOREACH(pl, &pp.pp_patch, linkage) { + if (pkg_to_live_filename(tmp, pl->filename, &pkg_live, + "pp_patch") != 0) { + err_count++; + continue; + } + if (access(tmp, F_OK) != 0) { + warnx("File should exist but doesn't: %s", tmp); + err_count++; + } + } if (err_count != 0) errx(1, "Found %u errors. Cannot continue.", err_count); /* Step 4 - backup the existing package */ if (pkg_backup(pp.source) != 0) err(1, "Cannot backup package: %s", pp.source); + + /* + * Step 5 - apply patches. This is done by generating new files from + * existing files and patches, then verifying all of them match expected + * checksum, then rename()-ing them to the expected files. + */ + snprintf(pext, sizeof(pext), ".p%u", getpid()); + fpvect = calloc(pplist_count(&pp.pp_patch), sizeof(*fpvect)); + n_patched_files = 0; + STAILQ_FOREACH(pl, &pp.pp_patch, linkage) { + char newfile[PATH_MAX], patchfile[PATH_MAX]; + + if (pl->filename[0] == '+') + continue; + if (pkg_to_live_filename(tmp, pl->filename, &pkg_live, + "pp_patch2") != 0) { + err_count++; + warnx("Cannot resolve %s on a live pkg", pl->filename); + } + snprintf(newfile, PATH_MAX, "%s%s", tmp, pext); + if (pl->method == PPMETHOD_CP) + snprintf(patchfile, PATH_MAX, "%s/%s", dpatch, + pl->filename); + else if (pl->method == PPMETHOD_BSDIFF) + snprintf(patchfile, PATH_MAX, "%s/%s.bsdiff", dpatch, + pl->filename); + else + errx(1, "Unknown patch method: %d", (int)(pl->method)); + if (Verbose > 2) + printf("Raw patching %s to %s using %s\n", tmp, newfile, + patchfile); + if (pl->method == PPMETHOD_BSDIFF) { + char cmd[3*PATH_MAX]; + + snprintf(cmd, sizeof(cmd), "%s \"%s\" \"%s\" \"%s\"", + _PATH_BSPATCH, tmp, newfile, patchfile); + fpvect[n_patched_files] = popen(cmd, "r+"); + if (fpvect[n_patched_files] == NULL) + err(1, "Cannot popen: %s", cmd); + n_patched_files++; + } else + if (cp(tmp, newfile) != 0) + err(1, "Cannot copy %s to %s", tmp, newfile); + } + for (i = 0; i < n_patched_files; i++) + if (pclose(fpvect[i]) != 0) + err(1, "pclose() failed"); + /* Verify patched files are correct */ + STAILQ_FOREACH(pl, &pp.pp_patch, linkage) { + char live_md5[33], target_md5[33], newfile[PATH_MAX]; + + if (pl->filename[0] == '+') + continue; + if (pkg_find_md5(pl->filename, &pkg_new, target_md5) != 0) { + warnx("Cannot find MD5 of %s in target metadata", + pl->filename); + continue; + } + if (pkg_to_live_filename(newfile, pl->filename, &pkg_live, + "pp_patch3") != 0) { + err_count++; + warnx("Cannot resolve %s on live pkg for verifying", + pl->filename); + break; + } + strncat(newfile, pext, PATH_MAX); + if (MD5File(newfile, live_md5) == NULL) + err(1, "Cannot MD5 file: %s", newfile); + if (strncmp(live_md5, target_md5, sizeof(live_md5)) != 0) + errx(1, "MD5 mismatch for %s: expected %s, got %s", + pl->filename, target_md5, live_md5); + snprintf(tmp, PATH_MAX, "%s/%s", dpatch, pl->filename); + if (copy_file_attrs(tmp, NULL, newfile) != 0) { + warn("Cannot copy file attributes from %s to %s", + tmp, newfile); + goto error_cleanup; + } + } + /* All is well, we can rename() the new files to the live ones. */ + STAILQ_FOREACH(pl, &pp.pp_patch, linkage) { + char newfile[PATH_MAX], livefile[PATH_MAX]; + + if (pl->filename[0] == '+') + continue; + if (pkg_to_live_filename(livefile, pl->filename, &pkg_live, + "pp_patch4") != 0) + errx(1, "Cannot resolve %s on live pkg", pl->filename); + snprintf(newfile, PATH_MAX, "%s%s", livefile, pext); + assert(access(newfile, F_OK) == 0); + if (rename(newfile, livefile) != 0) { + warn("rename(%s,%s) failed", newfile, livefile); + goto error_cleanup; + } + } + /* Step 6 - apply other classes - files to add, remove, dirs to rmdir */ + STAILQ_FOREACH(pl, &pp.pp_add, linkage) { + } + STAILQ_FOREACH(pl, &pp.pp_remove, linkage) { + } + STAILQ_FOREACH(pl, &pp.pp_rmdir, linkage) { + } + /* Step 7 - fixup metadata */ + return; +error_cleanup: + /* Remove temp patch files, restore backed-up package */ + warnx("Error detected! Rolling back package."); } ==== //depot/projects/soc2010/pkg_patch/src/patch/applypatch.h#7 (text+ko) ==== ==== //depot/projects/soc2010/pkg_patch/src/patch/hashjob.c#16 (text+ko) ==== ==== //depot/projects/soc2010/pkg_patch/src/patch/hashjob.h#16 (text+ko) ==== ==== //depot/projects/soc2010/pkg_patch/src/patch/main.c#17 (text+ko) ==== ==== //depot/projects/soc2010/pkg_patch/src/patch/mkpatch.c#15 (text+ko) ==== @@ -243,10 +243,10 @@ * I've observed linear or better processing time improvments * with this simple trick. */ - FILE **fplist = calloc(n_changed_files, sizeof(*fplist)); + FILE **fpvect = calloc(n_changed_files, sizeof(*fpvect)); int n = 0; - if (fplist == NULL) + if (fpvect == NULL) err(1, "calloc() failed"); /* Start jobs */ SLIST_FOREACH(fl, &flchanged, linkage) { @@ -254,13 +254,14 @@ continue; if (Verbose > 1) printf("bsdiff for %s\n", fl->filename); - snprintf(tmp, PATH_MAX, "%s %s/%s %s/%s %s/%s.bsdiff", + snprintf(tmp, PATH_MAX, + "%s \"%s/%s\" \"%s/%s\" \"%s/%s.bsdiff\"", _PATH_BSDIFF, dold, fl->filename, dnew, fl->filename, dpatch, fl->filename); - fplist[n] = popen(tmp, "r+"); - if (fplist[n] == NULL) + fpvect[n] = popen(tmp, "r+"); + if (fpvect[n] == NULL) err(1, "Cannot popen bsdiff for %s", fl->filename); n++; @@ -270,7 +271,7 @@ SLIST_FOREACH(fl, &flchanged, linkage) { if (fl->filename[0] == '+') continue; - if (pclose(fplist[n]) < 0) + if (pclose(fpvect[n]) < 0) err(1, "pclose() failed for bsdiff of %s", fl->filename); n++; @@ -281,7 +282,7 @@ err(1, "copy_file_attrs(%s,%s) failed", tmp, tmp2); } - free(fplist); + free(fpvect); } /* Finally, create the patch archive and call it a day. */ ==== //depot/projects/soc2010/pkg_patch/src/patch/mkpatch.h#15 (text+ko) ==== ==== //depot/projects/soc2010/pkg_patch/src/patch/pkg_patch.h#15 (text+ko) ==== @@ -26,6 +26,9 @@ #ifndef _PATH_BSDIFF #define _PATH_BSDIFF "/usr/bin/bsdiff" #endif +#ifndef _PATH_BSPATCH +#define _PATH_BSPATCH "/usr/bin/bspatch" +#endif #ifndef _PATH_PKG_CREATE #define _PATH_PKG_CREATE "/usr/sbin/pkg_create" #endif @@ -100,6 +103,7 @@ int rm_rf(char *dir); +int cp(char *from, char *to); int pkgxjob_start(struct pkgxjob *job, char *dir, char *filename); int pkgxjob_finish(struct pkgxjob *job); int filelist_gather(char *dir, struct filelist_head *head); ==== //depot/projects/soc2010/pkg_patch/src/patch/support.c#14 (text+ko) ==== @@ -37,6 +37,9 @@ #include "pkg_patch.h" +/* + * Removes a directory hierarchy. + */ int rm_rf(char *dir) { @@ -48,6 +51,56 @@ } +/* + * Simple file copy. + */ +int +cp(char *from, char *to) +{ + int fd1, fd2, rval = 0; + size_t bs = 1*1024*1024; + char *buf; + + fd1 = open(from, O_RDONLY); + if (fd1 < 0) + return (-1); + fd2 = open(to, O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (fd2 < 0) { + close(fd1); + return (-1); + } + buf = malloc(bs); + if (buf == NULL) { + rval = -1; + goto end; + } + while (bs > 0) { + bs = read(fd1, buf, bs); + if (bs > 0) { + ssize_t written = 0, wr; + + while (bs - written > 0) { + wr = write(fd2, buf + written, bs - written); + if (wr < 0) { + rval = -1; + goto end; + } + written += wr; + } + } + } +end: + if (buf != NULL) + free(buf); + close(fd2); + close(fd1); + return (rval); +} + + +/* + * Starts a package extract (tar) job as a separate process. + */ int pkgxjob_start(struct pkgxjob *job, char *dir, char *filename) { @@ -65,6 +118,9 @@ } +/* + * Finish (cleanup) a tar job. + */ int pkgxjob_finish(struct pkgxjob *job) { @@ -72,6 +128,9 @@ } +/* + * Gather files in a file hierarchy into the given filelist_head. + */ int filelist_gather(char *dir, struct filelist_head *head) { @@ -108,6 +167,9 @@ } +/* + * Returns a list of differences between filelists. + */ int filelist_diff(struct filelist_head *flist1, struct filelist_head *flist2, struct filelist_head *fldiff) @@ -133,6 +195,9 @@ } +/* + * Return a list consisting of the intersection of two given filelists. + */ int filelist_intersect(struct filelist_head *flist1, struct filelist_head *flist2, struct filelist_head *flintersect) @@ -158,6 +223,9 @@ } +/* + * Returns the number of elements in the given filelist. + */ unsigned int filelist_count(struct filelist_head *flist) {
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201006280012.o5S0CTXu008122>