From owner-svn-src-all@FreeBSD.ORG Fri Jan 9 03:42:42 2009 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 84FE51065672; Fri, 9 Jan 2009 03:42:42 +0000 (UTC) (envelope-from murray@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 70EF28FC08; Fri, 9 Jan 2009 03:42:42 +0000 (UTC) (envelope-from murray@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n093ggrY041862; Fri, 9 Jan 2009 03:42:42 GMT (envelope-from murray@svn.freebsd.org) Received: (from murray@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n093gfug041855; Fri, 9 Jan 2009 03:42:41 GMT (envelope-from murray@svn.freebsd.org) Message-Id: <200901090342.n093gfug041855@svn.freebsd.org> From: Murray Stokely Date: Fri, 9 Jan 2009 03:42:41 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-7@freebsd.org X-SVN-Group: stable-7 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r186920 - in stable/7: lib/libfetch usr.bin/fetch X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 09 Jan 2009 03:42:43 -0000 Author: murray Date: Fri Jan 9 03:42:41 2009 New Revision: 186920 URL: http://svn.freebsd.org/changeset/base/186920 Log: MFC r186043, 186124, 186241 Add support for If-Modified-Since to fetch with new -i option. Consistently return 1 on error. Improve usage and SYNOPSIS. _M usr.bin/fetch M usr.bin/fetch/fetch.1 M usr.bin/fetch/fetch.c _M lib/libfetch M lib/libfetch/fetch.h M lib/libfetch/http.c M lib/libfetch/fetch.3 M lib/libfetch/fetch.c Modified: stable/7/lib/libfetch/ (props changed) stable/7/lib/libfetch/fetch.3 stable/7/lib/libfetch/fetch.c stable/7/lib/libfetch/fetch.h stable/7/lib/libfetch/http.c stable/7/usr.bin/fetch/ (props changed) stable/7/usr.bin/fetch/fetch.1 stable/7/usr.bin/fetch/fetch.c Modified: stable/7/lib/libfetch/fetch.3 ============================================================================== --- stable/7/lib/libfetch/fetch.3 Fri Jan 9 02:31:51 2009 (r186919) +++ stable/7/lib/libfetch/fetch.3 Fri Jan 9 03:42:41 2009 (r186920) @@ -25,7 +25,7 @@ .\" .\" $FreeBSD$ .\" -.Dd December 18, 2007 +.Dd December 14, 2008 .Dt FETCH 3 .Os .Sh NAME @@ -165,9 +165,16 @@ struct url { char *doc; off_t offset; size_t length; + time_t ims_time; }; .Ed .Pp +The +.Va ims_time +field stores the time value for +.Li If-Modified-Since +HTTP requests. +.Pp The pointer returned by .Fn fetchMakeURL or @@ -353,6 +360,22 @@ and .Fn fetchPutHTTP will use a direct connection even if a proxy server is defined. .Pp +If the +.Ql i +(if-modified-since) flag is specified, and +the +.Va ims_time +field is set in +.Vt "struct url" , +then +.Fn fetchXGetHTTP +and +.Fn fetchGetHTTP +will send a conditional +.Li If-Modified-Since +HTTP header to only fetch the content if it is newer than +.Va ims_time . +.Pp Since there seems to be no good way of implementing the HTTP PUT method in a manner consistent with the rest of the .Nm fetch Modified: stable/7/lib/libfetch/fetch.c ============================================================================== --- stable/7/lib/libfetch/fetch.c Fri Jan 9 02:31:51 2009 (r186919) +++ stable/7/lib/libfetch/fetch.c Fri Jan 9 03:42:41 2009 (r186920) @@ -74,9 +74,7 @@ static struct fetcherr url_errlist[] = { FILE * fetchXGet(struct url *URL, struct url_stat *us, const char *flags) { - int direct; - direct = CHECK_FLAG('d'); if (us != NULL) { us->size = -1; us->atime = us->mtime = 0; @@ -110,9 +108,7 @@ fetchGet(struct url *URL, const char *fl FILE * fetchPut(struct url *URL, const char *flags) { - int direct; - direct = CHECK_FLAG('d'); if (strcasecmp(URL->scheme, SCHEME_FILE) == 0) return (fetchPutFile(URL, flags)); else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0) @@ -132,9 +128,7 @@ fetchPut(struct url *URL, const char *fl int fetchStat(struct url *URL, struct url_stat *us, const char *flags) { - int direct; - direct = CHECK_FLAG('d'); if (us != NULL) { us->size = -1; us->atime = us->mtime = 0; @@ -158,9 +152,7 @@ fetchStat(struct url *URL, struct url_st struct url_ent * fetchList(struct url *URL, const char *flags) { - int direct; - direct = CHECK_FLAG('d'); if (strcasecmp(URL->scheme, SCHEME_FILE) == 0) return (fetchListFile(URL, flags)); else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0) Modified: stable/7/lib/libfetch/fetch.h ============================================================================== --- stable/7/lib/libfetch/fetch.h Fri Jan 9 02:31:51 2009 (r186919) +++ stable/7/lib/libfetch/fetch.h Fri Jan 9 03:42:41 2009 (r186920) @@ -46,6 +46,7 @@ struct url { char *doc; off_t offset; size_t length; + time_t ims_time; }; struct url_stat { Modified: stable/7/lib/libfetch/http.c ============================================================================== --- stable/7/lib/libfetch/http.c Fri Jan 9 02:31:51 2009 (r186919) +++ stable/7/lib/libfetch/http.c Fri Jan 9 03:42:41 2009 (r186920) @@ -63,6 +63,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include @@ -92,6 +93,7 @@ __FBSDID("$FreeBSD$"); #define HTTP_MOVED_PERM 301 #define HTTP_MOVED_TEMP 302 #define HTTP_SEE_OTHER 303 +#define HTTP_NOT_MODIFIED 304 #define HTTP_TEMP_REDIRECT 307 #define HTTP_NEED_AUTH 401 #define HTTP_NEED_PROXY_AUTH 407 @@ -797,20 +799,23 @@ FILE * http_request(struct url *URL, const char *op, struct url_stat *us, struct url *purl, const char *flags) { + char timebuf[80]; + char hbuf[MAXHOSTNAMELEN + 7], *host; conn_t *conn; struct url *url, *new; - int chunked, direct, need_auth, noredirect, verbose; + int chunked, direct, ims, need_auth, noredirect, verbose; int e, i, n, val; off_t offset, clength, length, size; time_t mtime; const char *p; FILE *f; hdr_t h; - char hbuf[MAXHOSTNAMELEN + 7], *host; + struct tm *timestruct; direct = CHECK_FLAG('d'); noredirect = CHECK_FLAG('A'); verbose = CHECK_FLAG('v'); + ims = CHECK_FLAG('i'); if (direct && purl) { fetchFreeURL(purl); @@ -879,6 +884,14 @@ http_request(struct url *URL, const char op, url->doc); } + if (ims && url->ims_time) { + timestruct = gmtime((time_t *)&url->ims_time); + (void)strftime(timebuf, 80, "%a, %d %b %Y %T GMT", + timestruct); + if (verbose) + fetch_info("If-Modified-Since: %s", timebuf); + http_cmd(conn, "If-Modified-Since: %s", timebuf); + } /* virtual host */ http_cmd(conn, "Host: %s", host); @@ -940,6 +953,7 @@ http_request(struct url *URL, const char switch (http_get_reply(conn)) { case HTTP_OK: case HTTP_PARTIAL: + case HTTP_NOT_MODIFIED: /* fine */ break; case HTTP_MOVED_PERM: @@ -1074,7 +1088,10 @@ http_request(struct url *URL, const char } /* we have a hit or an error */ - if (conn->err == HTTP_OK || conn->err == HTTP_PARTIAL || HTTP_ERROR(conn->err)) + if (conn->err == HTTP_OK + || conn->err == HTTP_NOT_MODIFIED + || conn->err == HTTP_PARTIAL + || HTTP_ERROR(conn->err)) break; /* all other cases: we got a redirect */ @@ -1102,6 +1119,11 @@ http_request(struct url *URL, const char (long long)offset, (long long)length, (long long)size, (long long)clength)); + if (conn->err == HTTP_NOT_MODIFIED) { + http_seterr(HTTP_NOT_MODIFIED); + return (NULL); + } + /* check for inconsistencies */ if (clength != -1 && length != -1 && clength != length) { http_seterr(HTTP_PROTOCOL_ERROR); Modified: stable/7/usr.bin/fetch/fetch.1 ============================================================================== --- stable/7/usr.bin/fetch/fetch.1 Fri Jan 9 02:31:51 2009 (r186919) +++ stable/7/usr.bin/fetch/fetch.1 Fri Jan 9 03:42:41 2009 (r186920) @@ -29,7 +29,7 @@ .\" .\" $FreeBSD$ .\" -.Dd March 11, 2003 +.Dd December 14, 2008 .Dt FETCH 1 .Os .Sh NAME @@ -37,17 +37,25 @@ .Nd retrieve a file by Uniform Resource Locator .Sh SYNOPSIS .Nm -.Op Fl 146AFMPRUadlmnpqrsv +.Op Fl 146AadFlMmnPpqRrsUv .Op Fl B Ar bytes +.Op Fl i Ar file +.Op Fl N Ar file +.Op Fl o Ar file .Op Fl S Ar bytes .Op Fl T Ar seconds +.Op Fl w Ar seconds +.Ar URL ... +.Nm +.Op Fl 146AadFlMmnPpqRrsUv +.Op Fl B Ar bytes +.Op Fl i Ar file .Op Fl N Ar file .Op Fl o Ar file +.Op Fl S Ar bytes +.Op Fl T Ar seconds .Op Fl w Ar seconds -.Op Fl h Ar host -.Op Fl c Ar dir -.Op Fl f Ar file -.Op Ar URL ... +.Fl h Ar host Fl f Ar file Oo Fl c Ar dir Oc .Sh DESCRIPTION The .Nm @@ -59,7 +67,7 @@ command line. .Pp The following options are available: .Bl -tag -width Fl -.It Fl \&1 +.It Fl 1 Stop and return exit code 0 at the first successfully retrieved file. .It Fl 4 Forces @@ -110,6 +118,12 @@ The file to retrieve is located on the h .Ar host . This option is deprecated and is provided for backward compatibility only. +.It Fl i Ar file +If-Modified-Since mode: the remote file will only be retrieved if it +is newer than +.Ar file +on the local host. +(HTTP only) .It Fl l If the target is a file-scheme URL, make a symbolic link to the target rather than trying to copy it. @@ -243,6 +257,12 @@ If multiple URLs are listed on the comma .Nm will attempt to retrieve each one of them in turn, and will return zero only if they were all successfully retrieved. +.Pp +If the +.Fl i +argument is used and the remote file is not newer than the +specified file then the command will still return success, +although no file is transferred. .Sh SEE ALSO .Xr fetch 3 .Sh HISTORY Modified: stable/7/usr.bin/fetch/fetch.c ============================================================================== --- stable/7/usr.bin/fetch/fetch.c Fri Jan 9 02:31:51 2009 (r186919) +++ stable/7/usr.bin/fetch/fetch.c Fri Jan 9 03:42:41 2009 (r186920) @@ -42,7 +42,6 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #include #include @@ -60,6 +59,8 @@ int d_flag; /* -d: direct connection int F_flag; /* -F: restart without checking mtime */ char *f_filename; /* -f: file to fetch */ char *h_hostname; /* -h: host to fetch from */ +int i_flag; /* -i: specify input file for mtime comparison */ +char *i_filename; /* name of input file */ int l_flag; /* -l: link rather than copy file: URLs */ int m_flag; /* -[Mm]: mirror mode */ char *N_filename; /* -N: netrc file name */ @@ -381,6 +382,14 @@ fetch(char *URL, const char *path) if (A_flag) strcat(flags, "A"); timeout = T_secs ? T_secs : http_timeout; + if (i_flag) { + if (stat(i_filename, &sb)) { + warn("%s: stat()", i_filename); + goto failure; + } + url->ims_time = sb.st_mtime; + strcat(flags, "i"); + } } /* set the protocol timeout. */ @@ -448,7 +457,14 @@ fetch(char *URL, const char *path) goto signal; if (f == NULL) { warnx("%s: %s", URL, fetchLastErrString); - goto failure; + if (i_flag && strcmp(url->scheme, SCHEME_HTTP) == 0 + && fetchLastErrCode == FETCH_OK + && strcmp(fetchLastErrString, "Not Modified") == 0) { + /* HTTP Not Modified Response, return OK. */ + r = 0; + goto done; + } else + goto failure; } if (sigint) goto signal; @@ -710,10 +726,11 @@ fetch(char *URL, const char *path) static void usage(void) { - fprintf(stderr, "%s\n%s\n%s\n", - "usage: fetch [-146AFMPRUadlmnpqrsv] [-N netrc] [-o outputfile]", - " [-S bytes] [-B bytes] [-T seconds] [-w seconds]", - " [-h host -f file [-c dir] | URL ...]"); + fprintf(stderr, "%s\n%s\n%s\n%s\n", +"usage: fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]", +" [-T seconds] [-w seconds] [-i file] URL ...", +" fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]", +" [-T seconds] [-w seconds] [-i file] -h host -f file [-c dir]"); } @@ -730,7 +747,7 @@ main(int argc, char *argv[]) int c, e, r; while ((c = getopt(argc, argv, - "146AaB:bc:dFf:Hh:lMmN:nPpo:qRrS:sT:tUvw:")) != -1) + "146AaB:bc:dFf:Hh:i:lMmN:nPpo:qRrS:sT:tUvw:")) != -1) switch (c) { case '1': once_flag = 1; @@ -775,6 +792,10 @@ main(int argc, char *argv[]) case 'h': h_hostname = optarg; break; + case 'i': + i_flag = 1; + i_filename = optarg; + break; case 'l': l_flag = 1; break; @@ -842,7 +863,7 @@ main(int argc, char *argv[]) break; default: usage(); - exit(EX_USAGE); + exit(1); } argc -= optind; @@ -851,7 +872,7 @@ main(int argc, char *argv[]) if (h_hostname || f_filename || c_dirname) { if (!h_hostname || !f_filename || argc) { usage(); - exit(EX_USAGE); + exit(1); } /* XXX this is a hack. */ if (strcspn(h_hostname, "@:/") != strlen(h_hostname)) @@ -864,7 +885,7 @@ main(int argc, char *argv[]) if (!argc) { usage(); - exit(EX_USAGE); + exit(1); } /* allocate buffer */ @@ -905,10 +926,10 @@ main(int argc, char *argv[]) } else if (stat(o_filename, &sb) == -1) { if (errno == ENOENT) { if (argc > 1) - errx(EX_USAGE, "%s is not a directory", + errx(1, "%s is not a directory", o_filename); } else { - err(EX_IOERR, "%s", o_filename); + err(1, "%s", o_filename); } } else { if (sb.st_mode & S_IFDIR)