Skip site navigation (1)Skip section navigation (2)
Date:      Fri, 9 Jan 2009 03:42:41 +0000 (UTC)
From:      Murray Stokely <murray@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-7@freebsd.org
Subject:   svn commit: r186920 - in stable/7: lib/libfetch usr.bin/fetch
Message-ID:  <200901090342.n093gfug041855@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: murray
Date: Fri Jan  9 03:42:41 2009
New Revision: 186920
URL: http://svn.freebsd.org/changeset/base/186920

Log:
  MFC r186043, 186124, 186241
  
  Add support for If-Modified-Since to fetch with new -i option.
  Consistently return 1 on error.
  Improve usage and SYNOPSIS.
  
  _M   usr.bin/fetch
  M    usr.bin/fetch/fetch.1
  M    usr.bin/fetch/fetch.c
  _M   lib/libfetch
  M    lib/libfetch/fetch.h
  M    lib/libfetch/http.c
  M    lib/libfetch/fetch.3
  M    lib/libfetch/fetch.c

Modified:
  stable/7/lib/libfetch/   (props changed)
  stable/7/lib/libfetch/fetch.3
  stable/7/lib/libfetch/fetch.c
  stable/7/lib/libfetch/fetch.h
  stable/7/lib/libfetch/http.c
  stable/7/usr.bin/fetch/   (props changed)
  stable/7/usr.bin/fetch/fetch.1
  stable/7/usr.bin/fetch/fetch.c

Modified: stable/7/lib/libfetch/fetch.3
==============================================================================
--- stable/7/lib/libfetch/fetch.3	Fri Jan  9 02:31:51 2009	(r186919)
+++ stable/7/lib/libfetch/fetch.3	Fri Jan  9 03:42:41 2009	(r186920)
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd December 18, 2007
+.Dd December 14, 2008
 .Dt FETCH 3
 .Os
 .Sh NAME
@@ -165,9 +165,16 @@ struct url {
     char	*doc;
     off_t	 offset;
     size_t	 length;
+    time_t	 ims_time;
 };
 .Ed
 .Pp
+The
+.Va ims_time
+field stores the time value for
+.Li If-Modified-Since
+HTTP requests.
+.Pp
 The pointer returned by
 .Fn fetchMakeURL
 or
@@ -353,6 +360,22 @@ and
 .Fn fetchPutHTTP
 will use a direct connection even if a proxy server is defined.
 .Pp
+If the
+.Ql i
+(if-modified-since) flag is specified, and
+the
+.Va ims_time
+field is set in 
+.Vt "struct url" ,
+then
+.Fn fetchXGetHTTP
+and
+.Fn fetchGetHTTP
+will send a conditional
+.Li If-Modified-Since
+HTTP header to only fetch the content if it is newer than
+.Va ims_time .
+.Pp
 Since there seems to be no good way of implementing the HTTP PUT
 method in a manner consistent with the rest of the
 .Nm fetch

Modified: stable/7/lib/libfetch/fetch.c
==============================================================================
--- stable/7/lib/libfetch/fetch.c	Fri Jan  9 02:31:51 2009	(r186919)
+++ stable/7/lib/libfetch/fetch.c	Fri Jan  9 03:42:41 2009	(r186920)
@@ -74,9 +74,7 @@ static struct fetcherr url_errlist[] = {
 FILE *
 fetchXGet(struct url *URL, struct url_stat *us, const char *flags)
 {
-	int direct;
 
-	direct = CHECK_FLAG('d');
 	if (us != NULL) {
 		us->size = -1;
 		us->atime = us->mtime = 0;
@@ -110,9 +108,7 @@ fetchGet(struct url *URL, const char *fl
 FILE *
 fetchPut(struct url *URL, const char *flags)
 {
-	int direct;
 
-	direct = CHECK_FLAG('d');
 	if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
 		return (fetchPutFile(URL, flags));
 	else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
@@ -132,9 +128,7 @@ fetchPut(struct url *URL, const char *fl
 int
 fetchStat(struct url *URL, struct url_stat *us, const char *flags)
 {
-	int direct;
 
-	direct = CHECK_FLAG('d');
 	if (us != NULL) {
 		us->size = -1;
 		us->atime = us->mtime = 0;
@@ -158,9 +152,7 @@ fetchStat(struct url *URL, struct url_st
 struct url_ent *
 fetchList(struct url *URL, const char *flags)
 {
-	int direct;
 
-	direct = CHECK_FLAG('d');
 	if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
 		return (fetchListFile(URL, flags));
 	else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)

Modified: stable/7/lib/libfetch/fetch.h
==============================================================================
--- stable/7/lib/libfetch/fetch.h	Fri Jan  9 02:31:51 2009	(r186919)
+++ stable/7/lib/libfetch/fetch.h	Fri Jan  9 03:42:41 2009	(r186920)
@@ -46,6 +46,7 @@ struct url {
 	char		*doc;
 	off_t		 offset;
 	size_t		 length;
+	time_t		 ims_time;
 };
 
 struct url_stat {

Modified: stable/7/lib/libfetch/http.c
==============================================================================
--- stable/7/lib/libfetch/http.c	Fri Jan  9 02:31:51 2009	(r186919)
+++ stable/7/lib/libfetch/http.c	Fri Jan  9 03:42:41 2009	(r186920)
@@ -63,6 +63,7 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/socket.h>
+#include <sys/time.h>
 
 #include <ctype.h>
 #include <err.h>
@@ -92,6 +93,7 @@ __FBSDID("$FreeBSD$");
 #define HTTP_MOVED_PERM		301
 #define HTTP_MOVED_TEMP		302
 #define HTTP_SEE_OTHER		303
+#define HTTP_NOT_MODIFIED	304
 #define HTTP_TEMP_REDIRECT	307
 #define HTTP_NEED_AUTH		401
 #define HTTP_NEED_PROXY_AUTH	407
@@ -797,20 +799,23 @@ FILE *
 http_request(struct url *URL, const char *op, struct url_stat *us,
     struct url *purl, const char *flags)
 {
+	char timebuf[80];
+	char hbuf[MAXHOSTNAMELEN + 7], *host;
 	conn_t *conn;
 	struct url *url, *new;
-	int chunked, direct, need_auth, noredirect, verbose;
+	int chunked, direct, ims, need_auth, noredirect, verbose;
 	int e, i, n, val;
 	off_t offset, clength, length, size;
 	time_t mtime;
 	const char *p;
 	FILE *f;
 	hdr_t h;
-	char hbuf[MAXHOSTNAMELEN + 7], *host;
+	struct tm *timestruct;
 
 	direct = CHECK_FLAG('d');
 	noredirect = CHECK_FLAG('A');
 	verbose = CHECK_FLAG('v');
+	ims = CHECK_FLAG('i');
 
 	if (direct && purl) {
 		fetchFreeURL(purl);
@@ -879,6 +884,14 @@ http_request(struct url *URL, const char
 			    op, url->doc);
 		}
 
+		if (ims && url->ims_time) {
+			timestruct = gmtime((time_t *)&url->ims_time);
+			(void)strftime(timebuf, 80, "%a, %d %b %Y %T GMT",
+			    timestruct);
+			if (verbose)
+				fetch_info("If-Modified-Since: %s", timebuf);
+			http_cmd(conn, "If-Modified-Since: %s", timebuf);
+		}
 		/* virtual host */
 		http_cmd(conn, "Host: %s", host);
 
@@ -940,6 +953,7 @@ http_request(struct url *URL, const char
 		switch (http_get_reply(conn)) {
 		case HTTP_OK:
 		case HTTP_PARTIAL:
+		case HTTP_NOT_MODIFIED:
 			/* fine */
 			break;
 		case HTTP_MOVED_PERM:
@@ -1074,7 +1088,10 @@ http_request(struct url *URL, const char
 		}
 
 		/* we have a hit or an error */
-		if (conn->err == HTTP_OK || conn->err == HTTP_PARTIAL || HTTP_ERROR(conn->err))
+		if (conn->err == HTTP_OK
+		    || conn->err == HTTP_NOT_MODIFIED
+		    || conn->err == HTTP_PARTIAL
+		    || HTTP_ERROR(conn->err))
 			break;
 
 		/* all other cases: we got a redirect */
@@ -1102,6 +1119,11 @@ http_request(struct url *URL, const char
 		  (long long)offset, (long long)length,
 		  (long long)size, (long long)clength));
 
+	if (conn->err == HTTP_NOT_MODIFIED) {
+		http_seterr(HTTP_NOT_MODIFIED);
+		return (NULL);
+	}
+
 	/* check for inconsistencies */
 	if (clength != -1 && length != -1 && clength != length) {
 		http_seterr(HTTP_PROTOCOL_ERROR);

Modified: stable/7/usr.bin/fetch/fetch.1
==============================================================================
--- stable/7/usr.bin/fetch/fetch.1	Fri Jan  9 02:31:51 2009	(r186919)
+++ stable/7/usr.bin/fetch/fetch.1	Fri Jan  9 03:42:41 2009	(r186920)
@@ -29,7 +29,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd March 11, 2003
+.Dd December 14, 2008
 .Dt FETCH 1
 .Os
 .Sh NAME
@@ -37,17 +37,25 @@
 .Nd retrieve a file by Uniform Resource Locator
 .Sh SYNOPSIS
 .Nm
-.Op Fl 146AFMPRUadlmnpqrsv
+.Op Fl 146AadFlMmnPpqRrsUv
 .Op Fl B Ar bytes
+.Op Fl i Ar file
+.Op Fl N Ar file
+.Op Fl o Ar file
 .Op Fl S Ar bytes
 .Op Fl T Ar seconds
+.Op Fl w Ar seconds
+.Ar URL ...
+.Nm
+.Op Fl 146AadFlMmnPpqRrsUv
+.Op Fl B Ar bytes
+.Op Fl i Ar file
 .Op Fl N Ar file
 .Op Fl o Ar file
+.Op Fl S Ar bytes
+.Op Fl T Ar seconds
 .Op Fl w Ar seconds
-.Op Fl h Ar host
-.Op Fl c Ar dir
-.Op Fl f Ar file
-.Op Ar URL ...
+.Fl h Ar host Fl f Ar file Oo Fl c Ar dir Oc
 .Sh DESCRIPTION
 The
 .Nm
@@ -59,7 +67,7 @@ command line.
 .Pp
 The following options are available:
 .Bl -tag -width Fl
-.It Fl \&1
+.It Fl 1
 Stop and return exit code 0 at the first successfully retrieved file.
 .It Fl 4
 Forces
@@ -110,6 +118,12 @@ The file to retrieve is located on the h
 .Ar host .
 This option is deprecated and is provided for backward compatibility
 only.
+.It Fl i Ar file
+If-Modified-Since mode: the remote file will only be retrieved if it
+is newer than
+.Ar file
+on the local host.
+(HTTP only)
 .It Fl l
 If the target is a file-scheme URL, make a symbolic link to the target
 rather than trying to copy it.
@@ -243,6 +257,12 @@ If multiple URLs are listed on the comma
 .Nm
 will attempt to retrieve each one of them in turn, and will return
 zero only if they were all successfully retrieved.
+.Pp
+If the
+.Fl i
+argument is used and the remote file is not newer than the
+specified file then the command will still return success,
+although no file is transferred.
 .Sh SEE ALSO
 .Xr fetch 3
 .Sh HISTORY

Modified: stable/7/usr.bin/fetch/fetch.c
==============================================================================
--- stable/7/usr.bin/fetch/fetch.c	Fri Jan  9 02:31:51 2009	(r186919)
+++ stable/7/usr.bin/fetch/fetch.c	Fri Jan  9 03:42:41 2009	(r186920)
@@ -42,7 +42,6 @@ __FBSDID("$FreeBSD$");
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <sysexits.h>
 #include <termios.h>
 #include <unistd.h>
 
@@ -60,6 +59,8 @@ int	 d_flag;	/*    -d: direct connection
 int	 F_flag;	/*    -F: restart without checking mtime  */
 char	*f_filename;	/*    -f: file to fetch */
 char	*h_hostname;	/*    -h: host to fetch from */
+int	 i_flag;	/*    -i: specify input file for mtime comparison */
+char	*i_filename;	/*        name of input file */
 int	 l_flag;	/*    -l: link rather than copy file: URLs */
 int	 m_flag;	/* -[Mm]: mirror mode */
 char	*N_filename;	/*    -N: netrc file name */
@@ -381,6 +382,14 @@ fetch(char *URL, const char *path)
 		if (A_flag)
 			strcat(flags, "A");
 		timeout = T_secs ? T_secs : http_timeout;
+		if (i_flag) {
+			if (stat(i_filename, &sb)) {
+				warn("%s: stat()", i_filename);
+				goto failure;
+			}
+			url->ims_time = sb.st_mtime;
+			strcat(flags, "i");
+		}
 	}
 
 	/* set the protocol timeout. */
@@ -448,7 +457,14 @@ fetch(char *URL, const char *path)
 		goto signal;
 	if (f == NULL) {
 		warnx("%s: %s", URL, fetchLastErrString);
-		goto failure;
+		if (i_flag && strcmp(url->scheme, SCHEME_HTTP) == 0
+		    && fetchLastErrCode == FETCH_OK
+		    && strcmp(fetchLastErrString, "Not Modified") == 0) {
+			/* HTTP Not Modified Response, return OK. */
+			r = 0;
+			goto done;
+		} else
+			goto failure;
 	}
 	if (sigint)
 		goto signal;
@@ -710,10 +726,11 @@ fetch(char *URL, const char *path)
 static void
 usage(void)
 {
-	fprintf(stderr, "%s\n%s\n%s\n",
-	    "usage: fetch [-146AFMPRUadlmnpqrsv] [-N netrc] [-o outputfile]",
-	    "             [-S bytes] [-B bytes] [-T seconds] [-w seconds]",
-	    "             [-h host -f file [-c dir] | URL ...]");
+	fprintf(stderr, "%s\n%s\n%s\n%s\n",
+"usage: fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]",
+"       [-T seconds] [-w seconds] [-i file] URL ...",
+"       fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]",
+"       [-T seconds] [-w seconds] [-i file] -h host -f file [-c dir]");
 }
 
 
@@ -730,7 +747,7 @@ main(int argc, char *argv[])
 	int c, e, r;
 
 	while ((c = getopt(argc, argv,
-	    "146AaB:bc:dFf:Hh:lMmN:nPpo:qRrS:sT:tUvw:")) != -1)
+	    "146AaB:bc:dFf:Hh:i:lMmN:nPpo:qRrS:sT:tUvw:")) != -1)
 		switch (c) {
 		case '1':
 			once_flag = 1;
@@ -775,6 +792,10 @@ main(int argc, char *argv[])
 		case 'h':
 			h_hostname = optarg;
 			break;
+		case 'i':
+			i_flag = 1;
+			i_filename = optarg;
+			break;
 		case 'l':
 			l_flag = 1;
 			break;
@@ -842,7 +863,7 @@ main(int argc, char *argv[])
 			break;
 		default:
 			usage();
-			exit(EX_USAGE);
+			exit(1);
 		}
 
 	argc -= optind;
@@ -851,7 +872,7 @@ main(int argc, char *argv[])
 	if (h_hostname || f_filename || c_dirname) {
 		if (!h_hostname || !f_filename || argc) {
 			usage();
-			exit(EX_USAGE);
+			exit(1);
 		}
 		/* XXX this is a hack. */
 		if (strcspn(h_hostname, "@:/") != strlen(h_hostname))
@@ -864,7 +885,7 @@ main(int argc, char *argv[])
 
 	if (!argc) {
 		usage();
-		exit(EX_USAGE);
+		exit(1);
 	}
 
 	/* allocate buffer */
@@ -905,10 +926,10 @@ main(int argc, char *argv[])
 		} else if (stat(o_filename, &sb) == -1) {
 			if (errno == ENOENT) {
 				if (argc > 1)
-					errx(EX_USAGE, "%s is not a directory",
+					errx(1, "%s is not a directory",
 					    o_filename);
 			} else {
-				err(EX_IOERR, "%s", o_filename);
+				err(1, "%s", o_filename);
 			}
 		} else {
 			if (sb.st_mode & S_IFDIR)



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200901090342.n093gfug041855>