From owner-freebsd-hackers Wed Sep 27 09:23:42 1995 Return-Path: owner-hackers Received: (from root@localhost) by freefall.freebsd.org (8.6.12/8.6.6) id JAA05960 for hackers-outgoing; Wed, 27 Sep 1995 09:23:42 -0700 Received: from gvr.win.tue.nl (root@gvr.win.tue.nl [131.155.210.19]) by freefall.freebsd.org (8.6.12/8.6.6) with ESMTP id JAA05954 for ; Wed, 27 Sep 1995 09:23:33 -0700 Received: by gvr.win.tue.nl (8.6.10/1.53) id RAA13429; Wed, 27 Sep 1995 17:23:26 +0100 From: guido@gvr.win.tue.nl (Guido van Rooij) Message-Id: <199509271623.RAA13429@gvr.win.tue.nl> Subject: Re: pwd_mkdb speed up patches: please review and comment To: guido@gvr.win.tue.nl (Guido van Rooij) Date: Wed, 27 Sep 1995 17:23:26 +0100 (MET) Cc: hackers@freebsd.org In-Reply-To: <199509242023.VAA06416@gvr.win.tue.nl> from "Guido van Rooij" at Sep 24, 95 09:23:24 pm X-Mailer: ELM [version 2.4 PL24] Content-Type: text Content-Length: 15462 Sender: owner-hackers@freebsd.org Precedence: bulk Guido van Rooij wrote: > > Underneath are patches that try to touch all passwd related utils as > little as possible while speeding up regular password updates > dramatically. > > I'd like you all to look at them and give comments. > > What it does not solve is speeding up vipw. Chpass, chsh, chfn, passwd, > yppasswd will be quicker: on a 486, 66MHz, a 10.000 line password > file update will be limited by the disk transfer time. In my case > it took 1.5 minute. Now it is 15 seconds. > > This isn't a real solution for the basic problem. But it makes the > current problems with large password files a bit less painfull. > > What I did not do (yet) is change adduser. > After checking my patches I found what ppl were already afraid of: a descrepancy between how the original pwd_mkdb handled multiple users with the same uid. This has been fixed beneath. So getpwuid() now indeed returns the pw struct belonging to the first user in the passwd file with the given uid. What is absolutely *not* handled properly is multiple users with the same username. This is indeed possible with the old scheme. One might argue about how usefull this is, but is was valid. It will still work as expected, because the only tools making use of the improved pwd_mkdb handling are the ones that do a getpwnam() first and thus they automatically end up with the first user in the passwd file with the given username... (I hope this is still making sense). -Guido --- ./usr.bin/passwd/local_passwd.c.orig Sun Nov 6 22:08:19 1994 +++ ./usr.bin/passwd/local_passwd.c Sun Sep 10 20:03:48 1995 @@ -154,7 +154,7 @@ pw->pw_change = 0; pw_copy(pfd, tfd, pw); - if (!pw_mkdb()) + if (!pw_mkdb(uname)) pw_error((char *)NULL, 0, 1); return (0); } --- ./usr.bin/chpass/chpass.c.orig Tue May 30 08:29:36 1995 +++ ./usr.bin/chpass/chpass.c Fri Sep 22 19:42:57 1995 @@ -187,7 +187,7 @@ pw_copy(pfd, tfd, pw); - if (!pw_mkdb()) + if (!pw_mkdb(pw->pw_name)) pw_error((char *)NULL, 0, 1); exit(0); } --- ./usr.sbin/vipw/vipw.c.orig Tue May 30 05:52:55 1995 +++ ./usr.sbin/vipw/vipw.c Sun Sep 10 20:17:53 1995 @@ -96,7 +96,7 @@ warnx("no changes made"); pw_error((char *)NULL, 0, 0); } - if (pw_mkdb()) + if (pw_mkdb((char *)NULL)) break; pw_prompt(); } --- ./usr.sbin/vipw/pw_util.c.orig Tue May 30 05:52:54 1995 +++ ./usr.sbin/vipw/pw_util.c Fri Sep 22 19:39:07 1995 @@ -138,7 +138,8 @@ } int -pw_mkdb() +pw_mkdb(username) +char *username; { int pstat; pid_t pid; @@ -146,7 +147,12 @@ warnx("rebuilding the database..."); (void)fflush(stderr); if (!(pid = vfork())) { - execl(_PATH_PWD_MKDB, "pwd_mkdb", "-p", tempname, NULL); + if(!username) { + execl(_PATH_PWD_MKDB, "pwd_mkdb", "-p", tempname, NULL); + } else { + execl(_PATH_PWD_MKDB, "pwd_mkdb", "-p", tempname, + "-u", username, NULL); + } pw_error(_PATH_PWD_MKDB, 1, 1); } pid = waitpid(pid, &pstat, 0); --- ./usr.sbin/vipw/pw_util.h.orig Thu May 26 07:23:31 1994 +++ ./usr.sbin/vipw/pw_util.h Sun Sep 10 20:04:57 1995 @@ -37,6 +37,6 @@ void pw_error __P((char *, int, int)); void pw_init __P((void)); int pw_lock __P((void)); -int pw_mkdb __P((void)); +int pw_mkdb __P((char *)); void pw_prompt __P((void)); int pw_tmp __P((void)); --- ./usr.sbin/pwd_mkdb/pwd_mkdb.c.orig Tue May 30 05:51:22 1995 +++ ./usr.sbin/pwd_mkdb/pwd_mkdb.c Tue Sep 26 22:45:25 1995 @@ -79,6 +79,7 @@ void cleanup __P((void)); void error __P((char *)); +void cp __P((char *, char *, mode_t mode)); void mv __P((char *, char *)); int scan __P((FILE *, struct passwd *)); void usage __P((void)); @@ -96,10 +97,14 @@ char *p, *t; char buf[MAX(MAXPATHLEN, LINE_MAX * 2)], tbuf[1024]; char buf2[MAXPATHLEN]; + char *username; + struct passwd *pw, *pw2; + u_int method, methoduid; strcpy(prefix, _PATH_PWD); makeold = 0; - while ((ch = getopt(argc, argv, "d:pv")) != EOF) + username = NULL; + while ((ch = getopt(argc, argv, "d:pu:v")) != EOF) switch(ch) { case 'd': strcpy(prefix, optarg); @@ -107,6 +112,9 @@ case 'p': /* create V7 "file.orig" */ makeold = 1; break; + case 'u': /* only update this record */ + username = optarg; + break; case 'v': /* backward compatible */ break; case '?': @@ -141,8 +149,24 @@ /* Open the temporary insecure password database. */ (void)snprintf(buf, sizeof(buf), "%s/%s.tmp", prefix, _MP_DB); - dp = dbopen(buf, - O_RDWR|O_CREAT|O_EXCL, PERM_INSECURE, DB_HASH, &openinfo); + if(username) { + (void)snprintf(buf2, sizeof(buf2), "%s/%s", prefix, _MP_DB); + cp(buf2, buf, PERM_INSECURE); + dp = dbopen(buf, + O_RDWR|O_EXCL, PERM_INSECURE, DB_HASH, &openinfo); + pw = getpwnam(username); + pw2 = getpwuid(pw->pw_uid); + if(!strcmp(pw2->pw_name, username)) + methoduid = 0; + else + methoduid = R_NOOVERWRITE; + method = 0; + } else { + dp = dbopen(buf, + O_RDWR|O_CREAT|O_EXCL, PERM_INSECURE, DB_HASH, &openinfo); + method = R_NOOVERWRITE; + methoduid = R_NOOVERWRITE; + } if (dp == NULL) error(buf); clean = FILE_INSECURE; @@ -182,47 +206,49 @@ if(pwd.pw_name[0] == '+' || pwd.pw_name[0] == '-') yp_enabled = 1; #define COMPACT(e) t = e; while (*p++ = *t++); - /* Create insecure data. */ - p = buf; - COMPACT(pwd.pw_name); - COMPACT("*"); - memmove(p, &pwd.pw_uid, sizeof(int)); - p += sizeof(int); - memmove(p, &pwd.pw_gid, sizeof(int)); - p += sizeof(int); - memmove(p, &pwd.pw_change, sizeof(time_t)); - p += sizeof(time_t); - COMPACT(pwd.pw_class); - COMPACT(pwd.pw_gecos); - COMPACT(pwd.pw_dir); - COMPACT(pwd.pw_shell); - memmove(p, &pwd.pw_expire, sizeof(time_t)); - p += sizeof(time_t); - memmove(p, &pwd.pw_fields, sizeof pwd.pw_fields); - p += sizeof pwd.pw_fields; - data.size = p - buf; - - /* Store insecure by name. */ - tbuf[0] = _PW_KEYBYNAME; - len = strlen(pwd.pw_name); - memmove(tbuf + 1, pwd.pw_name, len); - key.size = len + 1; - if ((dp->put)(dp, &key, &data, R_NOOVERWRITE) == -1) - error("put"); + if(!username || (strcmp(username, pwd.pw_name) == 0)) { + /* Create insecure data. */ + p = buf; + COMPACT(pwd.pw_name); + COMPACT("*"); + memmove(p, &pwd.pw_uid, sizeof(int)); + p += sizeof(int); + memmove(p, &pwd.pw_gid, sizeof(int)); + p += sizeof(int); + memmove(p, &pwd.pw_change, sizeof(time_t)); + p += sizeof(time_t); + COMPACT(pwd.pw_class); + COMPACT(pwd.pw_gecos); + COMPACT(pwd.pw_dir); + COMPACT(pwd.pw_shell); + memmove(p, &pwd.pw_expire, sizeof(time_t)); + p += sizeof(time_t); + memmove(p, &pwd.pw_fields, sizeof pwd.pw_fields); + p += sizeof pwd.pw_fields; + data.size = p - buf; + + /* Store insecure by name. */ + tbuf[0] = _PW_KEYBYNAME; + len = strlen(pwd.pw_name); + memmove(tbuf + 1, pwd.pw_name, len); + key.size = len + 1; + if ((dp->put)(dp, &key, &data, method) == -1) + error("put"); - /* Store insecure by number. */ - tbuf[0] = _PW_KEYBYNUM; - memmove(tbuf + 1, &cnt, sizeof(cnt)); - key.size = sizeof(cnt) + 1; - if ((dp->put)(dp, &key, &data, R_NOOVERWRITE) == -1) - error("put"); + /* Store insecure by number. */ + tbuf[0] = _PW_KEYBYNUM; + memmove(tbuf + 1, &cnt, sizeof(cnt)); + key.size = sizeof(cnt) + 1; + if ((dp->put)(dp, &key, &data, method) == -1) + error("put"); - /* Store insecure by uid. */ - tbuf[0] = _PW_KEYBYUID; - memmove(tbuf + 1, &pwd.pw_uid, sizeof(pwd.pw_uid)); - key.size = sizeof(pwd.pw_uid) + 1; - if ((dp->put)(dp, &key, &data, R_NOOVERWRITE) == -1) - error("put"); + /* Store insecure by uid. */ + tbuf[0] = _PW_KEYBYUID; + memmove(tbuf + 1, &pwd.pw_uid, sizeof(pwd.pw_uid)); + key.size = sizeof(pwd.pw_uid) + 1; + if ((dp->put)(dp, &key, &data, methoduid) == -1) + error("put"); + } /* Store insecure special plus and special minus */ if (pwd.pw_name[0] == '+' || pwd.pw_name[0] == '-') { @@ -235,7 +261,7 @@ else minuscnt++; key.size = sizeof(cnt) + 1; - if ((dp->put)(dp, &key, &data, R_NOOVERWRITE) == -1) + if ((dp->put)(dp, &key, &data, method) == -1) error("put"); } @@ -251,7 +277,7 @@ data.size = 1; tbuf[0] = _PW_KEYYPENABLED; key.size = 1; - if ((dp->put)(dp, &key, &data, R_NOOVERWRITE) == -1) + if ((dp->put)(dp, &key, &data, method) == -1) error("put"); } /* If we have +@netgroup entries, store the plus counter */ @@ -260,7 +286,7 @@ data.size = sizeof(pluscnt); tbuf[0] = _PW_KEYPLUSCNT; key.size = 1; - if ((dp->put)(dp, &key, &data, R_NOOVERWRITE) == -1) + if ((dp->put)(dp, &key, &data, method) == -1) error("put"); } /* If we have -@netgroup entries, store the minus counter */ @@ -269,7 +295,7 @@ data.size = sizeof(minuscnt); tbuf[0] = _PW_KEYMINUSCNT; key.size = 1; - if ((dp->put)(dp, &key, &data, R_NOOVERWRITE) == -1) + if ((dp->put)(dp, &key, &data, method) == -1) error("put"); } @@ -281,8 +307,15 @@ /* Open the temporary encrypted password database. */ (void)snprintf(buf, sizeof(buf), "%s/%s.tmp", prefix, _SMP_DB); - edp = dbopen(buf, - O_RDWR|O_CREAT|O_EXCL, PERM_SECURE, DB_HASH, &openinfo); + if(username) { + (void)snprintf(buf2, sizeof(buf2), "%s/%s", prefix, _SMP_DB); + cp(buf2, buf, PERM_SECURE); + edp = dbopen(buf, + O_RDWR|O_EXCL, PERM_SECURE, DB_HASH, &openinfo); + } else { + edp = dbopen(buf, + O_RDWR|O_CREAT|O_EXCL, PERM_SECURE, DB_HASH, &openinfo); + } if (!edp) error(buf); clean = FILE_SECURE; @@ -291,61 +324,63 @@ minuscnt = pluscnt = 0; for (cnt = 1; scan(fp, &pwd); ++cnt) { - /* Create secure data. */ - p = buf; - COMPACT(pwd.pw_name); - COMPACT(pwd.pw_passwd); - memmove(p, &pwd.pw_uid, sizeof(int)); - p += sizeof(int); - memmove(p, &pwd.pw_gid, sizeof(int)); - p += sizeof(int); - memmove(p, &pwd.pw_change, sizeof(time_t)); - p += sizeof(time_t); - COMPACT(pwd.pw_class); - COMPACT(pwd.pw_gecos); - COMPACT(pwd.pw_dir); - COMPACT(pwd.pw_shell); - memmove(p, &pwd.pw_expire, sizeof(time_t)); - p += sizeof(time_t); - memmove(p, &pwd.pw_fields, sizeof pwd.pw_fields); - p += sizeof pwd.pw_fields; - data.size = p - buf; - - /* Store secure by name. */ - tbuf[0] = _PW_KEYBYNAME; - len = strlen(pwd.pw_name); - memmove(tbuf + 1, pwd.pw_name, len); - key.size = len + 1; - if ((dp->put)(edp, &key, &data, R_NOOVERWRITE) == -1) - error("put"); - - /* Store secure by number. */ - tbuf[0] = _PW_KEYBYNUM; - memmove(tbuf + 1, &cnt, sizeof(cnt)); - key.size = sizeof(cnt) + 1; - if ((dp->put)(edp, &key, &data, R_NOOVERWRITE) == -1) - error("put"); - - /* Store secure by uid. */ - tbuf[0] = _PW_KEYBYUID; - memmove(tbuf + 1, &pwd.pw_uid, sizeof(pwd.pw_uid)); - key.size = sizeof(pwd.pw_uid) + 1; - if ((dp->put)(edp, &key, &data, R_NOOVERWRITE) == -1) - error("put"); + if(!username || (strcmp(username, pwd.pw_name) == 0)) { + /* Create secure data. */ + p = buf; + COMPACT(pwd.pw_name); + COMPACT(pwd.pw_passwd); + memmove(p, &pwd.pw_uid, sizeof(int)); + p += sizeof(int); + memmove(p, &pwd.pw_gid, sizeof(int)); + p += sizeof(int); + memmove(p, &pwd.pw_change, sizeof(time_t)); + p += sizeof(time_t); + COMPACT(pwd.pw_class); + COMPACT(pwd.pw_gecos); + COMPACT(pwd.pw_dir); + COMPACT(pwd.pw_shell); + memmove(p, &pwd.pw_expire, sizeof(time_t)); + p += sizeof(time_t); + memmove(p, &pwd.pw_fields, sizeof pwd.pw_fields); + p += sizeof pwd.pw_fields; + data.size = p - buf; + + /* Store secure by name. */ + tbuf[0] = _PW_KEYBYNAME; + len = strlen(pwd.pw_name); + memmove(tbuf + 1, pwd.pw_name, len); + key.size = len + 1; + if ((dp->put)(edp, &key, &data, method) == -1) + error("put"); - /* Store secure special plus and special minus */ - if (pwd.pw_name[0] == '+' || pwd.pw_name[0] == '-') { - tbuf[0] = (pwd.pw_name[0] == '+') ? - _PW_KEYPLUSBYNUM : _PW_KEYMINUSBYNUM; - memmove(tbuf + 1, (pwd.pw_name[0] == '+') ? - &pluscnt : &minuscnt, sizeof(cnt)); - if (pwd.pw_name[0] == '+') - pluscnt++; - else - minuscnt++; + /* Store secure by number. */ + tbuf[0] = _PW_KEYBYNUM; + memmove(tbuf + 1, &cnt, sizeof(cnt)); key.size = sizeof(cnt) + 1; - if ((dp->put)(edp, &key, &data, R_NOOVERWRITE) == -1) + if ((dp->put)(edp, &key, &data, method) == -1) error("put"); + + /* Store secure by uid. */ + tbuf[0] = _PW_KEYBYUID; + memmove(tbuf + 1, &pwd.pw_uid, sizeof(pwd.pw_uid)); + key.size = sizeof(pwd.pw_uid) + 1; + if ((dp->put)(edp, &key, &data, methoduid) == -1) + error("put"); + + /* Store secure special plus and special minus */ + if (pwd.pw_name[0] == '+' || pwd.pw_name[0] == '-') { + tbuf[0] = (pwd.pw_name[0] == '+') ? + _PW_KEYPLUSBYNUM : _PW_KEYMINUSBYNUM; + memmove(tbuf + 1, (pwd.pw_name[0] == '+') ? + &pluscnt : &minuscnt, sizeof(cnt)); + if (pwd.pw_name[0] == '+') + pluscnt++; + else + minuscnt++; + key.size = sizeof(cnt) + 1; + if ((dp->put)(edp, &key, &data, method) == -1) + error("put"); + } } } /* If YP enabled, set flag. */ @@ -354,7 +389,7 @@ data.size = 1; tbuf[0] = _PW_KEYYPENABLED; key.size = 1; - if ((edp->put)(edp, &key, &data, R_NOOVERWRITE) == -1) + if ((edp->put)(edp, &key, &data, method) == -1) error("put"); } /* If we have +@netgroup entries, store the plus counter */ @@ -363,7 +398,7 @@ data.size = sizeof(pluscnt); tbuf[0] = _PW_KEYPLUSCNT; key.size = 1; - if ((edp->put)(edp, &key, &data, R_NOOVERWRITE) == -1) + if ((edp->put)(edp, &key, &data, method) == -1) error("put"); } /* If we have -@netgroup entries, store the minus counter */ @@ -372,7 +407,7 @@ data.size = sizeof(minuscnt); tbuf[0] = _PW_KEYMINUSCNT; key.size = 1; - if ((edp->put)(edp, &key, &data, R_NOOVERWRITE) == -1) + if ((edp->put)(edp, &key, &data, method) == -1) error("put"); } (void)(edp->close)(edp); @@ -437,6 +472,37 @@ } void +cp(from, to, mode) + char *from, *to; + mode_t mode; +{ + static char buf[MAXBSIZE]; + int from_fd, rcount, to_fd, wcount; + + if ((from_fd = open(from, O_RDONLY, 0)) < 0) + error(from); + if ((to_fd = open(to, O_WRONLY|O_CREAT|O_EXCL, mode)) < 0) + error(to); + while ((rcount = read(from_fd, buf, MAXBSIZE)) > 0) { + wcount = write(to_fd, buf, rcount); + if (rcount != wcount || wcount == -1) { + int sverrno = errno; + + (void)snprintf(buf, sizeof(buf), "%s to %s", from, to); + errno = sverrno; + error(buf); + } + } + if (rcount < 0) { + int sverrno = errno; + + (void)snprintf(buf, sizeof(buf), "%s to %s", from, to); + errno = sverrno; + error(buf); + } +} + +void mv(from, to) char *from, *to; { @@ -484,6 +550,6 @@ usage() { - (void)fprintf(stderr, "usage: pwd_mkdb [-p] [-d ] file\n"); + (void)fprintf(stderr, "usage: pwd_mkdb [-p] [-d ] [-u username] file\n"); exit(1); } --- ./gnu/usr.sbin/yppasswdd/pw_util.c.orig Tue May 30 07:05:28 1995 +++ ./gnu/usr.sbin/yppasswdd/pw_util.c Sun Sep 24 21:25:13 1995 @@ -132,7 +132,8 @@ } int -pw_mkdb() +pw_mkdb(username) +char *username; { int pstat; pid_t pid; @@ -140,7 +141,12 @@ warnx("rebuilding the database..."); (void)fflush(stderr); if (!(pid = vfork())) { - execl(_PATH_PWD_MKDB, "pwd_mkdb", "-p", tempname, NULL); + if(!username) { + execl(_PATH_PWD_MKDB, "pwd_mkdb", "-p", tempname, NULL); + } else { + execl(_PATH_PWD_MKDB, "pwd_mkdb", "-p", tempname, + "-u", username, NULL); + } pw_error(_PATH_PWD_MKDB, 1, 1); } pid = waitpid(pid, &pstat, 0);