From owner-svn-src-vendor@FreeBSD.ORG Mon Feb 18 11:48:09 2013 Return-Path: Delivered-To: svn-src-vendor@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by hub.freebsd.org (Postfix) with ESMTP id B661C3FA; Mon, 18 Feb 2013 11:48:09 +0000 (UTC) (envelope-from mm@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) by mx1.freebsd.org (Postfix) with ESMTP id 9A96A27B; Mon, 18 Feb 2013 11:48:09 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.5/8.14.5) with ESMTP id r1IBm9Le045535; Mon, 18 Feb 2013 11:48:09 GMT (envelope-from mm@svn.freebsd.org) Received: (from mm@localhost) by svn.freebsd.org (8.14.5/8.14.5/Submit) id r1IBm9kV045532; Mon, 18 Feb 2013 11:48:09 GMT (envelope-from mm@svn.freebsd.org) Message-Id: <201302181148.r1IBm9kV045532@svn.freebsd.org> From: Martin Matuska Date: Mon, 18 Feb 2013 11:48:09 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r246940 - vendor-sys/illumos/dist/uts/common/fs/zfs vendor-sys/illumos/dist/uts/common/fs/zfs/sys vendor/illumos/dist/lib/libzpool/common vendor/illumos/dist/lib/libzpool/common/sys X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 18 Feb 2013 11:48:09 -0000 Author: mm Date: Mon Feb 18 11:48:08 2013 New Revision: 246940 URL: http://svnweb.freebsd.org/changeset/base/246940 Log: Update vendor/illumos/dist and vendor-sys/illumos/dist to illumos-gate 13957:512faafc0eaf Illumos ZFS issues: 3537 want pool io kstats Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/spa_misc.c vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa_impl.h vendor-sys/illumos/dist/uts/common/fs/zfs/vdev_queue.c Changes in other areas also in this revision: Modified: vendor/illumos/dist/lib/libzpool/common/kernel.c vendor/illumos/dist/lib/libzpool/common/sys/zfs_context.h Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/spa_misc.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/spa_misc.c Mon Feb 18 09:07:59 2013 (r246939) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/spa_misc.c Mon Feb 18 11:48:08 2013 (r246940) @@ -480,6 +480,7 @@ spa_add(const char *name, nvlist_t *conf mutex_init(&spa->spa_scrub_lock, NULL, MUTEX_DEFAULT, NULL); mutex_init(&spa->spa_suspend_lock, NULL, MUTEX_DEFAULT, NULL); mutex_init(&spa->spa_vdev_top_lock, NULL, MUTEX_DEFAULT, NULL); + mutex_init(&spa->spa_iokstat_lock, NULL, MUTEX_DEFAULT, NULL); cv_init(&spa->spa_async_cv, NULL, CV_DEFAULT, NULL); cv_init(&spa->spa_proc_cv, NULL, CV_DEFAULT, NULL); @@ -559,6 +560,13 @@ spa_add(const char *name, nvlist_t *conf KM_SLEEP) == 0); } + spa->spa_iokstat = kstat_create("zfs", 0, name, + "disk", KSTAT_TYPE_IO, 1, 0); + if (spa->spa_iokstat) { + spa->spa_iokstat->ks_lock = &spa->spa_iokstat_lock; + kstat_install(spa->spa_iokstat); + } + return (spa); } @@ -608,6 +616,9 @@ spa_remove(spa_t *spa) spa_config_lock_destroy(spa); + kstat_delete(spa->spa_iokstat); + spa->spa_iokstat = NULL; + for (int t = 0; t < TXG_SIZE; t++) bplist_destroy(&spa->spa_free_bplist[t]); @@ -625,6 +636,7 @@ spa_remove(spa_t *spa) mutex_destroy(&spa->spa_scrub_lock); mutex_destroy(&spa->spa_suspend_lock); mutex_destroy(&spa->spa_vdev_top_lock); + mutex_destroy(&spa->spa_iokstat_lock); kmem_free(spa, sizeof (spa_t)); } Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa_impl.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa_impl.h Mon Feb 18 09:07:59 2013 (r246939) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa_impl.h Mon Feb 18 11:48:08 2013 (r246940) @@ -231,6 +231,8 @@ struct spa { uint64_t spa_deadman_calls; /* number of deadman calls */ uint64_t spa_sync_starttime; /* starting time fo spa_sync */ uint64_t spa_deadman_synctime; /* deadman expiration timer */ + kmutex_t spa_iokstat_lock; /* protects spa_iokstat_* */ + struct kstat *spa_iokstat; /* kstat of io to this pool */ /* * spa_refcnt & spa_config_lock must be the last elements * because refcount_t changes size based on compilation options. Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/vdev_queue.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/vdev_queue.c Mon Feb 18 09:07:59 2013 (r246939) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/vdev_queue.c Mon Feb 18 11:48:08 2013 (r246940) @@ -29,6 +29,7 @@ #include #include +#include #include #include @@ -142,15 +143,62 @@ vdev_queue_fini(vdev_t *vd) static void vdev_queue_io_add(vdev_queue_t *vq, zio_t *zio) { + spa_t *spa = zio->io_spa; avl_add(&vq->vq_deadline_tree, zio); avl_add(zio->io_vdev_tree, zio); + + if (spa->spa_iokstat != NULL) { + mutex_enter(&spa->spa_iokstat_lock); + kstat_waitq_enter(spa->spa_iokstat->ks_data); + mutex_exit(&spa->spa_iokstat_lock); + } } static void vdev_queue_io_remove(vdev_queue_t *vq, zio_t *zio) { + spa_t *spa = zio->io_spa; avl_remove(&vq->vq_deadline_tree, zio); avl_remove(zio->io_vdev_tree, zio); + + if (spa->spa_iokstat != NULL) { + mutex_enter(&spa->spa_iokstat_lock); + kstat_waitq_exit(spa->spa_iokstat->ks_data); + mutex_exit(&spa->spa_iokstat_lock); + } +} + +static void +vdev_queue_pending_add(vdev_queue_t *vq, zio_t *zio) +{ + spa_t *spa = zio->io_spa; + avl_add(&vq->vq_pending_tree, zio); + if (spa->spa_iokstat != NULL) { + mutex_enter(&spa->spa_iokstat_lock); + kstat_runq_enter(spa->spa_iokstat->ks_data); + mutex_exit(&spa->spa_iokstat_lock); + } +} + +static void +vdev_queue_pending_remove(vdev_queue_t *vq, zio_t *zio) +{ + spa_t *spa = zio->io_spa; + avl_remove(&vq->vq_pending_tree, zio); + if (spa->spa_iokstat != NULL) { + kstat_io_t *ksio = spa->spa_iokstat->ks_data; + + mutex_enter(&spa->spa_iokstat_lock); + kstat_runq_exit(spa->spa_iokstat->ks_data); + if (zio->io_type == ZIO_TYPE_READ) { + ksio->reads++; + ksio->nread += zio->io_size; + } else if (zio->io_type == ZIO_TYPE_WRITE) { + ksio->writes++; + ksio->nwritten += zio->io_size; + } + mutex_exit(&spa->spa_iokstat_lock); + } } static void @@ -317,7 +365,7 @@ again: zio_execute(dio); } while (dio != lio); - avl_add(&vq->vq_pending_tree, aio); + vdev_queue_pending_add(vq, aio); return (aio); } @@ -339,7 +387,7 @@ again: goto again; } - avl_add(&vq->vq_pending_tree, fio); + vdev_queue_pending_add(vq, fio); return (fio); } @@ -395,7 +443,7 @@ vdev_queue_io_done(zio_t *zio) mutex_enter(&vq->vq_lock); - avl_remove(&vq->vq_pending_tree, zio); + vdev_queue_pending_remove(vq, zio); vq->vq_io_complete_ts = ddi_get_lbolt64(); vq->vq_io_delta_ts = vq->vq_io_complete_ts - zio->io_timestamp; From owner-svn-src-vendor@FreeBSD.ORG Mon Feb 18 11:48:10 2013 Return-Path: Delivered-To: svn-src-vendor@freebsd.org Received: from mx1.freebsd.org (mx1.FreeBSD.org [8.8.178.115]) by hub.freebsd.org (Postfix) with ESMTP id 4ECAF3FB; Mon, 18 Feb 2013 11:48:10 +0000 (UTC) (envelope-from mm@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) by mx1.freebsd.org (Postfix) with ESMTP id 285A827C; Mon, 18 Feb 2013 11:48:10 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.5/8.14.5) with ESMTP id r1IBmAlT045542; Mon, 18 Feb 2013 11:48:10 GMT (envelope-from mm@svn.freebsd.org) Received: (from mm@localhost) by svn.freebsd.org (8.14.5/8.14.5/Submit) id r1IBm93g045540; Mon, 18 Feb 2013 11:48:09 GMT (envelope-from mm@svn.freebsd.org) Message-Id: <201302181148.r1IBm93g045540@svn.freebsd.org> From: Martin Matuska Date: Mon, 18 Feb 2013 11:48:09 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r246940 - vendor-sys/illumos/dist/uts/common/fs/zfs vendor-sys/illumos/dist/uts/common/fs/zfs/sys vendor/illumos/dist/lib/libzpool/common vendor/illumos/dist/lib/libzpool/common/sys X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 18 Feb 2013 11:48:10 -0000 Author: mm Date: Mon Feb 18 11:48:08 2013 New Revision: 246940 URL: http://svnweb.freebsd.org/changeset/base/246940 Log: Update vendor/illumos/dist and vendor-sys/illumos/dist to illumos-gate 13957:512faafc0eaf Illumos ZFS issues: 3537 want pool io kstats Modified: vendor/illumos/dist/lib/libzpool/common/kernel.c vendor/illumos/dist/lib/libzpool/common/sys/zfs_context.h Changes in other areas also in this revision: Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/spa_misc.c vendor-sys/illumos/dist/uts/common/fs/zfs/sys/spa_impl.h vendor-sys/illumos/dist/uts/common/fs/zfs/vdev_queue.c Modified: vendor/illumos/dist/lib/libzpool/common/kernel.c ============================================================================== --- vendor/illumos/dist/lib/libzpool/common/kernel.c Mon Feb 18 09:07:59 2013 (r246939) +++ vendor/illumos/dist/lib/libzpool/common/kernel.c Mon Feb 18 11:48:08 2013 (r246940) @@ -80,8 +80,8 @@ zk_thread_create(void (*func)(), void *a */ /*ARGSUSED*/ kstat_t * -kstat_create(char *module, int instance, char *name, char *class, - uchar_t type, ulong_t ndata, uchar_t ks_flag) +kstat_create(const char *module, int instance, const char *name, + const char *class, uchar_t type, ulong_t ndata, uchar_t ks_flag) { return (NULL); } @@ -96,6 +96,36 @@ void kstat_delete(kstat_t *ksp) {} +/*ARGSUSED*/ +void +kstat_waitq_enter(kstat_io_t *kiop) +{} + +/*ARGSUSED*/ +void +kstat_waitq_exit(kstat_io_t *kiop) +{} + +/*ARGSUSED*/ +void +kstat_runq_enter(kstat_io_t *kiop) +{} + +/*ARGSUSED*/ +void +kstat_runq_exit(kstat_io_t *kiop) +{} + +/*ARGSUSED*/ +void +kstat_waitq_to_runq(kstat_io_t *kiop) +{} + +/*ARGSUSED*/ +void +kstat_runq_back_to_waitq(kstat_io_t *kiop) +{} + /* * ========================================================================= * mutexes Modified: vendor/illumos/dist/lib/libzpool/common/sys/zfs_context.h ============================================================================== --- vendor/illumos/dist/lib/libzpool/common/sys/zfs_context.h Mon Feb 18 09:07:59 2013 (r246939) +++ vendor/illumos/dist/lib/libzpool/common/sys/zfs_context.h Mon Feb 18 11:48:08 2013 (r246940) @@ -255,10 +255,16 @@ extern void cv_broadcast(kcondvar_t *cv) /* * kstat creation, installation and deletion */ -extern kstat_t *kstat_create(char *, int, - char *, char *, uchar_t, ulong_t, uchar_t); +extern kstat_t *kstat_create(const char *, int, + const char *, const char *, uchar_t, ulong_t, uchar_t); extern void kstat_install(kstat_t *); extern void kstat_delete(kstat_t *); +extern void kstat_waitq_enter(kstat_io_t *); +extern void kstat_waitq_exit(kstat_io_t *); +extern void kstat_runq_enter(kstat_io_t *); +extern void kstat_runq_exit(kstat_io_t *); +extern void kstat_waitq_to_runq(kstat_io_t *); +extern void kstat_runq_back_to_waitq(kstat_io_t *); /* * Kernel memory From owner-svn-src-vendor@FreeBSD.ORG Mon Feb 18 19:23:42 2013 Return-Path: Delivered-To: svn-src-vendor@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by hub.freebsd.org (Postfix) with ESMTP id 8F8A37D2; Mon, 18 Feb 2013 19:23:42 +0000 (UTC) (envelope-from gshapiro@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) by mx1.freebsd.org (Postfix) with ESMTP id 7763110B; Mon, 18 Feb 2013 19:23:42 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.5/8.14.5) with ESMTP id r1IJNgG8086021; Mon, 18 Feb 2013 19:23:42 GMT (envelope-from gshapiro@svn.freebsd.org) Received: (from gshapiro@localhost) by svn.freebsd.org (8.14.5/8.14.5/Submit) id r1IJNgdQ086020; Mon, 18 Feb 2013 19:23:42 GMT (envelope-from gshapiro@svn.freebsd.org) Message-Id: <201302181923.r1IJNgdQ086020@svn.freebsd.org> From: Gregory Neil Shapiro Date: Mon, 18 Feb 2013 19:23:42 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r246946 - vendor/sendmail/dist/src X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 18 Feb 2013 19:23:42 -0000 Author: gshapiro Date: Mon Feb 18 19:23:41 2013 New Revision: 246946 URL: http://svnweb.freebsd.org/changeset/base/246946 Log: Fix regression in sendmail 8.14.6 release based on errata published at: http://www.sendmail.com/sm/open_source/download/8.14.6/ Modified: vendor/sendmail/dist/src/milter.c Modified: vendor/sendmail/dist/src/milter.c ============================================================================== --- vendor/sendmail/dist/src/milter.c Mon Feb 18 18:39:15 2013 (r246945) +++ vendor/sendmail/dist/src/milter.c Mon Feb 18 19:23:41 2013 (r246946) @@ -4048,7 +4048,7 @@ milter_helo(helo, e, state) } response = milter_command(SMFIC_HELO, helo, strlen(helo) + 1, - SMFIM_EOH, e, state, "helo", false); + SMFIM_HELO, e, state, "helo", false); milter_per_connection_check(e); return response; } From owner-svn-src-vendor@FreeBSD.ORG Thu Feb 21 17:16:06 2013 Return-Path: Delivered-To: svn-src-vendor@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by hub.freebsd.org (Postfix) with ESMTP id 06F88B50; Thu, 21 Feb 2013 17:16:06 +0000 (UTC) (envelope-from brooks@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) by mx1.freebsd.org (Postfix) with ESMTP id DE8A082E; Thu, 21 Feb 2013 17:16:05 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.5/8.14.5) with ESMTP id r1LHG5uf000610; Thu, 21 Feb 2013 17:16:05 GMT (envelope-from brooks@svn.freebsd.org) Received: (from brooks@localhost) by svn.freebsd.org (8.14.5/8.14.5/Submit) id r1LHG5fT000602; Thu, 21 Feb 2013 17:16:05 GMT (envelope-from brooks@svn.freebsd.org) Message-Id: <201302211716.r1LHG5fT000602@svn.freebsd.org> From: Brooks Davis Date: Thu, 21 Feb 2013 17:16:05 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r247106 - vendor/NetBSD/libc-vis/dist X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 21 Feb 2013 17:16:06 -0000 Author: brooks Date: Thu Feb 21 17:16:04 2013 New Revision: 247106 URL: http://svnweb.freebsd.org/changeset/base/247106 Log: Vendor import of NetBSD's (un)vis(3) at 2013-02-21 Modified: vendor/NetBSD/libc-vis/dist/unvis.3 vendor/NetBSD/libc-vis/dist/unvis.c vendor/NetBSD/libc-vis/dist/vis.3 vendor/NetBSD/libc-vis/dist/vis.c vendor/NetBSD/libc-vis/dist/vis.h Modified: vendor/NetBSD/libc-vis/dist/unvis.3 ============================================================================== --- vendor/NetBSD/libc-vis/dist/unvis.3 Thu Feb 21 17:00:37 2013 (r247105) +++ vendor/NetBSD/libc-vis/dist/unvis.3 Thu Feb 21 17:16:04 2013 (r247106) @@ -1,4 +1,4 @@ -.\" $NetBSD: unvis.3,v 1.23 2011/03/17 14:06:29 wiz Exp $ +.\" $NetBSD: unvis.3,v 1.27 2012/12/15 07:34:36 wiz Exp $ .\" .\" Copyright (c) 1989, 1991, 1993 .\" The Regents of the University of California. All rights reserved. @@ -125,15 +125,17 @@ The function has several return codes that must be handled properly. They are: .Bl -tag -width UNVIS_VALIDPUSH -.It Li \&0 (zero) +.It Li \&0 No (zero) Another character is necessary; nothing has been recognized yet. .It Dv UNVIS_VALID A valid character has been recognized and is available at the location -pointed to by cp. +pointed to by +.Fa cp . .It Dv UNVIS_VALIDPUSH A valid character has been recognized and is available at the location -pointed to by cp; however, the character currently passed in should -be passed in again. +pointed to by +.Fa cp ; +however, the character currently passed in should be passed in again. .It Dv UNVIS_NOCHAR A valid sequence was detected, but no character was produced. This return code is necessary to indicate a logical break between characters. @@ -149,7 +151,7 @@ one more time with flag set to to extract any remaining character (the character passed in is ignored). .Pp The -.Ar flag +.Fa flag argument is also used to specify the encoding style of the source. If set to .Dv VIS_HTTPSTYLE @@ -160,7 +162,8 @@ will decode URI strings as specified in If set to .Dv VIS_HTTP1866 , .Fn unvis -will decode URI strings as specified in RFC 1866. +will decode entity references and numeric character references +as specified in RFC 1866. If set to .Dv VIS_MIMESTYLE , .Fn unvis @@ -168,7 +171,9 @@ will decode MIME Quoted-Printable string If set to .Dv VIS_NOESCAPE , .Fn unvis -will not decode \e quoted characters. +will not decode +.Ql \e +quoted characters. .Pp The following code fragment illustrates a proper use of .Fn unvis . @@ -203,7 +208,7 @@ The functions and .Fn strnunvisx will return \-1 on error and set -.Va errno +.Va errno to: .Bl -tag -width Er .It Bq Er EINVAL @@ -211,7 +216,7 @@ An invalid escape sequence was detected, .El .Pp In addition the functions -.Fn strnunvis +.Fn strnunvis and .Fn strnunvisx will can also set @@ -242,3 +247,13 @@ and .Fn strnunvisx functions appeared in .Nx 6.0 . +.Sh BUGS +The names +.Dv VIS_HTTP1808 +and +.Dv VIS_HTTP1866 +are wrong. +Percent-encoding was defined in RFC 1738, the original RFC for URL. +RFC 1866 defines HTML 2.0, an application of SGML, from which it +inherits concepts of numeric character references and entity +references. Modified: vendor/NetBSD/libc-vis/dist/unvis.c ============================================================================== --- vendor/NetBSD/libc-vis/dist/unvis.c Thu Feb 21 17:00:37 2013 (r247105) +++ vendor/NetBSD/libc-vis/dist/unvis.c Thu Feb 21 17:16:04 2013 (r247106) @@ -1,4 +1,4 @@ -/* $NetBSD: unvis.c,v 1.40 2012/12/14 21:31:01 christos Exp $ */ +/* $NetBSD: unvis.c,v 1.41 2012/12/15 04:29:53 matt Exp $ */ /*- * Copyright (c) 1989, 1993 @@ -34,7 +34,7 @@ #if 0 static char sccsid[] = "@(#)unvis.c 8.1 (Berkeley) 6/4/93"; #else -__RCSID("$NetBSD: unvis.c,v 1.40 2012/12/14 21:31:01 christos Exp $"); +__RCSID("$NetBSD: unvis.c,v 1.41 2012/12/15 04:29:53 matt Exp $"); #endif #endif /* LIBC_SCCS and not lint */ @@ -81,7 +81,7 @@ __weak_alias(strnunvisx,_strnunvisx) * RFC 1866 */ static const struct nv { - const char name[7]; + char name[7]; uint8_t value; } nv[] = { { "AElig", 198 }, /* capital AE diphthong (ligature) */ Modified: vendor/NetBSD/libc-vis/dist/vis.3 ============================================================================== --- vendor/NetBSD/libc-vis/dist/vis.3 Thu Feb 21 17:00:37 2013 (r247105) +++ vendor/NetBSD/libc-vis/dist/vis.3 Thu Feb 21 17:16:04 2013 (r247106) @@ -1,4 +1,4 @@ -.\" $NetBSD: vis.3,v 1.29 2012/12/14 22:55:59 christos Exp $ +.\" $NetBSD: vis.3,v 1.39 2013/02/20 20:05:26 christos Exp $ .\" .\" Copyright (c) 1989, 1991, 1993 .\" The Regents of the University of California. All rights reserved. @@ -29,7 +29,7 @@ .\" .\" @(#)vis.3 8.1 (Berkeley) 6/9/93 .\" -.Dd December 14, 2012 +.Dd February 19, 2013 .Dt VIS 3 .Os .Sh NAME @@ -39,12 +39,14 @@ .Nm strnvis , .Nm strvisx , .Nm strnvisx , +.Nm strenvisx , .Nm svis , .Nm snvis , .Nm strsvis , .Nm strsnvis , -.Nm strsvisx -.Nm strsnvisx +.Nm strsvisx , +.Nm strsnvisx , +.Nm strsenvisx .Nd visually encode characters .Sh LIBRARY .Lb libc @@ -62,6 +64,8 @@ .Fn strvisx "char *dst" "const char *src" "size_t len" "int flag" .Ft int .Fn strnvisx "char *dst" "size_t dlen" "const char *src" "size_t len" "int flag" +.Ft int +.Fn strenvisx "char *dst" "size_t dlen" "const char *src" "size_t len" "int flag" "int *cerr_ptr" .Ft char * .Fn svis "char *dst" "int c" "int flag" "int nextc" "const char *extra" .Ft char * @@ -74,6 +78,8 @@ .Fn strsvisx "char *dst" "const char *src" "size_t len" "int flag" "const char *extra" .Ft int .Fn strsnvisx "char *dst" "size_t dlen" "const char *src" "size_t len" "int flag" "const char *extra" +.Ft int +.Fn strsenvisx "char *dst" "size_t dlen" "const char *src" "size_t len" "int flag" "const char *extra" "int *cerr_ptr" .Sh DESCRIPTION The .Fn vis @@ -88,11 +94,11 @@ needs no encoding, it is copied in unalt The string is null terminated, and a pointer to the end of the string is returned. The maximum length of any encoding is four -characters (not including the trailing +bytes (not including the trailing .Dv NUL ) ; thus, when encoding a set of characters into a buffer, the size of the buffer should -be four times the number of characters encoded, plus one for the trailing +be four times the number of bytes encoded, plus one for the trailing .Dv NUL . The flag parameter is used for altering the default range of characters considered for encoding and for altering the visual @@ -141,16 +147,17 @@ terminate The size of .Fa dst must be four times the number -of characters encoded from +of bytes encoded from .Fa src (plus one for the .Dv NUL ) . Both -forms return the number of characters in dst (not including -the trailing +forms return the number of characters in +.Fa dst +(not including the trailing .Dv NUL ) . The -.Dq n +.Dq Nm n versions of the functions also take an additional argument .Fa dlen that indicates the length of the @@ -158,7 +165,7 @@ that indicates the length of the buffer. If .Fa dlen -is not large enough to fix the converted string then the +is not large enough to fit the converted string then the .Fn strnvis and .Fn strnvisx @@ -166,6 +173,14 @@ functions return \-1 and set .Va errno to .Dv ENOSPC . +The +.Fn strenvisx +function takes an additional argument, +.Fa cerr_ptr , +that is used to pass in and out a multibyte conversion error flag. +This is useful when processing single characters at a time when +it is possible that the locale may be set to something other +than the locale of the characters in the input data. .Pp The functions .Fn svis , @@ -173,16 +188,18 @@ The functions .Fn strsvis , .Fn strsnvis , .Fn strsvisx , +.Fn strsnvisx , and -.Fn strsnvisx +.Fn strsenvisx correspond to .Fn vis , .Fn nvis , .Fn strvis , .Fn strnvis , .Fn strvisx , +.Fn strnvisx , and -.Fn strnvisx +.Fn strenvisx but have an additional argument .Fa extra , pointing to a @@ -213,14 +230,13 @@ and .Fn strnvisx ) , and the type of representation used. By default, all non-graphic characters, -except space, tab, and newline are encoded. -(See -.Xr isgraph 3 . ) +except space, tab, and newline are encoded (see +.Xr isgraph 3 ) . The following flags alter this: .Bl -tag -width VIS_WHITEX .It Dv VIS_GLOB -Also encode magic characters +Also encode the magic characters .Ql ( * , .Ql \&? , .Ql \&[ @@ -242,11 +258,13 @@ Synonym for \&| .Dv VIS_NL . .It Dv VIS_SAFE -Only encode "unsafe" characters. +Only encode +.Dq unsafe +characters. Unsafe means control characters which may cause common terminals to perform unexpected functions. Currently this form allows space, tab, newline, backspace, bell, and -return - in addition to all graphic characters - unencoded. +return \(em in addition to all graphic characters \(em unencoded. .El .Pp (The above flags have no effect for @@ -286,8 +304,8 @@ Use an to represent meta characters (characters with the 8th bit set), and use caret .Ql ^ -to represent control characters see -.Pf ( Xr iscntrl 3 ) . +to represent control characters (see +.Xr iscntrl 3 ) . The following formats are used: .Bl -tag -width xxxxx .It Dv \e^C @@ -334,19 +352,20 @@ Use C-style backslash sequences to repre characters. The following sequences are used to represent the indicated characters: .Bd -unfilled -offset indent -.Li \ea Tn - BEL No (007) -.Li \eb Tn - BS No (010) -.Li \ef Tn - NP No (014) -.Li \en Tn - NL No (012) -.Li \er Tn - CR No (015) -.Li \es Tn - SP No (040) -.Li \et Tn - HT No (011) -.Li \ev Tn - VT No (013) -.Li \e0 Tn - NUL No (000) +.Li \ea Tn \(em BEL No (007) +.Li \eb Tn \(em BS No (010) +.Li \ef Tn \(em NP No (014) +.Li \en Tn \(em NL No (012) +.Li \er Tn \(em CR No (015) +.Li \es Tn \(em SP No (040) +.Li \et Tn \(em HT No (011) +.Li \ev Tn \(em VT No (013) +.Li \e0 Tn \(em NUL No (000) .Ed .Pp -When using this format, the nextc parameter is looked at to determine -if a +When using this format, the +.Fa nextc +parameter is looked at to determine if a .Dv NUL character can be encoded as .Ql \e0 @@ -373,8 +392,8 @@ represents a lower case hexadecimal digi .It Dv VIS_MIMESTYLE Use MIME Quoted-Printable encoding as described in RFC 2045, only don't break lines and don't handle CRLF. -The form is: -.Ql %XX +The form is +.Ql =XX where .Em X represents an upper case hexadecimal digit. @@ -391,6 +410,41 @@ meta characters as .Ql M-C ) . With this flag set, the encoding is ambiguous and non-invertible. +.Sh MULTIBYTE CHARACTER SUPPORT +These functions support multibyte character input. +The encoding conversion is influenced by the setting of the +.Ev LC_CTYPE +environment variable which defines the set of characters +that can be copied without encoding. +.Pp +When 8-bit data is present in the input, +.Ev LC_CTYPE +must be set to the correct locale or to the C locale. +If the locales of the data and the conversion are mismatched, +multibyte character recognition may fail and encoding will be performed +byte-by-byte instead. +.Pp +As noted above, +.Fa dst +must be four times the number of bytes processed from +.Fa src . +But note that each multibyte character can be up to +.Dv MB_LEN_MAX +bytes +.\" (see +.\" .Xr multibyte 3 ) +so in terms of multibyte characters, +.Fa dst +must be four times +.Dv MB_LEN_MAX +times the number of characters processed from +.Fa src . +.Sh ENVIRONMENT +.Bl -tag -width ".Ev LC_CTYPE" +.It Ev LC_CTYPE +Specify the locale of the input data. +Set to C if the input data locale is unknown. +.El .Sh ERRORS The functions .Fn nvis @@ -406,11 +460,11 @@ and .Fn strsnvisx , will return \-1 when the .Fa dlen -destination buffer length size is not enough to perform the conversion while +destination buffer size is not enough to perform the conversion while setting .Va errno to: -.Bl -tag -width Er +.Bl -tag -width ".Bq Er ENOSPC" .It Bq Er ENOSPC The destination buffer size is not large enough to perform the conversion. .El @@ -418,18 +472,23 @@ The destination buffer size is not large .Xr unvis 1 , .Xr vis 1 , .Xr glob 3 , +.\" .Xr multibyte 3 , .Xr unvis 3 .Rs .%A T. Berners-Lee .%T Uniform Resource Locators (URL) -.%O RFC1738 +.%O "RFC 1738" +.Re +.Rs +.%T "Multipurpose Internet Mail Extensions (MIME) Part One: Format of Internet Message Bodies" +.%O "RFC 2045" .Re .Sh HISTORY The .Fn vis , .Fn strvis , and -.Fa strvisx +.Fn strvisx functions first appeared in .Bx 4.4 . The @@ -448,4 +507,10 @@ The buffer size limited versions of the and .Fn strsnvisx Pc appeared in -.Nx 6.0 . +.Nx 6.0 +and +.Fx 9.2 . +Myltibyte character support was added in +.Nx 7.0 +and +.Fx 9.2 . Modified: vendor/NetBSD/libc-vis/dist/vis.c ============================================================================== --- vendor/NetBSD/libc-vis/dist/vis.c Thu Feb 21 17:00:37 2013 (r247105) +++ vendor/NetBSD/libc-vis/dist/vis.c Thu Feb 21 17:16:04 2013 (r247106) @@ -1,4 +1,4 @@ -/* $NetBSD: vis.c,v 1.45 2012/12/14 21:38:18 christos Exp $ */ +/* $NetBSD: vis.c,v 1.60 2013/02/21 16:21:20 joerg Exp $ */ /*- * Copyright (c) 1989, 1993 @@ -57,16 +57,23 @@ #include #if defined(LIBC_SCCS) && !defined(lint) -__RCSID("$NetBSD: vis.c,v 1.45 2012/12/14 21:38:18 christos Exp $"); +__RCSID("$NetBSD: vis.c,v 1.60 2013/02/21 16:21:20 joerg Exp $"); #endif /* LIBC_SCCS and not lint */ +#ifdef __FBSDID +__FBSDID("$FreeBSD$"); +#define _DIAGASSERT(x) assert(x) +#endif #include "namespace.h" #include +#include #include #include #include #include +#include +#include #ifdef __weak_alias __weak_alias(strvisx,_strvisx) @@ -78,65 +85,66 @@ __weak_alias(strvisx,_strvisx) #include #include -static char *do_svis(char *, size_t *, int, int, int, const char *); +/* + * The reason for going through the trouble to deal with character encodings + * in vis(3), is that we use this to safe encode output of commands. This + * safe encoding varies depending on the character set. For example if we + * display ps output in French, we don't want to display French characters + * as M-foo. + */ + +static wchar_t *do_svis(wchar_t *, wint_t, int, wint_t, const wchar_t *); #undef BELL -#define BELL '\a' +#define BELL L'\a' + +#define iswoctal(c) (((u_char)(c)) >= L'0' && ((u_char)(c)) <= L'7') +#define iswwhite(c) (c == L' ' || c == L'\t' || c == L'\n') +#define iswsafe(c) (c == L'\b' || c == BELL || c == L'\r') +#define xtoa(c) L"0123456789abcdef"[c] +#define XTOA(c) L"0123456789ABCDEF"[c] + +#define MAXEXTRAS 10 -#define isoctal(c) (((u_char)(c)) >= '0' && ((u_char)(c)) <= '7') -#define iswhite(c) (c == ' ' || c == '\t' || c == '\n') -#define issafe(c) (c == '\b' || c == BELL || c == '\r') -#define xtoa(c) "0123456789abcdef"[c] -#define XTOA(c) "0123456789ABCDEF"[c] - -#define MAXEXTRAS 9 - -#define MAKEEXTRALIST(flag, extra, orig_str) \ -do { \ - const char *orig = orig_str; \ - const char *o = orig; \ - char *e; \ - while (*o++) \ - continue; \ - extra = malloc((size_t)((o - orig) + MAXEXTRAS)); \ - if (!extra) break; \ - for (o = orig, e = extra; (*e++ = *o++) != '\0';) \ - continue; \ - e--; \ - if (flag & VIS_GLOB) { \ - *e++ = '*'; \ - *e++ = '?'; \ - *e++ = '['; \ - *e++ = '#'; \ - } \ - if (flag & VIS_SP) *e++ = ' '; \ - if (flag & VIS_TAB) *e++ = '\t'; \ - if (flag & VIS_NL) *e++ = '\n'; \ - if ((flag & VIS_NOSLASH) == 0) *e++ = '\\'; \ - *e = '\0'; \ -} while (/*CONSTCOND*/0) +#if !HAVE_NBTOOL_CONFIG_H +#ifndef __NetBSD__ +/* + * On NetBSD MB_LEN_MAX is currently 32 which does not fit on any integer + * integral type and it is probably wrong, since currently the maximum + * number of bytes and character needs is 6. Until this is fixed, the + * loops below are using sizeof(uint64_t) - 1 instead of MB_LEN_MAX, and + * the assertion is commented out. + */ +#ifdef __FreeBSD__ +/* + * On FreeBSD including for CTASSERT only works in kernel + * mode. + */ +#ifndef CTASSERT +#define CTASSERT(x) _CTASSERT(x, __LINE__) +#define _CTASSERT(x, y) __CTASSERT(x, y) +#define __CTASSERT(x, y) typedef char __assert ## y[(x) ? 1 : -1] +#endif +#endif /* __FreeBSD__ */ +CTASSERT(MB_LEN_MAX <= sizeof(uint64_t)); +#endif /* !__NetBSD__ */ +#endif /* * This is do_hvis, for HTTP style (RFC 1808) */ -static char * -do_hvis(char *dst, size_t *dlen, int c, int flag, int nextc, const char *extra) +static wchar_t * +do_hvis(wchar_t *dst, wint_t c, int flags, wint_t nextc, const wchar_t *extra) { - - if ((isascii(c) && isalnum(c)) + if (iswalnum(c) /* safe */ - || c == '$' || c == '-' || c == '_' || c == '.' || c == '+' + || c == L'$' || c == L'-' || c == L'_' || c == L'.' || c == L'+' /* extra */ - || c == '!' || c == '*' || c == '\'' || c == '(' || c == ')' - || c == ',') { - dst = do_svis(dst, dlen, c, flag, nextc, extra); - } else { - if (dlen) { - if (*dlen < 3) - return NULL; - *dlen -= 3; - } - *dst++ = '%'; + || c == L'!' || c == L'*' || c == L'\'' || c == L'(' || c == L')' + || c == L',') + dst = do_svis(dst, c, flags, nextc, extra); + else { + *dst++ = L'%'; *dst++ = xtoa(((unsigned int)c >> 4) & 0xf); *dst++ = xtoa((unsigned int)c & 0xf); } @@ -148,312 +156,448 @@ do_hvis(char *dst, size_t *dlen, int c, * This is do_mvis, for Quoted-Printable MIME (RFC 2045) * NB: No handling of long lines or CRLF. */ -static char * -do_mvis(char *dst, size_t *dlen, int c, int flag, int nextc, const char *extra) +static wchar_t * +do_mvis(wchar_t *dst, wint_t c, int flags, wint_t nextc, const wchar_t *extra) { - if ((c != '\n') && + if ((c != L'\n') && /* Space at the end of the line */ - ((isspace(c) && (nextc == '\r' || nextc == '\n')) || + ((iswspace(c) && (nextc == L'\r' || nextc == L'\n')) || /* Out of range */ - (!isspace(c) && (c < 33 || (c > 60 && c < 62) || c > 126)) || - /* Specific char to be escaped */ - strchr("#$@[\\]^`{|}~", c) != NULL)) { - if (dlen) { - if (*dlen < 3) - return NULL; - *dlen -= 3; - } - *dst++ = '='; + (!iswspace(c) && (c < 33 || (c > 60 && c < 62) || c > 126)) || + /* Specific char to be escaped */ + wcschr(L"#$@[\\]^`{|}~", c) != NULL)) { + *dst++ = L'='; *dst++ = XTOA(((unsigned int)c >> 4) & 0xf); *dst++ = XTOA((unsigned int)c & 0xf); - } else { - dst = do_svis(dst, dlen, c, flag, nextc, extra); - } + } else + dst = do_svis(dst, c, flags, nextc, extra); return dst; } /* - * This is do_vis, the central code of vis. - * dst: Pointer to the destination buffer - * c: Character to encode - * flag: Flag word - * nextc: The character following 'c' - * extra: Pointer to the list of extra characters to be - * backslash-protected. + * Output single byte of multibyte character. */ -static char * -do_svis(char *dst, size_t *dlen, int c, int flag, int nextc, const char *extra) +static wchar_t * +do_mbyte(wchar_t *dst, wint_t c, int flags, wint_t nextc, int iswextra) { - int isextra; - size_t odlen = dlen ? *dlen : 0; - - isextra = strchr(extra, c) != NULL; -#define HAVE(x) \ - do { \ - if (dlen) { \ - if (*dlen < (x)) \ - goto out; \ - *dlen -= (x); \ - } \ - } while (/*CONSTCOND*/0) - if (!isextra && isascii(c) && (isgraph(c) || iswhite(c) || - ((flag & VIS_SAFE) && issafe(c)))) { - HAVE(1); - *dst++ = c; - return dst; - } - if (flag & VIS_CSTYLE) { - HAVE(2); + if (flags & VIS_CSTYLE) { switch (c) { - case '\n': - *dst++ = '\\'; *dst++ = 'n'; + case L'\n': + *dst++ = L'\\'; *dst++ = L'n'; return dst; - case '\r': - *dst++ = '\\'; *dst++ = 'r'; + case L'\r': + *dst++ = L'\\'; *dst++ = L'r'; return dst; - case '\b': - *dst++ = '\\'; *dst++ = 'b'; + case L'\b': + *dst++ = L'\\'; *dst++ = L'b'; return dst; case BELL: - *dst++ = '\\'; *dst++ = 'a'; + *dst++ = L'\\'; *dst++ = L'a'; return dst; - case '\v': - *dst++ = '\\'; *dst++ = 'v'; + case L'\v': + *dst++ = L'\\'; *dst++ = L'v'; return dst; - case '\t': - *dst++ = '\\'; *dst++ = 't'; + case L'\t': + *dst++ = L'\\'; *dst++ = L't'; return dst; - case '\f': - *dst++ = '\\'; *dst++ = 'f'; + case L'\f': + *dst++ = L'\\'; *dst++ = L'f'; return dst; - case ' ': - *dst++ = '\\'; *dst++ = 's'; + case L' ': + *dst++ = L'\\'; *dst++ = L's'; return dst; - case '\0': - *dst++ = '\\'; *dst++ = '0'; - if (isoctal(nextc)) { - HAVE(2); - *dst++ = '0'; - *dst++ = '0'; + case L'\0': + *dst++ = L'\\'; *dst++ = L'0'; + if (iswoctal(nextc)) { + *dst++ = L'0'; + *dst++ = L'0'; } return dst; default: - if (isgraph(c)) { - *dst++ = '\\'; *dst++ = c; + if (iswgraph(c)) { + *dst++ = L'\\'; + *dst++ = c; return dst; } - if (dlen) - *dlen = odlen; } } - if (isextra || ((c & 0177) == ' ') || (flag & VIS_OCTAL)) { - HAVE(4); - *dst++ = '\\'; - *dst++ = (u_char)(((u_int32_t)(u_char)c >> 6) & 03) + '0'; - *dst++ = (u_char)(((u_int32_t)(u_char)c >> 3) & 07) + '0'; - *dst++ = (c & 07) + '0'; + if (iswextra || ((c & 0177) == L' ') || (flags & VIS_OCTAL)) { + *dst++ = L'\\'; + *dst++ = (u_char)(((u_int32_t)(u_char)c >> 6) & 03) + L'0'; + *dst++ = (u_char)(((u_int32_t)(u_char)c >> 3) & 07) + L'0'; + *dst++ = (c & 07) + L'0'; } else { - if ((flag & VIS_NOSLASH) == 0) { - HAVE(1); - *dst++ = '\\'; - } + if ((flags & VIS_NOSLASH) == 0) + *dst++ = L'\\'; if (c & 0200) { - HAVE(1); - c &= 0177; *dst++ = 'M'; + c &= 0177; + *dst++ = L'M'; } - if (iscntrl(c)) { - HAVE(2); - *dst++ = '^'; + if (iswcntrl(c)) { + *dst++ = L'^'; if (c == 0177) - *dst++ = '?'; + *dst++ = L'?'; else - *dst++ = c + '@'; + *dst++ = c + L'@'; } else { - HAVE(2); - *dst++ = '-'; *dst++ = c; + *dst++ = L'-'; + *dst++ = c; } } + + return dst; +} + +/* + * This is do_vis, the central code of vis. + * dst: Pointer to the destination buffer + * c: Character to encode + * flags: Flags word + * nextc: The character following 'c' + * extra: Pointer to the list of extra characters to be + * backslash-protected. + */ +static wchar_t * +do_svis(wchar_t *dst, wint_t c, int flags, wint_t nextc, const wchar_t *extra) +{ + int iswextra, i, shft; + uint64_t bmsk, wmsk; + + iswextra = wcschr(extra, c) != NULL; + if (!iswextra && (iswgraph(c) || iswwhite(c) || + ((flags & VIS_SAFE) && iswsafe(c)))) { + *dst++ = c; + return dst; + } + + /* See comment in istrsenvisx() output loop, below. */ + wmsk = 0; + for (i = sizeof(wmsk) - 1; i >= 0; i--) { + shft = i * NBBY; + bmsk = (uint64_t)0xffLL << shft; + wmsk |= bmsk; + if ((c & wmsk) || i == 0) + dst = do_mbyte(dst, (wint_t)( + (uint64_t)(c & bmsk) >> shft), + flags, nextc, iswextra); + } + return dst; -out: - *dlen = odlen; - return NULL; } -typedef char *(*visfun_t)(char *, size_t *, int, int, int, const char *); +typedef wchar_t *(*visfun_t)(wchar_t *, wint_t, int, wint_t, const wchar_t *); /* * Return the appropriate encoding function depending on the flags given. */ static visfun_t -getvisfun(int flag) +getvisfun(int flags) { - if (flag & VIS_HTTPSTYLE) + if (flags & VIS_HTTPSTYLE) return do_hvis; - if (flag & VIS_MIMESTYLE) + if (flags & VIS_MIMESTYLE) return do_mvis; return do_svis; } /* - * isnvis - visually encode characters, also encoding the characters - * pointed to by `extra' + * Expand list of extra characters to not visually encode. */ -static char * -isnvis(char *dst, size_t *dlen, int c, int flag, int nextc, const char *extra) +static wchar_t * +makeextralist(int flags, const char *src) { - char *nextra = NULL; - visfun_t f; + wchar_t *dst, *d; + size_t len; - _DIAGASSERT(dst != NULL); - _DIAGASSERT(extra != NULL); - MAKEEXTRALIST(flag, nextra, extra); - if (!nextra) { - if (dlen && *dlen == 0) { - errno = ENOSPC; - return NULL; - } - *dst = '\0'; /* can't create nextra, return "" */ - return dst; - } - f = getvisfun(flag); - dst = (*f)(dst, dlen, c, flag, nextc, nextra); - free(nextra); - if (dst == NULL || (dlen && *dlen == 0)) { - errno = ENOSPC; + len = strlen(src); + if ((dst = calloc(len + MAXEXTRAS, sizeof(*dst))) == NULL) return NULL; - } - *dst = '\0'; - return dst; -} -char * -svis(char *dst, int c, int flag, int nextc, const char *extra) -{ - return isnvis(dst, NULL, c, flag, nextc, extra); -} + if (mbstowcs(dst, src, len) == (size_t)-1) { + size_t i; + for (i = 0; i < len; i++) + dst[i] = (wint_t)(u_char)src[i]; + d = dst + len; + } else + d = dst + wcslen(dst); + + if (flags & VIS_GLOB) { + *d++ = L'*'; + *d++ = L'?'; + *d++ = L'['; + *d++ = L'#'; + } + + if (flags & VIS_SP) *d++ = L' '; + if (flags & VIS_TAB) *d++ = L'\t'; + if (flags & VIS_NL) *d++ = L'\n'; + if ((flags & VIS_NOSLASH) == 0) *d++ = L'\\'; + *d = L'\0'; -char * -snvis(char *dst, size_t dlen, int c, int flag, int nextc, const char *extra) -{ - return isnvis(dst, &dlen, c, flag, nextc, extra); + return dst; } - /* - * strsvis, strsvisx - visually encode characters from src into dst - * - * Extra is a pointer to a \0-terminated list of characters to - * be encoded, too. These functions are useful e. g. to - * encode strings in such a way so that they are not interpreted - * by a shell. - * - * Dst must be 4 times the size of src to account for possible - * expansion. The length of dst, not including the trailing NULL, - * is returned. - * - * Strsvisx encodes exactly len bytes from src into dst. - * This is useful for encoding a block of data. + * istrsenvisx() + * The main internal function. + * All user-visible functions call this one. */ static int -istrsnvis(char *dst, size_t *dlen, const char *csrc, int flag, const char *extra) +istrsenvisx(char *mbdst, size_t *dlen, const char *mbsrc, size_t mblength, + int flags, const char *mbextra, int *cerr_ptr) { - int c; - char *start; - char *nextra = NULL; - const unsigned char *src = (const unsigned char *)csrc; + wchar_t *dst, *src, *pdst, *psrc, *start, *extra; + size_t len, olen; + uint64_t bmsk, wmsk; + wint_t c; visfun_t f; + int clen = 0, cerr = 0, error = -1, i, shft; + ssize_t mbslength, maxolen; + + _DIAGASSERT(mbdst != NULL); + _DIAGASSERT(mbsrc != NULL); + _DIAGASSERT(mbextra != NULL); + + /* + * Input (mbsrc) is a char string considered to be multibyte + * characters. The input loop will read this string pulling + * one character, possibly multiple bytes, from mbsrc and + * converting each to wchar_t in src. + * + * The vis conversion will be done using the wide char + * wchar_t string. + * + * This will then be converted back to a multibyte string to + * return to the caller. + */ + + /* Allocate space for the wide char strings */ + psrc = pdst = extra = NULL; + if (!mblength) + mblength = strlen(mbsrc); + if ((psrc = calloc(mblength + 1, sizeof(*psrc))) == NULL) + return -1; + if ((pdst = calloc((4 * mblength) + 1, sizeof(*pdst))) == NULL) + goto out; + dst = pdst; + src = psrc; + + /* Use caller's multibyte conversion error flag. */ + if (cerr_ptr) + cerr = *cerr_ptr; + + /* + * Input loop. + * Handle up to mblength characters (not bytes). We do not + * stop at NULs because we may be processing a block of data + * that includes NULs. + */ + mbslength = (ssize_t)mblength; + /* + * When inputing a single character, must also read in the + * next character for nextc, the look-ahead character. + */ + if (mbslength == 1) + mbslength++; + while (mbslength > 0) { + /* Convert one multibyte character to wchar_t. */ + if (!cerr) + clen = mbtowc(src, mbsrc, MB_LEN_MAX); + if (cerr || clen < 0) { + /* Conversion error, process as a byte instead. */ + *src = (wint_t)(u_char)*mbsrc; + clen = 1; + cerr = 1; + } + if (clen == 0) + /* + * NUL in input gives 0 return value. process + * as single NUL byte and keep going. + */ + clen = 1; + /* Advance buffer character pointer. */ + src++; + /* Advance input pointer by number of bytes read. */ *** DIFF OUTPUT TRUNCATED AT 1000 LINES *** From owner-svn-src-vendor@FreeBSD.ORG Thu Feb 21 17:17:14 2013 Return-Path: Delivered-To: svn-src-vendor@freebsd.org Received: from mx1.freebsd.org (mx1.FreeBSD.org [8.8.178.115]) by hub.freebsd.org (Postfix) with ESMTP id 9650CCD7; Thu, 21 Feb 2013 17:17:14 +0000 (UTC) (envelope-from brooks@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) by mx1.freebsd.org (Postfix) with ESMTP id 5AFE6849; Thu, 21 Feb 2013 17:17:14 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.5/8.14.5) with ESMTP id r1LHHEQR000791; Thu, 21 Feb 2013 17:17:14 GMT (envelope-from brooks@svn.freebsd.org) Received: (from brooks@localhost) by svn.freebsd.org (8.14.5/8.14.5/Submit) id r1LHHEw7000790; Thu, 21 Feb 2013 17:17:14 GMT (envelope-from brooks@svn.freebsd.org) Message-Id: <201302211717.r1LHHEw7000790@svn.freebsd.org> From: Brooks Davis Date: Thu, 21 Feb 2013 17:17:14 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r247107 - vendor/NetBSD/libc-vis/20132102 X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 21 Feb 2013 17:17:14 -0000 Author: brooks Date: Thu Feb 21 17:17:13 2013 New Revision: 247107 URL: http://svnweb.freebsd.org/changeset/base/247107 Log: Tag 2013-02-21 import of NetBSD's (un)vis(3) Added: vendor/NetBSD/libc-vis/20132102/ - copied from r247106, vendor/NetBSD/libc-vis/dist/ From owner-svn-src-vendor@FreeBSD.ORG Thu Feb 21 20:44:54 2013 Return-Path: Delivered-To: svn-src-vendor@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by hub.freebsd.org (Postfix) with ESMTP id E9291CA2; Thu, 21 Feb 2013 20:44:54 +0000 (UTC) (envelope-from brooks@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) by mx1.freebsd.org (Postfix) with ESMTP id CCDD39E0; Thu, 21 Feb 2013 20:44:54 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.5/8.14.5) with ESMTP id r1LKiso5065294; Thu, 21 Feb 2013 20:44:54 GMT (envelope-from brooks@svn.freebsd.org) Received: (from brooks@localhost) by svn.freebsd.org (8.14.5/8.14.5/Submit) id r1LKirvg065285; Thu, 21 Feb 2013 20:44:53 GMT (envelope-from brooks@svn.freebsd.org) Message-Id: <201302212044.r1LKirvg065285@svn.freebsd.org> From: Brooks Davis Date: Thu, 21 Feb 2013 20:44:53 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r247128 - in vendor/NetBSD/vis: . dist X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 21 Feb 2013 20:44:55 -0000 Author: brooks Date: Thu Feb 21 20:44:53 2013 New Revision: 247128 URL: http://svnweb.freebsd.org/changeset/base/247128 Log: Vendor import of NetBSD's vis(1) at 2013-02-21 Added: vendor/NetBSD/vis/ vendor/NetBSD/vis/FreeBSD-Upgrade vendor/NetBSD/vis/dist/ vendor/NetBSD/vis/dist/Makefile (contents, props changed) vendor/NetBSD/vis/dist/extern.h (contents, props changed) vendor/NetBSD/vis/dist/foldit.c (contents, props changed) vendor/NetBSD/vis/dist/vis.1 (contents, props changed) vendor/NetBSD/vis/dist/vis.c (contents, props changed) Added: vendor/NetBSD/vis/FreeBSD-Upgrade ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/NetBSD/vis/FreeBSD-Upgrade Thu Feb 21 20:44:53 2013 (r247128) @@ -0,0 +1,23 @@ +Instructions for updating vis(1): + +1) checkout the head of the vendor branch: + + REPO=svn+ssh://svn.freebsd.org/ + svn co $REPO/base/vendor/NetBSD/vis/dist + +2) Update sources: + + rm dist/* + export CVSROOT=anoncvs@anoncvs.NetBSD.org:/cvsroot + cvs export -r HEAD -d dist src/usr.bin/vis + +3) Commit the new versions: + + svn commit -m "Vendor import of NetBSD's vis(1) at `date +%F`" dist + svn cp -m "Tag `date +%F` import of NetBSD's vis(1)" \ + $REPO/base/vendor/NetBSD/vis/dist \ + $REPO/base/vendor/NetBSD/vis/`date +%Y%d%m` + +4) Update the files in src/contrib/vis: + + Added: vendor/NetBSD/vis/dist/Makefile ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/NetBSD/vis/dist/Makefile Thu Feb 21 20:44:53 2013 (r247128) @@ -0,0 +1,7 @@ +# $NetBSD: Makefile,v 1.3 1994/11/17 07:55:57 jtc Exp $ +# @(#)Makefile 8.1 (Berkeley) 6/6/93 + +PROG= vis +SRCS= vis.c foldit.c + +.include Added: vendor/NetBSD/vis/dist/extern.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/NetBSD/vis/dist/extern.h Thu Feb 21 20:44:53 2013 (r247128) @@ -0,0 +1,39 @@ +/* $NetBSD: extern.h,v 1.1 2009/02/10 23:06:31 christos Exp $ */ + +/*- + * Copyright (c) 2009 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Christos Zoulas. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the NetBSD + * Foundation, Inc. and its contributors. + * 4. Neither the name of The NetBSD Foundation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +int foldit(const char *, int, int, int); Added: vendor/NetBSD/vis/dist/foldit.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/NetBSD/vis/dist/foldit.c Thu Feb 21 20:44:53 2013 (r247128) @@ -0,0 +1,78 @@ +/* $NetBSD: foldit.c,v 1.7 2009/02/10 23:06:31 christos Exp $ */ + +/*- + * Copyright (c) 1990, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#ifndef lint +#if 0 +static char sccsid[] = "@(#)foldit.c 8.1 (Berkeley) 6/6/93"; +#endif +__RCSID("$NetBSD: foldit.c,v 1.7 2009/02/10 23:06:31 christos Exp $"); +#endif /* not lint */ + +#include +#include +#include "extern.h" + +int +foldit(const char *chunk, int col, int max, int flags) +{ + const char *cp; + + /* + * Keep track of column position. Insert hidden newline + * if this chunk puts us over the limit. + */ +again: + cp = chunk; + while (*cp) { + switch(*cp) { + case '\n': + case '\r': + col = 0; + break; + case '\t': + col = (col + 8) &~ 07; + break; + case '\b': + col = col ? col - 1 : 0; + break; + default: + col++; + } + if (col > (max - 2)) { + printf(flags & VIS_MIMESTYLE ? "=\n" : "\\\n"); + col = 0; + goto again; + } + cp++; + } + return (col); +} Added: vendor/NetBSD/vis/dist/vis.1 ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/NetBSD/vis/dist/vis.1 Thu Feb 21 20:44:53 2013 (r247128) @@ -0,0 +1,169 @@ +.\" $NetBSD: vis.1,v 1.17 2013/02/14 14:00:00 christos Exp $ +.\" +.\" Copyright (c) 1989, 1991, 1993, 1994 +.\" The Regents of the University of California. All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" 3. Neither the name of the University nor the names of its contributors +.\" may be used to endorse or promote products derived from this software +.\" without specific prior written permission. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" @(#)vis.1 8.4 (Berkeley) 4/19/94 +.\" +.Dd February 13, 2013 +.Dt VIS 1 +.Os +.Sh NAME +.Nm vis +.Nd display non-printable characters in a visual format +.Sh SYNOPSIS +.Nm +.Op Fl bcfhlmnostw +.Op Fl e Ar extra +.Op Fl F Ar foldwidth +.Op Ar file ... +.Sh DESCRIPTION +.Nm +is a filter for converting non-printable characters +into a visual representation. +It differs from +.Ql cat -v +in that +the form is unique and invertible. +By default, all non-graphic +characters except space, tab, and newline are encoded. +A detailed description of the +various visual formats is given in +.Xr vis 3 . +.Pp +The options are as follows: +.Bl -tag -width Ds +.It Fl b +Turns off prepending of backslash before up-arrow control sequences +and meta characters, and disables the doubling of backslashes. +This +produces output which is neither invertible or precise, but does +represent a minimum of change to the input. +It is similar to +.Dq Li cat -v . +.Pq Dv VIS_NOSLASH +.It Fl c +Request a format which displays a small subset of the +non-printable characters using C-style backslash sequences. +.Pq Dv VIS_CSTYLE +.It Fl e Ar extra +Also encode characters in +.Ar extra , +per +.Xr svis 3 . +.It Fl F Ar foldwidth +Causes +.Nm +to fold output lines to foldwidth columns (default 80), like +.Xr fold 1 , +except +that a hidden newline sequence is used, (which is removed +when inverting the file back to its original form with +.Xr unvis 1 ) . +If the last character in the encoded file does not end in a newline, +a hidden newline sequence is appended to the output. +This makes +the output usable with various editors and other utilities which +typically don't work with partial lines. +.It Fl f +Same as +.Fl F . +.It Fl h +Encode using the URI encoding from RFC 1808. +.Pq Dv VIS_HTTPSTYLE +.It Fl l +Mark newlines with the visible sequence +.Ql \e$ , +followed by the newline. +.It Fl m +Encode using the MIME Quoted-Printable encoding from RFC 2045. +.Pq Dv VIS_MIMESTYLE +.It Fl n +Turns off any encoding, except for the fact that backslashes are +still doubled and hidden newline sequences inserted if +.Fl f +or +.Fl F +is selected. +When combined with the +.Fl f +flag, +.Nm +becomes like +an invertible version of the +.Xr fold 1 +utility. +That is, the output can be unfolded by running the output through +.Xr unvis 1 . +.It Fl o +Request a format which displays non-printable characters as +an octal number, \eddd. +.Pq Dv VIS_OCTAL +.It Fl s +Only characters considered unsafe to send to a terminal are encoded. +This flag allows backspace, bell, and carriage return in addition +to the default space, tab and newline. +.Pq Dv VIS_SAFE +.It Fl t +Tabs are also encoded. +.Pq Dv VIS_TAB +.It Fl w +White space (space-tab-newline) is also encoded. +.Pq Dv VIS_WHITE +.El +.Sh MULTIBYTE CHARACTER SUPPORT +.Nm +supports multibyte character input. +The encoding conversion is influenced by the setting of the +.Ev LC_CTYPE +environment variable which defines the set of characters that can be +copied without encoding. +.Pp +When 8-bit data is present in the input, +.Ev LC_CTYPE +must be set to the correct locale or to the C locale. +If the locales of the data and the conversion are mismatched, multibyte +character recognition may fail and encoding will be performed byte-by-byte +instead. +.Sh ENVIRONMENT +.Bl -tag -width ".Ev LC_CTYPE" +.It Ev LC_CTYPE +Specify the locale of the input data. +Set to C if the input data locale is unknown. +.Sh SEE ALSO +.Xr unvis 1 , +.Xr svis 3 , +.Xr vis 3 +.Sh HISTORY +The +.Nm +command appears in +.Bx 4.4 . +Myltibyte character support was added in +.Nx 6.1 +and +.Fx 9.2 . Added: vendor/NetBSD/vis/dist/vis.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/NetBSD/vis/dist/vis.c Thu Feb 21 20:44:53 2013 (r247128) @@ -0,0 +1,276 @@ +/* $NetBSD: vis.c,v 1.22 2013/02/20 17:04:45 christos Exp $ */ + +/*- + * Copyright (c) 1989, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#ifndef lint +__COPYRIGHT("@(#) Copyright (c) 1989, 1993\ + The Regents of the University of California. All rights reserved."); +#endif /* not lint */ + +#ifndef lint +#if 0 +static char sccsid[] = "@(#)vis.c 8.1 (Berkeley) 6/6/93"; +#endif +__RCSID("$NetBSD: vis.c,v 1.22 2013/02/20 17:04:45 christos Exp $"); +#endif /* not lint */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "extern.h" + +static int eflags, fold, foldwidth = 80, none, markeol; +#ifdef DEBUG +int debug; +#endif +static const char *extra = ""; + +static void process(FILE *); + +int +main(int argc, char *argv[]) +{ + FILE *fp; + int ch; + int rval; + + while ((ch = getopt(argc, argv, "bcde:F:fhlmnostw")) != -1) + switch((char)ch) { + case 'b': + eflags |= VIS_NOSLASH; + break; + case 'c': + eflags |= VIS_CSTYLE; + break; +#ifdef DEBUG + case 'd': + debug++; + break; +#endif + case 'e': + extra = optarg; + break; + case 'F': + if ((foldwidth = atoi(optarg)) < 5) { + errx(1, "can't fold lines to less than 5 cols"); + /* NOTREACHED */ + } + markeol++; + break; + case 'f': + fold++; /* fold output lines to 80 cols */ + break; /* using hidden newline */ + case 'h': + eflags |= VIS_HTTPSTYLE; + break; + case 'l': + markeol++; /* mark end of line with \$ */ + break; + case 'm': + eflags |= VIS_MIMESTYLE; + if (foldwidth == 80) + foldwidth = 76; + break; + case 'n': + none++; + break; + case 'o': + eflags |= VIS_OCTAL; + break; + case 's': + eflags |= VIS_SAFE; + break; + case 't': + eflags |= VIS_TAB; + break; + case 'w': + eflags |= VIS_WHITE; + break; + case '?': + default: + (void)fprintf(stderr, + "Usage: %s [-bcfhlmnostw] [-e extra]" + " [-F foldwidth] [file ...]\n", getprogname()); + return 1; + } + + if ((eflags & (VIS_HTTPSTYLE|VIS_MIMESTYLE)) == + (VIS_HTTPSTYLE|VIS_MIMESTYLE)) + errx(1, "Can't specify -m and -h at the same time"); + + argc -= optind; + argv += optind; + + rval = 0; + + if (*argv) + while (*argv) { + if ((fp = fopen(*argv, "r")) != NULL) { + process(fp); + (void)fclose(fp); + } else { + warn("%s", *argv); + rval = 1; + } + argv++; + } + else + process(stdin); + return rval; +} + +static void +process(FILE *fp) +{ + static int col = 0; + static char nul[] = "\0"; + char *cp = nul + 1; /* so *(cp-1) starts out != '\n' */ + wint_t c, c1, rachar; + char mbibuff[2 * MB_LEN_MAX + 1]; /* max space for 2 wchars */ + char buff[4 * MB_LEN_MAX + 1]; /* max encoding length for one char */ + int mbilen, cerr = 0, raerr = 0; + + /* + * The input stream is considered to be multibyte characters. + * The input loop will read this data inputing one character, + * possibly multiple bytes, at a time and converting each to + * a wide character wchar_t. + * + * The vis(3) functions, however, require single either bytes + * or a multibyte string as their arguments. So we convert + * our input wchar_t and the following look-ahead wchar_t to + * a multibyte string for processing by vis(3). + */ + + /* Read one multibyte character, store as wchar_t */ + c = getwc(fp); + if (c == WEOF && errno == EILSEQ) { + /* Error in multibyte data. Read one byte. */ + c = (wint_t)getc(fp); + cerr = 1; + } + while (c != WEOF) { + /* Clear multibyte input buffer. */ + memset(mbibuff, 0, sizeof(mbibuff)); + /* Read-ahead next multibyte character. */ + if (!cerr) + rachar = getwc(fp); + if (cerr || (rachar == WEOF && errno == EILSEQ)) { + /* Error in multibyte data. Read one byte. */ + rachar = (wint_t)getc(fp); + raerr = 1; + } + if (none) { + /* Handle -n flag. */ + cp = buff; + *cp++ = c; + if (c == '\\') + *cp++ = '\\'; + *cp = '\0'; + } else if (markeol && c == '\n') { + /* Handle -l flag. */ + cp = buff; + if ((eflags & VIS_NOSLASH) == 0) + *cp++ = '\\'; + *cp++ = '$'; + *cp++ = '\n'; + *cp = '\0'; + } else { + /* + * Convert character using vis(3) library. + * At this point we will process one character. + * But we must pass the vis(3) library this + * character plus the next one because the next + * one is used as a look-ahead to decide how to + * encode this one under certain circumstances. + * + * Since our characters may be multibyte, e.g., + * in the UTF-8 locale, we cannot use vis() and + * svis() which require byte input, so we must + * create a multibyte string and use strvisx(). + */ + /* Treat EOF as a NUL char. */ + c1 = rachar; + if (c1 == WEOF) + c1 = L'\0'; + /* + * If we hit a multibyte conversion error above, + * insert byte directly into string buff because + * wctomb() will fail. Else convert wchar_t to + * multibyte using wctomb(). + */ + if (cerr) { + *mbibuff = (char)c; + mbilen = 1; + } else + mbilen = wctomb(mbibuff, c); + /* Same for look-ahead character. */ + if (raerr) + mbibuff[mbilen] = (char)c1; + else + wctomb(mbibuff + mbilen, c1); + /* Perform encoding on just first character. */ + (void) strsenvisx(buff, 4 * MB_LEN_MAX, mbibuff, + 1, eflags, extra, &cerr); + } + + cp = buff; + if (fold) { +#ifdef DEBUG + if (debug) + (void)printf("<%02d,", col); +#endif + col = foldit(cp, col, foldwidth, eflags); +#ifdef DEBUG + if (debug) + (void)printf("%02d>", col); +#endif + } + do { + (void)putchar(*cp); + } while (*++cp); + c = rachar; + cerr = raerr; + } + /* + * terminate partial line with a hidden newline + */ + if (fold && *(cp - 1) != '\n') + (void)printf(eflags & VIS_MIMESTYLE ? "=\n" : "\\\n"); +} From owner-svn-src-vendor@FreeBSD.ORG Thu Feb 21 20:45:25 2013 Return-Path: Delivered-To: svn-src-vendor@freebsd.org Received: from mx1.freebsd.org (mx1.FreeBSD.org [8.8.178.115]) by hub.freebsd.org (Postfix) with ESMTP id 8DC43E1E; Thu, 21 Feb 2013 20:45:25 +0000 (UTC) (envelope-from brooks@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) by mx1.freebsd.org (Postfix) with ESMTP id 687BE9EF; Thu, 21 Feb 2013 20:45:25 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.5/8.14.5) with ESMTP id r1LKjPHh065456; Thu, 21 Feb 2013 20:45:25 GMT (envelope-from brooks@svn.freebsd.org) Received: (from brooks@localhost) by svn.freebsd.org (8.14.5/8.14.5/Submit) id r1LKjPZt065455; Thu, 21 Feb 2013 20:45:25 GMT (envelope-from brooks@svn.freebsd.org) Message-Id: <201302212045.r1LKjPZt065455@svn.freebsd.org> From: Brooks Davis Date: Thu, 21 Feb 2013 20:45:25 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r247130 - vendor/NetBSD/vis/20132102 X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 21 Feb 2013 20:45:25 -0000 Author: brooks Date: Thu Feb 21 20:45:24 2013 New Revision: 247130 URL: http://svnweb.freebsd.org/changeset/base/247130 Log: Tag 2013-02-21 import of NetBSD's vis(1) Added: vendor/NetBSD/vis/20132102/ - copied from r247129, vendor/NetBSD/vis/dist/ From owner-svn-src-vendor@FreeBSD.ORG Thu Feb 21 21:09:57 2013 Return-Path: Delivered-To: svn-src-vendor@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by hub.freebsd.org (Postfix) with ESMTP id 24EC197C; Thu, 21 Feb 2013 21:09:57 +0000 (UTC) (envelope-from brooks@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) by mx1.freebsd.org (Postfix) with ESMTP id F359EB6A; Thu, 21 Feb 2013 21:09:56 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.5/8.14.5) with ESMTP id r1LL9uk0072557; Thu, 21 Feb 2013 21:09:56 GMT (envelope-from brooks@svn.freebsd.org) Received: (from brooks@localhost) by svn.freebsd.org (8.14.5/8.14.5/Submit) id r1LL9ufW072552; Thu, 21 Feb 2013 21:09:56 GMT (envelope-from brooks@svn.freebsd.org) Message-Id: <201302212109.r1LL9ufW072552@svn.freebsd.org> From: Brooks Davis Date: Thu, 21 Feb 2013 21:09:56 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r247131 - in vendor/NetBSD/unvis: . dist X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 21 Feb 2013 21:09:57 -0000 Author: brooks Date: Thu Feb 21 21:09:55 2013 New Revision: 247131 URL: http://svnweb.freebsd.org/changeset/base/247131 Log: Vendor import of NetBSD's unvis(1) at 2013-02-21 Added: vendor/NetBSD/unvis/ vendor/NetBSD/unvis/FreeBSD-Upgrade - copied, changed from r247128, vendor/NetBSD/vis/FreeBSD-Upgrade vendor/NetBSD/unvis/dist/ vendor/NetBSD/unvis/dist/Makefile (contents, props changed) vendor/NetBSD/unvis/dist/unvis.1 (contents, props changed) vendor/NetBSD/unvis/dist/unvis.c (contents, props changed) Copied and modified: vendor/NetBSD/unvis/FreeBSD-Upgrade (from r247128, vendor/NetBSD/vis/FreeBSD-Upgrade) ============================================================================== --- vendor/NetBSD/vis/FreeBSD-Upgrade Thu Feb 21 20:44:53 2013 (r247128, copy source) +++ vendor/NetBSD/unvis/FreeBSD-Upgrade Thu Feb 21 21:09:55 2013 (r247131) @@ -1,23 +1,23 @@ -Instructions for updating vis(1): +Instructions for updating unvis(1): 1) checkout the head of the vendor branch: REPO=svn+ssh://svn.freebsd.org/ - svn co $REPO/base/vendor/NetBSD/vis/dist + svn co $REPO/base/vendor/NetBSD/unvis/dist 2) Update sources: rm dist/* export CVSROOT=anoncvs@anoncvs.NetBSD.org:/cvsroot - cvs export -r HEAD -d dist src/usr.bin/vis + cvs export -r HEAD -d dist src/usr.bin/unvis 3) Commit the new versions: - svn commit -m "Vendor import of NetBSD's vis(1) at `date +%F`" dist - svn cp -m "Tag `date +%F` import of NetBSD's vis(1)" \ - $REPO/base/vendor/NetBSD/vis/dist \ - $REPO/base/vendor/NetBSD/vis/`date +%Y%d%m` + svn commit -m "Vendor import of NetBSD's unvis(1) at `date +%F`" dist + svn cp -m "Tag `date +%F` import of NetBSD's unvis(1)" \ + $REPO/base/vendor/NetBSD/unvis/dist \ + $REPO/base/vendor/NetBSD/unvis/`date +%Y%d%m` -4) Update the files in src/contrib/vis: +4) Update the files in src/contrib/unvis: Added: vendor/NetBSD/unvis/dist/Makefile ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/NetBSD/unvis/dist/Makefile Thu Feb 21 21:09:55 2013 (r247131) @@ -0,0 +1,6 @@ +# $NetBSD: Makefile,v 1.3 1994/12/06 07:36:07 jtc Exp $ +# @(#)Makefile 8.1 (Berkeley) 6/6/93 + +PROG= unvis + +.include Added: vendor/NetBSD/unvis/dist/unvis.1 ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/NetBSD/unvis/dist/unvis.1 Thu Feb 21 21:09:55 2013 (r247131) @@ -0,0 +1,80 @@ +.\" $NetBSD: unvis.1,v 1.10 2012/12/15 02:08:13 uwe Exp $ +.\" +.\" Copyright (c) 1989, 1990, 1993 +.\" The Regents of the University of California. All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" 3. Neither the name of the University nor the names of its contributors +.\" may be used to endorse or promote products derived from this software +.\" without specific prior written permission. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" @(#)unvis.1 8.1 (Berkeley) 6/6/93 +.\" +.Dd November 27, 2010 +.Dt UNVIS 1 +.Os +.Sh NAME +.Nm unvis +.Nd "revert a visual representation of data back to original form" +.Sh SYNOPSIS +.Nm +.Op Fl e +.Op Fl Hh | Fl m +.Op Ar file ... +.Sh DESCRIPTION +.Nm +is the inverse function of +.Xr vis 1 . +It reverts +a visual representation of data back to its original form on standard output. +.Pp +The options are as follows: +.Bl -tag -width Ds +.It Fl e +Don't decode \e escaped sequences. +.It Fl H +Decode entity references and numeric character references from RFC 1866. +.Pq Dv VIS_HTTP1866 +.It Fl h +Decode using the URI encoding from RFC 1808. +.Pq Dv VIS_HTTP1808 +.It Fl m +Decode using mime style. +.Pq Dv VIS_MIMESTYLE +.El +.Pp +Mixing +.Fl h +or +.Fl H +with +.Fl m +is not supported. +.Sh SEE ALSO +.Xr vis 1 , +.Xr unvis 3 , +.Xr vis 3 +.Sh HISTORY +The +.Nm +command appears in +.Bx 4.4 . Added: vendor/NetBSD/unvis/dist/unvis.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/NetBSD/unvis/dist/unvis.c Thu Feb 21 21:09:55 2013 (r247131) @@ -0,0 +1,138 @@ +/* $NetBSD: unvis.c,v 1.13 2010/11/27 19:46:25 christos Exp $ */ + +/*- + * Copyright (c) 1989, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#ifndef lint +__COPYRIGHT("@(#) Copyright (c) 1989, 1993\ + The Regents of the University of California. All rights reserved."); +#endif /* not lint */ + +#ifndef lint +#if 0 +static char sccsid[] = "@(#)unvis.c 8.1 (Berkeley) 6/6/93"; +#endif +__RCSID("$NetBSD: unvis.c,v 1.13 2010/11/27 19:46:25 christos Exp $"); +#endif /* not lint */ + +#include +#include +#include +#include +#include + +static void process(FILE *, const char *, int); + +int +main(int argc, char *argv[]) +{ + FILE *fp; + int ch, eflags = 0; + + setprogname(argv[0]); + while ((ch = getopt(argc, argv, "eHhm")) != -1) + switch((char)ch) { + case 'e': + eflags |= VIS_NOESCAPE; + break; + case 'H': + eflags |= VIS_HTTP1866; + break; + case 'h': + eflags |= VIS_HTTP1808; + break; + case 'm': + eflags |= VIS_MIMESTYLE; + break; + case '?': + default: + (void)fprintf(stderr, + "Usage: %s [-e] [-Hh | -m] [file...]\n", + getprogname()); + return EXIT_FAILURE; + } + argc -= optind; + argv += optind; + + switch (eflags & (VIS_HTTP1808|VIS_HTTP1866|VIS_MIMESTYLE)) { + case VIS_HTTP1808|VIS_MIMESTYLE: + case VIS_HTTP1866|VIS_MIMESTYLE: + case VIS_HTTP1808|VIS_HTTP1866|VIS_MIMESTYLE: + errx(EXIT_FAILURE, "Can't mix -m with -h and/or -H"); + /*NOTREACHED*/ + default: + break; + } + + if (*argv) + while (*argv) { + if ((fp = fopen(*argv, "r")) != NULL) + process(fp, *argv, eflags); + else + warn("%s", *argv); + argv++; + } + else + process(stdin, "", eflags); + return EXIT_SUCCESS; +} + +static void +process(FILE *fp, const char *filename, int eflags) +{ + int offset = 0, c, ret; + int state = 0; + char outc; + + while ((c = getc(fp)) != EOF) { + offset++; + again: + switch(ret = unvis(&outc, (char)c, &state, eflags)) { + case UNVIS_VALID: + (void)putchar(outc); + break; + case UNVIS_VALIDPUSH: + (void)putchar(outc); + goto again; + case UNVIS_SYNBAD: + warnx("%s: offset: %d: can't decode", filename, offset); + state = 0; + break; + case 0: + case UNVIS_NOCHAR: + break; + default: + errx(1, "bad return value (%d), can't happen", ret); + /* NOTREACHED */ + } + } + if (unvis(&outc, (char)0, &state, eflags | UNVIS_END) == UNVIS_VALID) + (void)putchar(outc); +} From owner-svn-src-vendor@FreeBSD.ORG Thu Feb 21 21:16:43 2013 Return-Path: Delivered-To: svn-src-vendor@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by hub.freebsd.org (Postfix) with ESMTP id 3CA6BC72; Thu, 21 Feb 2013 21:16:43 +0000 (UTC) (envelope-from brooks@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) by mx1.freebsd.org (Postfix) with ESMTP id 0B8CAC27; Thu, 21 Feb 2013 21:16:43 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.5/8.14.5) with ESMTP id r1LLGgno075253; Thu, 21 Feb 2013 21:16:42 GMT (envelope-from brooks@svn.freebsd.org) Received: (from brooks@localhost) by svn.freebsd.org (8.14.5/8.14.5/Submit) id r1LLGgjr075252; Thu, 21 Feb 2013 21:16:42 GMT (envelope-from brooks@svn.freebsd.org) Message-Id: <201302212116.r1LLGgjr075252@svn.freebsd.org> From: Brooks Davis Date: Thu, 21 Feb 2013 21:16:42 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r247132 - vendor/NetBSD/unvis/20132102 X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 21 Feb 2013 21:16:43 -0000 Author: brooks Date: Thu Feb 21 21:16:42 2013 New Revision: 247132 URL: http://svnweb.freebsd.org/changeset/base/247132 Log: Tag 2013-02-21 import of NetBSD's unvis(1) Added: vendor/NetBSD/unvis/20132102/ - copied from r247131, vendor/NetBSD/unvis/dist/ From owner-svn-src-vendor@FreeBSD.ORG Sat Feb 23 08:57:48 2013 Return-Path: Delivered-To: svn-src-vendor@freebsd.org Received: from mx1.freebsd.org (mx1.FreeBSD.org [8.8.178.115]) by hub.freebsd.org (Postfix) with ESMTP id 6DBD9873; Sat, 23 Feb 2013 08:57:48 +0000 (UTC) (envelope-from mm@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) by mx1.freebsd.org (Postfix) with ESMTP id 55BCE218; Sat, 23 Feb 2013 08:57:48 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.5/8.14.5) with ESMTP id r1N8vlHj028991; Sat, 23 Feb 2013 08:57:47 GMT (envelope-from mm@svn.freebsd.org) Received: (from mm@localhost) by svn.freebsd.org (8.14.5/8.14.5/Submit) id r1N8vliD028989; Sat, 23 Feb 2013 08:57:47 GMT (envelope-from mm@svn.freebsd.org) Message-Id: <201302230857.r1N8vliD028989@svn.freebsd.org> From: Martin Matuska Date: Sat, 23 Feb 2013 08:57:47 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r247174 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 23 Feb 2013 08:57:48 -0000 Author: mm Date: Sat Feb 23 08:57:47 2013 New Revision: 247174 URL: http://svnweb.freebsd.org/changeset/base/247174 Log: Update vendor-sys/illumos/dist to illumos-gate 13958:1fd91513472c Illumos ZFS issues: 3561 arc_meta_limit should be exposed via kstats 3116 zpool reguid may log negative guids to internal SPA history Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c vendor-sys/illumos/dist/uts/common/fs/zfs/spa.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c Sat Feb 23 08:19:24 2013 (r247173) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/arc.c Sat Feb 23 08:57:47 2013 (r247174) @@ -294,6 +294,9 @@ typedef struct arc_stats { kstat_named_t arcstat_duplicate_buffers; kstat_named_t arcstat_duplicate_buffers_size; kstat_named_t arcstat_duplicate_reads; + kstat_named_t arcstat_meta_used; + kstat_named_t arcstat_meta_limit; + kstat_named_t arcstat_meta_max; } arc_stats_t; static arc_stats_t arc_stats = { @@ -352,7 +355,10 @@ static arc_stats_t arc_stats = { { "memory_throttle_count", KSTAT_DATA_UINT64 }, { "duplicate_buffers", KSTAT_DATA_UINT64 }, { "duplicate_buffers_size", KSTAT_DATA_UINT64 }, - { "duplicate_reads", KSTAT_DATA_UINT64 } + { "duplicate_reads", KSTAT_DATA_UINT64 }, + { "arc_meta_used", KSTAT_DATA_UINT64 }, + { "arc_meta_limit", KSTAT_DATA_UINT64 }, + { "arc_meta_max", KSTAT_DATA_UINT64 } }; #define ARCSTAT(stat) (arc_stats.stat.value.ui64) @@ -414,13 +420,13 @@ static arc_state_t *arc_l2c_only; #define arc_c ARCSTAT(arcstat_c) /* target size of cache */ #define arc_c_min ARCSTAT(arcstat_c_min) /* min target cache size */ #define arc_c_max ARCSTAT(arcstat_c_max) /* max target cache size */ +#define arc_meta_limit ARCSTAT(arcstat_meta_limit) /* max size for metadata */ +#define arc_meta_used ARCSTAT(arcstat_meta_used) /* size of metadata */ +#define arc_meta_max ARCSTAT(arcstat_meta_max) /* max size of metadata */ static int arc_no_grow; /* Don't try to grow cache size */ static uint64_t arc_tempreserve; static uint64_t arc_loaned_bytes; -static uint64_t arc_meta_used; -static uint64_t arc_meta_limit; -static uint64_t arc_meta_max = 0; typedef struct l2arc_buf_hdr l2arc_buf_hdr_t; @@ -1218,7 +1224,7 @@ arc_space_consume(uint64_t space, arc_sp break; } - atomic_add_64(&arc_meta_used, space); + ARCSTAT_INCR(arcstat_meta_used, space); atomic_add_64(&arc_size, space); } @@ -1245,7 +1251,7 @@ arc_space_return(uint64_t space, arc_spa ASSERT(arc_meta_used >= space); if (arc_meta_max < arc_meta_used) arc_meta_max = arc_meta_used; - atomic_add_64(&arc_meta_used, -space); + ARCSTAT_INCR(arcstat_meta_used, -space); ASSERT(arc_size >= space); atomic_add_64(&arc_size, -space); } Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/spa.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/spa.c Sat Feb 23 08:19:24 2013 (r247173) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/spa.c Sat Feb 23 08:57:47 2013 (r247174) @@ -715,7 +715,7 @@ spa_change_guid_sync(void *arg1, void *a vdev_config_dirty(rvd); spa_config_exit(spa, SCL_STATE, FTAG); - spa_history_log_internal(spa, "guid change", tx, "old=%lld new=%lld", + spa_history_log_internal(spa, "guid change", tx, "old=%llu new=%llu", oldguid, *newguid); } From owner-svn-src-vendor@FreeBSD.ORG Sat Feb 23 09:00:36 2013 Return-Path: Delivered-To: svn-src-vendor@freebsd.org Received: from mx1.freebsd.org (mx1.FreeBSD.org [8.8.178.115]) by hub.freebsd.org (Postfix) with ESMTP id 7C669B04; Sat, 23 Feb 2013 09:00:36 +0000 (UTC) (envelope-from mm@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) by mx1.freebsd.org (Postfix) with ESMTP id 6074E22C; Sat, 23 Feb 2013 09:00:36 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.5/8.14.5) with ESMTP id r1N90asN029597; Sat, 23 Feb 2013 09:00:36 GMT (envelope-from mm@svn.freebsd.org) Received: (from mm@localhost) by svn.freebsd.org (8.14.5/8.14.5/Submit) id r1N90Zbc029591; Sat, 23 Feb 2013 09:00:35 GMT (envelope-from mm@svn.freebsd.org) Message-Id: <201302230900.r1N90Zbc029591@svn.freebsd.org> From: Martin Matuska Date: Sat, 23 Feb 2013 09:00:35 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r247176 - vendor-sys/illumos/dist/uts/common/fs/zfs vendor-sys/illumos/dist/uts/common/fs/zfs/sys vendor/illumos/dist/cmd/zdb X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 23 Feb 2013 09:00:36 -0000 Author: mm Date: Sat Feb 23 09:00:35 2013 New Revision: 247176 URL: http://svnweb.freebsd.org/changeset/base/247176 Log: Update vendor/illumos/dist and vendor-sys/illumos/dist to illumos-gate 13959:e03e14ddfb4c Illumos ZFS issues: 3552 condensing one space map burns 3 seconds of CPU in spa_sync() thread 3564 spa_sync() spends 5-10% of its time in metaslab_sync() (when not condensing) Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/metaslab.c vendor-sys/illumos/dist/uts/common/fs/zfs/space_map.c vendor-sys/illumos/dist/uts/common/fs/zfs/sys/metaslab_impl.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/space_map.h vendor-sys/illumos/dist/uts/common/fs/zfs/vdev.c Changes in other areas also in this revision: Modified: vendor/illumos/dist/cmd/zdb/zdb.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/metaslab.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/metaslab.c Sat Feb 23 08:59:07 2013 (r247175) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/metaslab.c Sat Feb 23 09:00:35 2013 (r247176) @@ -48,6 +48,14 @@ uint64_t metaslab_aliquot = 512ULL << 10 uint64_t metaslab_gang_bang = SPA_MAXBLOCKSIZE + 1; /* force gang blocks */ /* + * The in-core space map representation is more compact than its on-disk form. + * The zfs_condense_pct determines how much more compact the in-core + * space_map representation must be before we compact it on-disk. + * Values should be greater than or equal to 100. + */ +int zfs_condense_pct = 200; + +/* * This value defines the number of allowed allocation failures per vdev. * If a device reaches this threshold in a given txg then we consider skipping * allocations on that device. @@ -206,9 +214,9 @@ metaslab_compare(const void *x1, const v /* * If the weights are identical, use the offset to force uniqueness. */ - if (m1->ms_map.sm_start < m2->ms_map.sm_start) + if (m1->ms_map->sm_start < m2->ms_map->sm_start) return (-1); - if (m1->ms_map.sm_start > m2->ms_map.sm_start) + if (m1->ms_map->sm_start > m2->ms_map->sm_start) return (1); ASSERT3P(m1, ==, m2); @@ -723,14 +731,15 @@ metaslab_init(metaslab_group_t *mg, spac * addition of new space; and for debugging, it ensures that we'd * data fault on any attempt to use this metaslab before it's ready. */ - space_map_create(&msp->ms_map, start, size, + msp->ms_map = kmem_zalloc(sizeof (space_map_t), KM_SLEEP); + space_map_create(msp->ms_map, start, size, vd->vdev_ashift, &msp->ms_lock); metaslab_group_add(mg, msp); if (metaslab_debug && smo->smo_object != 0) { mutex_enter(&msp->ms_lock); - VERIFY(space_map_load(&msp->ms_map, mg->mg_class->mc_ops, + VERIFY(space_map_load(msp->ms_map, mg->mg_class->mc_ops, SM_FREE, smo, spa_meta_objset(vd->vdev_spa)) == 0); mutex_exit(&msp->ms_lock); } @@ -758,22 +767,27 @@ metaslab_fini(metaslab_t *msp) metaslab_group_t *mg = msp->ms_group; vdev_space_update(mg->mg_vd, - -msp->ms_smo.smo_alloc, 0, -msp->ms_map.sm_size); + -msp->ms_smo.smo_alloc, 0, -msp->ms_map->sm_size); metaslab_group_remove(mg, msp); mutex_enter(&msp->ms_lock); - space_map_unload(&msp->ms_map); - space_map_destroy(&msp->ms_map); + space_map_unload(msp->ms_map); + space_map_destroy(msp->ms_map); + kmem_free(msp->ms_map, sizeof (*msp->ms_map)); for (int t = 0; t < TXG_SIZE; t++) { - space_map_destroy(&msp->ms_allocmap[t]); - space_map_destroy(&msp->ms_freemap[t]); + space_map_destroy(msp->ms_allocmap[t]); + space_map_destroy(msp->ms_freemap[t]); + kmem_free(msp->ms_allocmap[t], sizeof (*msp->ms_allocmap[t])); + kmem_free(msp->ms_freemap[t], sizeof (*msp->ms_freemap[t])); } - for (int t = 0; t < TXG_DEFER_SIZE; t++) - space_map_destroy(&msp->ms_defermap[t]); + for (int t = 0; t < TXG_DEFER_SIZE; t++) { + space_map_destroy(msp->ms_defermap[t]); + kmem_free(msp->ms_defermap[t], sizeof (*msp->ms_defermap[t])); + } ASSERT0(msp->ms_deferspace); @@ -792,7 +806,7 @@ static uint64_t metaslab_weight(metaslab_t *msp) { metaslab_group_t *mg = msp->ms_group; - space_map_t *sm = &msp->ms_map; + space_map_t *sm = msp->ms_map; space_map_obj_t *smo = &msp->ms_smo; vdev_t *vd = mg->mg_vd; uint64_t weight, space; @@ -852,7 +866,7 @@ metaslab_prefetch(metaslab_group_t *mg) * Prefetch the next potential metaslabs */ for (msp = avl_first(t), m = 0; msp; msp = AVL_NEXT(t, msp), m++) { - space_map_t *sm = &msp->ms_map; + space_map_t *sm = msp->ms_map; space_map_obj_t *smo = &msp->ms_smo; /* If we have reached our prefetch limit then we're done */ @@ -873,7 +887,7 @@ static int metaslab_activate(metaslab_t *msp, uint64_t activation_weight) { metaslab_group_t *mg = msp->ms_group; - space_map_t *sm = &msp->ms_map; + space_map_t *sm = msp->ms_map; space_map_ops_t *sm_ops = msp->ms_group->mg_class->mc_ops; ASSERT(MUTEX_HELD(&msp->ms_lock)); @@ -890,7 +904,7 @@ metaslab_activate(metaslab_t *msp, uint6 return (error); } for (int t = 0; t < TXG_DEFER_SIZE; t++) - space_map_walk(&msp->ms_defermap[t], + space_map_walk(msp->ms_defermap[t], space_map_claim, sm); } @@ -921,12 +935,158 @@ metaslab_passivate(metaslab_t *msp, uint * this metaslab again. In that case, it had better be empty, * or we would be leaving space on the table. */ - ASSERT(size >= SPA_MINBLOCKSIZE || msp->ms_map.sm_space == 0); + ASSERT(size >= SPA_MINBLOCKSIZE || msp->ms_map->sm_space == 0); metaslab_group_sort(msp->ms_group, msp, MIN(msp->ms_weight, size)); ASSERT((msp->ms_weight & METASLAB_ACTIVE_MASK) == 0); } /* + * Determine if the in-core space map representation can be condensed on-disk. + * We would like to use the following criteria to make our decision: + * + * 1. The size of the space map object should not dramatically increase as a + * result of writing out our in-core free map. + * + * 2. The minimal on-disk space map representation is zfs_condense_pct/100 + * times the size than the in-core representation (i.e. zfs_condense_pct = 110 + * and in-core = 1MB, minimal = 1.1.MB). + * + * Checking the first condition is tricky since we don't want to walk + * the entire AVL tree calculating the estimated on-disk size. Instead we + * use the size-ordered AVL tree in the space map and calculate the + * size required for the largest segment in our in-core free map. If the + * size required to represent that segment on disk is larger than the space + * map object then we avoid condensing this map. + * + * To determine the second criterion we use a best-case estimate and assume + * each segment can be represented on-disk as a single 64-bit entry. We refer + * to this best-case estimate as the space map's minimal form. + */ +static boolean_t +metaslab_should_condense(metaslab_t *msp) +{ + space_map_t *sm = msp->ms_map; + space_map_obj_t *smo = &msp->ms_smo_syncing; + space_seg_t *ss; + uint64_t size, entries, segsz; + + ASSERT(MUTEX_HELD(&msp->ms_lock)); + ASSERT(sm->sm_loaded); + + /* + * Use the sm_pp_root AVL tree, which is ordered by size, to obtain + * the largest segment in the in-core free map. If the tree is + * empty then we should condense the map. + */ + ss = avl_last(sm->sm_pp_root); + if (ss == NULL) + return (B_TRUE); + + /* + * Calculate the number of 64-bit entries this segment would + * require when written to disk. If this single segment would be + * larger on-disk than the entire current on-disk structure, then + * clearly condensing will increase the on-disk structure size. + */ + size = (ss->ss_end - ss->ss_start) >> sm->sm_shift; + entries = size / (MIN(size, SM_RUN_MAX)); + segsz = entries * sizeof (uint64_t); + + return (segsz <= smo->smo_objsize && + smo->smo_objsize >= (zfs_condense_pct * + sizeof (uint64_t) * avl_numnodes(&sm->sm_root)) / 100); +} + +/* + * Condense the on-disk space map representation to its minimized form. + * The minimized form consists of a small number of allocations followed by + * the in-core free map. + */ +static void +metaslab_condense(metaslab_t *msp, uint64_t txg, dmu_tx_t *tx) +{ + spa_t *spa = msp->ms_group->mg_vd->vdev_spa; + space_map_t *freemap = msp->ms_freemap[txg & TXG_MASK]; + space_map_t condense_map; + space_map_t *sm = msp->ms_map; + objset_t *mos = spa_meta_objset(spa); + space_map_obj_t *smo = &msp->ms_smo_syncing; + + ASSERT(MUTEX_HELD(&msp->ms_lock)); + ASSERT3U(spa_sync_pass(spa), ==, 1); + ASSERT(sm->sm_loaded); + + spa_dbgmsg(spa, "condensing: txg %llu, msp[%llu] %p, " + "smo size %llu, segments %lu", txg, + (msp->ms_map->sm_start / msp->ms_map->sm_size), msp, + smo->smo_objsize, avl_numnodes(&sm->sm_root)); + + /* + * Create an map that is a 100% allocated map. We remove segments + * that have been freed in this txg, any deferred frees that exist, + * and any allocation in the future. Removing segments should be + * a relatively inexpensive operation since we expect these maps to + * a small number of nodes. + */ + space_map_create(&condense_map, sm->sm_start, sm->sm_size, + sm->sm_shift, sm->sm_lock); + space_map_add(&condense_map, condense_map.sm_start, + condense_map.sm_size); + + /* + * Remove what's been freed in this txg from the condense_map. + * Since we're in sync_pass 1, we know that all the frees from + * this txg are in the freemap. + */ + space_map_walk(freemap, space_map_remove, &condense_map); + + for (int t = 0; t < TXG_DEFER_SIZE; t++) + space_map_walk(msp->ms_defermap[t], + space_map_remove, &condense_map); + + for (int t = 1; t < TXG_CONCURRENT_STATES; t++) + space_map_walk(msp->ms_allocmap[(txg + t) & TXG_MASK], + space_map_remove, &condense_map); + + /* + * We're about to drop the metaslab's lock thus allowing + * other consumers to change it's content. Set the + * space_map's sm_condensing flag to ensure that + * allocations on this metaslab do not occur while we're + * in the middle of committing it to disk. This is only critical + * for the ms_map as all other space_maps use per txg + * views of their content. + */ + sm->sm_condensing = B_TRUE; + + mutex_exit(&msp->ms_lock); + space_map_truncate(smo, mos, tx); + mutex_enter(&msp->ms_lock); + + /* + * While we would ideally like to create a space_map representation + * that consists only of allocation records, doing so can be + * prohibitively expensive because the in-core free map can be + * large, and therefore computationally expensive to subtract + * from the condense_map. Instead we sync out two maps, a cheap + * allocation only map followed by the in-core free map. While not + * optimal, this is typically close to optimal, and much cheaper to + * compute. + */ + space_map_sync(&condense_map, SM_ALLOC, smo, mos, tx); + space_map_vacate(&condense_map, NULL, NULL); + space_map_destroy(&condense_map); + + space_map_sync(sm, SM_FREE, smo, mos, tx); + sm->sm_condensing = B_FALSE; + + spa_dbgmsg(spa, "condensed: txg %llu, msp[%llu] %p, " + "smo size %llu", txg, + (msp->ms_map->sm_start / msp->ms_map->sm_size), msp, + smo->smo_objsize); +} + +/* * Write a metaslab to disk in the context of the specified transaction group. */ void @@ -935,17 +1095,29 @@ metaslab_sync(metaslab_t *msp, uint64_t vdev_t *vd = msp->ms_group->mg_vd; spa_t *spa = vd->vdev_spa; objset_t *mos = spa_meta_objset(spa); - space_map_t *allocmap = &msp->ms_allocmap[txg & TXG_MASK]; - space_map_t *freemap = &msp->ms_freemap[txg & TXG_MASK]; - space_map_t *freed_map = &msp->ms_freemap[TXG_CLEAN(txg) & TXG_MASK]; - space_map_t *sm = &msp->ms_map; + space_map_t *allocmap = msp->ms_allocmap[txg & TXG_MASK]; + space_map_t **freemap = &msp->ms_freemap[txg & TXG_MASK]; + space_map_t **freed_map = &msp->ms_freemap[TXG_CLEAN(txg) & TXG_MASK]; + space_map_t *sm = msp->ms_map; space_map_obj_t *smo = &msp->ms_smo_syncing; dmu_buf_t *db; dmu_tx_t *tx; ASSERT(!vd->vdev_ishole); - if (allocmap->sm_space == 0 && freemap->sm_space == 0) + /* + * This metaslab has just been added so there's no work to do now. + */ + if (*freemap == NULL) { + ASSERT3P(allocmap, ==, NULL); + return; + } + + ASSERT3P(allocmap, !=, NULL); + ASSERT3P(*freemap, !=, NULL); + ASSERT3P(*freed_map, !=, NULL); + + if (allocmap->sm_space == 0 && (*freemap)->sm_space == 0) return; /* @@ -973,49 +1145,36 @@ metaslab_sync(metaslab_t *msp, uint64_t mutex_enter(&msp->ms_lock); - space_map_walk(freemap, space_map_add, freed_map); - - if (sm->sm_loaded && spa_sync_pass(spa) == 1 && smo->smo_objsize >= - 2 * sizeof (uint64_t) * avl_numnodes(&sm->sm_root)) { - /* - * The in-core space map representation is twice as compact - * as the on-disk one, so it's time to condense the latter - * by generating a pure allocmap from first principles. - * - * This metaslab is 100% allocated, - * minus the content of the in-core map (sm), - * minus what's been freed this txg (freed_map), - * minus deferred frees (ms_defermap[]), - * minus allocations from txgs in the future - * (because they haven't been committed yet). - */ - space_map_vacate(allocmap, NULL, NULL); - space_map_vacate(freemap, NULL, NULL); - - space_map_add(allocmap, allocmap->sm_start, allocmap->sm_size); - - space_map_walk(sm, space_map_remove, allocmap); - space_map_walk(freed_map, space_map_remove, allocmap); - - for (int t = 0; t < TXG_DEFER_SIZE; t++) - space_map_walk(&msp->ms_defermap[t], - space_map_remove, allocmap); + if (sm->sm_loaded && spa_sync_pass(spa) == 1 && + metaslab_should_condense(msp)) { + metaslab_condense(msp, txg, tx); + } else { + space_map_sync(allocmap, SM_ALLOC, smo, mos, tx); + space_map_sync(*freemap, SM_FREE, smo, mos, tx); + } - for (int t = 1; t < TXG_CONCURRENT_STATES; t++) - space_map_walk(&msp->ms_allocmap[(txg + t) & TXG_MASK], - space_map_remove, allocmap); + space_map_vacate(allocmap, NULL, NULL); - mutex_exit(&msp->ms_lock); - space_map_truncate(smo, mos, tx); - mutex_enter(&msp->ms_lock); + /* + * For sync pass 1, we avoid walking the entire space map and + * instead will just swap the pointers for freemap and + * freed_map. We can safely do this since the freed_map is + * guaranteed to be empty on the initial pass. + */ + if (spa_sync_pass(spa) == 1) { + ASSERT0((*freed_map)->sm_space); + ASSERT0(avl_numnodes(&(*freed_map)->sm_root)); + space_map_swap(freemap, freed_map); + } else { + space_map_vacate(*freemap, space_map_add, *freed_map); } - space_map_sync(allocmap, SM_ALLOC, smo, mos, tx); - space_map_sync(freemap, SM_FREE, smo, mos, tx); + ASSERT0(msp->ms_allocmap[txg & TXG_MASK]->sm_space); + ASSERT0(msp->ms_freemap[txg & TXG_MASK]->sm_space); mutex_exit(&msp->ms_lock); - VERIFY(0 == dmu_bonus_hold(mos, smo->smo_object, FTAG, &db)); + VERIFY0(dmu_bonus_hold(mos, smo->smo_object, FTAG, &db)); dmu_buf_will_dirty(db, tx); ASSERT3U(db->db_size, >=, sizeof (*smo)); bcopy(smo, db->db_data, sizeof (*smo)); @@ -1033,9 +1192,9 @@ metaslab_sync_done(metaslab_t *msp, uint { space_map_obj_t *smo = &msp->ms_smo; space_map_obj_t *smosync = &msp->ms_smo_syncing; - space_map_t *sm = &msp->ms_map; - space_map_t *freed_map = &msp->ms_freemap[TXG_CLEAN(txg) & TXG_MASK]; - space_map_t *defer_map = &msp->ms_defermap[txg % TXG_DEFER_SIZE]; + space_map_t *sm = msp->ms_map; + space_map_t *freed_map = msp->ms_freemap[TXG_CLEAN(txg) & TXG_MASK]; + space_map_t *defer_map = msp->ms_defermap[txg % TXG_DEFER_SIZE]; metaslab_group_t *mg = msp->ms_group; vdev_t *vd = mg->mg_vd; int64_t alloc_delta, defer_delta; @@ -1046,19 +1205,30 @@ metaslab_sync_done(metaslab_t *msp, uint /* * If this metaslab is just becoming available, initialize its - * allocmaps and freemaps and add its capacity to the vdev. + * allocmaps, freemaps, and defermap and add its capacity to the vdev. */ - if (freed_map->sm_size == 0) { + if (freed_map == NULL) { + ASSERT(defer_map == NULL); for (int t = 0; t < TXG_SIZE; t++) { - space_map_create(&msp->ms_allocmap[t], sm->sm_start, + msp->ms_allocmap[t] = kmem_zalloc(sizeof (space_map_t), + KM_SLEEP); + space_map_create(msp->ms_allocmap[t], sm->sm_start, sm->sm_size, sm->sm_shift, sm->sm_lock); - space_map_create(&msp->ms_freemap[t], sm->sm_start, + msp->ms_freemap[t] = kmem_zalloc(sizeof (space_map_t), + KM_SLEEP); + space_map_create(msp->ms_freemap[t], sm->sm_start, sm->sm_size, sm->sm_shift, sm->sm_lock); } - for (int t = 0; t < TXG_DEFER_SIZE; t++) - space_map_create(&msp->ms_defermap[t], sm->sm_start, + for (int t = 0; t < TXG_DEFER_SIZE; t++) { + msp->ms_defermap[t] = kmem_zalloc(sizeof (space_map_t), + KM_SLEEP); + space_map_create(msp->ms_defermap[t], sm->sm_start, sm->sm_size, sm->sm_shift, sm->sm_lock); + } + + freed_map = msp->ms_freemap[TXG_CLEAN(txg) & TXG_MASK]; + defer_map = msp->ms_defermap[txg % TXG_DEFER_SIZE]; vdev_space_update(vd, 0, 0, sm->sm_size); } @@ -1068,8 +1238,8 @@ metaslab_sync_done(metaslab_t *msp, uint vdev_space_update(vd, alloc_delta + defer_delta, defer_delta, 0); - ASSERT(msp->ms_allocmap[txg & TXG_MASK].sm_space == 0); - ASSERT(msp->ms_freemap[txg & TXG_MASK].sm_space == 0); + ASSERT(msp->ms_allocmap[txg & TXG_MASK]->sm_space == 0); + ASSERT(msp->ms_freemap[txg & TXG_MASK]->sm_space == 0); /* * If there's a space_map_load() in progress, wait for it to complete @@ -1103,7 +1273,7 @@ metaslab_sync_done(metaslab_t *msp, uint int evictable = 1; for (int t = 1; t < TXG_CONCURRENT_STATES; t++) - if (msp->ms_allocmap[(txg + t) & TXG_MASK].sm_space) + if (msp->ms_allocmap[(txg + t) & TXG_MASK]->sm_space) evictable = 0; if (evictable && !metaslab_debug) @@ -1128,7 +1298,7 @@ metaslab_sync_reassess(metaslab_group_t for (int m = 0; m < vd->vdev_ms_count; m++) { metaslab_t *msp = vd->vdev_ms[m]; - if (msp->ms_map.sm_start > mg->mg_bonus_area) + if (msp->ms_map->sm_start > mg->mg_bonus_area) break; mutex_enter(&msp->ms_lock); @@ -1149,7 +1319,7 @@ metaslab_distance(metaslab_t *msp, dva_t { uint64_t ms_shift = msp->ms_group->mg_vd->vdev_ms_shift; uint64_t offset = DVA_GET_OFFSET(dva) >> ms_shift; - uint64_t start = msp->ms_map.sm_start >> ms_shift; + uint64_t start = msp->ms_map->sm_start >> ms_shift; if (msp->ms_group->mg_vd->vdev_id != DVA_GET_VDEV(dva)) return (1ULL << 63); @@ -1237,6 +1407,16 @@ metaslab_group_alloc(metaslab_group_t *m mutex_enter(&msp->ms_lock); /* + * If this metaslab is currently condensing then pick again as + * we can't manipulate this metaslab until it's committed + * to disk. + */ + if (msp->ms_map->sm_condensing) { + mutex_exit(&msp->ms_lock); + continue; + } + + /* * Ensure that the metaslab we have selected is still * capable of handling our request. It's possible that * another thread may have changed the weight while we @@ -1262,20 +1442,20 @@ metaslab_group_alloc(metaslab_group_t *m continue; } - if ((offset = space_map_alloc(&msp->ms_map, asize)) != -1ULL) + if ((offset = space_map_alloc(msp->ms_map, asize)) != -1ULL) break; atomic_inc_64(&mg->mg_alloc_failures); - metaslab_passivate(msp, space_map_maxsize(&msp->ms_map)); + metaslab_passivate(msp, space_map_maxsize(msp->ms_map)); mutex_exit(&msp->ms_lock); } - if (msp->ms_allocmap[txg & TXG_MASK].sm_space == 0) + if (msp->ms_allocmap[txg & TXG_MASK]->sm_space == 0) vdev_dirty(mg->mg_vd, VDD_METASLAB, msp, txg); - space_map_add(&msp->ms_allocmap[txg & TXG_MASK], offset, asize); + space_map_add(msp->ms_allocmap[txg & TXG_MASK], offset, asize); mutex_exit(&msp->ms_lock); @@ -1507,13 +1687,13 @@ metaslab_free_dva(spa_t *spa, const dva_ mutex_enter(&msp->ms_lock); if (now) { - space_map_remove(&msp->ms_allocmap[txg & TXG_MASK], + space_map_remove(msp->ms_allocmap[txg & TXG_MASK], offset, size); - space_map_free(&msp->ms_map, offset, size); + space_map_free(msp->ms_map, offset, size); } else { - if (msp->ms_freemap[txg & TXG_MASK].sm_space == 0) + if (msp->ms_freemap[txg & TXG_MASK]->sm_space == 0) vdev_dirty(vd, VDD_METASLAB, msp, txg); - space_map_add(&msp->ms_freemap[txg & TXG_MASK], offset, size); + space_map_add(msp->ms_freemap[txg & TXG_MASK], offset, size); } mutex_exit(&msp->ms_lock); @@ -1548,10 +1728,10 @@ metaslab_claim_dva(spa_t *spa, const dva mutex_enter(&msp->ms_lock); - if ((txg != 0 && spa_writeable(spa)) || !msp->ms_map.sm_loaded) + if ((txg != 0 && spa_writeable(spa)) || !msp->ms_map->sm_loaded) error = metaslab_activate(msp, METASLAB_WEIGHT_SECONDARY); - if (error == 0 && !space_map_contains(&msp->ms_map, offset, size)) + if (error == 0 && !space_map_contains(msp->ms_map, offset, size)) error = ENOENT; if (error || txg == 0) { /* txg == 0 indicates dry run */ @@ -1559,12 +1739,12 @@ metaslab_claim_dva(spa_t *spa, const dva return (error); } - space_map_claim(&msp->ms_map, offset, size); + space_map_claim(msp->ms_map, offset, size); if (spa_writeable(spa)) { /* don't dirty if we're zdb(1M) */ - if (msp->ms_allocmap[txg & TXG_MASK].sm_space == 0) + if (msp->ms_allocmap[txg & TXG_MASK]->sm_space == 0) vdev_dirty(vd, VDD_METASLAB, msp, txg); - space_map_add(&msp->ms_allocmap[txg & TXG_MASK], offset, size); + space_map_add(msp->ms_allocmap[txg & TXG_MASK], offset, size); } mutex_exit(&msp->ms_lock); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/space_map.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/space_map.c Sat Feb 23 08:59:07 2013 (r247175) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/space_map.c Sat Feb 23 09:00:35 2013 (r247176) @@ -107,6 +107,7 @@ space_map_add(space_map_t *sm, uint64_t int merge_before, merge_after; ASSERT(MUTEX_HELD(sm->sm_lock)); + VERIFY(!sm->sm_condensing); VERIFY(size != 0); VERIFY3U(start, >=, sm->sm_start); VERIFY3U(end, <=, sm->sm_start + sm->sm_size); @@ -175,6 +176,7 @@ space_map_remove(space_map_t *sm, uint64 int left_over, right_over; ASSERT(MUTEX_HELD(sm->sm_lock)); + VERIFY(!sm->sm_condensing); VERIFY(size != 0); VERIFY(P2PHASE(start, 1ULL << sm->sm_shift) == 0); VERIFY(P2PHASE(size, 1ULL << sm->sm_shift) == 0); @@ -244,6 +246,20 @@ space_map_contains(space_map_t *sm, uint } void +space_map_swap(space_map_t **msrc, space_map_t **mdst) +{ + space_map_t *sm; + + ASSERT(MUTEX_HELD((*msrc)->sm_lock)); + ASSERT0((*mdst)->sm_space); + ASSERT0(avl_numnodes(&(*mdst)->sm_root)); + + sm = *msrc; + *msrc = *mdst; + *mdst = sm; +} + +void space_map_vacate(space_map_t *sm, space_map_func_t *func, space_map_t *mdest) { space_seg_t *ss; @@ -424,9 +440,9 @@ space_map_sync(space_map_t *sm, uint8_t space_map_obj_t *smo, objset_t *os, dmu_tx_t *tx) { spa_t *spa = dmu_objset_spa(os); - void *cookie = NULL; + avl_tree_t *t = &sm->sm_root; space_seg_t *ss; - uint64_t bufsize, start, size, run_len, delta, sm_space; + uint64_t bufsize, start, size, run_len, total, sm_space, nodes; uint64_t *entry, *entry_map, *entry_map_end; ASSERT(MUTEX_HELD(sm->sm_lock)); @@ -455,13 +471,14 @@ space_map_sync(space_map_t *sm, uint8_t SM_DEBUG_SYNCPASS_ENCODE(spa_sync_pass(spa)) | SM_DEBUG_TXG_ENCODE(dmu_tx_get_txg(tx)); - delta = 0; + total = 0; + nodes = avl_numnodes(&sm->sm_root); sm_space = sm->sm_space; - while ((ss = avl_destroy_nodes(&sm->sm_root, &cookie)) != NULL) { + for (ss = avl_first(t); ss != NULL; ss = AVL_NEXT(t, ss)) { size = ss->ss_end - ss->ss_start; start = (ss->ss_start - sm->sm_start) >> sm->sm_shift; - delta += size; + total += size; size >>= sm->sm_shift; while (size) { @@ -483,7 +500,6 @@ space_map_sync(space_map_t *sm, uint8_t start += run_len; size -= run_len; } - kmem_cache_free(space_seg_cache, ss); } if (entry != entry_map) { @@ -499,12 +515,11 @@ space_map_sync(space_map_t *sm, uint8_t * Ensure that the space_map's accounting wasn't changed * while we were in the middle of writing it out. */ + VERIFY3U(nodes, ==, avl_numnodes(&sm->sm_root)); VERIFY3U(sm->sm_space, ==, sm_space); + VERIFY3U(sm->sm_space, ==, total); zio_buf_free(entry_map, bufsize); - - sm->sm_space -= delta; - VERIFY0(sm->sm_space); } void Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/metaslab_impl.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/metaslab_impl.h Sat Feb 23 08:59:07 2013 (r247175) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/metaslab_impl.h Sat Feb 23 09:00:35 2013 (r247176) @@ -66,20 +66,38 @@ struct metaslab_group { }; /* - * Each metaslab's free space is tracked in space map object in the MOS, - * which is only updated in syncing context. Each time we sync a txg, + * Each metaslab maintains an in-core free map (ms_map) that contains the + * current list of free segments. As blocks are allocated, the allocated + * segment is removed from the ms_map and added to a per txg allocation map. + * As blocks are freed, they are added to the per txg free map. These per + * txg maps allow us to process all allocations and frees in syncing context + * where it is safe to update the on-disk space maps. + * + * Each metaslab's free space is tracked in a space map object in the MOS, + * which is only updated in syncing context. Each time we sync a txg, * we append the allocs and frees from that txg to the space map object. * When the txg is done syncing, metaslab_sync_done() updates ms_smo - * to ms_smo_syncing. Everything in ms_smo is always safe to allocate. + * to ms_smo_syncing. Everything in ms_smo is always safe to allocate. + * + * To load the in-core free map we read the space map object from disk. + * This object contains a series of alloc and free records that are + * combined to make up the list of all free segments in this metaslab. These + * segments are represented in-core by the ms_map and are stored in an + * AVL tree. + * + * As the space map objects grows (as a result of the appends) it will + * eventually become space-inefficient. When the space map object is + * zfs_condense_pct/100 times the size of the minimal on-disk representation, + * we rewrite it in its minimized form. */ struct metaslab { kmutex_t ms_lock; /* metaslab lock */ space_map_obj_t ms_smo; /* synced space map object */ space_map_obj_t ms_smo_syncing; /* syncing space map object */ - space_map_t ms_allocmap[TXG_SIZE]; /* allocated this txg */ - space_map_t ms_freemap[TXG_SIZE]; /* freed this txg */ - space_map_t ms_defermap[TXG_DEFER_SIZE]; /* deferred frees */ - space_map_t ms_map; /* in-core free space map */ + space_map_t *ms_allocmap[TXG_SIZE]; /* allocated this txg */ + space_map_t *ms_freemap[TXG_SIZE]; /* freed this txg */ + space_map_t *ms_defermap[TXG_DEFER_SIZE]; /* deferred frees */ + space_map_t *ms_map; /* in-core free space map */ int64_t ms_deferspace; /* sum of ms_defermap[] space */ uint64_t ms_weight; /* weight vs. others in group */ metaslab_group_t *ms_group; /* metaslab group */ Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/space_map.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/space_map.h Sat Feb 23 08:59:07 2013 (r247175) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/space_map.h Sat Feb 23 09:00:35 2013 (r247176) @@ -40,17 +40,17 @@ extern "C" { typedef struct space_map_ops space_map_ops_t; typedef struct space_map { - avl_tree_t sm_root; /* AVL tree of map segments */ + avl_tree_t sm_root; /* offset-ordered segment AVL tree */ uint64_t sm_space; /* sum of all segments in the map */ uint64_t sm_start; /* start of map */ uint64_t sm_size; /* size of map */ uint8_t sm_shift; /* unit shift */ - uint8_t sm_pad[3]; /* unused */ uint8_t sm_loaded; /* map loaded? */ uint8_t sm_loading; /* map loading? */ + uint8_t sm_condensing; /* map condensing? */ kcondvar_t sm_load_cv; /* map load completion */ space_map_ops_t *sm_ops; /* space map block picker ops vector */ - avl_tree_t *sm_pp_root; /* picker-private AVL tree */ + avl_tree_t *sm_pp_root; /* size-ordered, picker-private tree */ void *sm_ppd; /* picker-private data */ kmutex_t *sm_lock; /* pointer to lock that protects map */ } space_map_t; @@ -149,6 +149,7 @@ extern void space_map_add(space_map_t *s extern void space_map_remove(space_map_t *sm, uint64_t start, uint64_t size); extern boolean_t space_map_contains(space_map_t *sm, uint64_t start, uint64_t size); +extern void space_map_swap(space_map_t **msrc, space_map_t **mdest); extern void space_map_vacate(space_map_t *sm, space_map_func_t *func, space_map_t *mdest); extern void space_map_walk(space_map_t *sm, Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/vdev.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/vdev.c Sat Feb 23 08:59:07 2013 (r247175) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/vdev.c Sat Feb 23 09:00:35 2013 (r247176) @@ -1836,6 +1836,7 @@ vdev_dtl_sync(vdev_t *vd, uint64_t txg) space_map_truncate(smo, mos, tx); space_map_sync(&smsync, SM_ALLOC, smo, mos, tx); + space_map_vacate(&smsync, NULL, NULL); space_map_destroy(&smsync); From owner-svn-src-vendor@FreeBSD.ORG Sat Feb 23 09:00:36 2013 Return-Path: Delivered-To: svn-src-vendor@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by hub.freebsd.org (Postfix) with ESMTP id DE5AAB05; Sat, 23 Feb 2013 09:00:36 +0000 (UTC) (envelope-from mm@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) by mx1.freebsd.org (Postfix) with ESMTP id B6B3822D; Sat, 23 Feb 2013 09:00:36 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.5/8.14.5) with ESMTP id r1N90ai6029603; Sat, 23 Feb 2013 09:00:36 GMT (envelope-from mm@svn.freebsd.org) Received: (from mm@localhost) by svn.freebsd.org (8.14.5/8.14.5/Submit) id r1N90a43029602; Sat, 23 Feb 2013 09:00:36 GMT (envelope-from mm@svn.freebsd.org) Message-Id: <201302230900.r1N90a43029602@svn.freebsd.org> From: Martin Matuska Date: Sat, 23 Feb 2013 09:00:36 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r247176 - vendor-sys/illumos/dist/uts/common/fs/zfs vendor-sys/illumos/dist/uts/common/fs/zfs/sys vendor/illumos/dist/cmd/zdb X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 23 Feb 2013 09:00:36 -0000 Author: mm Date: Sat Feb 23 09:00:35 2013 New Revision: 247176 URL: http://svnweb.freebsd.org/changeset/base/247176 Log: Update vendor/illumos/dist and vendor-sys/illumos/dist to illumos-gate 13959:e03e14ddfb4c Illumos ZFS issues: 3552 condensing one space map burns 3 seconds of CPU in spa_sync() thread 3564 spa_sync() spends 5-10% of its time in metaslab_sync() (when not condensing) Modified: vendor/illumos/dist/cmd/zdb/zdb.c Changes in other areas also in this revision: Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/metaslab.c vendor-sys/illumos/dist/uts/common/fs/zfs/space_map.c vendor-sys/illumos/dist/uts/common/fs/zfs/sys/metaslab_impl.h vendor-sys/illumos/dist/uts/common/fs/zfs/sys/space_map.h vendor-sys/illumos/dist/uts/common/fs/zfs/vdev.c Modified: vendor/illumos/dist/cmd/zdb/zdb.c ============================================================================== --- vendor/illumos/dist/cmd/zdb/zdb.c Sat Feb 23 08:59:07 2013 (r247175) +++ vendor/illumos/dist/cmd/zdb/zdb.c Sat Feb 23 09:00:35 2013 (r247176) @@ -571,7 +571,7 @@ static void dump_metaslab_stats(metaslab_t *msp) { char maxbuf[32]; - space_map_t *sm = &msp->ms_map; + space_map_t *sm = msp->ms_map; avl_tree_t *t = sm->sm_pp_root; int free_pct = sm->sm_space * 100 / sm->sm_size; @@ -587,7 +587,7 @@ dump_metaslab(metaslab_t *msp) { vdev_t *vd = msp->ms_group->mg_vd; spa_t *spa = vd->vdev_spa; - space_map_t *sm = &msp->ms_map; + space_map_t *sm = msp->ms_map; space_map_obj_t *smo = &msp->ms_smo; char freebuf[32]; @@ -2216,11 +2216,11 @@ zdb_leak_init(spa_t *spa, zdb_cb_t *zcb) for (int m = 0; m < vd->vdev_ms_count; m++) { metaslab_t *msp = vd->vdev_ms[m]; mutex_enter(&msp->ms_lock); - space_map_unload(&msp->ms_map); - VERIFY(space_map_load(&msp->ms_map, + space_map_unload(msp->ms_map); + VERIFY(space_map_load(msp->ms_map, &zdb_space_map_ops, SM_ALLOC, &msp->ms_smo, spa->spa_meta_objset) == 0); - msp->ms_map.sm_ppd = vd; + msp->ms_map->sm_ppd = vd; mutex_exit(&msp->ms_lock); } } @@ -2243,7 +2243,7 @@ zdb_leak_fini(spa_t *spa) for (int m = 0; m < vd->vdev_ms_count; m++) { metaslab_t *msp = vd->vdev_ms[m]; mutex_enter(&msp->ms_lock); - space_map_unload(&msp->ms_map); + space_map_unload(msp->ms_map); mutex_exit(&msp->ms_lock); } } From owner-svn-src-vendor@FreeBSD.ORG Sat Feb 23 09:02:47 2013 Return-Path: Delivered-To: svn-src-vendor@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by hub.freebsd.org (Postfix) with ESMTP id 13A9CF05; Sat, 23 Feb 2013 09:02:47 +0000 (UTC) (envelope-from mm@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) by mx1.freebsd.org (Postfix) with ESMTP id 05F2B25C; Sat, 23 Feb 2013 09:02:47 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.5/8.14.5) with ESMTP id r1N92k7W031644; Sat, 23 Feb 2013 09:02:46 GMT (envelope-from mm@svn.freebsd.org) Received: (from mm@localhost) by svn.freebsd.org (8.14.5/8.14.5/Submit) id r1N92kHA031643; Sat, 23 Feb 2013 09:02:46 GMT (envelope-from mm@svn.freebsd.org) Message-Id: <201302230902.r1N92kHA031643@svn.freebsd.org> From: Martin Matuska Date: Sat, 23 Feb 2013 09:02:46 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r247178 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 23 Feb 2013 09:02:47 -0000 Author: mm Date: Sat Feb 23 09:02:46 2013 New Revision: 247178 URL: http://svnweb.freebsd.org/changeset/base/247178 Log: Update vendor-sys/illumos/dist to illumos-gate 13966:0e1d84ebb004 Illumos ZFS issues: 3578 transferring the freed map to the defer map should be constant time 3579 ztest trips assertion in metaslab_weight() Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/metaslab.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/metaslab.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/metaslab.c Sat Feb 23 09:00:44 2013 (r247177) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/metaslab.c Sat Feb 23 09:02:46 2013 (r247178) @@ -814,6 +814,16 @@ metaslab_weight(metaslab_t *msp) ASSERT(MUTEX_HELD(&msp->ms_lock)); /* + * This vdev is in the process of being removed so there is nothing + * for us to do here. + */ + if (vd->vdev_removing) { + ASSERT0(smo->smo_alloc); + ASSERT0(vd->vdev_ms_shift); + return (0); + } + + /* * The baseline weight is the metaslab's free space. */ space = sm->sm_size - smo->smo_alloc; @@ -1193,8 +1203,8 @@ metaslab_sync_done(metaslab_t *msp, uint space_map_obj_t *smo = &msp->ms_smo; space_map_obj_t *smosync = &msp->ms_smo_syncing; space_map_t *sm = msp->ms_map; - space_map_t *freed_map = msp->ms_freemap[TXG_CLEAN(txg) & TXG_MASK]; - space_map_t *defer_map = msp->ms_defermap[txg % TXG_DEFER_SIZE]; + space_map_t **freed_map = &msp->ms_freemap[TXG_CLEAN(txg) & TXG_MASK]; + space_map_t **defer_map = &msp->ms_defermap[txg % TXG_DEFER_SIZE]; metaslab_group_t *mg = msp->ms_group; vdev_t *vd = mg->mg_vd; int64_t alloc_delta, defer_delta; @@ -1207,8 +1217,8 @@ metaslab_sync_done(metaslab_t *msp, uint * If this metaslab is just becoming available, initialize its * allocmaps, freemaps, and defermap and add its capacity to the vdev. */ - if (freed_map == NULL) { - ASSERT(defer_map == NULL); + if (*freed_map == NULL) { + ASSERT(*defer_map == NULL); for (int t = 0; t < TXG_SIZE; t++) { msp->ms_allocmap[t] = kmem_zalloc(sizeof (space_map_t), KM_SLEEP); @@ -1227,14 +1237,14 @@ metaslab_sync_done(metaslab_t *msp, uint sm->sm_size, sm->sm_shift, sm->sm_lock); } - freed_map = msp->ms_freemap[TXG_CLEAN(txg) & TXG_MASK]; - defer_map = msp->ms_defermap[txg % TXG_DEFER_SIZE]; + freed_map = &msp->ms_freemap[TXG_CLEAN(txg) & TXG_MASK]; + defer_map = &msp->ms_defermap[txg % TXG_DEFER_SIZE]; vdev_space_update(vd, 0, 0, sm->sm_size); } alloc_delta = smosync->smo_alloc - smo->smo_alloc; - defer_delta = freed_map->sm_space - defer_map->sm_space; + defer_delta = (*freed_map)->sm_space - (*defer_map)->sm_space; vdev_space_update(vd, alloc_delta + defer_delta, defer_delta, 0); @@ -1244,12 +1254,18 @@ metaslab_sync_done(metaslab_t *msp, uint /* * If there's a space_map_load() in progress, wait for it to complete * so that we have a consistent view of the in-core space map. - * Then, add defer_map (oldest deferred frees) to this map and - * transfer freed_map (this txg's frees) to defer_map. */ space_map_load_wait(sm); - space_map_vacate(defer_map, sm->sm_loaded ? space_map_free : NULL, sm); - space_map_vacate(freed_map, space_map_add, defer_map); + + /* + * Move the frees from the defer_map to this map (if it's loaded). + * Swap the freed_map and the defer_map -- this is safe to do + * because we've just emptied out the defer_map. + */ + space_map_vacate(*defer_map, sm->sm_loaded ? space_map_free : NULL, sm); + ASSERT0((*defer_map)->sm_space); + ASSERT0(avl_numnodes(&(*defer_map)->sm_root)); + space_map_swap(freed_map, defer_map); *smo = *smosync; From owner-svn-src-vendor@FreeBSD.ORG Sat Feb 23 09:06:37 2013 Return-Path: Delivered-To: svn-src-vendor@freebsd.org Received: from mx1.freebsd.org (mx1.FreeBSD.org [8.8.178.115]) by hub.freebsd.org (Postfix) with ESMTP id 48989253; Sat, 23 Feb 2013 09:06:37 +0000 (UTC) (envelope-from mm@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) by mx1.freebsd.org (Postfix) with ESMTP id 2DD55271; Sat, 23 Feb 2013 09:06:37 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.5/8.14.5) with ESMTP id r1N96b1f032418; Sat, 23 Feb 2013 09:06:37 GMT (envelope-from mm@svn.freebsd.org) Received: (from mm@localhost) by svn.freebsd.org (8.14.5/8.14.5/Submit) id r1N96arJ032415; Sat, 23 Feb 2013 09:06:36 GMT (envelope-from mm@svn.freebsd.org) Message-Id: <201302230906.r1N96arJ032415@svn.freebsd.org> From: Martin Matuska Date: Sat, 23 Feb 2013 09:06:36 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r247180 - vendor-sys/illumos/dist/uts/common/fs/zfs vendor-sys/illumos/dist/uts/common/fs/zfs/sys vendor/illumos/dist/cmd/zfs vendor/illumos/dist/lib/libzfs/common X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 23 Feb 2013 09:06:37 -0000 Author: mm Date: Sat Feb 23 09:06:36 2013 New Revision: 247180 URL: http://svnweb.freebsd.org/changeset/base/247180 Log: Update vendor/illumos/dist and vendor-sys/illumos/dist to illumos-gate 13967:92bec6d87f59 Illumos ZFS issues: 3557 dumpvp_size is not updated correctly when a dump zvol's size is changed 3558 setting the volsize on a dump device does not return back ENOSPC 3559 setting a volsize larger than the space available sometimes succeeds Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zvol.h vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ioctl.c vendor-sys/illumos/dist/uts/common/fs/zfs/zvol.c Changes in other areas also in this revision: Modified: vendor/illumos/dist/cmd/zfs/zfs_main.c vendor/illumos/dist/lib/libzfs/common/libzfs_dataset.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zvol.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zvol.h Sat Feb 23 09:02:55 2013 (r247179) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zvol.h Sat Feb 23 09:06:36 2013 (r247180) @@ -43,7 +43,7 @@ extern void zvol_create_cb(objset_t *os, extern int zvol_create_minor(const char *); extern int zvol_remove_minor(const char *); extern void zvol_remove_minors(const char *); -extern int zvol_set_volsize(const char *, major_t, uint64_t); +extern int zvol_set_volsize(const char *, uint64_t); extern int zvol_open(dev_t *devp, int flag, int otyp, cred_t *cr); extern int zvol_dump(dev_t dev, caddr_t addr, daddr_t offset, int nblocks); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ioctl.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ioctl.c Sat Feb 23 09:02:55 2013 (r247179) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ioctl.c Sat Feb 23 09:06:36 2013 (r247180) @@ -2368,8 +2368,7 @@ zfs_prop_set_special(const char *dsname, err = dsl_dataset_set_reservation(dsname, source, intval); break; case ZFS_PROP_VOLSIZE: - err = zvol_set_volsize(dsname, ddi_driver_major(zfs_dip), - intval); + err = zvol_set_volsize(dsname, intval); break; case ZFS_PROP_VERSION: { Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zvol.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zvol.c Sat Feb 23 09:02:55 2013 (r247179) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zvol.c Sat Feb 23 09:06:36 2013 (r247180) @@ -145,10 +145,11 @@ static int zvol_dump_fini(zvol_state_t * static int zvol_dump_init(zvol_state_t *zv, boolean_t resize); static void -zvol_size_changed(uint64_t volsize, major_t maj, minor_t min) +zvol_size_changed(zvol_state_t *zv, uint64_t volsize) { - dev_t dev = makedevice(maj, min); + dev_t dev = makedevice(ddi_driver_major(zfs_dip), zv->zv_minor); + zv->zv_volsize = volsize; VERIFY(ddi_prop_update_int64(dev, zfs_dip, "Size", volsize) == DDI_SUCCESS); VERIFY(ddi_prop_update_int64(dev, zfs_dip, @@ -610,22 +611,22 @@ zvol_first_open(zvol_state_t *zv) if (error) return (error); + zv->zv_objset = os; error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize); if (error) { ASSERT(error == 0); dmu_objset_disown(os, zvol_tag); return (error); } - zv->zv_objset = os; + error = dmu_bonus_hold(os, ZVOL_OBJ, zvol_tag, &zv->zv_dbuf); if (error) { dmu_objset_disown(os, zvol_tag); return (error); } - zv->zv_volsize = volsize; + + zvol_size_changed(zv, volsize); zv->zv_zilog = zil_open(os, zvol_get_data); - zvol_size_changed(zv->zv_volsize, ddi_driver_major(zfs_dip), - zv->zv_minor); VERIFY(dsl_prop_get_integer(zv->zv_name, "readonly", &readonly, NULL) == 0); @@ -747,56 +748,37 @@ zvol_remove_minors(const char *name) mutex_exit(&zfsdev_state_lock); } -int -zvol_set_volsize(const char *name, major_t maj, uint64_t volsize) +static int +zvol_set_volsize_impl(objset_t *os, zvol_state_t *zv, uint64_t volsize) { - zvol_state_t *zv = NULL; - objset_t *os; - int error; - dmu_object_info_t doi; uint64_t old_volsize = 0ULL; - uint64_t readonly; - - mutex_enter(&zfsdev_state_lock); - zv = zvol_minor_lookup(name); - if ((error = dmu_objset_hold(name, FTAG, &os)) != 0) { - mutex_exit(&zfsdev_state_lock); - return (error); - } - - if ((error = dmu_object_info(os, ZVOL_OBJ, &doi)) != 0 || - (error = zvol_check_volsize(volsize, - doi.doi_data_block_size)) != 0) - goto out; - - VERIFY(dsl_prop_get_integer(name, "readonly", &readonly, - NULL) == 0); - if (readonly) { - error = EROFS; - goto out; - } + int error; + ASSERT(MUTEX_HELD(&zfsdev_state_lock)); error = zvol_update_volsize(os, volsize); + /* * Reinitialize the dump area to the new size. If we * failed to resize the dump area then restore it back to - * its original size. + * its original size. We must set the new volsize prior + * to calling dumpvp_resize() to ensure that the devices' + * size(9P) is not visible by the dump subsystem. */ if (zv && error == 0) { + old_volsize = zv->zv_volsize; + zvol_size_changed(zv, volsize); + if (zv->zv_flags & ZVOL_DUMPIFIED) { - old_volsize = zv->zv_volsize; - zv->zv_volsize = volsize; if ((error = zvol_dumpify(zv)) != 0 || (error = dumpvp_resize()) != 0) { + int dumpify_error; + (void) zvol_update_volsize(os, old_volsize); - zv->zv_volsize = old_volsize; - error = zvol_dumpify(zv); + zvol_size_changed(zv, old_volsize); + dumpify_error = zvol_dumpify(zv); + error = dumpify_error ? dumpify_error : error; } } - if (error == 0) { - zv->zv_volsize = volsize; - zvol_size_changed(volsize, maj, zv->zv_minor); - } } /* @@ -819,12 +801,41 @@ zvol_set_volsize(const char *name, major nvlist_free(attr); kmem_free(physpath, MAXPATHLEN); } + return (error); +} +int +zvol_set_volsize(const char *name, uint64_t volsize) +{ + zvol_state_t *zv = NULL; + objset_t *os; + int error; + dmu_object_info_t doi; + uint64_t readonly; + + mutex_enter(&zfsdev_state_lock); + zv = zvol_minor_lookup(name); + if ((error = dmu_objset_hold(name, FTAG, &os)) != 0) { + mutex_exit(&zfsdev_state_lock); + return (error); + } + + if ((error = dmu_object_info(os, ZVOL_OBJ, &doi)) != 0 || + (error = zvol_check_volsize(volsize, + doi.doi_data_block_size)) != 0) + goto out; + + VERIFY3U(dsl_prop_get_integer(name, + zfs_prop_to_name(ZFS_PROP_READONLY), &readonly, NULL), ==, 0); + if (readonly) { + error = EROFS; + goto out; + } + + error = zvol_set_volsize_impl(os, zv, volsize); out: dmu_objset_rele(os, FTAG); - mutex_exit(&zfsdev_state_lock); - return (error); } From owner-svn-src-vendor@FreeBSD.ORG Sat Feb 23 09:06:37 2013 Return-Path: Delivered-To: svn-src-vendor@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by hub.freebsd.org (Postfix) with ESMTP id CBD29254; Sat, 23 Feb 2013 09:06:37 +0000 (UTC) (envelope-from mm@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) by mx1.freebsd.org (Postfix) with ESMTP id B199D272; Sat, 23 Feb 2013 09:06:37 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.5/8.14.5) with ESMTP id r1N96bj7032426; Sat, 23 Feb 2013 09:06:37 GMT (envelope-from mm@svn.freebsd.org) Received: (from mm@localhost) by svn.freebsd.org (8.14.5/8.14.5/Submit) id r1N96bjZ032423; Sat, 23 Feb 2013 09:06:37 GMT (envelope-from mm@svn.freebsd.org) Message-Id: <201302230906.r1N96bjZ032423@svn.freebsd.org> From: Martin Matuska Date: Sat, 23 Feb 2013 09:06:37 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r247180 - vendor-sys/illumos/dist/uts/common/fs/zfs vendor-sys/illumos/dist/uts/common/fs/zfs/sys vendor/illumos/dist/cmd/zfs vendor/illumos/dist/lib/libzfs/common X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 23 Feb 2013 09:06:37 -0000 Author: mm Date: Sat Feb 23 09:06:36 2013 New Revision: 247180 URL: http://svnweb.freebsd.org/changeset/base/247180 Log: Update vendor/illumos/dist and vendor-sys/illumos/dist to illumos-gate 13967:92bec6d87f59 Illumos ZFS issues: 3557 dumpvp_size is not updated correctly when a dump zvol's size is changed 3558 setting the volsize on a dump device does not return back ENOSPC 3559 setting a volsize larger than the space available sometimes succeeds Modified: vendor/illumos/dist/cmd/zfs/zfs_main.c vendor/illumos/dist/lib/libzfs/common/libzfs_dataset.c Changes in other areas also in this revision: Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zvol.h vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ioctl.c vendor-sys/illumos/dist/uts/common/fs/zfs/zvol.c Modified: vendor/illumos/dist/cmd/zfs/zfs_main.c ============================================================================== --- vendor/illumos/dist/cmd/zfs/zfs_main.c Sat Feb 23 09:02:55 2013 (r247179) +++ vendor/illumos/dist/cmd/zfs/zfs_main.c Sat Feb 23 09:06:36 2013 (r247180) @@ -775,10 +775,12 @@ zfs_do_create(int argc, char **argv) if (type == ZFS_TYPE_VOLUME && !noreserve) { zpool_handle_t *zpool_handle; + nvlist_t *real_props; uint64_t spa_version; char *p; zfs_prop_t resv_prop; char *strval; + char msg[1024]; if (p = strchr(argv[0], '/')) *p = '\0'; @@ -794,7 +796,15 @@ zfs_do_create(int argc, char **argv) resv_prop = ZFS_PROP_REFRESERVATION; else resv_prop = ZFS_PROP_RESERVATION; - volsize = zvol_volsize_to_reservation(volsize, props); + + (void) snprintf(msg, sizeof (msg), + gettext("cannot create '%s'"), argv[0]); + if (props && (real_props = zfs_valid_proplist(g_zfs, type, + props, 0, NULL, msg)) == NULL) + goto error; + + volsize = zvol_volsize_to_reservation(volsize, real_props); + nvlist_free(real_props); if (nvlist_lookup_string(props, zfs_prop_to_name(resv_prop), &strval) != 0) { Modified: vendor/illumos/dist/lib/libzfs/common/libzfs_dataset.c ============================================================================== --- vendor/illumos/dist/lib/libzfs/common/libzfs_dataset.c Sat Feb 23 09:02:55 2013 (r247179) +++ vendor/illumos/dist/lib/libzfs/common/libzfs_dataset.c Sat Feb 23 09:06:36 2013 (r247180) @@ -1283,6 +1283,7 @@ zfs_add_synthetic_resv(zfs_handle_t *zhp uint64_t old_reservation; uint64_t new_reservation; zfs_prop_t resv_prop; + nvlist_t *props; /* * If this is an existing volume, and someone is setting the volsize, @@ -1292,16 +1293,25 @@ zfs_add_synthetic_resv(zfs_handle_t *zhp if (zfs_which_resv_prop(zhp, &resv_prop) < 0) return (-1); old_reservation = zfs_prop_get_int(zhp, resv_prop); - if ((zvol_volsize_to_reservation(old_volsize, zhp->zfs_props) != - old_reservation) || nvlist_lookup_uint64(nvl, - zfs_prop_to_name(resv_prop), &new_reservation) != ENOENT) { + + props = fnvlist_alloc(); + fnvlist_add_uint64(props, zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE), + zfs_prop_get_int(zhp, ZFS_PROP_VOLBLOCKSIZE)); + + if ((zvol_volsize_to_reservation(old_volsize, props) != + old_reservation) || nvlist_exists(nvl, + zfs_prop_to_name(resv_prop))) { + fnvlist_free(props); return (0); } if (nvlist_lookup_uint64(nvl, zfs_prop_to_name(ZFS_PROP_VOLSIZE), - &new_volsize) != 0) + &new_volsize) != 0) { + fnvlist_free(props); return (-1); - new_reservation = zvol_volsize_to_reservation(new_volsize, - zhp->zfs_props); + } + new_reservation = zvol_volsize_to_reservation(new_volsize, props); + fnvlist_free(props); + if (nvlist_add_uint64(nvl, zfs_prop_to_name(resv_prop), new_reservation) != 0) { (void) no_memory(zhp->zfs_hdl);