From owner-svn-src-all@FreeBSD.ORG Thu Jan 16 14:34:55 2014 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id 56263BE7; Thu, 16 Jan 2014 14:34:55 +0000 (UTC) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mx1.freebsd.org (Postfix) with ESMTPS id 3E71912A7; Thu, 16 Jan 2014 14:34:55 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.7/8.14.7) with ESMTP id s0GEYtHb036580; Thu, 16 Jan 2014 14:34:55 GMT (envelope-from avg@svn.freebsd.org) Received: (from avg@localhost) by svn.freebsd.org (8.14.7/8.14.7/Submit) id s0GEYr62036568; Thu, 16 Jan 2014 14:34:53 GMT (envelope-from avg@svn.freebsd.org) Message-Id: <201401161434.s0GEYr62036568@svn.freebsd.org> From: Andriy Gapon Date: Thu, 16 Jan 2014 14:34:53 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org Subject: svn commit: r260742 - in stable/10: cddl/contrib/opensolaris/lib/libzpool/common cddl/contrib/opensolaris/lib/libzpool/common/sys sys/cddl/compat/opensolaris/kern sys/cddl/compat/opensolaris/sys sy... X-SVN-Group: stable-10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.17 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 16 Jan 2014 14:34:55 -0000 Author: avg Date: Thu Jan 16 14:34:53 2014 New Revision: 260742 URL: http://svnweb.freebsd.org/changeset/base/260742 Log: MFC r258630: 734 taskq_dispatch_prealloc() desired Deleted: stable/10/sys/cddl/compat/opensolaris/sys/taskq.h Modified: stable/10/cddl/contrib/opensolaris/lib/libzpool/common/sys/zfs_context.h stable/10/cddl/contrib/opensolaris/lib/libzpool/common/taskq.c stable/10/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c stable/10/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c stable/10/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h stable/10/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c stable/10/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h Directory Properties: stable/10/ (props changed) Modified: stable/10/cddl/contrib/opensolaris/lib/libzpool/common/sys/zfs_context.h ============================================================================== --- stable/10/cddl/contrib/opensolaris/lib/libzpool/common/sys/zfs_context.h Thu Jan 16 14:30:46 2014 (r260741) +++ stable/10/cddl/contrib/opensolaris/lib/libzpool/common/sys/zfs_context.h Thu Jan 16 14:34:53 2014 (r260742) @@ -23,6 +23,9 @@ * Copyright (c) 2013 by Delphix. All rights reserved. * Copyright (c) 2012, Joyent, Inc. All rights reserved. */ +/* + * Copyright 2011 Nexenta Systems, Inc. All rights reserved. + */ #ifndef _SYS_ZFS_CONTEXT_H #define _SYS_ZFS_CONTEXT_H @@ -367,6 +370,16 @@ typedef struct taskq taskq_t; typedef uintptr_t taskqid_t; typedef void (task_func_t)(void *); +typedef struct taskq_ent { + struct taskq_ent *tqent_next; + struct taskq_ent *tqent_prev; + task_func_t *tqent_func; + void *tqent_arg; + uintptr_t tqent_flags; +} taskq_ent_t; + +#define TQENT_FLAG_PREALLOC 0x1 /* taskq_dispatch_ent used */ + #define TASKQ_PREPOPULATE 0x0001 #define TASKQ_CPR_SAFE 0x0002 /* Use CPR safe protocol */ #define TASKQ_DYNAMIC 0x0004 /* Use dynamic thread scheduling */ @@ -378,6 +391,7 @@ typedef void (task_func_t)(void *); #define TQ_NOQUEUE 0x02 /* Do not enqueue if can't dispatch */ #define TQ_FRONT 0x08 /* Queue in front */ + extern taskq_t *system_taskq; extern taskq_t *taskq_create(const char *, int, pri_t, int, int, uint_t); @@ -386,6 +400,8 @@ extern taskq_t *taskq_create(const char #define taskq_create_sysdc(a, b, d, e, p, dc, f) \ (taskq_create(a, b, maxclsyspri, d, e, f)) extern taskqid_t taskq_dispatch(taskq_t *, task_func_t, void *, uint_t); +extern void taskq_dispatch_ent(taskq_t *, task_func_t, void *, uint_t, + taskq_ent_t *); extern void taskq_destroy(taskq_t *); extern void taskq_wait(taskq_t *); extern int taskq_member(taskq_t *, void *); Modified: stable/10/cddl/contrib/opensolaris/lib/libzpool/common/taskq.c ============================================================================== --- stable/10/cddl/contrib/opensolaris/lib/libzpool/common/taskq.c Thu Jan 16 14:30:46 2014 (r260741) +++ stable/10/cddl/contrib/opensolaris/lib/libzpool/common/taskq.c Thu Jan 16 14:34:53 2014 (r260742) @@ -22,19 +22,15 @@ * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ +/* + * Copyright 2011 Nexenta Systems, Inc. All rights reserved. + */ #include int taskq_now; taskq_t *system_taskq; -typedef struct task { - struct task *task_next; - struct task *task_prev; - task_func_t *task_func; - void *task_arg; -} task_t; - #define TASKQ_ACTIVE 0x00010000 struct taskq { @@ -51,18 +47,18 @@ struct taskq { int tq_maxalloc; kcondvar_t tq_maxalloc_cv; int tq_maxalloc_wait; - task_t *tq_freelist; - task_t tq_task; + taskq_ent_t *tq_freelist; + taskq_ent_t tq_task; }; -static task_t * +static taskq_ent_t * task_alloc(taskq_t *tq, int tqflags) { - task_t *t; + taskq_ent_t *t; int rv; again: if ((t = tq->tq_freelist) != NULL && tq->tq_nalloc >= tq->tq_minalloc) { - tq->tq_freelist = t->task_next; + tq->tq_freelist = t->tqent_next; } else { if (tq->tq_nalloc >= tq->tq_maxalloc) { if (!(tqflags & KM_SLEEP)) @@ -87,7 +83,7 @@ again: if ((t = tq->tq_freelist) != NULL } mutex_exit(&tq->tq_lock); - t = kmem_alloc(sizeof (task_t), tqflags & KM_SLEEP); + t = kmem_alloc(sizeof (taskq_ent_t), tqflags & KM_SLEEP); mutex_enter(&tq->tq_lock); if (t != NULL) @@ -97,15 +93,15 @@ again: if ((t = tq->tq_freelist) != NULL } static void -task_free(taskq_t *tq, task_t *t) +task_free(taskq_t *tq, taskq_ent_t *t) { if (tq->tq_nalloc <= tq->tq_minalloc) { - t->task_next = tq->tq_freelist; + t->tqent_next = tq->tq_freelist; tq->tq_freelist = t; } else { tq->tq_nalloc--; mutex_exit(&tq->tq_lock); - kmem_free(t, sizeof (task_t)); + kmem_free(t, sizeof (taskq_ent_t)); mutex_enter(&tq->tq_lock); } @@ -116,7 +112,7 @@ task_free(taskq_t *tq, task_t *t) taskqid_t taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t tqflags) { - task_t *t; + taskq_ent_t *t; if (taskq_now) { func(arg); @@ -130,26 +126,58 @@ taskq_dispatch(taskq_t *tq, task_func_t return (0); } if (tqflags & TQ_FRONT) { - t->task_next = tq->tq_task.task_next; - t->task_prev = &tq->tq_task; + t->tqent_next = tq->tq_task.tqent_next; + t->tqent_prev = &tq->tq_task; } else { - t->task_next = &tq->tq_task; - t->task_prev = tq->tq_task.task_prev; + t->tqent_next = &tq->tq_task; + t->tqent_prev = tq->tq_task.tqent_prev; } - t->task_next->task_prev = t; - t->task_prev->task_next = t; - t->task_func = func; - t->task_arg = arg; + t->tqent_next->tqent_prev = t; + t->tqent_prev->tqent_next = t; + t->tqent_func = func; + t->tqent_arg = arg; cv_signal(&tq->tq_dispatch_cv); mutex_exit(&tq->tq_lock); return (1); } void +taskq_dispatch_ent(taskq_t *tq, task_func_t func, void *arg, uint_t flags, + taskq_ent_t *t) +{ + ASSERT(func != NULL); + ASSERT(!(tq->tq_flags & TASKQ_DYNAMIC)); + + /* + * Mark it as a prealloc'd task. This is important + * to ensure that we don't free it later. + */ + t->tqent_flags |= TQENT_FLAG_PREALLOC; + /* + * Enqueue the task to the underlying queue. + */ + mutex_enter(&tq->tq_lock); + + if (flags & TQ_FRONT) { + t->tqent_next = tq->tq_task.tqent_next; + t->tqent_prev = &tq->tq_task; + } else { + t->tqent_next = &tq->tq_task; + t->tqent_prev = tq->tq_task.tqent_prev; + } + t->tqent_next->tqent_prev = t; + t->tqent_prev->tqent_next = t; + t->tqent_func = func; + t->tqent_arg = arg; + cv_signal(&tq->tq_dispatch_cv); + mutex_exit(&tq->tq_lock); +} + +void taskq_wait(taskq_t *tq) { mutex_enter(&tq->tq_lock); - while (tq->tq_task.task_next != &tq->tq_task || tq->tq_active != 0) + while (tq->tq_task.tqent_next != &tq->tq_task || tq->tq_active != 0) cv_wait(&tq->tq_wait_cv, &tq->tq_lock); mutex_exit(&tq->tq_lock); } @@ -158,27 +186,32 @@ static void * taskq_thread(void *arg) { taskq_t *tq = arg; - task_t *t; + taskq_ent_t *t; + boolean_t prealloc; mutex_enter(&tq->tq_lock); while (tq->tq_flags & TASKQ_ACTIVE) { - if ((t = tq->tq_task.task_next) == &tq->tq_task) { + if ((t = tq->tq_task.tqent_next) == &tq->tq_task) { if (--tq->tq_active == 0) cv_broadcast(&tq->tq_wait_cv); cv_wait(&tq->tq_dispatch_cv, &tq->tq_lock); tq->tq_active++; continue; } - t->task_prev->task_next = t->task_next; - t->task_next->task_prev = t->task_prev; + t->tqent_prev->tqent_next = t->tqent_next; + t->tqent_next->tqent_prev = t->tqent_prev; + t->tqent_next = NULL; + t->tqent_prev = NULL; + prealloc = t->tqent_flags & TQENT_FLAG_PREALLOC; mutex_exit(&tq->tq_lock); rw_enter(&tq->tq_threadlock, RW_READER); - t->task_func(t->task_arg); + t->tqent_func(t->tqent_arg); rw_exit(&tq->tq_threadlock); mutex_enter(&tq->tq_lock); - task_free(tq, t); + if (!prealloc) + task_free(tq, t); } tq->tq_nthreads--; cv_broadcast(&tq->tq_wait_cv); @@ -217,8 +250,8 @@ taskq_create(const char *name, int nthre tq->tq_nthreads = nthreads; tq->tq_minalloc = minalloc; tq->tq_maxalloc = maxalloc; - tq->tq_task.task_next = &tq->tq_task; - tq->tq_task.task_prev = &tq->tq_task; + tq->tq_task.tqent_next = &tq->tq_task; + tq->tq_task.tqent_prev = &tq->tq_task; tq->tq_threadlist = kmem_alloc(nthreads * sizeof (thread_t), KM_SLEEP); if (flags & TASKQ_PREPOPULATE) { Modified: stable/10/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c ============================================================================== --- stable/10/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c Thu Jan 16 14:30:46 2014 (r260741) +++ stable/10/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c Thu Jan 16 14:34:53 2014 (r260742) @@ -46,7 +46,7 @@ static void system_taskq_init(void *arg) { - taskq_zone = uma_zcreate("taskq_zone", sizeof(struct ostask), + taskq_zone = uma_zcreate("taskq_zone", sizeof(taskq_ent_t), NULL, NULL, NULL, NULL, 0, 0); system_taskq = taskq_create("system_taskq", mp_ncpus, 0, 0, 0, 0); } @@ -104,9 +104,9 @@ taskq_member(taskq_t *tq, kthread_t *thr static void taskq_run(void *arg, int pending __unused) { - struct ostask *task = arg; + taskq_ent_t *task = arg; - task->ost_func(task->ost_arg); + task->tqent_func(task->tqent_arg); uma_zfree(taskq_zone, task); } @@ -114,7 +114,7 @@ taskq_run(void *arg, int pending __unuse taskqid_t taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags) { - struct ostask *task; + taskq_ent_t *task; int mflag, prio; if ((flags & (TQ_SLEEP | TQ_NOQUEUE)) == TQ_SLEEP) @@ -131,26 +131,26 @@ taskq_dispatch(taskq_t *tq, task_func_t if (task == NULL) return (0); - task->ost_func = func; - task->ost_arg = arg; + task->tqent_func = func; + task->tqent_arg = arg; - TASK_INIT(&task->ost_task, prio, taskq_run, task); - taskqueue_enqueue(tq->tq_queue, &task->ost_task); + TASK_INIT(&task->tqent_task, prio, taskq_run, task); + taskqueue_enqueue(tq->tq_queue, &task->tqent_task); return ((taskqid_t)(void *)task); } static void -taskq_run_safe(void *arg, int pending __unused) +taskq_run_ent(void *arg, int pending __unused) { - struct ostask *task = arg; + taskq_ent_t *task = arg; - task->ost_func(task->ost_arg); + task->tqent_func(task->tqent_arg); } -taskqid_t -taskq_dispatch_safe(taskq_t *tq, task_func_t func, void *arg, u_int flags, - struct ostask *task) +void +taskq_dispatch_ent(taskq_t *tq, task_func_t func, void *arg, u_int flags, + taskq_ent_t *task) { int prio; @@ -160,11 +160,9 @@ taskq_dispatch_safe(taskq_t *tq, task_fu */ prio = !!(flags & TQ_FRONT); - task->ost_func = func; - task->ost_arg = arg; + task->tqent_func = func; + task->tqent_arg = arg; - TASK_INIT(&task->ost_task, prio, taskq_run_safe, task); - taskqueue_enqueue(tq->tq_queue, &task->ost_task); - - return ((taskqid_t)(void *)task); + TASK_INIT(&task->tqent_task, prio, taskq_run_ent, task); + taskqueue_enqueue(tq->tq_queue, &task->tqent_task); } Modified: stable/10/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c ============================================================================== --- stable/10/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c Thu Jan 16 14:30:46 2014 (r260741) +++ stable/10/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c Thu Jan 16 14:34:53 2014 (r260742) @@ -825,7 +825,7 @@ static taskq_t * spa_taskq_create(spa_t *spa, const char *name, enum zti_modes mode, uint_t value) { - uint_t flags = TASKQ_PREPOPULATE; + uint_t flags = 0; boolean_t batch = B_FALSE; switch (mode) { Modified: stable/10/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h ============================================================================== --- stable/10/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h Thu Jan 16 14:30:46 2014 (r260741) +++ stable/10/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h Thu Jan 16 14:34:53 2014 (r260742) @@ -24,6 +24,7 @@ * Copyright (c) 2012 by Delphix. All rights reserved. * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. * Copyright (c) 2013, Joyent, Inc. All rights reserved. + * Copyright 2011 Nexenta Systems, Inc. All rights reserved. */ #ifndef _ZIO_H @@ -474,10 +475,9 @@ struct zio { zio_cksum_report_t *io_cksum_report; uint64_t io_ena; -#ifdef _KERNEL - /* FreeBSD only. */ - struct ostask io_task; -#endif + /* Taskq dispatching state */ + taskq_ent_t io_tqent; + avl_node_t io_trim_node; list_node_t io_trim_link; }; Modified: stable/10/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c ============================================================================== --- stable/10/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c Thu Jan 16 14:30:46 2014 (r260741) +++ stable/10/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c Thu Jan 16 14:34:53 2014 (r260742) @@ -21,6 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2013 by Delphix. All rights reserved. + * Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved. */ #include @@ -1224,7 +1225,7 @@ zio_taskq_dispatch(zio_t *zio, enum zio_ { spa_t *spa = zio->io_spa; zio_type_t t = zio->io_type; - int flags = TQ_SLEEP | (cutinline ? TQ_FRONT : 0); + int flags = (cutinline ? TQ_FRONT : 0); ASSERT(q == ZIO_TASKQ_ISSUE || q == ZIO_TASKQ_INTERRUPT); @@ -1250,13 +1251,19 @@ zio_taskq_dispatch(zio_t *zio, enum zio_ q++; ASSERT3U(q, <, ZIO_TASKQ_TYPES); -#ifdef _KERNEL - (void) taskq_dispatch_safe(spa->spa_zio_taskq[t][q], - (task_func_t *)zio_execute, zio, flags, &zio->io_task); + + /* + * NB: We are assuming that the zio can only be dispatched + * to a single taskq at a time. It would be a grievous error + * to dispatch the zio to another taskq at the same time. + */ +#if defined(illumos) || !defined(_KERNEL) + ASSERT(zio->io_tqent.tqent_next == NULL); #else - (void) taskq_dispatch(spa->spa_zio_taskq[t][q], - (task_func_t *)zio_execute, zio, flags); + ASSERT(zio->io_tqent.tqent_task.ta_pending == 0); #endif + taskq_dispatch_ent(spa->spa_zio_taskq[t][q], + (task_func_t *)zio_execute, zio, flags, &zio->io_tqent); } static boolean_t @@ -3174,16 +3181,15 @@ zio_done(zio_t *zio) * Reexecution is potentially a huge amount of work. * Hand it off to the otherwise-unused claim taskq. */ -#ifdef _KERNEL - (void) taskq_dispatch_safe( - spa->spa_zio_taskq[ZIO_TYPE_CLAIM][ZIO_TASKQ_ISSUE], - (task_func_t *)zio_reexecute, zio, TQ_SLEEP, - &zio->io_task); +#if defined(illumos) || !defined(_KERNEL) + ASSERT(zio->io_tqent.tqent_next == NULL); #else - (void) taskq_dispatch( - spa->spa_zio_taskq[ZIO_TYPE_CLAIM][ZIO_TASKQ_ISSUE], - (task_func_t *)zio_reexecute, zio, TQ_SLEEP); + ASSERT(zio->io_tqent.tqent_task.ta_pending == 0); #endif + (void) taskq_dispatch_ent( + spa->spa_zio_taskq[ZIO_TYPE_CLAIM][ZIO_TASKQ_ISSUE], + (task_func_t *)zio_reexecute, zio, 0, + &zio->io_tqent); } return (ZIO_PIPELINE_STOP); } Modified: stable/10/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h ============================================================================== --- stable/10/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h Thu Jan 16 14:30:46 2014 (r260741) +++ stable/10/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h Thu Jan 16 14:34:53 2014 (r260742) @@ -45,6 +45,12 @@ typedef struct taskq taskq_t; typedef uintptr_t taskqid_t; typedef void (task_func_t)(void *); +typedef struct taskq_ent { + struct task tqent_task; + task_func_t *tqent_func; + void *tqent_arg; +} taskq_ent_t; + struct proc; /* @@ -80,6 +86,8 @@ taskq_t *taskq_create_proc(const char *, taskq_t *taskq_create_sysdc(const char *, int, int, int, struct proc *, uint_t, uint_t); taskqid_t taskq_dispatch(taskq_t *, task_func_t, void *, uint_t); +void taskq_dispatch_ent(taskq_t *, task_func_t, void *, uint_t, + taskq_ent_t *); void nulltask(void *); void taskq_destroy(taskq_t *); void taskq_wait(taskq_t *);