Date: Sat, 24 Aug 2013 08:06:57 +0000 (UTC) From: Alexander Motin <mav@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r254763 - projects/camlock/sys/kern Message-ID: <201308240806.r7O86vTL032134@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: mav Date: Sat Aug 24 08:06:57 2013 New Revision: 254763 URL: http://svnweb.freebsd.org/changeset/base/254763 Log: Move tq_enqueue() call out of the queue lock for known handlers (actually I have found no others in the base system). This reduces queue lock hold time and congestion spinning under active multithreaded enqueuing. According to hwpmc, for ZFS benchmark doing 100K disk IOPS this moved 10% of CPU time spent on lock spinning from this lock to UMA locks. Hope that side also can be addressed somehow sometimes. Modified: projects/camlock/sys/kern/subr_taskqueue.c Modified: projects/camlock/sys/kern/subr_taskqueue.c ============================================================================== --- projects/camlock/sys/kern/subr_taskqueue.c Sat Aug 24 07:19:57 2013 (r254762) +++ projects/camlock/sys/kern/subr_taskqueue.c Sat Aug 24 08:06:57 2013 (r254763) @@ -46,6 +46,9 @@ __FBSDID("$FreeBSD$"); static MALLOC_DEFINE(M_TASKQUEUE, "taskqueue", "Task Queues"); static void *taskqueue_giant_ih; static void *taskqueue_ih; +static void taskqueue_fast_enqueue(void *); +static void taskqueue_swi_enqueue(void *); +static void taskqueue_swi_giant_enqueue(void *); struct taskqueue_busy { struct task *tb_running; @@ -69,6 +72,7 @@ struct taskqueue { #define TQ_FLAGS_ACTIVE (1 << 0) #define TQ_FLAGS_BLOCKED (1 << 1) +#define TQ_FLAGS_UNLOCKED_ENQUEUE (1 << 2) #define DT_CALLOUT_ARMED (1 << 0) @@ -96,7 +100,8 @@ _timeout_task_init(struct taskqueue *que { TASK_INIT(&timeout_task->t, priority, func, context); - callout_init_mtx(&timeout_task->c, &queue->tq_mutex, 0); + callout_init_mtx(&timeout_task->c, &queue->tq_mutex, + CALLOUT_RETURNUNLOCKED); timeout_task->q = queue; timeout_task->f = 0; } @@ -127,6 +132,11 @@ _taskqueue_create(const char *name __unu queue->tq_context = context; queue->tq_spin = (mtxflags & MTX_SPIN) != 0; queue->tq_flags |= TQ_FLAGS_ACTIVE; + if (enqueue == taskqueue_fast_enqueue || + enqueue == taskqueue_swi_enqueue || + enqueue == taskqueue_swi_giant_enqueue || + enqueue == taskqueue_thread_enqueue) + queue->tq_flags |= TQ_FLAGS_UNLOCKED_ENQUEUE; mtx_init(&queue->tq_mutex, mtxname, NULL, mtxflags); return queue; @@ -196,6 +206,7 @@ taskqueue_enqueue_locked(struct taskqueu if (task->ta_pending) { if (task->ta_pending < USHRT_MAX) task->ta_pending++; + TQ_UNLOCK(queue); return (0); } @@ -219,8 +230,12 @@ taskqueue_enqueue_locked(struct taskqueu } task->ta_pending = 1; + if ((queue->tq_flags & TQ_FLAGS_UNLOCKED_ENQUEUE) != 0) + TQ_UNLOCK(queue); if ((queue->tq_flags & TQ_FLAGS_BLOCKED) == 0) queue->tq_enqueue(queue->tq_context); + if ((queue->tq_flags & TQ_FLAGS_UNLOCKED_ENQUEUE) == 0) + TQ_UNLOCK(queue); return (0); } @@ -231,7 +246,6 @@ taskqueue_enqueue(struct taskqueue *queu TQ_LOCK(queue); res = taskqueue_enqueue_locked(queue, task); - TQ_UNLOCK(queue); return (res); } @@ -277,8 +291,8 @@ taskqueue_enqueue_timeout(struct taskque callout_reset(&timeout_task->c, ticks, taskqueue_timeout_func, timeout_task); } + TQ_UNLOCK(queue); } - TQ_UNLOCK(queue); return (res); } @@ -563,7 +577,6 @@ taskqueue_thread_enqueue(void *context) tqp = context; tq = *tqp; - TQ_ASSERT_LOCKED(tq); wakeup_one(tq); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201308240806.r7O86vTL032134>