Skip site navigation (1)Skip section navigation (2) 
Date:      Sun, 20 Mar 2016 14:37:37 +0000 (UTC)
From:      Alexander Motin <mav@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org
Subject:   svn commit: r297066 - stable/10/sys/kern
Message-ID:  <201603201437.u2KEbb13046504@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help 
Author: mav
Date: Sun Mar 20 14:37:37 2016
New Revision: 297066
URL: https://svnweb.freebsd.org/changeset/base/297066

Log:
  MFC r256613, r256862: MFprojects/camlock r254763:
  Move tq_enqueue() call out of the queue lock for known handlers (actually
  I have found no others in the base system).  This reduces queue lock hold
  time and congestion spinning under active multithreaded enqueuing.

Modified:
  stable/10/sys/kern/subr_taskqueue.c
Directory Properties:
  stable/10/   (props changed)

Modified: stable/10/sys/kern/subr_taskqueue.c
==============================================================================
--- stable/10/sys/kern/subr_taskqueue.c	Sun Mar 20 14:21:07 2016	(r297065)
+++ stable/10/sys/kern/subr_taskqueue.c	Sun Mar 20 14:37:37 2016	(r297066)
@@ -46,6 +46,9 @@ __FBSDID("$FreeBSD$");
 static MALLOC_DEFINE(M_TASKQUEUE, "taskqueue", "Task Queues");
 static void	*taskqueue_giant_ih;
 static void	*taskqueue_ih;
+static void	 taskqueue_fast_enqueue(void *);
+static void	 taskqueue_swi_enqueue(void *);
+static void	 taskqueue_swi_giant_enqueue(void *);
 
 struct taskqueue_busy {
 	struct task	*tb_running;
@@ -69,6 +72,7 @@ struct taskqueue {
 
 #define	TQ_FLAGS_ACTIVE		(1 << 0)
 #define	TQ_FLAGS_BLOCKED	(1 << 1)
+#define	TQ_FLAGS_UNLOCKED_ENQUEUE	(1 << 2)
 
 #define	DT_CALLOUT_ARMED	(1 << 0)
 
@@ -96,7 +100,8 @@ _timeout_task_init(struct taskqueue *que
 {
 
 	TASK_INIT(&timeout_task->t, priority, func, context);
-	callout_init_mtx(&timeout_task->c, &queue->tq_mutex, 0);
+	callout_init_mtx(&timeout_task->c, &queue->tq_mutex,
+	    CALLOUT_RETURNUNLOCKED);
 	timeout_task->q = queue;
 	timeout_task->f = 0;
 }
@@ -127,6 +132,11 @@ _taskqueue_create(const char *name __unu
 	queue->tq_context = context;
 	queue->tq_spin = (mtxflags & MTX_SPIN) != 0;
 	queue->tq_flags |= TQ_FLAGS_ACTIVE;
+	if (enqueue == taskqueue_fast_enqueue ||
+	    enqueue == taskqueue_swi_enqueue ||
+	    enqueue == taskqueue_swi_giant_enqueue ||
+	    enqueue == taskqueue_thread_enqueue)
+		queue->tq_flags |= TQ_FLAGS_UNLOCKED_ENQUEUE;
 	mtx_init(&queue->tq_mutex, mtxname, NULL, mtxflags);
 
 	return queue;
@@ -196,6 +206,7 @@ taskqueue_enqueue_locked(struct taskqueu
 	if (task->ta_pending) {
 		if (task->ta_pending < USHRT_MAX)
 			task->ta_pending++;
+		TQ_UNLOCK(queue);
 		return (0);
 	}
 
@@ -219,9 +230,14 @@ taskqueue_enqueue_locked(struct taskqueu
 	}
 
 	task->ta_pending = 1;
+	if ((queue->tq_flags & TQ_FLAGS_UNLOCKED_ENQUEUE) != 0)
+		TQ_UNLOCK(queue);
 	if ((queue->tq_flags & TQ_FLAGS_BLOCKED) == 0)
 		queue->tq_enqueue(queue->tq_context);
+	if ((queue->tq_flags & TQ_FLAGS_UNLOCKED_ENQUEUE) == 0)
+		TQ_UNLOCK(queue);
 
+	/* Return with lock released. */
 	return (0);
 }
 int
@@ -231,7 +247,7 @@ taskqueue_enqueue(struct taskqueue *queu
 
 	TQ_LOCK(queue);
 	res = taskqueue_enqueue_locked(queue, task);
-	TQ_UNLOCK(queue);
+	/* The lock is released inside. */
 
 	return (res);
 }
@@ -248,6 +264,7 @@ taskqueue_timeout_func(void *arg)
 	timeout_task->f &= ~DT_CALLOUT_ARMED;
 	queue->tq_callouts--;
 	taskqueue_enqueue_locked(timeout_task->q, &timeout_task->t);
+	/* The lock is released inside. */
 }
 
 int
@@ -264,6 +281,7 @@ taskqueue_enqueue_timeout(struct taskque
 	res = timeout_task->t.ta_pending;
 	if (ticks == 0) {
 		taskqueue_enqueue_locked(queue, &timeout_task->t);
+		/* The lock is released inside. */
 	} else {
 		if ((timeout_task->f & DT_CALLOUT_ARMED) != 0) {
 			res++;
@@ -277,8 +295,8 @@ taskqueue_enqueue_timeout(struct taskque
 			callout_reset(&timeout_task->c, ticks,
 			    taskqueue_timeout_func, timeout_task);
 		}
+		TQ_UNLOCK(queue);
 	}
-	TQ_UNLOCK(queue);
 	return (res);
 }
 
@@ -591,7 +609,6 @@ taskqueue_thread_enqueue(void *context)
 	tqp = context;
 	tq = *tqp;
 
-	TQ_ASSERT_LOCKED(tq);
 	wakeup_one(tq);
 }

Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201603201437.u2KEbb13046504>