Skip site navigation (1)Skip section navigation (2)
Date:      Sat, 2 Oct 2004 20:31:20 +0200
From:      Peter Holm <peter@holm.cc>
To:        Julian Elischer <julian@elischer.org>
Cc:        "freebsd-arch@freebsd.org" <freebsd-arch@freebsd.org>
Subject:   Re: scheduler (sched_4bsd) questions
Message-ID:  <20041002183120.GA1202@peter.osted.lan>
In-Reply-To: <415EEFFE.5080309@elischer.org>
References:  <1096489576.3733.1868.camel@palm.tree.com> <200409291652.29990.jhb@FreeBSD.org> <1096496057.3733.2163.camel@palm.tree.com> <1096603981.21577.195.camel@palm.tree.com> <1096608201.21577.203.camel@palm.tree.com> <20041001141040.GA1556@peter.osted.lan> <1096647194.27811.12.camel@palm.tree.com> <20041001192551.GA3381@peter.osted.lan> <20041002053351.GA6259@peter.osted.lan> <415EEFFE.5080309@elischer.org>

next in thread | previous in thread | raw e-mail | index | archive | help

--T4sUOijqQbZv57TR
Content-Type: text/plain; charset=iso-8859-1
Content-Disposition: inline
Content-Transfer-Encoding: 8bit

On Sat, Oct 02, 2004 at 11:14:22AM -0700, Julian Elischer wrote:
> Peter Holm wrote:
> >On Fri, Oct 01, 2004 at 09:25:51PM +0200, Peter Holm wrote:
> >
> >>>>For once I'm the bearer of good news. The switch_patch_v2 + the
> >>>>sched_4bsd patch ran the tests for more than one hour without
> >>>>any freeze. The sched_4bsd alone did not stop the freezes. I'm
> >>>>now testing the switch_patch_v2 alone and it's looking good for
> >>>>55+ minutes of testing.
> >>>
> >>>Great !
> >>>I guess I should roll a cleaned up cumulative patch soon.
> >>>
> >>>	Stephan
> >>
> >>I have now been running the stress test for more than 3½ hours, without
> >>any freezes. I have included the two of your changes I have been using.
> >>
> >>- Peter
> >
> >
> >After more testing, I'm sad to report that the freeze is still there.
> >The patch has however decreased the number of freezes dramatically:
> >
> >During 14 hours of testing 3 separate freezes has been seen:
> >
> >24 Giant held for more than 60 sec by td 0xc244e900, pid 27683
> >31 Giant held for more than 60 sec by td 0xc1b7b600, pid 12098
> >79 Giant held for more than 60 sec by td 0xc25f3180, pid 75531
> >
> >- Peter
> >_______________________________________________
> >freebsd-arch@freebsd.org mailing list
> >http://lists.freebsd.org/mailman/listinfo/freebsd-arch
> >To unsubscribe, send any mail to "freebsd-arch-unsubscribe@freebsd.org"
> When this happes, drop to debugger..
> 
> using:
> 
>   kdb_enter("Giant too long");
> 
> and dump out teh thread backtrace, and the output of show ktr
> iff you have ktr enabled.. (as we discussed before)

OK, right now I'm testing with all of Stephan's patches + the 
MUTEX_WAKE_ALL flag. Uptime is 3 3/4 hour and looking good.

-- 
Peter Holm

--T4sUOijqQbZv57TR
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="changes.diff"

Index: sys/kern/kern_mutex.c
===================================================================
RCS file: /home/ncvs/src/sys/kern/kern_mutex.c,v
retrieving revision 1.149
diff -u -r1.149 kern_mutex.c
--- sys/kern/kern_mutex.c	2 Sep 2004 18:59:15 -0000	1.149
+++ sys/kern/kern_mutex.c	2 Oct 2004 14:46:26 -0000
@@ -492,7 +492,9 @@
 		if (v == MTX_CONTESTED) {
 			MPASS(ts != NULL);
 			m->mtx_lock = (uintptr_t)td | MTX_CONTESTED;
+			critical_enter();
 			turnstile_claim(ts);
+			critical_exit();
 			break;
 		}
 #endif
@@ -651,6 +653,9 @@
 #else
 	MPASS(ts != NULL);
 #endif
+
+	critical_enter();
+
 #ifndef PREEMPTION
 	/* XXX */
 	td1 = turnstile_head(ts);
@@ -671,6 +676,7 @@
 	}
 #endif
 	turnstile_unpend(ts);
+	critical_exit();
 
 #ifndef PREEMPTION
 	/*
Index: sys/kern/kern_switch.c
===================================================================
RCS file: /home/ncvs/src/sys/kern/kern_switch.c,v
retrieving revision 1.95
diff -u -r1.95 kern_switch.c
--- sys/kern/kern_switch.c	19 Sep 2004 18:34:17 -0000	1.95
+++ sys/kern/kern_switch.c	2 Oct 2004 14:46:27 -0000
@@ -315,6 +315,106 @@
 	td->td_priority = newpri;
 	setrunqueue(td, SRQ_BORING);
 }
+
+
+/*
+ * This function is called when a thread is about to be put on a
+ * ksegrp run queue because it has been made runnable or its 
+ * priority has been adjusted and the ksegrp does not have a 
+ * free kse slot.  It determines if a thread from the same ksegrp
+ * should be preempted.  If so, it tries to switch threads
+ * if the thread is on the same cpu or notifies another cpu that
+ * it should switch threads. 
+ */
+
+static void
+maybe_preempt_in_ksegrp(struct thread *td)
+{
+#if  defined(SMP)
+	int highest_pri;
+	struct ksegrp *kg;
+	cpumask_t cpumask,dontuse;
+	struct pcpu *pc;
+	struct pcpu *highest_pcpu;
+	struct thread *running_thread;
+
+#ifndef FULL_PREEMPTION
+	int pri;
+
+	pri = td->td_priority;
+
+	if (!(pri >= PRI_MIN_ITHD && pri <= PRI_MAX_ITHD))
+	  return;
+#endif
+
+	mtx_assert(&sched_lock, MA_OWNED);
+
+	running_thread = curthread;
+
+#if !defined(KSEG_PEEMPT_BEST_CPU)
+	if(running_thread->td_ksegrp != td->td_ksegrp)
+#endif
+		{
+			kg = td->td_ksegrp;
+
+			/* Anyone waiting in front ? */
+			if(td != TAILQ_FIRST(&kg->kg_runq))  {
+				return; /* Yes - wait your turn*/
+			}
+			highest_pri  = td->td_priority;
+			highest_pcpu = NULL;
+			dontuse      = stopped_cpus | idle_cpus_mask;
+
+			/* Find a cpu with the worst priority that runs at thread from the
+			 * same  ksegrp - if multiple exist give first the last run cpu and then
+			 * the current cpu priority 
+			 */
+
+			SLIST_FOREACH(pc, &cpuhead, pc_allcpu) {
+				cpumask = pc->pc_cpumask;
+				if ( (cpumask & dontuse) == 0 && 
+				     pc->pc_curthread->td_ksegrp == kg) {
+					if (pc->pc_curthread->td_priority > highest_pri) {
+						highest_pri  = pc->pc_curthread->td_priority;
+						highest_pcpu = pc;
+					} else if (pc->pc_curthread->td_priority == highest_pri &&
+						   highest_pcpu != NULL) {
+						if (td->td_lastcpu == pc->pc_cpuid ||
+						    (PCPU_GET(cpumask) == cpumask &&
+						     td->td_lastcpu != highest_pcpu->pc_cpuid)) {
+							highest_pcpu = pc;
+						}
+					}
+				}
+			}
+			
+			/* Check if we need to preempt someone */
+			if (highest_pcpu == NULL) return;
+
+			if (PCPU_GET(cpuid) != highest_pcpu->pc_cpuid) {
+				highest_pcpu->pc_curthread->td_flags |= TDF_NEEDRESCHED;
+				ipi_selected(highest_pcpu->pc_cpumask, IPI_AST);
+				return;
+			}
+		}
+#else
+	KASSERT(running_thread->td_ksegrp == td->td_ksegrp,("maybe_preempt_in_ksegrp: No chance to run thread"));
+#endif
+
+	if  (td->td_priority > running_thread->td_priority)
+		return;
+#ifdef PREEMPTION
+	if (running_thread->td_critnest > 1) {
+		running_thread->td_pflags |= TDP_OWEPREEMPT;
+	} else {
+		mi_switch(SW_INVOL, NULL);
+	}
+#else
+	running_thread->td_flags |= TDF_NEEDRESCHED;
+#endif
+	return;
+}
+
 int limitcount;
 void
 setrunqueue(struct thread *td, int flags)
@@ -422,6 +522,7 @@
 	} else {
 		CTR3(KTR_RUNQ, "setrunqueue: held: td%p kg%p pid%d",
 			td, td->td_ksegrp, td->td_proc->p_pid);
+		maybe_preempt_in_ksegrp(td);
 	}
 }
 
Index: sys/kern/sched_4bsd.c
===================================================================
RCS file: /home/ncvs/src/sys/kern/sched_4bsd.c,v
retrieving revision 1.65
diff -u -r1.65 sched_4bsd.c
--- sys/kern/sched_4bsd.c	16 Sep 2004 07:12:59 -0000	1.65
+++ sys/kern/sched_4bsd.c	2 Oct 2004 14:46:29 -0000
@@ -823,6 +823,7 @@
 		TD_SET_CAN_RUN(td);
 	else {
 		td->td_ksegrp->kg_avail_opennings++;
+		critical_enter();
 		if (TD_IS_RUNNING(td)) {
 			/* Put us back on the run queue (kse and all). */
 			setrunqueue(td, SRQ_OURSELF|SRQ_YIELDING);
@@ -834,6 +835,8 @@
 			 */
 			slot_fill(td->td_ksegrp);
 		}
+		critical_exit();
+		td->td_pflags &= ~TDP_OWEPREEMPT;
 	}
 	if (newtd == NULL)
 		newtd = choosethread();
--- sys/i386/conf/GENERIC	Sun Sep 19 02:52:22 2004
+++ sys/i386/conf/PHO	Sat Oct  2 16:06:19 2004
@@ -66,6 +66,7 @@
 options 	KDB			# Enable kernel debugger support.
 options 	DDB			# Support DDB.
 options 	GDB			# Support remote GDB.
+options         BREAK_TO_DEBUGGER
 options 	INVARIANTS		# Enable calls of extra sanity checking
 options 	INVARIANT_SUPPORT	# Extra sanity checks of internal structures, required by INVARIANTS
 options 	WITNESS			# Enable checks to detect deadlocks and cycles
@@ -285,3 +286,4 @@
 device		firewire	# FireWire bus code
 device		sbp		# SCSI over FireWire (Requires scbus and da)
 device		fwe		# Ethernet over FireWire (non-standard!)
+options           MUTEX_WAKE_ALL          # Needed do not remove

--T4sUOijqQbZv57TR--



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20041002183120.GA1202>