Skip site navigation (1)Skip section navigation (2) 
Date:      Sat, 3 Nov 2018 03:43:32 +0000 (UTC)
From:      Matt Macy <mmacy@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r340097 - in head/sys: kern sys
Message-ID:  <201811030343.wA33hXRD067832@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help 
Author: mmacy
Date: Sat Nov  3 03:43:32 2018
New Revision: 340097
URL: https://svnweb.freebsd.org/changeset/base/340097

Log:
  Convert epoch to read / write records per cpu
  
  In discussing D17503 "Run epoch calls sooner and more reliably" with
  sbahra@ we came to the conclusion that epoch is currently misusing the
  ck_epoch API. It isn't safe to do a "write side" operation (ck_epoch_call
  or ck_epoch_poll) in the middle of a "read side" section. Since, by definition,
  it's possible to be preempted during the middle of an EPOCH_PREEMPT
  epoch the GC task might call ck_epoch_poll or another thread might call
  ck_epoch_call on the same section. The right solution is ultimately to change
  the way that ck_epoch works for this use case. However, as a stopgap for
  12 we agreed to simply have separate records for each use case.
  
  Tested by: pho@
  
  MFC after:	3 days

Modified:
  head/sys/kern/subr_epoch.c
  head/sys/sys/epoch_private.h

Modified: head/sys/kern/subr_epoch.c
==============================================================================
--- head/sys/kern/subr_epoch.c	Sat Nov  3 03:10:06 2018	(r340096)
+++ head/sys/kern/subr_epoch.c	Sat Nov  3 03:43:32 2018	(r340097)
@@ -150,7 +150,8 @@ epoch_ctor(epoch_t epoch)
 	CPU_FOREACH(cpu) {
 		er = zpcpu_get_cpu(epoch->e_pcpu_record, cpu);
 		bzero(er, sizeof(*er));
-		ck_epoch_register(&epoch->e_epoch, &er->er_record, NULL);
+		ck_epoch_register(&epoch->e_epoch, &er->er_read_record, NULL);
+		ck_epoch_register(&epoch->e_epoch, &er->er_write_record, NULL);
 		TAILQ_INIT((struct threadlist *)(uintptr_t)&er->er_tdlist);
 		er->er_cpuid = cpu;
 	}
@@ -235,7 +236,7 @@ epoch_block_handler_preempt(struct ck_epoch *global __
 	int spincount, gen;
 	int locksheld __unused;
 
-	record = __containerof(cr, struct epoch_record, er_record);
+	record = __containerof(cr, struct epoch_record, er_read_record);
 	td = curthread;
 	locksheld = td->td_locks;
 	spincount = 0;
@@ -461,7 +462,7 @@ epoch_call(epoch_t epoch, epoch_context_t ctx, void (*
 	critical_enter();
 	*DPCPU_PTR(epoch_cb_count) += 1;
 	er = epoch_currecord(epoch);
-	ck_epoch_call(&er->er_record, cb, (ck_epoch_cb_t *)callback);
+	ck_epoch_call(&er->er_write_record, cb, (ck_epoch_cb_t *)callback);
 	critical_exit();
 	return;
 boottime:
@@ -485,7 +486,7 @@ epoch_call_task(void *arg __unused)
 		if (__predict_false((epoch = allepochs[i]) == NULL))
 			continue;
 		er = epoch_currecord(epoch);
-		record = &er->er_record;
+		record = &er->er_write_record;
 		if ((npending = record->n_pending) == 0)
 			continue;
 		ck_epoch_poll_deferred(record, &cb_stack);

Modified: head/sys/sys/epoch_private.h
==============================================================================
--- head/sys/sys/epoch_private.h	Sat Nov  3 03:10:06 2018	(r340096)
+++ head/sys/sys/epoch_private.h	Sat Nov  3 03:43:32 2018	(r340097)
@@ -89,7 +89,8 @@ typedef struct epoch_thread {
 TAILQ_HEAD (epoch_tdlist, epoch_thread);
 
 typedef struct epoch_record {
-	ck_epoch_record_t er_record;
+	ck_epoch_record_t er_read_record;
+	ck_epoch_record_t er_write_record;
 	volatile struct epoch_tdlist er_tdlist;
 	volatile uint32_t er_gen;
 	uint32_t er_cpuid;
@@ -138,7 +139,7 @@ epoch_enter_preempt(epoch_t epoch, epoch_tracker_t et)
 	td->td_pre_epoch_prio = td->td_priority;
 	er = epoch_currecord(epoch);
 	TAILQ_INSERT_TAIL(&er->er_tdlist, etd, et_link);
-	ck_epoch_begin(&er->er_record, (ck_epoch_section_t *)&etd->et_section);
+	ck_epoch_begin(&er->er_read_record, (ck_epoch_section_t *)&etd->et_section);
 	critical_exit_sa(td);
 }
 
@@ -155,7 +156,7 @@ epoch_enter(epoch_t epoch)
 	td->td_epochnest++;
 	critical_enter_sa(td);
 	er = epoch_currecord(epoch);
-	ck_epoch_begin(&er->er_record, NULL);
+	ck_epoch_begin(&er->er_read_record, NULL);
 }
 
 static __inline void
@@ -183,7 +184,7 @@ epoch_exit_preempt(epoch_t epoch, epoch_tracker_t et)
 	etd->et_magic_post = 0;
 #endif
 	etd->et_td = (void*)0xDEADBEEF;
-	ck_epoch_end(&er->er_record,
+	ck_epoch_end(&er->er_read_record,
 		(ck_epoch_section_t *)&etd->et_section);
 	TAILQ_REMOVE(&er->er_tdlist, etd, et_link);
 	er->er_gen++;
@@ -203,7 +204,7 @@ epoch_exit(epoch_t epoch)
 	MPASS(td->td_epochnest);
 	td->td_epochnest--;
 	er = epoch_currecord(epoch);
-	ck_epoch_end(&er->er_record, NULL);
+	ck_epoch_end(&er->er_read_record, NULL);
 	critical_exit_sa(td);
 }
 #endif /* _KERNEL */

Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201811030343.wA33hXRD067832>