Skip site navigation (1)Skip section navigation (2)
Date:      Thu, 16 Sep 2010 03:46:18 +0000 (UTC)
From:      Nathan Whitehorn <nwhitehorn@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r212722 - in head/sys/powerpc: aim include
Message-ID:  <201009160346.o8G3kIFt060149@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: nwhitehorn
Date: Thu Sep 16 03:46:17 2010
New Revision: 212722
URL: http://svn.freebsd.org/changeset/base/212722

Log:
  Split the SLB mirror cache into two kinds of object, one for kernel maps
  which are similar to the previous ones, and one for user maps, which
  are arrays of pointers into the SLB tree. This changes makes user SLB
  updates atomic, closing a window for memory corruption. While here,
  rearrange the allocation functions to make context switches faster.

Modified:
  head/sys/powerpc/aim/mmu_oea64.c
  head/sys/powerpc/aim/slb.c
  head/sys/powerpc/aim/trap.c
  head/sys/powerpc/aim/trap_subr64.S
  head/sys/powerpc/include/pcpu.h
  head/sys/powerpc/include/pmap.h
  head/sys/powerpc/include/sr.h

Modified: head/sys/powerpc/aim/mmu_oea64.c
==============================================================================
--- head/sys/powerpc/aim/mmu_oea64.c	Thu Sep 16 02:59:25 2010	(r212721)
+++ head/sys/powerpc/aim/mmu_oea64.c	Thu Sep 16 03:46:17 2010	(r212722)
@@ -838,7 +838,7 @@ moea64_bootstrap_slb_prefault(vm_offset_
 	if (large)
 		entry.slbv |= SLBV_L;
 
-	slb_insert(kernel_pmap, cache, &entry);
+	slb_insert_kernel(entry.slbe, entry.slbv);
 }
 #endif
 
@@ -2099,6 +2099,7 @@ moea64_pinit(mmu_t mmu, pmap_t pmap)
 
 	pmap->pm_slb_tree_root = slb_alloc_tree();
 	pmap->pm_slb = slb_alloc_user_cache();
+	pmap->pm_slb_len = 0;
 }
 #else
 void

Modified: head/sys/powerpc/aim/slb.c
==============================================================================
--- head/sys/powerpc/aim/slb.c	Thu Sep 16 02:59:25 2010	(r212721)
+++ head/sys/powerpc/aim/slb.c	Thu Sep 16 03:46:17 2010	(r212722)
@@ -263,13 +263,14 @@ va_to_vsid(pmap_t pm, vm_offset_t va)
 	entry = user_va_to_slb_entry(pm, va);
 
 	if (entry == NULL)
-		return (allocate_vsid(pm, (uintptr_t)va >> ADDR_SR_SHFT, 0));
+		return (allocate_user_vsid(pm,
+		    (uintptr_t)va >> ADDR_SR_SHFT, 0));
 
 	return ((entry->slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT);
 }
 
 uint64_t
-allocate_vsid(pmap_t pm, uint64_t esid, int large)
+allocate_user_vsid(pmap_t pm, uint64_t esid, int large)
 {
 	uint64_t vsid, slbv;
 	struct slbtnode *ua, *next, *inter;
@@ -327,7 +328,7 @@ allocate_vsid(pmap_t pm, uint64_t esid, 
 	 * SLB mapping, so pre-spill this entry.
 	 */
 	eieio();
-	slb_insert(pm, pm->pm_slb, slb);
+	slb_insert_user(pm, slb);
 
 	return (vsid);
 }
@@ -410,57 +411,68 @@ slb_alloc_tree(void)
 	    (slbe & SLBE_ESID_MASK) > 16*SEGMENT_LENGTH) || \
 	    (slbe & SLBE_ESID_MASK) > VM_MAX_KERNEL_ADDRESS)
 void
-slb_insert(pmap_t pm, struct slb *slbcache, struct slb *slb_entry)
+slb_insert_kernel(uint64_t slbe, uint64_t slbv)
 {
-	uint64_t slbe, slbv;
-	int i, j, to_spill;
+	struct slb *slbcache;
+	int i, j;
 
 	/* We don't want to be preempted while modifying the kernel map */
 	critical_enter();
 
-	to_spill = -1;
-	slbv = slb_entry->slbv;
-	slbe = slb_entry->slbe;
+	slbcache = PCPU_GET(slb);
 
-	/* Hunt for a likely candidate */
+	/* Check for an unused slot, abusing the USER_SR slot as a full flag */
+	if (slbcache[USER_SR].slbe == 0) {
+		for (i = 0; i < USER_SR; i++) {
+			if (!(slbcache[i].slbe & SLBE_VALID)) 
+				goto fillkernslb;
+		}
+
+		if (i == USER_SR)
+			slbcache[USER_SR].slbe = 1;
+	}
 
 	for (i = mftb() % 64, j = 0; j < 64; j++, i = (i+1) % 64) {
-		if (pm == kernel_pmap && i == USER_SR)
-				continue;
+		if (i == USER_SR)
+			continue;
 
-		if (!(slbcache[i].slbe & SLBE_VALID)) {
-			to_spill = i;
+		if (SLB_SPILLABLE(slbcache[i].slbe))
 			break;
-		}
-
-		if (to_spill < 0 && (pm != kernel_pmap ||
-		    SLB_SPILLABLE(slbcache[i].slbe)))
-			to_spill = i;
-	}
-
-	if (to_spill < 0)
-		panic("SLB spill on ESID %#lx, but no available candidates!\n",
-		   (slbe & SLBE_ESID_MASK) >> SLBE_ESID_SHIFT);
-
-	if (slbcache[to_spill].slbe & SLBE_VALID) {
-		/* Invalidate this first to avoid races */
-		slbcache[to_spill].slbe = 0;
-		mb();
 	}
-	slbcache[to_spill].slbv = slbv;
-	slbcache[to_spill].slbe = slbe | (uint64_t)to_spill;
+
+	KASSERT(j < 64, ("All kernel SLB slots locked!"));
+
+fillkernslb:
+	slbcache[i].slbv = slbv;
+	slbcache[i].slbe = slbe | (uint64_t)i;
 
 	/* If it is for this CPU, put it in the SLB right away */
-	if (pm == kernel_pmap && pmap_bootstrapped) {
+	if (pmap_bootstrapped) {
 		/* slbie not required */
 		__asm __volatile ("slbmte %0, %1" :: 
-		    "r"(slbcache[to_spill].slbv),
-		    "r"(slbcache[to_spill].slbe)); 
+		    "r"(slbcache[i].slbv), "r"(slbcache[i].slbe)); 
 	}
 
 	critical_exit();
 }
 
+void
+slb_insert_user(pmap_t pm, struct slb *slb)
+{
+	int i;
+
+	PMAP_LOCK_ASSERT(pm, MA_OWNED);
+
+	if (pm->pm_slb_len < 64) {
+		i = pm->pm_slb_len;
+		pm->pm_slb_len++;
+	} else {
+		i = mftb() % 64;
+	}
+
+	/* Note that this replacement is atomic with respect to trap_subr */
+	pm->pm_slb[i] = slb;
+}
 
 static void
 slb_zone_init(void *dummy)
@@ -468,18 +480,18 @@ slb_zone_init(void *dummy)
 
 	slbt_zone = uma_zcreate("SLB tree node", sizeof(struct slbtnode),
 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM);
-	slb_cache_zone = uma_zcreate("SLB cache", 64*sizeof(struct slb),
+	slb_cache_zone = uma_zcreate("SLB cache", 64*sizeof(struct slb *),
 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM);
 }
 
-struct slb *
+struct slb **
 slb_alloc_user_cache(void)
 {
 	return (uma_zalloc(slb_cache_zone, M_ZERO));
 }
 
 void
-slb_free_user_cache(struct slb *slb)
+slb_free_user_cache(struct slb **slb)
 {
 	uma_zfree(slb_cache_zone, slb);
 }

Modified: head/sys/powerpc/aim/trap.c
==============================================================================
--- head/sys/powerpc/aim/trap.c	Thu Sep 16 02:59:25 2010	(r212721)
+++ head/sys/powerpc/aim/trap.c	Thu Sep 16 03:46:17 2010	(r212722)
@@ -445,17 +445,15 @@ syscall(struct trapframe *frame)
 static int 
 handle_slb_spill(pmap_t pm, vm_offset_t addr)
 {
-	struct slb kern_entry, *user_entry;
+	struct slb *user_entry;
 	uint64_t esid;
 	int i;
 
 	esid = (uintptr_t)addr >> ADDR_SR_SHFT;
 
 	if (pm == kernel_pmap) {
-		kern_entry.slbv = kernel_va_to_slbv(addr);
-		kern_entry.slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID;
-
-		slb_insert(pm, PCPU_GET(slb), &kern_entry);
+		slb_insert_kernel((esid << SLBE_ESID_SHIFT) | SLBE_VALID,
+		    kernel_va_to_slbv(addr));
 		return (0);
 	}
 
@@ -464,18 +462,18 @@ handle_slb_spill(pmap_t pm, vm_offset_t 
 
 	if (user_entry == NULL) {
 		/* allocate_vsid auto-spills it */
-		(void)allocate_vsid(pm, esid, 0);
+		(void)allocate_user_vsid(pm, esid, 0);
 	} else {
 		/*
 		 * Check that another CPU has not already mapped this.
 		 * XXX: Per-thread SLB caches would be better.
 		 */
-		for (i = 0; i < 64; i++)
-			if (pm->pm_slb[i].slbe == (user_entry->slbe | i))
+		for (i = 0; i < pm->pm_slb_len; i++)
+			if (pm->pm_slb[i] == user_entry)
 				break;
 
-		if (i == 64)
-			slb_insert(pm, pm->pm_slb, user_entry);
+		if (i == pm->pm_slb_len)
+			slb_insert_user(pm, user_entry);
 	}
 	PMAP_UNLOCK(pm);
 

Modified: head/sys/powerpc/aim/trap_subr64.S
==============================================================================
--- head/sys/powerpc/aim/trap_subr64.S	Thu Sep 16 02:59:25 2010	(r212721)
+++ head/sys/powerpc/aim/trap_subr64.S	Thu Sep 16 03:46:17 2010	(r212722)
@@ -49,45 +49,59 @@
  * Requires that r28-r31 be scratch, with r28 initialized to the SLB cache
  */
 
-restoresrs: 
+/*
+ * User SRs are loaded through a pointer to the current pmap.
+ */
+restore_usersrs:
+	GET_CPUINFO(%r28);
+	ld	%r28,PC_USERSLB(%r28);
 	li	%r29, 0			/* Set the counter to zero */
 
 	slbia
 	slbmfee	%r31,%r29		
 	clrrdi	%r31,%r31,28
 	slbie	%r31
-instslb:
-	ld	%r31, 8(%r28);		/* Load SLBE */
-
-	cmpli	0, %r31, 0;		/* If SLBE is not valid, get the next */
-	beq	nslb;
-
-	ld	%r30, 0(%r28)		/* Load SLBV */
+instuserslb:
+	ld	%r31, 0(%r28);		/* Load SLB entry pointer */
+	cmpli	0, %r31, 0;		/* If NULL, stop */
+	beqlr;
+
+	ld	%r30, 0(%r31)		/* Load SLBV */
+	ld	%r31, 8(%r31)		/* Load SLBE */
+	or	%r31, %r31, %r29	/*  Set SLBE slot */
 	slbmte	%r30, %r31;		/* Install SLB entry */
 
-nslb:
-	addi	%r28, %r28, 16;		/* Advance */
+	addi	%r28, %r28, 8;		/* Advance pointer */
 	addi	%r29, %r29, 1;
 	cmpli	0, %r29, 64;		/* Repeat if we are not at the end */
-	blt instslb;
-
+	blt instuserslb;
 	blr;
 
 /*
- * User SRs are loaded through a pointer to the current pmap.
+ * Kernel SRs are loaded directly from the PCPU fields
  */
-#define RESTORE_USER_SRS() \
-	GET_CPUINFO(%r28); \
-	ld	%r28,PC_USERSLB(%r28); \
-	bl	restoresrs;
+restore_kernsrs:
+	GET_CPUINFO(%r28);
+	addi	%r28,%r28,PC_KERNSLB;
+	li	%r29, 0			/* Set the counter to zero */
 
-/*
- * Kernel SRs are loaded directly from kernel_pmap_
- */
-#define RESTORE_KERN_SRS() \
-	GET_CPUINFO(%r28); \
-	addi	%r28,%r28,PC_KERNSLB; \
-	bl	restoresrs;
+	slbia
+	slbmfee	%r31,%r29		
+	clrrdi	%r31,%r31,28
+	slbie	%r31
+instkernslb:
+	ld	%r31, 8(%r28);		/* Load SLBE */
+
+	cmpli	0, %r31, 0;		/* If SLBE is not valid, stop */
+	beqlr;
+	ld	%r30, 0(%r28)		/* Load SLBV  */
+	slbmte	%r30, %r31;		/* Install SLB entry */
+
+	addi	%r28, %r28, 16;		/* Advance pointer */
+	addi	%r29, %r29, 1;
+	cmpli	0, %r29, USER_SR;	/* Repeat if we are not at the end */
+	blt instkernslb;
+	blr;
 
 /*
  * FRAME_SETUP assumes:
@@ -237,7 +251,7 @@ nslb:
 	std	%r30,(savearea+CPUSAVE_R30)(%r3);			\
 	std	%r31,(savearea+CPUSAVE_R31)(%r3);			\
 	mflr	%r27;			/* preserve LR */		\
-	RESTORE_USER_SRS();		/* uses r28-r31 */		\
+	bl	restore_usersrs;	/* uses r28-r31 */		\
 	mtlr	%r27;							\
 	ld	%r31,(savearea+CPUSAVE_R31)(%r3);			\
 	ld	%r30,(savearea+CPUSAVE_R30)(%r3);			\
@@ -432,7 +446,7 @@ realtrap:
 	ld	%r1,PC_CURPCB(%r1)
 	mr	%r27,%r28		/* Save LR, r29 */
 	mtsprg2	%r29
-	RESTORE_KERN_SRS()		/* enable kernel mapping */
+	bl	restore_kernsrs		/* enable kernel mapping */
 	mfsprg2	%r29
 	mr	%r28,%r27
 	ba s_trap
@@ -482,7 +496,7 @@ u_trap:
 	ld	%r1,PC_CURPCB(%r1)
 	mr	%r27,%r28		/* Save LR, r29 */
 	mtsprg2	%r29
-	RESTORE_KERN_SRS() /* enable kernel mapping */
+	bl	restore_kernsrs		/* enable kernel mapping */
 	mfsprg2	%r29
 	mr	%r28,%r27
 

Modified: head/sys/powerpc/include/pcpu.h
==============================================================================
--- head/sys/powerpc/include/pcpu.h	Thu Sep 16 02:59:25 2010	(r212721)
+++ head/sys/powerpc/include/pcpu.h	Thu Sep 16 03:46:17 2010	(r212722)
@@ -55,7 +55,7 @@ struct pmap;
 
 #define PCPU_MD_AIM64_FIELDS						\
 	struct slb	pc_slb[64];					\
-	struct slb	*pc_userslb;
+	struct slb	**pc_userslb;
 
 #ifdef __powerpc64__
 #define PCPU_MD_AIM_FIELDS	PCPU_MD_AIM64_FIELDS

Modified: head/sys/powerpc/include/pmap.h
==============================================================================
--- head/sys/powerpc/include/pmap.h	Thu Sep 16 02:59:25 2010	(r212721)
+++ head/sys/powerpc/include/pmap.h	Thu Sep 16 03:46:17 2010	(r212722)
@@ -93,7 +93,8 @@ struct	pmap {
 	
     #ifdef __powerpc64__
 	struct slbtnode	*pm_slb_tree_root;
-	struct slb	*pm_slb;
+	struct slb	**pm_slb;
+	int		pm_slb_len;
     #else
 	register_t	pm_sr[16];
     #endif
@@ -142,14 +143,15 @@ uint64_t va_to_vsid(pmap_t pm, vm_offset
 uint64_t kernel_va_to_slbv(vm_offset_t va);
 struct slb *user_va_to_slb_entry(pmap_t pm, vm_offset_t va);
 
-uint64_t allocate_vsid(pmap_t pm, uint64_t esid, int large);
+uint64_t allocate_user_vsid(pmap_t pm, uint64_t esid, int large);
 void	free_vsid(pmap_t pm, uint64_t esid, int large);
-void	slb_insert(pmap_t pm, struct slb *dst, struct slb *);
+void	slb_insert_user(pmap_t pm, struct slb *slb);
+void	slb_insert_kernel(uint64_t slbe, uint64_t slbv);
 
 struct slbtnode *slb_alloc_tree(void);
 void     slb_free_tree(pmap_t pm);
-struct slb *slb_alloc_user_cache(void);
-void	slb_free_user_cache(struct slb *);
+struct slb **slb_alloc_user_cache(void);
+void	slb_free_user_cache(struct slb **);
 
 #else
 

Modified: head/sys/powerpc/include/sr.h
==============================================================================
--- head/sys/powerpc/include/sr.h	Thu Sep 16 02:59:25 2010	(r212721)
+++ head/sys/powerpc/include/sr.h	Thu Sep 16 03:46:17 2010	(r212722)
@@ -42,7 +42,11 @@
 #define	SR_VSID_MASK	0x00ffffff	/* Virtual Segment ID mask */
 
 /* Kernel segment register usage */
+#ifdef __powerpc64__
+#define	USER_SR		63
+#else
 #define	USER_SR		12
+#endif
 #define	KERNEL_SR	13
 #define	KERNEL2_SR	14
 #define	KERNEL3_SR	15



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201009160346.o8G3kIFt060149>