Skip site navigation (1)Skip section navigation (2)
Date:      Sat, 29 Aug 2009 13:28:02 +0000 (UTC)
From:      Konstantin Belousov <kib@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r196640 - in head/sys: arm/arm kern sys vm
Message-ID:  <200908291328.n7TDS2Ih065462@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: kib
Date: Sat Aug 29 13:28:02 2009
New Revision: 196640
URL: http://svn.freebsd.org/changeset/base/196640

Log:
  Remove the altkstacks, instead instantiate threads with kernel stack
  allocated with the right size from the start. For the thread that has
  kernel stack cached, verify that requested stack size is equial to the
  actual, and reallocate the stack if sizes differ [1].
  
  This fixes the bug introduced by r173361 that was committed several days
  after r173004 and consisted of kthread_add(9) ignoring the non-default
  kernel stack size.
  
  Also, r173361 removed the caching of the kernel stacks for a non-first
  thread in the process. Introduce separate kernel stack cache that keeps
  some limited amount of preallocated kernel stacks to lower the latency
  of thread allocation. Add vm_lowmem handler to prune the cache on
  low memory condition. This way, system with reasonable amount of the
  threads get lower latency of thread creation, while still not exhausting
  significant portion of KVA for unused kstacks.
  
  Submitted by:	peter [1]
  Discussed with:	jhb, julian, peter
  Reviewed by:	jhb
  Tested by:	pho
  MFC after:	1 week

Modified:
  head/sys/arm/arm/vm_machdep.c
  head/sys/kern/kern_fork.c
  head/sys/kern/kern_kthread.c
  head/sys/kern/kern_proc.c
  head/sys/kern/kern_thr.c
  head/sys/kern/kern_thread.c
  head/sys/sys/proc.h
  head/sys/vm/vm_extern.h
  head/sys/vm/vm_glue.c

Modified: head/sys/arm/arm/vm_machdep.c
==============================================================================
--- head/sys/arm/arm/vm_machdep.c	Sat Aug 29 04:15:37 2009	(r196639)
+++ head/sys/arm/arm/vm_machdep.c	Sat Aug 29 13:28:02 2009	(r196640)
@@ -119,9 +119,6 @@ cpu_fork(register struct thread *td1, re
 #ifdef __XSCALE__
 #ifndef CPU_XSCALE_CORE3
 	pmap_use_minicache(td2->td_kstack, td2->td_kstack_pages * PAGE_SIZE);
-	if (td2->td_altkstack)
-		pmap_use_minicache(td2->td_altkstack, td2->td_altkstack_pages *
-		    PAGE_SIZE);
 #endif
 #endif
 	td2->td_pcb = pcb2;

Modified: head/sys/kern/kern_fork.c
==============================================================================
--- head/sys/kern/kern_fork.c	Sat Aug 29 04:15:37 2009	(r196639)
+++ head/sys/kern/kern_fork.c	Sat Aug 29 13:28:02 2009	(r196640)
@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_kdtrace.h"
 #include "opt_ktrace.h"
+#include "opt_kstack_pages.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -276,25 +277,29 @@ norfproc_fail:
 
 	mem_charged = 0;
 	vm2 = NULL;
+	if (pages == 0)
+		pages = KSTACK_PAGES;
 	/* Allocate new proc. */
 	newproc = uma_zalloc(proc_zone, M_WAITOK);
-	if (TAILQ_EMPTY(&newproc->p_threads)) {
-		td2 = thread_alloc();
+	td2 = FIRST_THREAD_IN_PROC(newproc);
+	if (td2 == NULL) {
+		td2 = thread_alloc(pages);
 		if (td2 == NULL) {
 			error = ENOMEM;
 			goto fail1;
 		}
 		proc_linkup(newproc, td2);
-	} else
-		td2 = FIRST_THREAD_IN_PROC(newproc);
-
-	/* Allocate and switch to an alternate kstack if specified. */
-	if (pages != 0) {
-		if (!vm_thread_new_altkstack(td2, pages)) {
-			error = ENOMEM;
-			goto fail1;
+	} else {
+		if (td2->td_kstack == 0 || td2->td_kstack_pages != pages) {
+			if (td2->td_kstack != 0)
+				vm_thread_dispose(td);
+			if (!thread_alloc_stack(td2, pages)) {
+				error = ENOMEM;
+				goto fail1;
+			}
 		}
 	}
+
 	if ((flags & RFMEM) == 0) {
 		vm2 = vmspace_fork(p1->p_vmspace, &mem_charged);
 		if (vm2 == NULL) {

Modified: head/sys/kern/kern_kthread.c
==============================================================================
--- head/sys/kern/kern_kthread.c	Sat Aug 29 04:15:37 2009	(r196639)
+++ head/sys/kern/kern_kthread.c	Sat Aug 29 13:28:02 2009	(r196640)
@@ -256,7 +256,7 @@ kthread_add(void (*func)(void *), void *
 	}
 
 	/* Initialize our new td  */
-	newtd = thread_alloc();
+	newtd = thread_alloc(pages);
 	if (newtd == NULL)
 		return (ENOMEM);
 
@@ -282,9 +282,6 @@ kthread_add(void (*func)(void *), void *
 
 	newtd->td_pflags |= TDP_KTHREAD;
 	newtd->td_ucred = crhold(p->p_ucred);
-	/* Allocate and switch to an alternate kstack if specified. */
-	if (pages != 0)
-		vm_thread_new_altkstack(newtd, pages);
 
 	/* this code almost the same as create_thread() in kern_thr.c */
 	PROC_LOCK(p);

Modified: head/sys/kern/kern_proc.c
==============================================================================
--- head/sys/kern/kern_proc.c	Sat Aug 29 04:15:37 2009	(r196639)
+++ head/sys/kern/kern_proc.c	Sat Aug 29 13:28:02 2009	(r196640)
@@ -203,14 +203,6 @@ proc_dtor(void *mem, int size, void *arg
 #endif
 		/* Free all OSD associated to this thread. */
 		osd_thread_exit(td);
-
-		/* Dispose of an alternate kstack, if it exists.
-		 * XXX What if there are more than one thread in the proc?
-		 *     The first thread in the proc is special and not
-		 *     freed, so you gotta do this here.
-		 */
-		if (((p->p_flag & P_KTHREAD) != 0) && (td->td_altkstack != 0))
-			vm_thread_dispose_altkstack(td);
 	}
 	EVENTHANDLER_INVOKE(process_dtor, p);
 	if (p->p_ksi != NULL)
@@ -767,8 +759,6 @@ fill_kinfo_proc_only(struct proc *p, str
 		FOREACH_THREAD_IN_PROC(p, td0) {
 			if (!TD_IS_SWAPPED(td0))
 				kp->ki_rssize += td0->td_kstack_pages;
-			if (td0->td_altkstack_obj != NULL)
-				kp->ki_rssize += td0->td_altkstack_pages;
 		}
 		kp->ki_swrss = vm->vm_swrss;
 		kp->ki_tsize = vm->vm_tsize;

Modified: head/sys/kern/kern_thr.c
==============================================================================
--- head/sys/kern/kern_thr.c	Sat Aug 29 04:15:37 2009	(r196639)
+++ head/sys/kern/kern_thr.c	Sat Aug 29 13:28:02 2009	(r196640)
@@ -176,7 +176,7 @@ create_thread(struct thread *td, mcontex
 	}
 
 	/* Initialize our td */
-	newtd = thread_alloc();
+	newtd = thread_alloc(0);
 	if (newtd == NULL)
 		return (ENOMEM);
 

Modified: head/sys/kern/kern_thread.c
==============================================================================
--- head/sys/kern/kern_thread.c	Sat Aug 29 04:15:37 2009	(r196639)
+++ head/sys/kern/kern_thread.c	Sat Aug 29 13:28:02 2009	(r196640)
@@ -283,7 +283,7 @@ thread_reap(void)
  * Allocate a thread.
  */
 struct thread *
-thread_alloc(void)
+thread_alloc(int pages)
 {
 	struct thread *td;
 
@@ -291,7 +291,7 @@ thread_alloc(void)
 
 	td = (struct thread *)uma_zalloc(thread_zone, M_WAITOK);
 	KASSERT(td->td_kstack == 0, ("thread_alloc got thread with kstack"));
-	if (!vm_thread_new(td, 0)) {
+	if (!vm_thread_new(td, pages)) {
 		uma_zfree(thread_zone, td);
 		return (NULL);
 	}
@@ -299,6 +299,17 @@ thread_alloc(void)
 	return (td);
 }
 
+int
+thread_alloc_stack(struct thread *td, int pages)
+{
+
+	KASSERT(td->td_kstack == 0,
+	    ("thread_alloc_stack called on a thread with kstack"));
+	if (!vm_thread_new(td, pages))
+		return (0);
+	cpu_thread_alloc(td);
+	return (1);
+}
 
 /*
  * Deallocate a thread.
@@ -312,8 +323,6 @@ thread_free(struct thread *td)
 		cpuset_rel(td->td_cpuset);
 	td->td_cpuset = NULL;
 	cpu_thread_free(td);
-	if (td->td_altkstack != 0)
-		vm_thread_dispose_altkstack(td);
 	if (td->td_kstack != 0)
 		vm_thread_dispose(td);
 	uma_zfree(thread_zone, td);

Modified: head/sys/sys/proc.h
==============================================================================
--- head/sys/sys/proc.h	Sat Aug 29 04:15:37 2009	(r196639)
+++ head/sys/sys/proc.h	Sat Aug 29 13:28:02 2009	(r196640)
@@ -267,9 +267,6 @@ struct thread {
 	struct vm_object *td_kstack_obj;/* (a) Kstack object. */
 	vm_offset_t	td_kstack;	/* (a) Kernel VA of kstack. */
 	int		td_kstack_pages; /* (a) Size of the kstack. */
-	struct vm_object *td_altkstack_obj;/* (a) Alternate kstack object. */
-	vm_offset_t	td_altkstack;	/* (a) Kernel VA of alternate kstack. */
-	int		td_altkstack_pages; /* (a) Size of alternate kstack. */
 	volatile u_int	td_critnest;	/* (k*) Critical section nest level. */
 	struct mdthread td_md;		/* (k) Any machine-dependent fields. */
 	struct td_sched	*td_sched;	/* (*) Scheduler-specific data. */
@@ -850,7 +847,8 @@ void	cpu_thread_exit(struct thread *);
 void	cpu_thread_free(struct thread *);
 void	cpu_thread_swapin(struct thread *);
 void	cpu_thread_swapout(struct thread *);
-struct	thread *thread_alloc(void);
+struct	thread *thread_alloc(int pages);
+int	thread_alloc_stack(struct thread *, int pages);
 void	thread_exit(void) __dead2;
 void	thread_free(struct thread *td);
 void	thread_link(struct thread *td, struct proc *p);

Modified: head/sys/vm/vm_extern.h
==============================================================================
--- head/sys/vm/vm_extern.h	Sat Aug 29 04:15:37 2009	(r196639)
+++ head/sys/vm/vm_extern.h	Sat Aug 29 13:28:02 2009	(r196640)
@@ -80,9 +80,7 @@ int vm_fault_quick(caddr_t v, int prot);
 struct sf_buf *vm_imgact_map_page(vm_object_t object, vm_ooffset_t offset);
 void vm_imgact_unmap_page(struct sf_buf *sf);
 void vm_thread_dispose(struct thread *td);
-void vm_thread_dispose_altkstack(struct thread *td);
 int vm_thread_new(struct thread *td, int pages);
-int vm_thread_new_altkstack(struct thread *td, int pages);
 void vm_thread_swapin(struct thread *td);
 void vm_thread_swapout(struct thread *td);
 #endif				/* _KERNEL */

Modified: head/sys/vm/vm_glue.c
==============================================================================
--- head/sys/vm/vm_glue.c	Sat Aug 29 04:15:37 2009	(r196639)
+++ head/sys/vm/vm_glue.c	Sat Aug 29 13:28:02 2009	(r196640)
@@ -77,6 +77,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sx.h>
 #include <sys/sysctl.h>
 
+#include <sys/eventhandler.h>
 #include <sys/kernel.h>
 #include <sys/ktr.h>
 #include <sys/unistd.h>
@@ -308,6 +309,20 @@ vm_imgact_unmap_page(struct sf_buf *sf)
 	vm_page_unlock_queues();
 }
 
+struct kstack_cache_entry {
+	vm_object_t ksobj;
+	struct kstack_cache_entry *next_ks_entry;
+};
+
+static struct kstack_cache_entry *kstack_cache;
+static int kstack_cache_size = 128;
+static int kstacks;
+static struct mtx kstack_cache_mtx;
+SYSCTL_INT(_vm, OID_AUTO, kstack_cache_size, CTLFLAG_RW, &kstack_cache_size, 0,
+    "");
+SYSCTL_INT(_vm, OID_AUTO, kstacks, CTLFLAG_RD, &kstacks, 0,
+    "");
+
 #ifndef KSTACK_MAX_PAGES
 #define KSTACK_MAX_PAGES 32
 #endif
@@ -323,6 +338,7 @@ vm_thread_new(struct thread *td, int pag
 	vm_object_t ksobj;
 	vm_offset_t ks;
 	vm_page_t m, ma[KSTACK_MAX_PAGES];
+	struct kstack_cache_entry *ks_ce;
 	int i;
 
 	/* Bounds check */
@@ -330,6 +346,22 @@ vm_thread_new(struct thread *td, int pag
 		pages = KSTACK_PAGES;
 	else if (pages > KSTACK_MAX_PAGES)
 		pages = KSTACK_MAX_PAGES;
+
+	if (pages == KSTACK_PAGES) {
+		mtx_lock(&kstack_cache_mtx);
+		if (kstack_cache != NULL) {
+			ks_ce = kstack_cache;
+			kstack_cache = ks_ce->next_ks_entry;
+			mtx_unlock(&kstack_cache_mtx);
+
+			td->td_kstack_obj = ks_ce->ksobj;
+			td->td_kstack = (vm_offset_t)ks_ce;
+			td->td_kstack_pages = KSTACK_PAGES;
+			return (1);
+		}
+		mtx_unlock(&kstack_cache_mtx);
+	}
+
 	/*
 	 * Allocate an object for the kstack.
 	 */
@@ -345,7 +377,8 @@ vm_thread_new(struct thread *td, int pag
 		vm_object_deallocate(ksobj);
 		return (0);
 	}
-	
+
+	atomic_add_int(&kstacks, 1);
 	if (KSTACK_GUARD_PAGES != 0) {
 		pmap_qremove(ks, KSTACK_GUARD_PAGES);
 		ks += KSTACK_GUARD_PAGES * PAGE_SIZE;
@@ -376,20 +409,13 @@ vm_thread_new(struct thread *td, int pag
 	return (1);
 }
 
-/*
- * Dispose of a thread's kernel stack.
- */
-void
-vm_thread_dispose(struct thread *td)
+static void
+vm_thread_stack_dispose(vm_object_t ksobj, vm_offset_t ks, int pages)
 {
-	vm_object_t ksobj;
-	vm_offset_t ks;
 	vm_page_t m;
-	int i, pages;
+	int i;
 
-	pages = td->td_kstack_pages;
-	ksobj = td->td_kstack_obj;
-	ks = td->td_kstack;
+	atomic_add_int(&kstacks, -1);
 	pmap_qremove(ks, pages);
 	VM_OBJECT_LOCK(ksobj);
 	for (i = 0; i < pages; i++) {
@@ -405,9 +431,66 @@ vm_thread_dispose(struct thread *td)
 	vm_object_deallocate(ksobj);
 	kmem_free(kernel_map, ks - (KSTACK_GUARD_PAGES * PAGE_SIZE),
 	    (pages + KSTACK_GUARD_PAGES) * PAGE_SIZE);
+}
+
+/*
+ * Dispose of a thread's kernel stack.
+ */
+void
+vm_thread_dispose(struct thread *td)
+{
+	vm_object_t ksobj;
+	vm_offset_t ks;
+	struct kstack_cache_entry *ks_ce;
+	int pages;
+
+	pages = td->td_kstack_pages;
+	ksobj = td->td_kstack_obj;
+	ks = td->td_kstack;
+	if (pages == KSTACK_PAGES && kstacks <= kstack_cache_size) {
+		ks_ce = (struct kstack_cache_entry *)ks;
+		ks_ce->ksobj = ksobj;
+		mtx_lock(&kstack_cache_mtx);
+		ks_ce->next_ks_entry = ks_ce;
+		kstack_cache = ks_ce;
+		mtx_unlock(&kstack_cache_mtx);
+		return;
+	}
+	vm_thread_stack_dispose(ksobj, ks, pages);
 	td->td_kstack = 0;
+	td->td_kstack_pages = 0;
+}
+
+static void
+vm_thread_stack_lowmem(void *nulll)
+{
+	struct kstack_cache_entry *ks_ce, *ks_ce1;
+
+	mtx_lock(&kstack_cache_mtx);
+	ks_ce = kstack_cache;
+	kstack_cache = NULL;
+	mtx_unlock(&kstack_cache_mtx);
+
+	while (ks_ce != NULL) {
+		ks_ce1 = ks_ce;
+		ks_ce = ks_ce->next_ks_entry;
+
+		vm_thread_stack_dispose(ks_ce1->ksobj, (vm_offset_t)ks_ce1,
+		    KSTACK_PAGES);
+	}
+}
+
+static void
+kstack_cache_init(void *nulll)
+{
+
+	EVENTHANDLER_REGISTER(vm_lowmem, vm_thread_stack_lowmem, NULL,
+	    EVENTHANDLER_PRI_ANY);
 }
 
+MTX_SYSINIT(kstack_cache, &kstack_cache_mtx, "kstkch", MTX_DEF);
+SYSINIT(vm_kstacks, SI_SUB_KTHREAD_INIT, SI_ORDER_ANY, kstack_cache_init, NULL);
+
 /*
  * Allow a thread's kernel stack to be paged out.
  */
@@ -468,37 +551,6 @@ vm_thread_swapin(struct thread *td)
 }
 
 /*
- * Set up a variable-sized alternate kstack.
- */
-int
-vm_thread_new_altkstack(struct thread *td, int pages)
-{
-
-	td->td_altkstack = td->td_kstack;
-	td->td_altkstack_obj = td->td_kstack_obj;
-	td->td_altkstack_pages = td->td_kstack_pages;
-
-	return (vm_thread_new(td, pages));
-}
-
-/*
- * Restore the original kstack.
- */
-void
-vm_thread_dispose_altkstack(struct thread *td)
-{
-
-	vm_thread_dispose(td);
-
-	td->td_kstack = td->td_altkstack;
-	td->td_kstack_obj = td->td_altkstack_obj;
-	td->td_kstack_pages = td->td_altkstack_pages;
-	td->td_altkstack = 0;
-	td->td_altkstack_obj = NULL;
-	td->td_altkstack_pages = 0;
-}
-
-/*
  * Implement fork's actions on an address space.
  * Here we arrange for the address space to be copied or referenced,
  * allocate a user struct (pcb and kernel stack), then call the



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200908291328.n7TDS2Ih065462>