Date: Fri, 21 Feb 2014 05:31:34 +0000 (UTC) From: Jeff Roberson <jeff@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r262279 - projects/numa/sys/vm Message-ID: <201402210531.s1L5VYPg063076@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jeff Date: Fri Feb 21 05:31:34 2014 New Revision: 262279 URL: http://svnweb.freebsd.org/changeset/base/262279 Log: - Add the start of a domain selection policy system Added: projects/numa/sys/vm/_vm_domain.h projects/numa/sys/vm/vm_domain.h Modified: projects/numa/sys/vm/vm_object.c projects/numa/sys/vm/vm_object.h projects/numa/sys/vm/vm_page.c projects/numa/sys/vm/vm_phys.c Added: projects/numa/sys/vm/_vm_domain.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/numa/sys/vm/_vm_domain.h Fri Feb 21 05:31:34 2014 (r262279) @@ -0,0 +1,78 @@ +/*- + * Copyright (c) 2014, Jeffrey Roberson <jeff@freebsd.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _VM__DOMAIN_H_ +#define _VM__DOMAIN_H_ + +#include <sys/_bitset.h> +#include <sys/bitset.h> + +#ifdef _KERNEL +#define VM_DOMAIN_SETSIZE MAXMEMDOM +#endif + +#define VM_DOMAIN_MAXSIZE 64 + +#ifndef VM_DOMAIN_SETSIZE +#define VM_DOMAIN_SETSIZE VM_DOMAIN_MAXSIZE +#endif + +#define _NVM_DOMAINBITS _BITSET_BITS +#define _NVM_DOMAINWORDS __bitset_words(VM_DOMAIN_SETSIZE) + +BITSET_DEFINE(_vm_domainset, VM_DOMAIN_SETSIZE); +typedef struct _vm_domainset vm_domainset_t; + +#define VM_DOMAIN_FSET BITSET_FSET(_NVM_DOMAINWORDS) +#define VM_DOMAIN_T_INITIALIZER BITSET_T_INITIALIZER +#define VM_DOMAIN_SETBUFSIZ ((2 + sizeof(long) * 2) * _NVM_DOMAINWORDS) + +#ifdef _KERNEL + +/* + * Valid memory domain (NUMA) policy values. + */ +enum vm_domain_policy { + ROUNDROBIN, /* Select between any in the set. */ + FIRSTTOUCH /* Select the current domain. */ +}; + +/* + * The select structure encapsulate domain allocation strategy with + * allocator information. + */ +struct vm_domain_select { + vm_domainset_t ds_mask; /* bitmask of valid domains. */ + enum vm_domain_policy ds_policy; /* Allocation policy. */ + int ds_cursor; /* Allocation cursor. */ + int ds_count; /* Domains in policy. */ +}; + +#endif /* _KERNEL */ + +#endif /* !_VM__DOMAIN_H_ */ Added: projects/numa/sys/vm/vm_domain.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/numa/sys/vm/vm_domain.h Fri Feb 21 05:31:34 2014 (r262279) @@ -0,0 +1,110 @@ +/*- + * Copyright (c) 2014, Jeffrey Roberson <jeff@freebsd.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _VM_DOMAIN_H_ +#define _VM_DOMAIN_H_ + +#include <vm/_vm_domain.h> +#include <sys/bitset.h> + +#define VM_DOMAIN_CLR(n, p) BIT_CLR(VM_DOMAIN_SETSIZE, n, p) +#define VM_DOMAIN_COPY(f, t) BIT_COPY(VM_DOMAIN_SETSIZE, f, t) +#define VM_DOMAIN_ISSET(n, p) BIT_ISSET(VM_DOMAIN_SETSIZE, n, p) +#define VM_DOMAIN_SET(n, p) BIT_SET(VM_DOMAIN_SETSIZE, n, p) +#define VM_DOMAIN_ZERO(p) BIT_ZERO(VM_DOMAIN_SETSIZE, p) +#define VM_DOMAIN_FILL(p) BIT_FILL(VM_DOMAIN_SETSIZE, p) +#define VM_DOMAIN_SETOF(n, p) BIT_SETOF(VM_DOMAIN_SETSIZE, n, p) +#define VM_DOMAIN_EMPTY(p) BIT_EMPTY(VM_DOMAIN_SETSIZE, p) +#define VM_DOMAIN_ISFULLSET(p) BIT_ISFULLSET(VM_DOMAIN_SETSIZE, p) +#define VM_DOMAIN_SUBSET(p, c) BIT_SUBSET(VM_DOMAIN_SETSIZE, p, c) +#define VM_DOMAIN_OVERLAP(p, c) BIT_OVERLAP(VM_DOMAIN_SETSIZE, p, c) +#define VM_DOMAIN_CMP(p, c) BIT_CMP(VM_DOMAIN_SETSIZE, p, c) +#define VM_DOMAIN_OR(d, s) BIT_OR(VM_DOMAIN_SETSIZE, d, s) +#define VM_DOMAIN_AND(d, s) BIT_AND(VM_DOMAIN_SETSIZE, d, s) +#define VM_DOMAIN_NAND(d, s) BIT_NAND(VM_DOMAIN_SETSIZE, d, s) +#define VM_DOMAIN_CLR_ATOMIC(n, p) BIT_CLR_ATOMIC(VM_DOMAIN_SETSIZE, n, p) +#define VM_DOMAIN_SET_ATOMIC(n, p) BIT_SET_ATOMIC(VM_DOMAIN_SETSIZE, n, p) +#define VM_DOMAIN_AND_ATOMIC(n, p) BIT_AND_ATOMIC(VM_DOMAIN_SETSIZE, n, p) +#define VM_DOMAIN_OR_ATOMIC(d, s) BIT_OR_ATOMIC(VM_DOMAIN_SETSIZE, d, s) +#define VM_DOMAIN_COPY_STORE_REL(f, t) BIT_COPY_STORE_REL(VM_DOMAIN_SETSIZE, f, t) +#define VM_DOMAIN_FFS(p) BIT_FFS(VM_DOMAIN_SETSIZE, p) + +#ifdef _KERNEL + +/* + * Domain sets. + */ +extern vm_domainset_t vm_alldomains; /* All domains. */ +extern vm_domainset_t vm_domset[MAXMEMDOM]; /* Specific domain bitmask. */ +extern int vm_ndomains; + +/* + * Domain allocation selectors. + */ +extern struct vm_domain_select vm_sel_def; /* default */ +extern struct vm_domain_select vm_sel_rr; /* round-robin */ +extern struct vm_domain_select vm_sel_ft; /* first-touch */ +extern struct vm_domain_select vm_sel_dom[MAXMEMDOM]; /* specific domain */ + +static inline int +vm_domain_select_first(struct vm_domain_select *sel) +{ + int domain; + + switch (sel->ds_policy) { + case FIRSTTOUCH: + domain = PCPU_GET(domain); + if (VM_DOMAIN_ISSET(domain, &sel->ds_mask)) + break; + /* FALLTHROUGH */ + case ROUNDROBIN: + domain = atomic_fetchadd_int(&sel->ds_cursor, 1) % vm_ndomains; + while (!VM_DOMAIN_ISSET(domain, &sel->ds_mask)) + domain = (domain + 1) % vm_ndomains; + } + return domain; +} + +static inline int +vm_domain_select_next(struct vm_domain_select *sel, int domain) +{ + + switch (sel->ds_policy) { + case FIRSTTOUCH: + /* FALLTHROUGH */ + case ROUNDROBIN: + do { + domain = (domain + 1) % vm_ndomains; + } while (!VM_DOMAIN_ISSET(domain, &sel->ds_mask)); + } + return domain; +} + +#endif /* _KERNEL */ + +#endif /* !_VM_DOMAIN_H_ */ Modified: projects/numa/sys/vm/vm_object.c ============================================================================== --- projects/numa/sys/vm/vm_object.c Fri Feb 21 05:17:30 2014 (r262278) +++ projects/numa/sys/vm/vm_object.c Fri Feb 21 05:31:34 2014 (r262279) @@ -86,6 +86,7 @@ __FBSDID("$FreeBSD$"); #include <vm/vm.h> #include <vm/vm_param.h> #include <vm/pmap.h> +#include <vm/vm_domain.h> #include <vm/vm_map.h> #include <vm/vm_object.h> #include <vm/vm_page.h> @@ -243,6 +244,9 @@ _vm_object_allocate(objtype_t type, vm_p panic("_vm_object_allocate: type %d is undefined", type); } object->size = size; +#if MAXMEMDOM > 1 + object->selector = vm_sel_def; +#endif object->generation = 1; object->ref_count = 1; object->memattr = VM_MEMATTR_DEFAULT; @@ -254,7 +258,6 @@ _vm_object_allocate(objtype_t type, vm_p #if VM_NRESERVLEVEL > 0 LIST_INIT(&object->rvq); #endif - mtx_lock(&vm_object_list_mtx); TAILQ_INSERT_TAIL(&vm_object_list, object, object_list); mtx_unlock(&vm_object_list_mtx); @@ -1254,6 +1257,9 @@ vm_object_shadow( result->pg_color = (source->pg_color + OFF_TO_IDX(*offset)) & ((1 << (VM_NFREEORDER - 1)) - 1); #endif +#if MAXMEMDOM > 0 + result->selector = source->selector; +#endif VM_OBJECT_WUNLOCK(source); } @@ -1295,6 +1301,9 @@ vm_object_split(vm_map_entry_t entry) * into a swap object. */ new_object = vm_object_allocate(OBJT_DEFAULT, size); +#if MAXMEMDOM > 0 + new_object->selector = orig_object->selector; +#endif /* * At this point, the new object is still private, so the order in @@ -2200,24 +2209,6 @@ vm_object_set_writeable_dirty(vm_object_ vm_object_set_flag(object, OBJ_MIGHTBEDIRTY); } -int -vm_object_domain(vm_object_t object) -{ -#if MAXMEMDOM > 1 - static volatile unsigned int noobj_domain; - - if (object == NULL) - return (atomic_fetchadd_int(&noobj_domain, 1) % vm_ndomains); - - object->domain = (object->domain + 1) % vm_ndomains; - - return (object->domain); -#else - return (0); -#endif - return (0); -} - #include "opt_ddb.h" #ifdef DDB #include <sys/kernel.h> Modified: projects/numa/sys/vm/vm_object.h ============================================================================== --- projects/numa/sys/vm/vm_object.h Fri Feb 21 05:17:30 2014 (r262278) +++ projects/numa/sys/vm/vm_object.h Fri Feb 21 05:31:34 2014 (r262279) @@ -73,6 +73,7 @@ #include <sys/_rwlock.h> #include <vm/_vm_radix.h> +#include <vm/_vm_domain.h> /* * Types defined: @@ -106,7 +107,7 @@ struct vm_object { struct vm_radix rtree; /* root of the resident page radix trie*/ vm_pindex_t size; /* Object size */ #if MAXMEMDOM > 1 - int domain; /* Last selected NUMA domain. */ + struct vm_domain_select selector; /* NUMA domain policy. */ #endif int generation; /* generation ID */ int ref_count; /* How many refs?? */ @@ -293,7 +294,6 @@ void vm_object_shadow (vm_object_t *, vm void vm_object_split(vm_map_entry_t); boolean_t vm_object_sync(vm_object_t, vm_ooffset_t, vm_size_t, boolean_t, boolean_t); -int vm_object_domain(vm_object_t); #endif /* _KERNEL */ Modified: projects/numa/sys/vm/vm_page.c ============================================================================== --- projects/numa/sys/vm/vm_page.c Fri Feb 21 05:17:30 2014 (r262278) +++ projects/numa/sys/vm/vm_page.c Fri Feb 21 05:31:34 2014 (r262279) @@ -104,6 +104,7 @@ __FBSDID("$FreeBSD$"); #include <vm/vm.h> #include <vm/pmap.h> #include <vm/vm_param.h> +#include <vm/vm_domain.h> #include <vm/vm_kern.h> #include <vm/vm_object.h> #include <vm/vm_page.h> @@ -1447,9 +1448,26 @@ vm_page_is_cached(vm_object_t object, vm vm_page_t vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req) { +#if MAXMEMDOM > 1 + struct vm_domain_select *sel; + vm_page_t m; + int i, dom; + + if (object == NULL) + sel = &vm_sel_def; + else + sel = &object->selector; - return vm_page_alloc_domain(object, pindex, vm_object_domain(object), - req); + for (i = 0, dom = vm_domain_select_first(sel); + i < sel->ds_count; i++, dom = vm_domain_select_next(sel, dom)) { + if ((m = vm_page_alloc_domain(object, pindex, dom, + req)) != NULL) + return (m); + } + return (NULL); +#else + return vm_page_alloc_domain(object, pindex, 0, req); +#endif } vm_page_t @@ -1706,9 +1724,28 @@ vm_page_alloc_contig(vm_object_t object, u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, vm_memattr_t memattr) { +#if MAXMEMDOM > 1 + struct vm_domain_select *sel; + vm_page_t m; + int i, dom; + + if (object == NULL) + sel = &vm_sel_def; + else + sel = &object->selector; + + for (i = 0, dom = vm_domain_select_first(sel); + i < sel->ds_count; i++, dom = vm_domain_select_next(sel, dom)) { + if ((m = vm_page_alloc_contig_domain(object, pindex, + dom, req, npages, low, high, + alignment, boundary, memattr)) != NULL) + return (m); + } + return (NULL); +#else return vm_page_alloc_contig_domain(object, pindex, - vm_object_domain(object), req, npages, low, high, - alignment, boundary, memattr); + 0, req, npages, low, high, alignment, boundary, memattr); +#endif } vm_page_t Modified: projects/numa/sys/vm/vm_phys.c ============================================================================== --- projects/numa/sys/vm/vm_phys.c Fri Feb 21 05:17:30 2014 (r262278) +++ projects/numa/sys/vm/vm_phys.c Fri Feb 21 05:31:34 2014 (r262279) @@ -60,6 +60,7 @@ __FBSDID("$FreeBSD$"); #include <vm/vm.h> #include <vm/vm_param.h> +#include <vm/vm_domain.h> #include <vm/vm_kern.h> #include <vm/vm_object.h> #include <vm/vm_page.h> @@ -71,6 +72,12 @@ _Static_assert(sizeof(long) * NBBY >= VM struct mem_affinity *mem_affinity; int vm_ndomains = 1; +vm_domainset_t vm_alldomains; +vm_domainset_t vm_domset[MAXMEMDOM]; +struct vm_domain_select vm_sel_def; +struct vm_domain_select vm_sel_rr; +struct vm_domain_select vm_sel_ft; +struct vm_domain_select vm_sel_dom[MAXMEMDOM]; struct vm_phys_seg vm_phys_segs[VM_PHYSSEG_MAX]; int vm_phys_nsegs; @@ -327,6 +334,11 @@ vm_phys_init(void) VM_FREELIST_DEFAULT); } for (dom = 0; dom < vm_ndomains; dom++) { + VM_DOMAIN_SET(dom, &vm_alldomains); + VM_DOMAIN_SET(dom, &vm_domset[dom]); + vm_sel_dom[dom].ds_mask = vm_domset[dom]; + vm_sel_dom[dom].ds_policy = ROUNDROBIN; + vm_sel_dom[dom].ds_count = 1; for (flind = 0; flind < vm_nfreelists; flind++) { for (pind = 0; pind < VM_NFREEPOOL; pind++) { fl = vm_phys_free_queues[dom][flind][pind]; @@ -335,6 +347,15 @@ vm_phys_init(void) } } } + vm_sel_def.ds_mask = vm_alldomains; + vm_sel_def.ds_policy = ROUNDROBIN; + vm_sel_def.ds_count = vm_ndomains; + vm_sel_rr.ds_mask = vm_alldomains; + vm_sel_rr.ds_policy = ROUNDROBIN; + vm_sel_rr.ds_count = vm_ndomains; + vm_sel_ft.ds_mask = vm_alldomains; + vm_sel_ft.ds_policy = FIRSTTOUCH; + vm_sel_ft.ds_count = vm_ndomains; mtx_init(&vm_phys_fictitious_reg_mtx, "vmfctr", NULL, MTX_DEF); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201402210531.s1L5VYPg063076>