Date: Thu, 1 Nov 2018 15:19:36 +0000 (UTC) From: Mark Johnston <markj@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org Subject: svn commit: r339998 - in stable/12/sys: sys vm Message-ID: <201811011519.wA1FJaxd009424@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: markj Date: Thu Nov 1 15:19:36 2018 New Revision: 339998 URL: https://svnweb.freebsd.org/changeset/base/339998 Log: MFC r339661, r339669: Refactor domainset iterators for use by malloc(9) and UMA. Approved by: re (gjb) Modified: stable/12/sys/sys/_domainset.h stable/12/sys/vm/vm_domainset.c stable/12/sys/vm/vm_domainset.h stable/12/sys/vm/vm_glue.c stable/12/sys/vm/vm_kern.c stable/12/sys/vm/vm_object.c stable/12/sys/vm/vm_page.c Directory Properties: stable/12/ (props changed) Modified: stable/12/sys/sys/_domainset.h ============================================================================== --- stable/12/sys/sys/_domainset.h Thu Nov 1 14:43:34 2018 (r339997) +++ stable/12/sys/sys/_domainset.h Thu Nov 1 15:19:36 2018 (r339998) @@ -54,7 +54,7 @@ typedef struct _domainset domainset_t; struct domainset; struct domainset_ref { struct domainset * volatile dr_policy; - unsigned int dr_iterator; + unsigned int dr_iter; }; #endif /* !_SYS__DOMAINSET_H_ */ Modified: stable/12/sys/vm/vm_domainset.c ============================================================================== --- stable/12/sys/vm/vm_domainset.c Thu Nov 1 14:43:34 2018 (r339997) +++ stable/12/sys/vm/vm_domainset.c Thu Nov 1 15:19:36 2018 (r339998) @@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$"); #include <sys/lock.h> #include <sys/mutex.h> #include <sys/malloc.h> +#include <sys/rwlock.h> #include <sys/vmmeter.h> #include <vm/vm.h> @@ -62,26 +63,13 @@ static int vm_domainset_default_stride = 64; * Determine which policy is to be used for this allocation. */ static void -vm_domainset_iter_init(struct vm_domainset_iter *di, struct vm_object *obj, - vm_pindex_t pindex) +vm_domainset_iter_init(struct vm_domainset_iter *di, struct domainset *ds, + int *iter, struct vm_object *obj, vm_pindex_t pindex) { - struct domainset *domain; - struct thread *td; - /* - * object policy takes precedence over thread policy. The policies - * are immutable and unsynchronized. Updates can race but pointer - * loads are assumed to be atomic. - */ - if (obj != NULL && (domain = obj->domain.dr_policy) != NULL) { - di->di_domain = domain; - di->di_iter = &obj->domain.dr_iterator; - } else { - td = curthread; - di->di_domain = td->td_domain.dr_policy; - di->di_iter = &td->td_domain.dr_iterator; - } - di->di_policy = di->di_domain->ds_policy; + di->di_domain = ds; + di->di_iter = iter; + di->di_policy = ds->ds_policy; if (di->di_policy == DOMAINSET_POLICY_INTERLEAVE) { #if VM_NRESERVLEVEL > 0 if (vm_object_reserv(obj)) { @@ -211,33 +199,39 @@ void vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj, vm_pindex_t pindex, int *domain, int *req) { + struct domainset_ref *dr; - vm_domainset_iter_init(di, obj, pindex); + /* + * Object policy takes precedence over thread policy. The policies + * are immutable and unsynchronized. Updates can race but pointer + * loads are assumed to be atomic. + */ + if (obj != NULL && obj->domain.dr_policy != NULL) + dr = &obj->domain; + else + dr = &curthread->td_domain; + vm_domainset_iter_init(di, dr->dr_policy, &dr->dr_iter, obj, pindex); di->di_flags = *req; *req = (di->di_flags & ~(VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) | VM_ALLOC_NOWAIT; vm_domainset_iter_first(di, domain); if (vm_page_count_min_domain(*domain)) - vm_domainset_iter_page(di, domain, req); + vm_domainset_iter_page(di, obj, domain); } int -vm_domainset_iter_page(struct vm_domainset_iter *di, int *domain, int *req) +vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj, + int *domain) { - /* - * If we exhausted all options with NOWAIT and did a WAITFAIL it - * is time to return an error to the caller. - */ - if ((*req & VM_ALLOC_WAITFAIL) != 0) - return (ENOMEM); - /* If there are more domains to visit we run the iterator. */ while (--di->di_n != 0) { vm_domainset_iter_next(di, domain); if (!di->di_minskip || !vm_page_count_min_domain(*domain)) return (0); } + + /* If we skipped domains below min restart the search. */ if (di->di_minskip) { di->di_minskip = false; vm_domainset_iter_first(di, domain); @@ -248,34 +242,53 @@ vm_domainset_iter_page(struct vm_domainset_iter *di, i if ((di->di_flags & (VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) == 0) return (ENOMEM); - /* - * We have visited all domains with non-blocking allocations, try - * from the beginning with a blocking allocation. - */ + /* Wait for one of the domains to accumulate some free pages. */ + if (obj != NULL) + VM_OBJECT_WUNLOCK(obj); + vm_wait_doms(&di->di_domain->ds_mask); + if (obj != NULL) + VM_OBJECT_WLOCK(obj); + if ((di->di_flags & VM_ALLOC_WAITFAIL) != 0) + return (ENOMEM); + + /* Restart the search. */ vm_domainset_iter_first(di, domain); - *req = di->di_flags; return (0); } - -void -vm_domainset_iter_malloc_init(struct vm_domainset_iter *di, - struct vm_object *obj, int *domain, int *flags) +static void +_vm_domainset_iter_policy_init(struct vm_domainset_iter *di, int *domain, + int *flags) { - vm_domainset_iter_init(di, obj, 0); - if (di->di_policy == DOMAINSET_POLICY_INTERLEAVE) - di->di_policy = DOMAINSET_POLICY_ROUNDROBIN; di->di_flags = *flags; *flags = (di->di_flags & ~M_WAITOK) | M_NOWAIT; vm_domainset_iter_first(di, domain); if (vm_page_count_min_domain(*domain)) - vm_domainset_iter_malloc(di, domain, flags); + vm_domainset_iter_policy(di, domain); } +void +vm_domainset_iter_policy_init(struct vm_domainset_iter *di, + struct domainset *ds, int *domain, int *flags) +{ + + vm_domainset_iter_init(di, ds, &curthread->td_domain.dr_iter, NULL, 0); + _vm_domainset_iter_policy_init(di, domain, flags); +} + +void +vm_domainset_iter_policy_ref_init(struct vm_domainset_iter *di, + struct domainset_ref *dr, int *domain, int *flags) +{ + + vm_domainset_iter_init(di, dr->dr_policy, &dr->dr_iter, NULL, 0); + _vm_domainset_iter_policy_init(di, domain, flags); +} + int -vm_domainset_iter_malloc(struct vm_domainset_iter *di, int *domain, int *flags) +vm_domainset_iter_policy(struct vm_domainset_iter *di, int *domain) { /* If there are more domains to visit we run the iterator. */ @@ -296,45 +309,54 @@ vm_domainset_iter_malloc(struct vm_domainset_iter *di, if ((di->di_flags & M_WAITOK) == 0) return (ENOMEM); - /* - * We have visited all domains with non-blocking allocations, try - * from the beginning with a blocking allocation. - */ + /* Wait for one of the domains to accumulate some free pages. */ + vm_wait_doms(&di->di_domain->ds_mask); + + /* Restart the search. */ vm_domainset_iter_first(di, domain); - *flags = di->di_flags; return (0); } #else /* !NUMA */ + int -vm_domainset_iter_page(struct vm_domainset_iter *di, int *domain, int *flags) +vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj, + int *domain) { return (EJUSTRETURN); } void -vm_domainset_iter_page_init(struct vm_domainset_iter *di, - struct vm_object *obj, vm_pindex_t pindex, int *domain, int *flags) +vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj, + vm_pindex_t pindex, int *domain, int *flags) { *domain = 0; } int -vm_domainset_iter_malloc(struct vm_domainset_iter *di, int *domain, int *flags) +vm_domainset_iter_policy(struct vm_domainset_iter *di, int *domain) { return (EJUSTRETURN); } void -vm_domainset_iter_malloc_init(struct vm_domainset_iter *di, - struct vm_object *obj, int *domain, int *flags) +vm_domainset_iter_policy_init(struct vm_domainset_iter *di, + struct domainset *ds, int *domain, int *flags) { *domain = 0; } -#endif +void +vm_domainset_iter_policy_ref_init(struct vm_domainset_iter *di, + struct domainset_ref *dr, int *domain, int *flags) +{ + + *domain = 0; +} + +#endif /* NUMA */ Modified: stable/12/sys/vm/vm_domainset.h ============================================================================== --- stable/12/sys/vm/vm_domainset.h Thu Nov 1 14:43:34 2018 (r339997) +++ stable/12/sys/vm/vm_domainset.h Thu Nov 1 15:19:36 2018 (r339998) @@ -40,12 +40,15 @@ struct vm_domainset_iter { bool di_minskip; }; -int vm_domainset_iter_page(struct vm_domainset_iter *, int *, int *); +int vm_domainset_iter_page(struct vm_domainset_iter *, struct vm_object *, + int *); void vm_domainset_iter_page_init(struct vm_domainset_iter *, struct vm_object *, vm_pindex_t, int *, int *); -int vm_domainset_iter_malloc(struct vm_domainset_iter *, int *, int *); -void vm_domainset_iter_malloc_init(struct vm_domainset_iter *, - struct vm_object *, int *, int *); +int vm_domainset_iter_policy(struct vm_domainset_iter *, int *); +void vm_domainset_iter_policy_init(struct vm_domainset_iter *, + struct domainset *, int *, int *); +void vm_domainset_iter_policy_ref_init(struct vm_domainset_iter *, + struct domainset_ref *, int *, int *); void vm_wait_doms(const domainset_t *); Modified: stable/12/sys/vm/vm_glue.c ============================================================================== --- stable/12/sys/vm/vm_glue.c Thu Nov 1 14:43:34 2018 (r339997) +++ stable/12/sys/vm/vm_glue.c Thu Nov 1 15:19:36 2018 (r339998) @@ -377,7 +377,7 @@ vm_thread_new(struct thread *td, int pages) */ if (vm_ndomains > 1) { ksobj->domain.dr_policy = DOMAINSET_RR(); - ksobj->domain.dr_iterator = + ksobj->domain.dr_iter = atomic_fetchadd_int(&kstack_domain_iter, 1); } Modified: stable/12/sys/vm/vm_kern.c ============================================================================== --- stable/12/sys/vm/vm_kern.c Thu Nov 1 14:43:34 2018 (r339997) +++ stable/12/sys/vm/vm_kern.c Thu Nov 1 15:19:36 2018 (r339998) @@ -235,13 +235,13 @@ kmem_alloc_attr(vm_size_t size, int flags, vm_paddr_t vm_offset_t addr; int domain; - vm_domainset_iter_malloc_init(&di, kernel_object, &domain, &flags); + vm_domainset_iter_policy_init(&di, DOMAINSET_RR(), &domain, &flags); do { addr = kmem_alloc_attr_domain(domain, size, flags, low, high, memattr); if (addr != 0) break; - } while (vm_domainset_iter_malloc(&di, &domain, &flags) == 0); + } while (vm_domainset_iter_policy(&di, &domain) == 0); return (addr); } @@ -319,13 +319,13 @@ kmem_alloc_contig(vm_size_t size, int flags, vm_paddr_ vm_offset_t addr; int domain; - vm_domainset_iter_malloc_init(&di, kernel_object, &domain, &flags); + vm_domainset_iter_policy_init(&di, DOMAINSET_RR(), &domain, &flags); do { addr = kmem_alloc_contig_domain(domain, size, flags, low, high, alignment, boundary, memattr); if (addr != 0) break; - } while (vm_domainset_iter_malloc(&di, &domain, &flags) == 0); + } while (vm_domainset_iter_policy(&di, &domain) == 0); return (addr); } @@ -406,12 +406,12 @@ kmem_malloc(vm_size_t size, int flags) vm_offset_t addr; int domain; - vm_domainset_iter_malloc_init(&di, kernel_object, &domain, &flags); + vm_domainset_iter_policy_init(&di, DOMAINSET_RR(), &domain, &flags); do { addr = kmem_malloc_domain(domain, size, flags); if (addr != 0) break; - } while (vm_domainset_iter_malloc(&di, &domain, &flags) == 0); + } while (vm_domainset_iter_policy(&di, &domain) == 0); return (addr); } Modified: stable/12/sys/vm/vm_object.c ============================================================================== --- stable/12/sys/vm/vm_object.c Thu Nov 1 14:43:34 2018 (r339997) +++ stable/12/sys/vm/vm_object.c Thu Nov 1 15:19:36 2018 (r339998) @@ -274,6 +274,7 @@ _vm_object_allocate(objtype_t type, vm_pindex_t size, panic("_vm_object_allocate: type %d is undefined", type); } object->size = size; + object->domain.dr_policy = NULL; object->generation = 1; object->ref_count = 1; object->memattr = VM_MEMATTR_DEFAULT; Modified: stable/12/sys/vm/vm_page.c ============================================================================== --- stable/12/sys/vm/vm_page.c Thu Nov 1 14:43:34 2018 (r339997) +++ stable/12/sys/vm/vm_page.c Thu Nov 1 15:19:36 2018 (r339998) @@ -1753,7 +1753,7 @@ vm_page_alloc_after(vm_object_t object, vm_pindex_t pi mpred); if (m != NULL) break; - } while (vm_domainset_iter_page(&di, &domain, &req) == 0); + } while (vm_domainset_iter_page(&di, object, &domain) == 0); return (m); } @@ -1990,7 +1990,7 @@ vm_page_alloc_contig(vm_object_t object, vm_pindex_t p npages, low, high, alignment, boundary, memattr); if (m != NULL) break; - } while (vm_domainset_iter_page(&di, &domain, &req) == 0); + } while (vm_domainset_iter_page(&di, object, &domain) == 0); return (m); } @@ -2191,7 +2191,7 @@ vm_page_alloc_freelist(int freelist, int req) m = vm_page_alloc_freelist_domain(domain, freelist, req); if (m != NULL) break; - } while (vm_domainset_iter_page(&di, &domain, &req) == 0); + } while (vm_domainset_iter_page(&di, NULL, &domain) == 0); return (m); } @@ -2830,7 +2830,7 @@ vm_page_reclaim_contig(int req, u_long npages, vm_padd high, alignment, boundary); if (ret) break; - } while (vm_domainset_iter_page(&di, &domain, &req) == 0); + } while (vm_domainset_iter_page(&di, NULL, &domain) == 0); return (ret); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201811011519.wA1FJaxd009424>