Date: Wed, 22 Feb 2017 19:39:54 +0000 (UTC) From: Hans Petter Selasky <hselasky@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r314106 - in head/sys: compat/linuxkpi/common/include/linux compat/linuxkpi/common/src conf modules/linuxkpi Message-ID: <201702221939.v1MJdslE035438@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: hselasky Date: Wed Feb 22 19:39:54 2017 New Revision: 314106 URL: https://svnweb.freebsd.org/changeset/base/314106 Log: Optimise unmapped LinuxKPI page allocations. When allocating unmapped pages, take advantage of the direct map on AMD64 to get the virtual address corresponding to a page. Else all pages allocated must be mapped because sometimes the virtual address of a page is requested. Move all page allocation and deallocation code into an own C-file. Add support for GFP_DMA32, GFP_KERNEL, GFP_ATOMIC and __GFP_ZERO allocation flags. Make a clear separation between mapped and unmapped allocations. Obtained from: kmacy @ MFC after: 1 week Sponsored by: Mellanox Technologies Added: head/sys/compat/linuxkpi/common/src/linux_page.c (contents, props changed) Modified: head/sys/compat/linuxkpi/common/include/linux/gfp.h head/sys/conf/files head/sys/modules/linuxkpi/Makefile Modified: head/sys/compat/linuxkpi/common/include/linux/gfp.h ============================================================================== --- head/sys/compat/linuxkpi/common/include/linux/gfp.h Wed Feb 22 19:31:02 2017 (r314105) +++ head/sys/compat/linuxkpi/common/include/linux/gfp.h Wed Feb 22 19:39:54 2017 (r314106) @@ -2,7 +2,7 @@ * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. - * Copyright (c) 2013 Mellanox Technologies, Ltd. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -67,94 +67,106 @@ #define GFP_TEMPORARY M_NOWAIT #define GFP_NATIVE_MASK (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_ZERO) -static inline void * -page_address(struct page *page) +/* + * Resolve a page into a virtual address: + * + * NOTE: This function only works for pages allocated by the kernel. + */ +extern void *linux_page_address(struct page *); + +#define page_address(page) linux_page_address(page) + +/* + * Page management for unmapped pages: + */ +extern vm_page_t linux_alloc_pages(gfp_t flags, unsigned int order); +extern void linux_free_pages(vm_page_t page, unsigned int order); + +static inline struct page * +alloc_page(gfp_t flags) { - if (page->object != kmem_object && page->object != kernel_object) - return (NULL); - return ((void *)(uintptr_t)(VM_MIN_KERNEL_ADDRESS + - IDX_TO_OFF(page->pindex))); + return (linux_alloc_pages(flags, 0)); } -static inline unsigned long -linux_get_page(gfp_t mask) +static inline struct page * +alloc_pages(gfp_t flags, unsigned int order) { - return kmem_malloc(kmem_arena, PAGE_SIZE, mask); + return (linux_alloc_pages(flags, order)); } -#define get_zeroed_page(mask) linux_get_page((mask) | M_ZERO) -#define alloc_page(mask) virt_to_page(linux_get_page((mask))) -#define __get_free_page(mask) linux_get_page((mask)) +static inline struct page * +alloc_pages_node(int node_id, gfp_t flags, unsigned int order) +{ + + return (linux_alloc_pages(flags, order)); +} static inline void -free_page(unsigned long page) +__free_pages(struct page *page, unsigned int order) { - if (page == 0) - return; - kmem_free(kmem_arena, page, PAGE_SIZE); + linux_free_pages(page, order); } static inline void -__free_page(struct page *m) +__free_page(struct page *page) { - if (m->object != kmem_object) - panic("__free_page: Freed page %p not allocated via wrappers.", - m); - kmem_free(kmem_arena, (vm_offset_t)page_address(m), PAGE_SIZE); + linux_free_pages(page, 0); } -static inline void -__free_pages(struct page *m, unsigned int order) +/* + * Page management for mapped pages: + */ +extern vm_offset_t linux_alloc_kmem(gfp_t flags, unsigned int order); +extern void linux_free_kmem(vm_offset_t, unsigned int order); + +static inline vm_offset_t +get_zeroed_page(gfp_t flags) { - size_t size; - if (m == NULL) - return; - size = PAGE_SIZE << order; - kmem_free(kmem_arena, (vm_offset_t)page_address(m), size); + return (linux_alloc_kmem(flags | __GFP_ZERO, 0)); } -static inline void free_pages(uintptr_t addr, unsigned int order) +static inline vm_offset_t +__get_free_page(gfp_t flags) { - if (addr == 0) - return; - __free_pages(virt_to_page((void *)addr), order); + + return (linux_alloc_kmem(flags, 0)); } -/* - * Alloc pages allocates directly from the buddy allocator on linux so - * order specifies a power of two bucket of pages and the results - * are expected to be aligned on the size as well. - */ -static inline struct page * -alloc_pages(gfp_t gfp_mask, unsigned int order) +static inline vm_offset_t +__get_free_pages(gfp_t flags, unsigned int order) { - unsigned long page; - size_t size; - size = PAGE_SIZE << order; - page = kmem_alloc_contig(kmem_arena, size, gfp_mask, - 0, ~(vm_paddr_t)0, size, 0, VM_MEMATTR_DEFAULT); - if (page == 0) - return (NULL); - return (virt_to_page(page)); + return (linux_alloc_kmem(flags, order)); } -static inline uintptr_t __get_free_pages(gfp_t gfp_mask, unsigned int order) +static inline void +free_pages(uintptr_t addr, unsigned int order) { - struct page *page; + if (addr == 0) + return; - page = alloc_pages(gfp_mask, order); - if (page == NULL) - return (0); - return ((uintptr_t)page_address(page)); + linux_free_kmem(addr, order); } -#define alloc_pages_node(node, mask, order) alloc_pages(mask, order) +static inline void +free_page(uintptr_t addr) +{ + if (addr == 0) + return; + + linux_free_kmem(addr, 0); +} + +static inline bool +gfpflags_allow_blocking(const gfp_t gfp_flags) +{ + return ((gfp_flags & (M_WAITOK | M_NOWAIT)) == M_WAITOK); +} #define kmalloc_node(chunk, mask, node) kmalloc(chunk, mask) Added: head/sys/compat/linuxkpi/common/src/linux_page.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/compat/linuxkpi/common/src/linux_page.c Wed Feb 22 19:39:54 2017 (r314106) @@ -0,0 +1,167 @@ +/*- + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2016 Matt Macy (mmacy@nextbsd.org) + * Copyright (c) 2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/malloc.h> +#include <sys/kernel.h> +#include <sys/sysctl.h> +#include <sys/lock.h> +#include <sys/mutex.h> +#include <sys/rwlock.h> +#include <sys/proc.h> +#include <sys/sched.h> + +#include <machine/bus.h> + +#include <linux/gfp.h> + +#include <vm/vm.h> +#include <vm/vm_page.h> +#include <vm/vm_pageout.h> + +void * +linux_page_address(struct page *page) +{ +#ifdef __amd64__ + return ((void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(page))); +#else + if (page->object != kmem_object && page->object != kernel_object) + return (NULL); + return ((void *)(uintptr_t)(VM_MIN_KERNEL_ADDRESS + + IDX_TO_OFF(page->pindex))); +#endif +} + +vm_page_t +linux_alloc_pages(gfp_t flags, unsigned int order) +{ +#ifdef __amd64__ + unsigned long npages = 1UL << order; + int req = (flags & M_ZERO) ? (VM_ALLOC_ZERO | VM_ALLOC_NOOBJ | + VM_ALLOC_NORMAL) : (VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL); + vm_page_t page; + + if (order == 0 && (flags & GFP_DMA32) == 0) { + page = vm_page_alloc(NULL, 0, req); + if (page == NULL) + return (NULL); + } else { + vm_paddr_t pmax = (flags & GFP_DMA32) ? + BUS_SPACE_MAXADDR_32BIT : BUS_SPACE_MAXADDR; +retry: + page = vm_page_alloc_contig(NULL, 0, req, + npages, 0, pmax, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); + + if (page == NULL) { + if (flags & M_WAITOK) { + if (!vm_page_reclaim_contig(req, + npages, 0, pmax, PAGE_SIZE, 0)) { + VM_WAIT; + } + flags &= ~M_WAITOK; + goto retry; + } + return (NULL); + } + } + if (flags & M_ZERO) { + unsigned long x; + + for (x = 0; x != npages; x++) { + vm_page_t pgo = page + x; + + if ((pgo->flags & PG_ZERO) == 0) + pmap_zero_page(pgo); + } + } +#else + vm_offset_t vaddr; + vm_page_t page; + + vaddr = linux_alloc_kmem(flags, order); + if (vaddr == 0) + return (NULL); + + page = PHYS_TO_VM_PAGE(vtophys((void *)vaddr)); + + KASSERT(vaddr == (vm_offset_t)page_address(page), + ("Page address mismatch")); +#endif + return (page); +} + +void +linux_free_pages(vm_page_t page, unsigned int order) +{ +#ifdef __amd64__ + unsigned long npages = 1UL << order; + unsigned long x; + + for (x = 0; x != npages; x++) { + vm_page_t pgo = page + x; + + vm_page_lock(pgo); + vm_page_free(pgo); + vm_page_unlock(pgo); + } +#else + vm_offset_t vaddr; + + vaddr = (vm_offset_t)page_address(page); + + linux_free_kmem(vaddr, order); +#endif +} + +vm_offset_t +linux_alloc_kmem(gfp_t flags, unsigned int order) +{ + size_t size = ((size_t)PAGE_SIZE) << order; + vm_offset_t addr; + + if ((flags & GFP_DMA32) == 0) { + addr = kmem_malloc(kmem_arena, size, flags & GFP_NATIVE_MASK); + } else { + addr = kmem_alloc_contig(kmem_arena, size, + flags & GFP_NATIVE_MASK, 0, BUS_SPACE_MAXADDR_32BIT, + PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); + } + return (addr); +} + +void +linux_free_kmem(vm_offset_t addr, unsigned int order) +{ + size_t size = ((size_t)PAGE_SIZE) << order; + + kmem_free(kmem_arena, addr, size); +} Modified: head/sys/conf/files ============================================================================== --- head/sys/conf/files Wed Feb 22 19:31:02 2017 (r314105) +++ head/sys/conf/files Wed Feb 22 19:39:54 2017 (r314106) @@ -4286,6 +4286,8 @@ compat/linuxkpi/common/src/linux_current compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_kthread.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" +compat/linuxkpi/common/src/linux_page.c optional compat_linuxkpi \ + compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_pci.c optional compat_linuxkpi pci \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_tasklet.c optional compat_linuxkpi \ Modified: head/sys/modules/linuxkpi/Makefile ============================================================================== --- head/sys/modules/linuxkpi/Makefile Wed Feb 22 19:31:02 2017 (r314105) +++ head/sys/modules/linuxkpi/Makefile Wed Feb 22 19:39:54 2017 (r314106) @@ -6,6 +6,7 @@ SRCS= linux_kmod.c \ linux_compat.c \ linux_current.c \ linux_kthread.c \ + linux_page.c \ linux_pci.c \ linux_radix.c \ linux_rcu.c \
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201702221939.v1MJdslE035438>