From owner-svn-src-head@freebsd.org Wed Mar 8 11:09:28 2017 Return-Path: Delivered-To: svn-src-head@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id C8345D02949; Wed, 8 Mar 2017 11:09:28 +0000 (UTC) (envelope-from hselasky@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id A2C461775; Wed, 8 Mar 2017 11:09:28 +0000 (UTC) (envelope-from hselasky@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v28B9RTn000623; Wed, 8 Mar 2017 11:09:27 GMT (envelope-from hselasky@FreeBSD.org) Received: (from hselasky@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v28B9RCD000618; Wed, 8 Mar 2017 11:09:27 GMT (envelope-from hselasky@FreeBSD.org) Message-Id: <201703081109.v28B9RCD000618@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: hselasky set sender to hselasky@FreeBSD.org using -f From: Hans Petter Selasky Date: Wed, 8 Mar 2017 11:09:27 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r314905 - in head/sys: compat/linuxkpi/common/include/linux compat/linuxkpi/common/src conf modules/linuxkpi X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 08 Mar 2017 11:09:28 -0000 Author: hselasky Date: Wed Mar 8 11:09:27 2017 New Revision: 314905 URL: https://svnweb.freebsd.org/changeset/base/314905 Log: Cleanup the LinuxKPI slab implementation. Put large functions into linux_slab.c instead of declaring them static inline. Add support for more memory allocation wrappers like kmalloc_array() and __vmalloc(). Make sure either the M_WAITOK or the M_NOWAIT flag is set and mask away unused memory allocation flags before calling FreeBSD's malloc() routine. Move kmalloc_node() definition to slab.h where it belongs. Implement support for the SLAB_DESTROY_BY_RCU feature when creating a kmem_cache which basically means kmem_cache memory is freed using call_rcu(). MFC after: 1 week Sponsored by: Mellanox Technologies Added: head/sys/compat/linuxkpi/common/src/linux_slab.c (contents, props changed) Modified: head/sys/compat/linuxkpi/common/include/linux/gfp.h head/sys/compat/linuxkpi/common/include/linux/slab.h head/sys/conf/files head/sys/modules/linuxkpi/Makefile Modified: head/sys/compat/linuxkpi/common/include/linux/gfp.h ============================================================================== --- head/sys/compat/linuxkpi/common/include/linux/gfp.h Wed Mar 8 09:53:20 2017 (r314904) +++ head/sys/compat/linuxkpi/common/include/linux/gfp.h Wed Mar 8 11:09:27 2017 (r314905) @@ -168,8 +168,6 @@ gfpflags_allow_blocking(const gfp_t gfp_ return ((gfp_flags & (M_WAITOK | M_NOWAIT)) == M_WAITOK); } -#define kmalloc_node(chunk, mask, node) kmalloc(chunk, mask) - #define SetPageReserved(page) do { } while (0) /* NOP */ #define ClearPageReserved(page) do { } while (0) /* NOP */ Modified: head/sys/compat/linuxkpi/common/include/linux/slab.h ============================================================================== --- head/sys/compat/linuxkpi/common/include/linux/slab.h Wed Mar 8 09:53:20 2017 (r314904) +++ head/sys/compat/linuxkpi/common/include/linux/slab.h Wed Mar 8 11:09:27 2017 (r314905) @@ -2,7 +2,7 @@ * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. - * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. + * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -41,20 +41,19 @@ MALLOC_DECLARE(M_KMALLOC); -#define kmalloc(size, flags) malloc((size), M_KMALLOC, (flags)) -#define kvmalloc(size) kmalloc((size), 0) -#define kzalloc(size, flags) kmalloc((size), M_ZERO | ((flags) ? (flags) : M_NOWAIT)) -#define kzalloc_node(size, flags, node) kzalloc(size, flags) -#define kfree(ptr) free(__DECONST(void *, (ptr)), M_KMALLOC) +#define kvmalloc(size) kmalloc(size, 0) +#define kzalloc(size, flags) kmalloc(size, (flags) | __GFP_ZERO) +#define kzalloc_node(size, flags, node) kmalloc(size, (flags) | __GFP_ZERO) #define kfree_const(ptr) kfree(ptr) -#define krealloc(ptr, size, flags) realloc((ptr), (size), M_KMALLOC, (flags)) -#define kcalloc(n, size, flags) kmalloc((n) * (size), flags | M_ZERO) -#define vzalloc(size) kzalloc(size, GFP_KERNEL | __GFP_NOWARN) +#define kcalloc(n, size, flags) kmalloc((n) * (size), (flags) | __GFP_ZERO) +#define vzalloc(size) __vmalloc(size, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO, 0) #define vfree(arg) kfree(arg) #define kvfree(arg) kfree(arg) -#define vmalloc(size) kmalloc(size, GFP_KERNEL) -#define vmalloc_node(size, node) kmalloc(size, GFP_KERNEL) - +#define vmalloc_node(size, node) __vmalloc(size, GFP_KERNEL, 0) +#define vmalloc_user(size) __vmalloc(size, GFP_KERNEL | __GFP_ZERO, 0) +#define vmalloc(size) __vmalloc(size, GFP_KERNEL, 0) +#define __kmalloc(...) kmalloc(__VA_ARGS__) +#define kmalloc_node(chunk, flags, n) kmalloc(chunk, flags) /* * Prefix some functions with linux_ to avoid namespace conflict @@ -66,59 +65,94 @@ MALLOC_DECLARE(M_KMALLOC); #define kmem_cache_free(...) linux_kmem_cache_free(__VA_ARGS__) #define kmem_cache_destroy(...) linux_kmem_cache_destroy(__VA_ARGS__) +typedef void linux_kmem_ctor_t (void *); + struct linux_kmem_cache { - uma_zone_t cache_zone; - void (*cache_ctor)(void *); + uma_zone_t cache_zone; + linux_kmem_ctor_t *cache_ctor; + unsigned cache_flags; + unsigned cache_size; }; -#define SLAB_HWCACHE_ALIGN 0x0001 +#define SLAB_HWCACHE_ALIGN (1 << 0) +#define SLAB_DESTROY_BY_RCU (1 << 1) +#define SLAB_RECLAIM_ACCOUNT (1 << 2) -static inline int -linux_kmem_ctor(void *mem, int size, void *arg, int flags) +static inline gfp_t +linux_check_m_flags(gfp_t flags) { - void (*ctor)(void *); + const gfp_t m = M_NOWAIT | M_WAITOK; - ctor = arg; - ctor(mem); + /* make sure either M_NOWAIT or M_WAITOK is set */ + if ((flags & m) == 0) + flags |= M_NOWAIT; + else if ((flags & m) == m) + flags &= ~M_WAITOK; - return (0); + /* mask away LinuxKPI specific flags */ + return (flags & GFP_NATIVE_MASK); } -static inline struct kmem_cache * -linux_kmem_cache_create(char *name, size_t size, size_t align, u_long flags, - void (*ctor)(void *)) +static inline void * +kmalloc(size_t size, gfp_t flags) { - struct kmem_cache *c; + return (malloc(size, M_KMALLOC, linux_check_m_flags(flags))); +} - c = malloc(sizeof(*c), M_KMALLOC, M_WAITOK); - if (align) - align--; - if (flags & SLAB_HWCACHE_ALIGN) - align = UMA_ALIGN_CACHE; - c->cache_zone = uma_zcreate(name, size, ctor ? linux_kmem_ctor : NULL, - NULL, NULL, NULL, align, 0); - c->cache_ctor = ctor; +static inline void * +__vmalloc(size_t size, gfp_t flags, int other) +{ + return (malloc(size, M_KMALLOC, linux_check_m_flags(flags))); +} - return c; +static inline void * +kmalloc_array(size_t n, size_t size, gfp_t flags) +{ + if (size != 0 && n > (SIZE_MAX / size)) + return (NULL); + return (malloc(n * size, M_KMALLOC, linux_check_m_flags(flags))); } static inline void * -linux_kmem_cache_alloc(struct kmem_cache *c, int flags) +krealloc(void *ptr, size_t size, gfp_t flags) { - return uma_zalloc_arg(c->cache_zone, c->cache_ctor, flags); + return (realloc(ptr, size, M_KMALLOC, linux_check_m_flags(flags))); } static inline void -linux_kmem_cache_free(struct kmem_cache *c, void *m) +kfree(const void *ptr) +{ + free(__DECONST(void *, ptr), M_KMALLOC); +} + +extern struct linux_kmem_cache *linux_kmem_cache_create(const char *name, + size_t size, size_t align, unsigned flags, linux_kmem_ctor_t *ctor); + +static inline void * +linux_kmem_cache_alloc(struct linux_kmem_cache *c, gfp_t flags) +{ + return (uma_zalloc_arg(c->cache_zone, c, + linux_check_m_flags(flags))); +} + +static inline void * +kmem_cache_zalloc(struct linux_kmem_cache *c, gfp_t flags) { - uma_zfree(c->cache_zone, m); + return (uma_zalloc_arg(c->cache_zone, c, + linux_check_m_flags(flags | M_ZERO))); } +extern void linux_kmem_cache_free_rcu(struct linux_kmem_cache *, void *); + static inline void -linux_kmem_cache_destroy(struct kmem_cache *c) +linux_kmem_cache_free(struct linux_kmem_cache *c, void *m) { - uma_zdestroy(c->cache_zone); - free(c, M_KMALLOC); + if (unlikely(c->cache_flags & SLAB_DESTROY_BY_RCU)) + linux_kmem_cache_free_rcu(c, m); + else + uma_zfree(c->cache_zone, m); } -#endif /* _LINUX_SLAB_H_ */ +extern void linux_kmem_cache_destroy(struct linux_kmem_cache *); + +#endif /* _LINUX_SLAB_H_ */ Added: head/sys/compat/linuxkpi/common/src/linux_slab.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/compat/linuxkpi/common/src/linux_slab.c Wed Mar 8 11:09:27 2017 (r314905) @@ -0,0 +1,128 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies, Ltd. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include + +struct linux_kmem_rcu { + struct rcu_head rcu_head; + struct linux_kmem_cache *cache; +}; + +#define LINUX_KMEM_TO_RCU(c, m) \ + ((struct linux_kmem_rcu *)((char *)(m) + \ + (c)->cache_size - sizeof(struct linux_kmem_rcu))) + +#define LINUX_RCU_TO_KMEM(r) \ + ((void *)((char *)(r) + sizeof(struct linux_kmem_rcu) - \ + (r)->cache->cache_size)) + +static int +linux_kmem_ctor(void *mem, int size, void *arg, int flags) +{ + struct linux_kmem_cache *c = arg; + + if (unlikely(c->cache_flags & SLAB_DESTROY_BY_RCU)) { + struct linux_kmem_rcu *rcu = LINUX_KMEM_TO_RCU(c, mem); + + /* duplicate cache pointer */ + rcu->cache = c; + } + + /* check for constructor */ + if (likely(c->cache_ctor != NULL)) + c->cache_ctor(mem); + + return (0); +} + +static void +linux_kmem_cache_free_rcu_callback(struct rcu_head *head) +{ + struct linux_kmem_rcu *rcu = + container_of(head, struct linux_kmem_rcu, rcu_head); + + uma_zfree(rcu->cache->cache_zone, LINUX_RCU_TO_KMEM(rcu)); +} + +struct linux_kmem_cache * +linux_kmem_cache_create(const char *name, size_t size, size_t align, + unsigned flags, linux_kmem_ctor_t *ctor) +{ + struct linux_kmem_cache *c; + + c = malloc(sizeof(*c), M_KMALLOC, M_WAITOK); + + if (flags & SLAB_HWCACHE_ALIGN) + align = UMA_ALIGN_CACHE; + else if (align != 0) + align--; + + if (flags & SLAB_DESTROY_BY_RCU) { + /* make room for RCU structure */ + size = ALIGN(size, sizeof(void *)); + size += sizeof(struct linux_kmem_rcu); + + /* create cache_zone */ + c->cache_zone = uma_zcreate(name, size, + linux_kmem_ctor, NULL, NULL, NULL, + align, UMA_ZONE_ZINIT); + } else { + /* create cache_zone */ + c->cache_zone = uma_zcreate(name, size, + ctor ? linux_kmem_ctor : NULL, NULL, + NULL, NULL, align, 0); + } + + c->cache_flags = flags; + c->cache_ctor = ctor; + c->cache_size = size; + return (c); +} + +void +linux_kmem_cache_free_rcu(struct linux_kmem_cache *c, void *m) +{ + struct linux_kmem_rcu *rcu = LINUX_KMEM_TO_RCU(c, m); + + call_rcu(&rcu->rcu_head, linux_kmem_cache_free_rcu_callback); +} + +void +linux_kmem_cache_destroy(struct linux_kmem_cache *c) +{ + if (unlikely(c->cache_flags & SLAB_DESTROY_BY_RCU)) { + /* make sure all free callbacks have been called */ + rcu_barrier(); + } + + uma_zdestroy(c->cache_zone); + free(c, M_KMALLOC); +} Modified: head/sys/conf/files ============================================================================== --- head/sys/conf/files Wed Mar 8 09:53:20 2017 (r314904) +++ head/sys/conf/files Wed Mar 8 11:09:27 2017 (r314905) @@ -4296,6 +4296,8 @@ compat/linuxkpi/common/src/linux_radix.c compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_rcu.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C} -I$S/contrib/ck/include" +compat/linuxkpi/common/src/linux_slab.c optional compat_linuxkpi \ + compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_usb.c optional compat_linuxkpi usb \ compile-with "${LINUXKPI_C}" compat/linuxkpi/common/src/linux_work.c optional compat_linuxkpi \ Modified: head/sys/modules/linuxkpi/Makefile ============================================================================== --- head/sys/modules/linuxkpi/Makefile Wed Mar 8 09:53:20 2017 (r314904) +++ head/sys/modules/linuxkpi/Makefile Wed Mar 8 11:09:27 2017 (r314905) @@ -10,6 +10,7 @@ SRCS= linux_kmod.c \ linux_pci.c \ linux_radix.c \ linux_rcu.c \ + linux_slab.c \ linux_tasklet.c \ linux_idr.c \ linux_usb.c \