From owner-svn-src-user@FreeBSD.ORG Sun Oct 21 08:46:16 2012 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [69.147.83.52]) by hub.freebsd.org (Postfix) with ESMTP id A28D2DFC; Sun, 21 Oct 2012 08:46:16 +0000 (UTC) (envelope-from andre@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 8A1AD8FC08; Sun, 21 Oct 2012 08:46:16 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.4/8.14.4) with ESMTP id q9L8kGVQ029656; Sun, 21 Oct 2012 08:46:16 GMT (envelope-from andre@svn.freebsd.org) Received: (from andre@localhost) by svn.freebsd.org (8.14.4/8.14.4/Submit) id q9L8kGX7029653; Sun, 21 Oct 2012 08:46:16 GMT (envelope-from andre@svn.freebsd.org) Message-Id: <201210210846.q9L8kGX7029653@svn.freebsd.org> From: Andre Oppermann Date: Sun, 21 Oct 2012 08:46:16 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r241798 - in user/andre/tcp_workqueue/sys: kern sys X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 21 Oct 2012 08:46:16 -0000 Author: andre Date: Sun Oct 21 08:46:15 2012 New Revision: 241798 URL: http://svn.freebsd.org/changeset/base/241798 Log: Make sure that global locks have their own CPU cache line in the .bss section of the kernel and do not end up sharing one by chance or the place they are defined in. Three new macros are added: MTX_GLOBAL(name) defining the mutex MTX_GLOBAL_STATIC(name) defining the mutex as static MTX_GLB(name) accessing the mutex in mtx_lock() This makes global locks cache deterministic. Changes to the .bss layout and ordering due to differences in completely unrelated parts of the kernel can no longer cause global locks to share the same CPU cache line. This is a proof of concept with only one global lock converted. I'm open to suggestions and improvements on the macros and their naming. Modified: user/andre/tcp_workqueue/sys/kern/uipc_socket.c user/andre/tcp_workqueue/sys/sys/mutex.h Modified: user/andre/tcp_workqueue/sys/kern/uipc_socket.c ============================================================================== --- user/andre/tcp_workqueue/sys/kern/uipc_socket.c Sun Oct 21 08:38:55 2012 (r241797) +++ user/andre/tcp_workqueue/sys/kern/uipc_socket.c Sun Oct 21 08:46:15 2012 (r241798) @@ -215,8 +215,8 @@ MTX_SYSINIT(accept_mtx, &accept_mtx, "ac * so_global_mtx protects so_gencnt, numopensockets, and the per-socket * so_gencnt field. */ -static struct mtx so_global_mtx; -MTX_SYSINIT(so_global_mtx, &so_global_mtx, "so_glabel", MTX_DEF); +MTX_GLOBAL_STATIC(so_global_mtx); +MTX_SYSINIT(so_global_mtx, MTX_GLB(so_global_mtx), "so_glabel", MTX_DEF); /* * General IPC sysctl name space, used by sockets and a variety of other IPC @@ -324,7 +324,7 @@ soalloc(struct vnet *vnet) sx_init(&so->so_snd.sb_sx, "so_snd_sx"); sx_init(&so->so_rcv.sb_sx, "so_rcv_sx"); TAILQ_INIT(&so->so_aiojobq); - mtx_lock(&so_global_mtx); + mtx_lock(MTX_GLB(so_global_mtx)); so->so_gencnt = ++so_gencnt; ++numopensockets; #ifdef VIMAGE @@ -333,7 +333,7 @@ soalloc(struct vnet *vnet) vnet->vnet_sockcnt++; so->so_vnet = vnet; #endif - mtx_unlock(&so_global_mtx); + mtx_unlock(MTX_GLB(so_global_mtx)); return (so); } @@ -349,7 +349,7 @@ sodealloc(struct socket *so) KASSERT(so->so_count == 0, ("sodealloc(): so_count %d", so->so_count)); KASSERT(so->so_pcb == NULL, ("sodealloc(): so_pcb != NULL")); - mtx_lock(&so_global_mtx); + mtx_lock(MTX_GLB(so_global_mtx)); so->so_gencnt = ++so_gencnt; --numopensockets; /* Could be below, but faster here. */ #ifdef VIMAGE @@ -357,7 +357,7 @@ sodealloc(struct socket *so) __func__, __LINE__, so)); so->so_vnet->vnet_sockcnt--; #endif - mtx_unlock(&so_global_mtx); + mtx_unlock(MTX_GLB(so_global_mtx)); if (so->so_rcv.sb_hiwat) (void)chgsbsize(so->so_cred->cr_uidinfo, &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY); Modified: user/andre/tcp_workqueue/sys/sys/mutex.h ============================================================================== --- user/andre/tcp_workqueue/sys/sys/mutex.h Sun Oct 21 08:38:55 2012 (r241797) +++ user/andre/tcp_workqueue/sys/sys/mutex.h Sun Oct 21 08:46:15 2012 (r241798) @@ -411,6 +411,18 @@ struct mtx_args { SYSUNINIT(name##_mtx_sysuninit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \ mtx_destroy, (mtx)) +#define MTX_GLOBAL(name) \ + struct { \ + struct mtx (name); \ + } __aligned(CACHE_LINE_SIZE) (name); + +#define MTX_GLOBAL_STATIC(name) \ + static struct { \ + struct mtx (name); \ + } __aligned(CACHE_LINE_SIZE) (name); + +#define MTX_GLB(name) (&(name.name)) + /* * The INVARIANTS-enabled mtx_assert() functionality. *