From owner-svn-src-all@freebsd.org Sun Feb 5 03:26:36 2017 Return-Path: Delivered-To: svn-src-all@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 0BB81CD17EE; Sun, 5 Feb 2017 03:26:36 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id D7E1D1224; Sun, 5 Feb 2017 03:26:35 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v153QY9m035292; Sun, 5 Feb 2017 03:26:34 GMT (envelope-from mjg@FreeBSD.org) Received: (from mjg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id v153QYhP035290; Sun, 5 Feb 2017 03:26:34 GMT (envelope-from mjg@FreeBSD.org) Message-Id: <201702050326.v153QYhP035290@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: mjg set sender to mjg@FreeBSD.org using -f From: Mateusz Guzik Date: Sun, 5 Feb 2017 03:26:34 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r313269 - in head/sys: kern sys X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 05 Feb 2017 03:26:36 -0000 Author: mjg Date: Sun Feb 5 03:26:34 2017 New Revision: 313269 URL: https://svnweb.freebsd.org/changeset/base/313269 Log: mtx: switch to fcmpset The found value is passed to locking routines in order to reduce cacheline accesses. mtx_unlock grows an explicit check for regular unlock. On ll/sc architectures the routine can fail even if the lock could have been handled by the inline primitive. Discussed with: jhb Tested by: pho (previous version) Modified: head/sys/kern/kern_mutex.c head/sys/sys/mutex.h Modified: head/sys/kern/kern_mutex.c ============================================================================== --- head/sys/kern/kern_mutex.c Sun Feb 5 03:23:16 2017 (r313268) +++ head/sys/kern/kern_mutex.c Sun Feb 5 03:26:34 2017 (r313269) @@ -455,12 +455,11 @@ _mtx_trylock_flags_(volatile uintptr_t * * sleep waiting for it), or if we need to recurse on it. */ void -__mtx_lock_sleep(volatile uintptr_t *c, uintptr_t tid, int opts, +__mtx_lock_sleep(volatile uintptr_t *c, uintptr_t v, uintptr_t tid, int opts, const char *file, int line) { struct mtx *m; struct turnstile *ts; - uintptr_t v; #ifdef ADAPTIVE_MUTEXES volatile struct thread *owner; #endif @@ -489,7 +488,6 @@ __mtx_lock_sleep(volatile uintptr_t *c, lock_delay_arg_init(&lda, NULL); #endif m = mtxlock2mtx(c); - v = MTX_READ_VALUE(m); if (__predict_false(lv_mtx_owner(v) == (struct thread *)tid)) { KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0 || @@ -520,9 +518,8 @@ __mtx_lock_sleep(volatile uintptr_t *c, for (;;) { if (v == MTX_UNOWNED) { - if (_mtx_obtain_lock(m, tid)) + if (_mtx_obtain_lock_fetch(m, &v, tid)) break; - v = MTX_READ_VALUE(m); continue; } #ifdef KDTRACE_HOOKS @@ -674,12 +671,11 @@ _mtx_lock_spin_failed(struct mtx *m) * is handled inline. */ void -_mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t tid, int opts, - const char *file, int line) +_mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t v, uintptr_t tid, + int opts, const char *file, int line) { struct mtx *m; struct lock_delay_arg lda; - uintptr_t v; #ifdef LOCK_PROFILING int contested = 0; uint64_t waittime = 0; @@ -706,12 +702,10 @@ _mtx_lock_spin_cookie(volatile uintptr_t #ifdef KDTRACE_HOOKS spin_time -= lockstat_nsecs(&m->lock_object); #endif - v = MTX_READ_VALUE(m); for (;;) { if (v == MTX_UNOWNED) { - if (_mtx_obtain_lock(m, tid)) + if (_mtx_obtain_lock_fetch(m, &v, tid)) break; - v = MTX_READ_VALUE(m); continue; } /* Give interrupts a chance while we spin. */ @@ -796,14 +790,11 @@ retry: m->lock_object.lo_name, file, line)); WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); - v = MTX_READ_VALUE(m); for (;;) { - if (v == MTX_UNOWNED) { - if (_mtx_obtain_lock(m, tid)) - break; - v = MTX_READ_VALUE(m); + if (_mtx_obtain_lock_fetch(m, &v, tid)) + break; + if (v == MTX_UNOWNED) continue; - } if (v == tid) { m->mtx_recurse++; break; @@ -896,11 +887,18 @@ __mtx_unlock_sleep(volatile uintptr_t *c { struct mtx *m; struct turnstile *ts; + uintptr_t v; if (SCHEDULER_STOPPED()) return; m = mtxlock2mtx(c); + v = MTX_READ_VALUE(m); + + if (v == (uintptr_t)curthread) { + if (_mtx_release_lock(m, (uintptr_t)curthread)) + return; + } if (mtx_recursed(m)) { if (--(m->mtx_recurse) == 0) Modified: head/sys/sys/mutex.h ============================================================================== --- head/sys/sys/mutex.h Sun Feb 5 03:23:16 2017 (r313268) +++ head/sys/sys/mutex.h Sun Feb 5 03:26:34 2017 (r313269) @@ -98,13 +98,13 @@ void mtx_sysinit(void *arg); int _mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file, int line); void mutex_init(void); -void __mtx_lock_sleep(volatile uintptr_t *c, uintptr_t tid, int opts, - const char *file, int line); +void __mtx_lock_sleep(volatile uintptr_t *c, uintptr_t v, uintptr_t tid, + int opts, const char *file, int line); void __mtx_unlock_sleep(volatile uintptr_t *c, int opts, const char *file, int line); #ifdef SMP -void _mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t tid, int opts, - const char *file, int line); +void _mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t v, uintptr_t tid, + int opts, const char *file, int line); #endif void __mtx_lock_flags(volatile uintptr_t *c, int opts, const char *file, int line); @@ -140,13 +140,13 @@ void thread_lock_flags_(struct thread *, _mtx_destroy(&(m)->mtx_lock) #define mtx_trylock_flags_(m, o, f, l) \ _mtx_trylock_flags_(&(m)->mtx_lock, o, f, l) -#define _mtx_lock_sleep(m, t, o, f, l) \ - __mtx_lock_sleep(&(m)->mtx_lock, t, o, f, l) +#define _mtx_lock_sleep(m, v, t, o, f, l) \ + __mtx_lock_sleep(&(m)->mtx_lock, v, t, o, f, l) #define _mtx_unlock_sleep(m, o, f, l) \ __mtx_unlock_sleep(&(m)->mtx_lock, o, f, l) #ifdef SMP -#define _mtx_lock_spin(m, t, o, f, l) \ - _mtx_lock_spin_cookie(&(m)->mtx_lock, t, o, f, l) +#define _mtx_lock_spin(m, v, t, o, f, l) \ + _mtx_lock_spin_cookie(&(m)->mtx_lock, v, t, o, f, l) #endif #define _mtx_lock_flags(m, o, f, l) \ __mtx_lock_flags(&(m)->mtx_lock, o, f, l) @@ -171,6 +171,11 @@ void thread_lock_flags_(struct thread *, #define _mtx_obtain_lock(mp, tid) \ atomic_cmpset_acq_ptr(&(mp)->mtx_lock, MTX_UNOWNED, (tid)) +#define _mtx_obtain_lock_fetch(mp, vp, tid) ({ \ + *vp = MTX_UNOWNED; \ + atomic_fcmpset_rel_ptr(&(mp)->mtx_lock, vp, (tid)); \ +}) + /* Try to release mtx_lock if it is unrecursed and uncontested. */ #define _mtx_release_lock(mp, tid) \ atomic_cmpset_rel_ptr(&(mp)->mtx_lock, (tid), MTX_UNOWNED) @@ -188,9 +193,10 @@ void thread_lock_flags_(struct thread *, /* Lock a normal mutex. */ #define __mtx_lock(mp, tid, opts, file, line) do { \ uintptr_t _tid = (uintptr_t)(tid); \ + uintptr_t _v; \ \ - if (((mp)->mtx_lock != MTX_UNOWNED || !_mtx_obtain_lock((mp), _tid)))\ - _mtx_lock_sleep((mp), _tid, (opts), (file), (line)); \ + if (!_mtx_obtain_lock_fetch((mp), &_v, _tid)) \ + _mtx_lock_sleep((mp), _v, _tid, (opts), (file), (line));\ else \ LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, \ mp, 0, 0, file, line); \ @@ -205,13 +211,14 @@ void thread_lock_flags_(struct thread *, #ifdef SMP #define __mtx_lock_spin(mp, tid, opts, file, line) do { \ uintptr_t _tid = (uintptr_t)(tid); \ + uintptr_t _v; \ \ spinlock_enter(); \ - if (((mp)->mtx_lock != MTX_UNOWNED || !_mtx_obtain_lock((mp), _tid))) {\ - if ((mp)->mtx_lock == _tid) \ + if (!_mtx_obtain_lock_fetch((mp), &_v, _tid)) { \ + if (_v == _tid) \ (mp)->mtx_recurse++; \ else \ - _mtx_lock_spin((mp), _tid, (opts), (file), (line)); \ + _mtx_lock_spin((mp), _v, _tid, (opts), (file), (line));\ } else \ LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(spin__acquire, \ mp, 0, 0, file, line); \ @@ -265,7 +272,7 @@ void thread_lock_flags_(struct thread *, \ if ((mp)->mtx_recurse == 0) \ LOCKSTAT_PROFILE_RELEASE_LOCK(adaptive__release, mp); \ - if ((mp)->mtx_lock != _tid || !_mtx_release_lock((mp), _tid)) \ + if (!_mtx_release_lock((mp), _tid)) \ _mtx_unlock_sleep((mp), (opts), (file), (line)); \ } while (0)