Date: Sun, 4 Mar 2018 19:12:54 +0000 (UTC) From: Mateusz Guzik <mjg@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r330400 - head/sys/kern Message-ID: <201803041912.w24JCsaa018130@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: mjg Date: Sun Mar 4 19:12:54 2018 New Revision: 330400 URL: https://svnweb.freebsd.org/changeset/base/330400 Log: lockmgr: start decomposing the main routine The main routine takes 8 args, 3 of which are almost the same for most uses. This in particular pushes it above the limit of 6 arguments passable through registers on amd64 making it impossible to tail call. This is a prerequisite for further cleanups. Tested by: pho Modified: head/sys/kern/kern_lock.c Modified: head/sys/kern/kern_lock.c ============================================================================== --- head/sys/kern/kern_lock.c Sun Mar 4 19:10:30 2018 (r330399) +++ head/sys/kern/kern_lock.c Sun Mar 4 19:12:54 2018 (r330400) @@ -160,11 +160,31 @@ struct lock_class lock_class_lockmgr = { #endif }; +struct lockmgr_wait { + const char *iwmesg; + int ipri; + int itimo; +}; + static bool __always_inline lockmgr_slock_try(struct lock *lk, uintptr_t *xp, int flags); -static bool __always_inline lockmgr_sunlock_try(struct lock *lk, uintptr_t x); +static bool __always_inline lockmgr_sunlock_try(struct lock *lk, uintptr_t *xp); static void +lockmgr_exit(u_int flags, struct lock_object *ilk, int wakeup_swapper) +{ + struct lock_class *class; + + if (flags & LK_INTERLOCK) { + class = LOCK_CLASS(ilk); + class->lc_unlock(ilk); + } + + if (__predict_false(wakeup_swapper)) + kick_proc0(); +} + +static void lockmgr_note_shared_acquire(struct lock *lk, int contested, uint64_t waittime, const char *file, int line, int flags) { @@ -295,7 +315,7 @@ wakeupshlk(struct lock *lk, const char *file, int line wakeup_swapper = 0; for (;;) { x = lk->lk_lock; - if (lockmgr_sunlock_try(lk, x)) + if (lockmgr_sunlock_try(lk, &x)) break; /* @@ -502,7 +522,7 @@ lockmgr_slock_try(struct lock *lk, uintptr_t *xp, int } static bool __always_inline -lockmgr_sunlock_try(struct lock *lk, uintptr_t x) +lockmgr_sunlock_try(struct lock *lk, uintptr_t *xp) { for (;;) { @@ -510,9 +530,9 @@ lockmgr_sunlock_try(struct lock *lk, uintptr_t x) * If there is more than one shared lock held, just drop one * and return. */ - if (LK_SHARERS(x) > 1) { - if (atomic_fcmpset_rel_ptr(&lk->lk_lock, &x, - x - LK_ONE_SHARER)) + if (LK_SHARERS(*xp) > 1) { + if (atomic_fcmpset_rel_ptr(&lk->lk_lock, xp, + *xp - LK_ONE_SHARER)) return (true); continue; } @@ -521,10 +541,10 @@ lockmgr_sunlock_try(struct lock *lk, uintptr_t x) * If there are not waiters on the exclusive queue, drop the * lock quickly. */ - if ((x & LK_ALL_WAITERS) == 0) { - MPASS((x & ~LK_EXCLUSIVE_SPINNERS) == + if ((*xp & LK_ALL_WAITERS) == 0) { + MPASS((*xp & ~LK_EXCLUSIVE_SPINNERS) == LK_SHARERS_LOCK(1)); - if (atomic_fcmpset_rel_ptr(&lk->lk_lock, &x, + if (atomic_fcmpset_rel_ptr(&lk->lk_lock, xp, LK_UNLOCKED)) return (true); continue; @@ -534,12 +554,373 @@ lockmgr_sunlock_try(struct lock *lk, uintptr_t x) return (false); } +static __noinline int +lockmgr_slock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, + const char *file, int line, struct lockmgr_wait *lwa) +{ + uintptr_t tid, x; + int error = 0; + const char *iwmesg; + int ipri, itimo; + +#ifdef LOCK_PROFILING + uint64_t waittime = 0; + int contested = 0; +#endif + + if (__predict_false(panicstr != NULL)) + goto out; + + tid = (uintptr_t)curthread; + + if (LK_CAN_WITNESS(flags)) + WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER, + file, line, flags & LK_INTERLOCK ? ilk : NULL); + for (;;) { + if (lockmgr_slock_try(lk, &x, flags)) + break; +#ifdef HWPMC_HOOKS + PMC_SOFT_CALL( , , lock, failed); +#endif + lock_profile_obtain_lock_failed(&lk->lock_object, + &contested, &waittime); + + /* + * If the lock is already held by curthread in + * exclusive way avoid a deadlock. + */ + if (LK_HOLDER(x) == tid) { + LOCK_LOG2(lk, + "%s: %p already held in exclusive mode", + __func__, lk); + error = EDEADLK; + break; + } + + /* + * If the lock is expected to not sleep just give up + * and return. + */ + if (LK_TRYOP(flags)) { + LOCK_LOG2(lk, "%s: %p fails the try operation", + __func__, lk); + error = EBUSY; + break; + } + + /* + * Acquire the sleepqueue chain lock because we + * probabilly will need to manipulate waiters flags. + */ + sleepq_lock(&lk->lock_object); + x = lk->lk_lock; + + /* + * if the lock can be acquired in shared mode, try + * again. + */ + if (LK_CAN_SHARE(x, flags)) { + sleepq_release(&lk->lock_object); + continue; + } + + /* + * Try to set the LK_SHARED_WAITERS flag. If we fail, + * loop back and retry. + */ + if ((x & LK_SHARED_WAITERS) == 0) { + if (!atomic_cmpset_acq_ptr(&lk->lk_lock, x, + x | LK_SHARED_WAITERS)) { + sleepq_release(&lk->lock_object); + continue; + } + LOCK_LOG2(lk, "%s: %p set shared waiters flag", + __func__, lk); + } + + if (lwa == NULL) { + iwmesg = lk->lock_object.lo_name; + ipri = lk->lk_pri; + itimo = lk->lk_timo; + } else { + iwmesg = lwa->iwmesg; + ipri = lwa->ipri; + itimo = lwa->itimo; + } + + /* + * As far as we have been unable to acquire the + * shared lock and the shared waiters flag is set, + * we will sleep. + */ + error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo, + SQ_SHARED_QUEUE); + flags &= ~LK_INTERLOCK; + if (error) { + LOCK_LOG3(lk, + "%s: interrupted sleep for %p with %d", + __func__, lk, error); + break; + } + LOCK_LOG2(lk, "%s: %p resuming from the sleep queue", + __func__, lk); + } + if (error == 0) { +#ifdef LOCK_PROFILING + lockmgr_note_shared_acquire(lk, contested, waittime, + file, line, flags); +#else + lockmgr_note_shared_acquire(lk, 0, 0, file, line, + flags); +#endif + } + +out: + lockmgr_exit(flags, ilk, 0); + return (error); +} + +static __noinline int +lockmgr_xlock_hard(struct lock *lk, u_int flags, struct lock_object *ilk, + const char *file, int line, struct lockmgr_wait *lwa) +{ + struct lock_class *class; + uintptr_t tid, x, v; + int error = 0; + const char *iwmesg; + int ipri, itimo; + +#ifdef LOCK_PROFILING + uint64_t waittime = 0; + int contested = 0; +#endif + + if (__predict_false(panicstr != NULL)) + goto out; + + tid = (uintptr_t)curthread; + + if (LK_CAN_WITNESS(flags)) + WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER | + LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ? + ilk : NULL); + + /* + * If curthread already holds the lock and this one is + * allowed to recurse, simply recurse on it. + */ + if (lockmgr_xlocked(lk)) { + if ((flags & LK_CANRECURSE) == 0 && + (lk->lock_object.lo_flags & LO_RECURSABLE) == 0) { + /* + * If the lock is expected to not panic just + * give up and return. + */ + if (LK_TRYOP(flags)) { + LOCK_LOG2(lk, + "%s: %p fails the try operation", + __func__, lk); + error = EBUSY; + goto out; + } + if (flags & LK_INTERLOCK) { + class = LOCK_CLASS(ilk); + class->lc_unlock(ilk); + } + panic("%s: recursing on non recursive lockmgr %s @ %s:%d\n", + __func__, iwmesg, file, line); + } + lk->lk_recurse++; + LOCK_LOG2(lk, "%s: %p recursing", __func__, lk); + LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, + lk->lk_recurse, file, line); + WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE | + LK_TRYWIT(flags), file, line); + TD_LOCKS_INC(curthread); + goto out; + } + + for (;;) { + if (lk->lk_lock == LK_UNLOCKED && + atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) + break; +#ifdef HWPMC_HOOKS + PMC_SOFT_CALL( , , lock, failed); +#endif + lock_profile_obtain_lock_failed(&lk->lock_object, + &contested, &waittime); + + /* + * If the lock is expected to not sleep just give up + * and return. + */ + if (LK_TRYOP(flags)) { + LOCK_LOG2(lk, "%s: %p fails the try operation", + __func__, lk); + error = EBUSY; + break; + } + + /* + * Acquire the sleepqueue chain lock because we + * probabilly will need to manipulate waiters flags. + */ + sleepq_lock(&lk->lock_object); + x = lk->lk_lock; + + /* + * if the lock has been released while we spun on + * the sleepqueue chain lock just try again. + */ + if (x == LK_UNLOCKED) { + sleepq_release(&lk->lock_object); + continue; + } + + /* + * The lock can be in the state where there is a + * pending queue of waiters, but still no owner. + * This happens when the lock is contested and an + * owner is going to claim the lock. + * If curthread is the one successfully acquiring it + * claim lock ownership and return, preserving waiters + * flags. + */ + v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS); + if ((x & ~v) == LK_UNLOCKED) { + v &= ~LK_EXCLUSIVE_SPINNERS; + if (atomic_cmpset_acq_ptr(&lk->lk_lock, x, + tid | v)) { + sleepq_release(&lk->lock_object); + LOCK_LOG2(lk, + "%s: %p claimed by a new writer", + __func__, lk); + break; + } + sleepq_release(&lk->lock_object); + continue; + } + + /* + * Try to set the LK_EXCLUSIVE_WAITERS flag. If we + * fail, loop back and retry. + */ + if ((x & LK_EXCLUSIVE_WAITERS) == 0) { + if (!atomic_cmpset_ptr(&lk->lk_lock, x, + x | LK_EXCLUSIVE_WAITERS)) { + sleepq_release(&lk->lock_object); + continue; + } + LOCK_LOG2(lk, "%s: %p set excl waiters flag", + __func__, lk); + } + + if (lwa == NULL) { + iwmesg = lk->lock_object.lo_name; + ipri = lk->lk_pri; + itimo = lk->lk_timo; + } else { + iwmesg = lwa->iwmesg; + ipri = lwa->ipri; + itimo = lwa->itimo; + } + + /* + * As far as we have been unable to acquire the + * exclusive lock and the exclusive waiters flag + * is set, we will sleep. + */ + error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo, + SQ_EXCLUSIVE_QUEUE); + flags &= ~LK_INTERLOCK; + if (error) { + LOCK_LOG3(lk, + "%s: interrupted sleep for %p with %d", + __func__, lk, error); + break; + } + LOCK_LOG2(lk, "%s: %p resuming from the sleep queue", + __func__, lk); + } + if (error == 0) { +#ifdef LOCK_PROFILING + lockmgr_note_exclusive_acquire(lk, contested, waittime, + file, line, flags); +#else + lockmgr_note_exclusive_acquire(lk, 0, 0, file, line, + flags); +#endif + } + +out: + lockmgr_exit(flags, ilk, 0); + return (error); +} + +static __noinline int +lockmgr_upgrade(struct lock *lk, u_int flags, struct lock_object *ilk, + const char *file, int line, struct lockmgr_wait *lwa) +{ + uintptr_t tid, x, v; + int error = 0; + int wakeup_swapper = 0; + int op; + + if (__predict_false(panicstr != NULL)) + goto out; + + tid = (uintptr_t)curthread; + + _lockmgr_assert(lk, KA_SLOCKED, file, line); + v = lk->lk_lock; + x = v & LK_ALL_WAITERS; + v &= LK_EXCLUSIVE_SPINNERS; + + /* + * Try to switch from one shared lock to an exclusive one. + * We need to preserve waiters flags during the operation. + */ + if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x | v, + tid | x)) { + LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file, + line); + WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE | + LK_TRYWIT(flags), file, line); + TD_SLOCKS_DEC(curthread); + goto out; + } + + op = flags & LK_TYPE_MASK; + + /* + * In LK_TRYUPGRADE mode, do not drop the lock, + * returning EBUSY instead. + */ + if (op == LK_TRYUPGRADE) { + LOCK_LOG2(lk, "%s: %p failed the nowait upgrade", + __func__, lk); + error = EBUSY; + goto out; + } + + /* + * We have been unable to succeed in upgrading, so just + * give up the shared lock. + */ + wakeup_swapper |= wakeupshlk(lk, file, line); + error = lockmgr_xlock_hard(lk, flags, ilk, file, line, lwa); + flags &= ~LK_INTERLOCK; +out: + lockmgr_exit(flags, ilk, wakeup_swapper); + return (error); +} + int lockmgr_lock_fast_path(struct lock *lk, u_int flags, struct lock_object *ilk, const char *file, int line) { struct lock_class *class; - uintptr_t x, v, tid; + uintptr_t x, tid; u_int op; bool locked; @@ -556,6 +937,9 @@ lockmgr_lock_fast_path(struct lock *lk, u_int flags, s lockmgr_note_shared_acquire(lk, 0, 0, file, line, flags); locked = true; + } else { + return (lockmgr_slock_hard(lk, flags, ilk, file, line, + NULL)); } break; case LK_EXCLUSIVE: @@ -569,21 +953,14 @@ lockmgr_lock_fast_path(struct lock *lk, u_int flags, s lockmgr_note_exclusive_acquire(lk, 0, 0, file, line, flags); locked = true; + } else { + return (lockmgr_xlock_hard(lk, flags, ilk, file, line, + NULL)); } break; case LK_UPGRADE: case LK_TRYUPGRADE: - _lockmgr_assert(lk, KA_SLOCKED, file, line); - tid = (uintptr_t)curthread; - v = lk->lk_lock; - x = v & LK_ALL_WAITERS; - v &= LK_EXCLUSIVE_SPINNERS; - if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x | v, - tid | x)) { - lockmgr_note_exclusive_upgrade(lk, file, line, flags); - locked = true; - } - break; + return (lockmgr_upgrade(lk, flags, ilk, file, line, NULL)); default: break; } @@ -599,6 +976,127 @@ lockmgr_lock_fast_path(struct lock *lk, u_int flags, s } } +static __noinline int +lockmgr_sunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk, + const char *file, int line) + +{ + int wakeup_swapper = 0; + + if (__predict_false(panicstr != NULL)) + goto out; + + wakeup_swapper = wakeupshlk(lk, file, line); + +out: + lockmgr_exit(flags, ilk, wakeup_swapper); + return (0); +} + +static __noinline int +lockmgr_xunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk, + const char *file, int line) +{ + uintptr_t tid, v; + int wakeup_swapper = 0; + u_int realexslp; + int queue; + + if (__predict_false(panicstr != NULL)) + goto out; + + tid = (uintptr_t)curthread; + + /* + * As first option, treact the lock as if it has not + * any waiter. + * Fix-up the tid var if the lock has been disowned. + */ + if (LK_HOLDER(x) == LK_KERNPROC) + tid = LK_KERNPROC; + else { + WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line); + TD_LOCKS_DEC(curthread); + } + LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line); + + /* + * The lock is held in exclusive mode. + * If the lock is recursed also, then unrecurse it. + */ + if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) { + LOCK_LOG2(lk, "%s: %p unrecursing", __func__, lk); + lk->lk_recurse--; + goto out; + } + if (tid != LK_KERNPROC) + lock_profile_release_lock(&lk->lock_object); + + if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) + goto out; + + sleepq_lock(&lk->lock_object); + x = lk->lk_lock; + v = LK_UNLOCKED; + + /* + * If the lock has exclusive waiters, give them + * preference in order to avoid deadlock with + * shared runners up. + * If interruptible sleeps left the exclusive queue + * empty avoid a starvation for the threads sleeping + * on the shared queue by giving them precedence + * and cleaning up the exclusive waiters bit anyway. + * Please note that lk_exslpfail count may be lying + * about the real number of waiters with the + * LK_SLEEPFAIL flag on because they may be used in + * conjunction with interruptible sleeps so + * lk_exslpfail might be considered an 'upper limit' + * bound, including the edge cases. + */ + MPASS((x & LK_EXCLUSIVE_SPINNERS) == 0); + realexslp = sleepq_sleepcnt(&lk->lock_object, SQ_EXCLUSIVE_QUEUE); + if ((x & LK_EXCLUSIVE_WAITERS) != 0 && realexslp != 0) { + if (lk->lk_exslpfail < realexslp) { + lk->lk_exslpfail = 0; + queue = SQ_EXCLUSIVE_QUEUE; + v |= (x & LK_SHARED_WAITERS); + } else { + lk->lk_exslpfail = 0; + LOCK_LOG2(lk, + "%s: %p has only LK_SLEEPFAIL sleepers", + __func__, lk); + LOCK_LOG2(lk, + "%s: %p waking up threads on the exclusive queue", + __func__, lk); + wakeup_swapper = sleepq_broadcast(&lk->lock_object, + SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE); + queue = SQ_SHARED_QUEUE; + } + } else { + + /* + * Exclusive waiters sleeping with LK_SLEEPFAIL + * on and using interruptible sleeps/timeout + * may have left spourious lk_exslpfail counts + * on, so clean it up anyway. + */ + lk->lk_exslpfail = 0; + queue = SQ_SHARED_QUEUE; + } + + LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue", + __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" : + "exclusive"); + atomic_store_rel_ptr(&lk->lk_lock, v); + wakeup_swapper |= sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue); + sleepq_release(&lk->lock_object); + +out: + lockmgr_exit(flags, ilk, wakeup_swapper); + return (0); +} + int lockmgr_unlock_fast_path(struct lock *lk, u_int flags, struct lock_object *ilk) { @@ -615,9 +1113,11 @@ lockmgr_unlock_fast_path(struct lock *lk, u_int flags, unlocked = false; x = lk->lk_lock; if (__predict_true(x & LK_SHARE) != 0) { - if (lockmgr_sunlock_try(lk, x)) { + if (lockmgr_sunlock_try(lk, &x)) { lockmgr_note_shared_release(lk, file, line); unlocked = true; + } else { + return (lockmgr_sunlock_hard(lk, x, flags, ilk, file, line)); } } else { tid = (uintptr_t)curthread; @@ -625,18 +1125,15 @@ lockmgr_unlock_fast_path(struct lock *lk, u_int flags, atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) { lockmgr_note_exclusive_release(lk, file, line); unlocked = true; + } else { + return (lockmgr_xunlock_hard(lk, x, flags, ilk, file, line)); } } - if (__predict_true(unlocked)) { - if (__predict_false(flags & LK_INTERLOCK)) { - class = LOCK_CLASS(ilk); - class->lc_unlock(ilk); - } - return (0); - } else { - return (__lockmgr_args(lk, flags | LK_RELEASE, ilk, LK_WMESG_DEFAULT, - LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, LOCK_FILE, LOCK_LINE)); + if (__predict_false(flags & LK_INTERLOCK)) { + class = LOCK_CLASS(ilk); + class->lc_unlock(ilk); } + return (0); } int @@ -644,6 +1141,7 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lo const char *wmesg, int pri, int timo, const char *file, int line) { GIANT_DECLARE; + struct lockmgr_wait lwa; struct lock_class *class; const char *iwmesg; uintptr_t tid, v, x; @@ -661,6 +1159,10 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lo ipri = (pri == LK_PRIO_DEFAULT) ? lk->lk_pri : pri; itimo = (timo == LK_TIMO_DEFAULT) ? lk->lk_timo : timo; + lwa.iwmesg = iwmesg; + lwa.ipri = ipri; + lwa.itimo = itimo; + MPASS((flags & ~LK_TOTAL_MASK) == 0); KASSERT((op & (op - 1)) == 0, ("%s: Invalid requested operation @ %s:%d", __func__, file, line)); @@ -701,278 +1203,14 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lo wakeup_swapper = 0; switch (op) { case LK_SHARED: - if (LK_CAN_WITNESS(flags)) - WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER, - file, line, flags & LK_INTERLOCK ? ilk : NULL); - for (;;) { - if (lockmgr_slock_try(lk, &x, flags)) - break; -#ifdef HWPMC_HOOKS - PMC_SOFT_CALL( , , lock, failed); -#endif - lock_profile_obtain_lock_failed(&lk->lock_object, - &contested, &waittime); - - /* - * If the lock is already held by curthread in - * exclusive way avoid a deadlock. - */ - if (LK_HOLDER(x) == tid) { - LOCK_LOG2(lk, - "%s: %p already held in exclusive mode", - __func__, lk); - error = EDEADLK; - break; - } - - /* - * If the lock is expected to not sleep just give up - * and return. - */ - if (LK_TRYOP(flags)) { - LOCK_LOG2(lk, "%s: %p fails the try operation", - __func__, lk); - error = EBUSY; - break; - } - - /* - * Acquire the sleepqueue chain lock because we - * probabilly will need to manipulate waiters flags. - */ - sleepq_lock(&lk->lock_object); - x = lk->lk_lock; - - /* - * if the lock can be acquired in shared mode, try - * again. - */ - if (LK_CAN_SHARE(x, flags)) { - sleepq_release(&lk->lock_object); - continue; - } - - /* - * Try to set the LK_SHARED_WAITERS flag. If we fail, - * loop back and retry. - */ - if ((x & LK_SHARED_WAITERS) == 0) { - if (!atomic_cmpset_acq_ptr(&lk->lk_lock, x, - x | LK_SHARED_WAITERS)) { - sleepq_release(&lk->lock_object); - continue; - } - LOCK_LOG2(lk, "%s: %p set shared waiters flag", - __func__, lk); - } - - /* - * As far as we have been unable to acquire the - * shared lock and the shared waiters flag is set, - * we will sleep. - */ - error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo, - SQ_SHARED_QUEUE); - flags &= ~LK_INTERLOCK; - if (error) { - LOCK_LOG3(lk, - "%s: interrupted sleep for %p with %d", - __func__, lk, error); - break; - } - LOCK_LOG2(lk, "%s: %p resuming from the sleep queue", - __func__, lk); - } - if (error == 0) { -#ifdef LOCK_PROFILING - lockmgr_note_shared_acquire(lk, contested, waittime, - file, line, flags); -#else - lockmgr_note_shared_acquire(lk, 0, 0, file, line, - flags); -#endif - } + return (lockmgr_slock_hard(lk, flags, ilk, file, line, &lwa)); break; case LK_UPGRADE: case LK_TRYUPGRADE: - _lockmgr_assert(lk, KA_SLOCKED, file, line); - v = lk->lk_lock; - x = v & LK_ALL_WAITERS; - v &= LK_EXCLUSIVE_SPINNERS; - - /* - * Try to switch from one shared lock to an exclusive one. - * We need to preserve waiters flags during the operation. - */ - if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x | v, - tid | x)) { - LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file, - line); - WITNESS_UPGRADE(&lk->lock_object, LOP_EXCLUSIVE | - LK_TRYWIT(flags), file, line); - TD_SLOCKS_DEC(curthread); - break; - } - - /* - * In LK_TRYUPGRADE mode, do not drop the lock, - * returning EBUSY instead. - */ - if (op == LK_TRYUPGRADE) { - LOCK_LOG2(lk, "%s: %p failed the nowait upgrade", - __func__, lk); - error = EBUSY; - break; - } - - /* - * We have been unable to succeed in upgrading, so just - * give up the shared lock. - */ - wakeup_swapper |= wakeupshlk(lk, file, line); - - /* FALLTHROUGH */ + return (lockmgr_upgrade(lk, flags, ilk, file, line, &lwa)); + break; case LK_EXCLUSIVE: - if (LK_CAN_WITNESS(flags)) - WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER | - LOP_EXCLUSIVE, file, line, flags & LK_INTERLOCK ? - ilk : NULL); - - /* - * If curthread already holds the lock and this one is - * allowed to recurse, simply recurse on it. - */ - if (lockmgr_xlocked(lk)) { - if ((flags & LK_CANRECURSE) == 0 && - (lk->lock_object.lo_flags & LO_RECURSABLE) == 0) { - - /* - * If the lock is expected to not panic just - * give up and return. - */ - if (LK_TRYOP(flags)) { - LOCK_LOG2(lk, - "%s: %p fails the try operation", - __func__, lk); - error = EBUSY; - break; - } - if (flags & LK_INTERLOCK) - class->lc_unlock(ilk); - panic("%s: recursing on non recursive lockmgr %s @ %s:%d\n", - __func__, iwmesg, file, line); - } - lk->lk_recurse++; - LOCK_LOG2(lk, "%s: %p recursing", __func__, lk); - LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0, - lk->lk_recurse, file, line); - WITNESS_LOCK(&lk->lock_object, LOP_EXCLUSIVE | - LK_TRYWIT(flags), file, line); - TD_LOCKS_INC(curthread); - break; - } - - for (;;) { - if (lk->lk_lock == LK_UNLOCKED && - atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) - break; -#ifdef HWPMC_HOOKS - PMC_SOFT_CALL( , , lock, failed); -#endif - lock_profile_obtain_lock_failed(&lk->lock_object, - &contested, &waittime); - - /* - * If the lock is expected to not sleep just give up - * and return. - */ - if (LK_TRYOP(flags)) { - LOCK_LOG2(lk, "%s: %p fails the try operation", - __func__, lk); - error = EBUSY; - break; - } - - /* - * Acquire the sleepqueue chain lock because we - * probabilly will need to manipulate waiters flags. - */ - sleepq_lock(&lk->lock_object); - x = lk->lk_lock; - - /* - * if the lock has been released while we spun on - * the sleepqueue chain lock just try again. - */ - if (x == LK_UNLOCKED) { - sleepq_release(&lk->lock_object); - continue; - } - - /* - * The lock can be in the state where there is a - * pending queue of waiters, but still no owner. - * This happens when the lock is contested and an - * owner is going to claim the lock. - * If curthread is the one successfully acquiring it - * claim lock ownership and return, preserving waiters - * flags. - */ - v = x & (LK_ALL_WAITERS | LK_EXCLUSIVE_SPINNERS); - if ((x & ~v) == LK_UNLOCKED) { - v &= ~LK_EXCLUSIVE_SPINNERS; - if (atomic_cmpset_acq_ptr(&lk->lk_lock, x, - tid | v)) { - sleepq_release(&lk->lock_object); - LOCK_LOG2(lk, - "%s: %p claimed by a new writer", - __func__, lk); - break; - } - sleepq_release(&lk->lock_object); - continue; - } - - /* - * Try to set the LK_EXCLUSIVE_WAITERS flag. If we - * fail, loop back and retry. - */ - if ((x & LK_EXCLUSIVE_WAITERS) == 0) { - if (!atomic_cmpset_ptr(&lk->lk_lock, x, - x | LK_EXCLUSIVE_WAITERS)) { - sleepq_release(&lk->lock_object); - continue; - } - LOCK_LOG2(lk, "%s: %p set excl waiters flag", - __func__, lk); - } - - /* - * As far as we have been unable to acquire the - * exclusive lock and the exclusive waiters flag - * is set, we will sleep. - */ - error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo, - SQ_EXCLUSIVE_QUEUE); - flags &= ~LK_INTERLOCK; - if (error) { - LOCK_LOG3(lk, - "%s: interrupted sleep for %p with %d", - __func__, lk, error); - break; - } - LOCK_LOG2(lk, "%s: %p resuming from the sleep queue", - __func__, lk); - } - if (error == 0) { -#ifdef LOCK_PROFILING - lockmgr_note_exclusive_acquire(lk, contested, waittime, - file, line, flags); -#else - lockmgr_note_exclusive_acquire(lk, 0, 0, file, line, - flags); -#endif - } + return (lockmgr_xlock_hard(lk, flags, ilk, file, line, &lwa)); break; case LK_DOWNGRADE: _lockmgr_assert(lk, KA_XLOCKED, file, line); @@ -1007,103 +1245,11 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lo _lockmgr_assert(lk, KA_LOCKED, file, line); x = lk->lk_lock; - if ((x & LK_SHARE) == 0) { - - /* - * As first option, treact the lock as if it has not - * any waiter. - * Fix-up the tid var if the lock has been disowned. - */ - if (LK_HOLDER(x) == LK_KERNPROC) - tid = LK_KERNPROC; - else { - WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, - file, line); - TD_LOCKS_DEC(curthread); - } - LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, - lk->lk_recurse, file, line); - - /* - * The lock is held in exclusive mode. - * If the lock is recursed also, then unrecurse it. - */ - if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) { - LOCK_LOG2(lk, "%s: %p unrecursing", __func__, - lk); - lk->lk_recurse--; - break; - } - if (tid != LK_KERNPROC) - lock_profile_release_lock(&lk->lock_object); - - if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid, - LK_UNLOCKED)) - break; - - sleepq_lock(&lk->lock_object); - x = lk->lk_lock; - v = LK_UNLOCKED; *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201803041912.w24JCsaa018130>