Date: Tue, 2 Aug 2016 02:32:00 +0000 (UTC) From: Mark Johnston <markj@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r303654 - in user/alc/PQ_LAUNDRY/sys: ddb dev/cxgbe dev/cxgbe/iw_cxgbe dev/pci kern net ofed/drivers/infiniband/ulp/ipoib opencrypto sys Message-ID: <201608020232.u722W09E009649@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: markj Date: Tue Aug 2 02:32:00 2016 New Revision: 303654 URL: https://svnweb.freebsd.org/changeset/base/303654 Log: MFH r303653 Modified: user/alc/PQ_LAUNDRY/sys/ddb/db_ps.c user/alc/PQ_LAUNDRY/sys/dev/cxgbe/adapter.h user/alc/PQ_LAUNDRY/sys/dev/cxgbe/iw_cxgbe/cq.c user/alc/PQ_LAUNDRY/sys/dev/cxgbe/iw_cxgbe/qp.c user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_main.c user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_netmap.c user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_sge.c user/alc/PQ_LAUNDRY/sys/dev/pci/pci_pci.c user/alc/PQ_LAUNDRY/sys/kern/kern_mutex.c user/alc/PQ_LAUNDRY/sys/kern/kern_rwlock.c user/alc/PQ_LAUNDRY/sys/kern/kern_sx.c user/alc/PQ_LAUNDRY/sys/kern/subr_lock.c user/alc/PQ_LAUNDRY/sys/net/route.h user/alc/PQ_LAUNDRY/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c user/alc/PQ_LAUNDRY/sys/opencrypto/xform_aes_icm.c user/alc/PQ_LAUNDRY/sys/sys/lock.h Directory Properties: user/alc/PQ_LAUNDRY/ (props changed) Modified: user/alc/PQ_LAUNDRY/sys/ddb/db_ps.c ============================================================================== --- user/alc/PQ_LAUNDRY/sys/ddb/db_ps.c Tue Aug 2 02:29:48 2016 (r303653) +++ user/alc/PQ_LAUNDRY/sys/ddb/db_ps.c Tue Aug 2 02:32:00 2016 (r303654) @@ -440,9 +440,16 @@ DB_SHOW_COMMAND(proc, db_show_proc) p->p_leader); if (p->p_sysent != NULL) db_printf(" ABI: %s\n", p->p_sysent->sv_name); - if (p->p_args != NULL) - db_printf(" arguments: %.*s\n", (int)p->p_args->ar_length, - p->p_args->ar_args); + if (p->p_args != NULL) { + db_printf(" arguments: "); + for (i = 0; i < (int)p->p_args->ar_length; i++) { + if (p->p_args->ar_args[i] == '\0') + db_printf(" "); + else + db_printf("%c", p->p_args->ar_args[i]); + } + db_printf("\n"); + } db_printf(" threads: %d\n", p->p_numthreads); FOREACH_THREAD_IN_PROC(p, td) { dumpthread(p, td, 1); Modified: user/alc/PQ_LAUNDRY/sys/dev/cxgbe/adapter.h ============================================================================== --- user/alc/PQ_LAUNDRY/sys/dev/cxgbe/adapter.h Tue Aug 2 02:29:48 2016 (r303653) +++ user/alc/PQ_LAUNDRY/sys/dev/cxgbe/adapter.h Tue Aug 2 02:32:00 2016 (r303654) @@ -781,6 +781,8 @@ struct adapter { struct sge_rxq *rxq; struct sge_nm_rxq *nm_rxq; } __aligned(CACHE_LINE_SIZE) *irq; + int sge_gts_reg; + int sge_kdoorbell_reg; bus_dma_tag_t dmat; /* Parent DMA tag */ Modified: user/alc/PQ_LAUNDRY/sys/dev/cxgbe/iw_cxgbe/cq.c ============================================================================== --- user/alc/PQ_LAUNDRY/sys/dev/cxgbe/iw_cxgbe/cq.c Tue Aug 2 02:29:48 2016 (r303653) +++ user/alc/PQ_LAUNDRY/sys/dev/cxgbe/iw_cxgbe/cq.c Tue Aug 2 02:32:00 2016 (r303654) @@ -172,7 +172,7 @@ create_cq(struct c4iw_rdev *rdev, struct cq->gen = 1; cq->gts = (void *)((unsigned long)rman_get_virtual(sc->regs_res) + - MYPF_REG(SGE_PF_GTS)); + sc->sge_gts_reg); cq->rdev = rdev; if (user) { Modified: user/alc/PQ_LAUNDRY/sys/dev/cxgbe/iw_cxgbe/qp.c ============================================================================== --- user/alc/PQ_LAUNDRY/sys/dev/cxgbe/iw_cxgbe/qp.c Tue Aug 2 02:29:48 2016 (r303653) +++ user/alc/PQ_LAUNDRY/sys/dev/cxgbe/iw_cxgbe/qp.c Tue Aug 2 02:32:00 2016 (r303654) @@ -182,9 +182,9 @@ static int create_qp(struct c4iw_rdev *r pci_unmap_addr_set(&wq->rq, mapping, wq->rq.dma_addr); wq->db = (void *)((unsigned long)rman_get_virtual(sc->regs_res) + - MYPF_REG(SGE_PF_KDOORBELL)); + sc->sge_kdoorbell_reg); wq->gts = (void *)((unsigned long)rman_get_virtual(rdev->adap->regs_res) - + MYPF_REG(SGE_PF_GTS)); + + sc->sge_gts_reg); if (user) { wq->sq.udb = (u64)((char*)rman_get_virtual(rdev->adap->udbs_res) + (wq->sq.qid << rdev->qpshift)); Modified: user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_main.c ============================================================================== --- user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_main.c Tue Aug 2 02:29:48 2016 (r303653) +++ user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_main.c Tue Aug 2 02:32:00 2016 (r303654) @@ -697,6 +697,8 @@ t4_attach(device_t dev) sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); } + sc->sge_gts_reg = MYPF_REG(A_SGE_PF_GTS); + sc->sge_kdoorbell_reg = MYPF_REG(A_SGE_PF_KDOORBELL); sc->traceq = -1; mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", Modified: user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_netmap.c ============================================================================== --- user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_netmap.c Tue Aug 2 02:29:48 2016 (r303653) +++ user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_netmap.c Tue Aug 2 02:32:00 2016 (r303654) @@ -204,7 +204,7 @@ alloc_nm_rxq_hwq(struct vi_info *vi, str } } - t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), + t4_write_reg(sc, sc->sge_gts_reg, V_INGRESSQID(nm_rxq->iq_cntxt_id) | V_SEINTARM(V_QINTR_TIMER_IDX(holdoff_tmr_idx))); @@ -364,7 +364,7 @@ cxgbe_netmap_on(struct adapter *sc, stru MPASS((j & 7) == 0); j /= 8; /* driver pidx to hardware pidx */ wmb(); - t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), + t4_write_reg(sc, sc->sge_kdoorbell_reg, nm_rxq->fl_db_val | V_PIDX(j)); atomic_cmpset_int(&irq->nm_state, NM_OFF, NM_ON); @@ -537,7 +537,7 @@ ring_nm_txq_db(struct adapter *sc, struc break; case DOORBELL_KDB: - t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), + t4_write_reg(sc, sc->sge_kdoorbell_reg, V_QID(nm_txq->cntxt_id) | V_PIDX(n)); break; } @@ -818,7 +818,7 @@ cxgbe_netmap_rxsync(struct netmap_kring } if (++dbinc == 8 && n >= 32) { wmb(); - t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), + t4_write_reg(sc, sc->sge_kdoorbell_reg, nm_rxq->fl_db_val | V_PIDX(dbinc)); dbinc = 0; } @@ -827,7 +827,7 @@ cxgbe_netmap_rxsync(struct netmap_kring if (dbinc > 0) { wmb(); - t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), + t4_write_reg(sc, sc->sge_kdoorbell_reg, nm_rxq->fl_db_val | V_PIDX(dbinc)); } } @@ -981,14 +981,14 @@ t4_nm_intr(void *arg) fl_credits /= 8; IDXINCR(nm_rxq->fl_pidx, fl_credits * 8, nm_rxq->fl_sidx); - t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), + t4_write_reg(sc, sc->sge_kdoorbell_reg, nm_rxq->fl_db_val | V_PIDX(fl_credits)); fl_credits = fl_cidx & 7; } else if (!black_hole) { netmap_rx_irq(ifp, nm_rxq->nid, &work); MPASS(work != 0); } - t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), + t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(n) | V_INGRESSQID(nm_rxq->iq_cntxt_id) | V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); n = 0; @@ -999,12 +999,12 @@ t4_nm_intr(void *arg) if (black_hole) { fl_credits /= 8; IDXINCR(nm_rxq->fl_pidx, fl_credits * 8, nm_rxq->fl_sidx); - t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), + t4_write_reg(sc, sc->sge_kdoorbell_reg, nm_rxq->fl_db_val | V_PIDX(fl_credits)); } else netmap_rx_irq(ifp, nm_rxq->nid, &work); - t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(n) | + t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(n) | V_INGRESSQID((u32)nm_rxq->iq_cntxt_id) | V_SEINTARM(V_QINTR_TIMER_IDX(holdoff_tmr_idx))); } Modified: user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_sge.c ============================================================================== --- user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_sge.c Tue Aug 2 02:29:48 2016 (r303653) +++ user/alc/PQ_LAUNDRY/sys/dev/cxgbe/t4_sge.c Tue Aug 2 02:32:00 2016 (r303654) @@ -1474,7 +1474,7 @@ service_iq(struct sge_iq *iq, int budget d = &iq->desc[0]; } if (__predict_false(++ndescs == limit)) { - t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), + t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) | V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); @@ -1529,7 +1529,7 @@ process_iql: } #endif - t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) | + t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params)); if (iq->flags & IQ_HAS_FL) { @@ -2793,7 +2793,7 @@ alloc_iq_fl(struct vi_info *vi, struct s /* Enable IQ interrupts */ atomic_store_rel_int(&iq->state, IQS_IDLE); - t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) | + t4_write_reg(sc, sc->sge_gts_reg, V_SEINTARM(iq->intr_params) | V_INGRESSQID(iq->cntxt_id)); return (0); @@ -3676,7 +3676,7 @@ ring_fl_db(struct adapter *sc, struct sg if (fl->udb) *fl->udb = htole32(v); else - t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v); + t4_write_reg(sc, sc->sge_kdoorbell_reg, v); IDXINCR(fl->dbidx, n, fl->sidx); } @@ -4409,7 +4409,7 @@ ring_eq_db(struct adapter *sc, struct sg break; case DOORBELL_KDB: - t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), + t4_write_reg(sc, sc->sge_kdoorbell_reg, V_QID(eq->cntxt_id) | V_PIDX(n)); break; } Modified: user/alc/PQ_LAUNDRY/sys/dev/pci/pci_pci.c ============================================================================== --- user/alc/PQ_LAUNDRY/sys/dev/pci/pci_pci.c Tue Aug 2 02:29:48 2016 (r303653) +++ user/alc/PQ_LAUNDRY/sys/dev/pci/pci_pci.c Tue Aug 2 02:32:00 2016 (r303654) @@ -932,6 +932,13 @@ pcib_probe_hotplug(struct pcib_softc *sc sc->pcie_link_cap = pcie_read_config(dev, PCIER_LINK_CAP, 4); sc->pcie_slot_cap = pcie_read_config(dev, PCIER_SLOT_CAP, 4); + /* + * XXX: Handling of slots with a power controller needs to be + * reexamined. Ignore hotplug on such slots for now. + */ + if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_PCP) + return; + if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_HPC) sc->flags |= PCIB_HOTPLUG; } Modified: user/alc/PQ_LAUNDRY/sys/kern/kern_mutex.c ============================================================================== --- user/alc/PQ_LAUNDRY/sys/kern/kern_mutex.c Tue Aug 2 02:29:48 2016 (r303653) +++ user/alc/PQ_LAUNDRY/sys/kern/kern_mutex.c Tue Aug 2 02:32:00 2016 (r303654) @@ -55,6 +55,7 @@ __FBSDID("$FreeBSD$"); #include <sys/resourcevar.h> #include <sys/sched.h> #include <sys/sbuf.h> +#include <sys/smp.h> #include <sys/sysctl.h> #include <sys/turnstile.h> #include <sys/vmmeter.h> @@ -138,6 +139,37 @@ struct lock_class lock_class_mtx_spin = #endif }; +#ifdef ADAPTIVE_MUTEXES +static SYSCTL_NODE(_debug, OID_AUTO, mtx, CTLFLAG_RD, NULL, "mtx debugging"); + +static struct lock_delay_config mtx_delay = { + .initial = 1000, + .step = 500, + .min = 100, + .max = 5000, +}; + +SYSCTL_INT(_debug_mtx, OID_AUTO, delay_initial, CTLFLAG_RW, &mtx_delay.initial, + 0, ""); +SYSCTL_INT(_debug_mtx, OID_AUTO, delay_step, CTLFLAG_RW, &mtx_delay.step, + 0, ""); +SYSCTL_INT(_debug_mtx, OID_AUTO, delay_min, CTLFLAG_RW, &mtx_delay.min, + 0, ""); +SYSCTL_INT(_debug_mtx, OID_AUTO, delay_max, CTLFLAG_RW, &mtx_delay.max, + 0, ""); + +static void +mtx_delay_sysinit(void *dummy) +{ + + mtx_delay.initial = mp_ncpus * 25; + mtx_delay.step = (mp_ncpus * 25) / 2; + mtx_delay.min = mp_ncpus * 5; + mtx_delay.max = mp_ncpus * 25 * 10; +} +LOCK_DELAY_SYSINIT(mtx_delay_sysinit); +#endif + /* * System-wide mutexes */ @@ -408,8 +440,10 @@ __mtx_lock_sleep(volatile uintptr_t *c, int contested = 0; uint64_t waittime = 0; #endif +#if defined(ADAPTIVE_MUTEXES) || defined(KDTRACE_HOOKS) + struct lock_delay_arg lda; +#endif #ifdef KDTRACE_HOOKS - u_int spin_cnt = 0; u_int sleep_cnt = 0; int64_t sleep_time = 0; int64_t all_time = 0; @@ -418,6 +452,9 @@ __mtx_lock_sleep(volatile uintptr_t *c, if (SCHEDULER_STOPPED()) return; +#if defined(ADAPTIVE_MUTEXES) || defined(KDTRACE_HOOKS) + lock_delay_arg_init(&lda, &mtx_delay); +#endif m = mtxlock2mtx(c); if (mtx_owned(m)) { @@ -451,7 +488,7 @@ __mtx_lock_sleep(volatile uintptr_t *c, if (m->mtx_lock == MTX_UNOWNED && _mtx_obtain_lock(m, tid)) break; #ifdef KDTRACE_HOOKS - spin_cnt++; + lda.spin_cnt++; #endif #ifdef ADAPTIVE_MUTEXES /* @@ -471,12 +508,8 @@ __mtx_lock_sleep(volatile uintptr_t *c, "spinning", "lockname:\"%s\"", m->lock_object.lo_name); while (mtx_owner(m) == owner && - TD_IS_RUNNING(owner)) { - cpu_spinwait(); -#ifdef KDTRACE_HOOKS - spin_cnt++; -#endif - } + TD_IS_RUNNING(owner)) + lock_delay(&lda); KTR_STATE0(KTR_SCHED, "thread", sched_tdname((struct thread *)tid), "running"); @@ -570,7 +603,7 @@ __mtx_lock_sleep(volatile uintptr_t *c, /* * Only record the loops spinning and not sleeping. */ - if (spin_cnt > sleep_cnt) + if (lda.spin_cnt > sleep_cnt) LOCKSTAT_RECORD1(adaptive__spin, m, all_time - sleep_time); #endif } Modified: user/alc/PQ_LAUNDRY/sys/kern/kern_rwlock.c ============================================================================== --- user/alc/PQ_LAUNDRY/sys/kern/kern_rwlock.c Tue Aug 2 02:29:48 2016 (r303653) +++ user/alc/PQ_LAUNDRY/sys/kern/kern_rwlock.c Tue Aug 2 02:32:00 2016 (r303654) @@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$"); #include <sys/proc.h> #include <sys/rwlock.h> #include <sys/sched.h> +#include <sys/smp.h> #include <sys/sysctl.h> #include <sys/systm.h> #include <sys/turnstile.h> @@ -65,15 +66,6 @@ PMC_SOFT_DECLARE( , , lock, failed); */ #define rwlock2rw(c) (__containerof(c, struct rwlock, rw_lock)) -#ifdef ADAPTIVE_RWLOCKS -static int rowner_retries = 10; -static int rowner_loops = 10000; -static SYSCTL_NODE(_debug, OID_AUTO, rwlock, CTLFLAG_RD, NULL, - "rwlock debugging"); -SYSCTL_INT(_debug_rwlock, OID_AUTO, retry, CTLFLAG_RW, &rowner_retries, 0, ""); -SYSCTL_INT(_debug_rwlock, OID_AUTO, loops, CTLFLAG_RW, &rowner_loops, 0, ""); -#endif - #ifdef DDB #include <ddb/ddb.h> @@ -100,6 +92,42 @@ struct lock_class lock_class_rw = { #endif }; +#ifdef ADAPTIVE_RWLOCKS +static int rowner_retries = 10; +static int rowner_loops = 10000; +static SYSCTL_NODE(_debug, OID_AUTO, rwlock, CTLFLAG_RD, NULL, + "rwlock debugging"); +SYSCTL_INT(_debug_rwlock, OID_AUTO, retry, CTLFLAG_RW, &rowner_retries, 0, ""); +SYSCTL_INT(_debug_rwlock, OID_AUTO, loops, CTLFLAG_RW, &rowner_loops, 0, ""); + +static struct lock_delay_config rw_delay = { + .initial = 1000, + .step = 500, + .min = 100, + .max = 5000, +}; + +SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_initial, CTLFLAG_RW, &rw_delay.initial, + 0, ""); +SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_step, CTLFLAG_RW, &rw_delay.step, + 0, ""); +SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_min, CTLFLAG_RW, &rw_delay.min, + 0, ""); +SYSCTL_INT(_debug_rwlock, OID_AUTO, delay_max, CTLFLAG_RW, &rw_delay.max, + 0, ""); + +static void +rw_delay_sysinit(void *dummy) +{ + + rw_delay.initial = mp_ncpus * 25; + rw_delay.step = (mp_ncpus * 25) / 2; + rw_delay.min = mp_ncpus * 5; + rw_delay.max = mp_ncpus * 25 * 10; +} +LOCK_DELAY_SYSINIT(rw_delay_sysinit); +#endif + /* * Return a pointer to the owning thread if the lock is write-locked or * NULL if the lock is unlocked or read-locked. @@ -355,9 +383,11 @@ __rw_rlock(volatile uintptr_t *c, const int contested = 0; #endif uintptr_t v; +#if defined(ADAPTIVE_RWLOCKS) || defined(KDTRACE_HOOKS) + struct lock_delay_arg lda; +#endif #ifdef KDTRACE_HOOKS uintptr_t state; - u_int spin_cnt = 0; u_int sleep_cnt = 0; int64_t sleep_time = 0; int64_t all_time = 0; @@ -366,6 +396,9 @@ __rw_rlock(volatile uintptr_t *c, const if (SCHEDULER_STOPPED()) return; +#if defined(ADAPTIVE_RWLOCKS) || defined(KDTRACE_HOOKS) + lock_delay_arg_init(&lda, &rw_delay); +#endif rw = rwlock2rw(c); KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), @@ -412,7 +445,7 @@ __rw_rlock(volatile uintptr_t *c, const continue; } #ifdef KDTRACE_HOOKS - spin_cnt++; + lda.spin_cnt++; #endif #ifdef HWPMC_HOOKS PMC_SOFT_CALL( , , lock, failed); @@ -437,12 +470,8 @@ __rw_rlock(volatile uintptr_t *c, const sched_tdname(curthread), "spinning", "lockname:\"%s\"", rw->lock_object.lo_name); while ((struct thread*)RW_OWNER(rw->rw_lock) == - owner && TD_IS_RUNNING(owner)) { - cpu_spinwait(); -#ifdef KDTRACE_HOOKS - spin_cnt++; -#endif - } + owner && TD_IS_RUNNING(owner)) + lock_delay(&lda); KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread), "running"); continue; @@ -459,7 +488,7 @@ __rw_rlock(volatile uintptr_t *c, const cpu_spinwait(); } #ifdef KDTRACE_HOOKS - spin_cnt += rowner_loops - i; + lda.spin_cnt += rowner_loops - i; #endif KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread), "running"); @@ -552,7 +581,7 @@ __rw_rlock(volatile uintptr_t *c, const (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); /* Record only the loops spinning and not sleeping. */ - if (spin_cnt > sleep_cnt) + if (lda.spin_cnt > sleep_cnt) LOCKSTAT_RECORD4(rw__spin, rw, all_time - sleep_time, LOCKSTAT_READER, (state & RW_LOCK_READ) == 0, (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); @@ -740,9 +769,11 @@ __rw_wlock_hard(volatile uintptr_t *c, u uint64_t waittime = 0; int contested = 0; #endif +#if defined(ADAPTIVE_RWLOCKS) || defined(KDTRACE_HOOKS) + struct lock_delay_arg lda; +#endif #ifdef KDTRACE_HOOKS uintptr_t state; - u_int spin_cnt = 0; u_int sleep_cnt = 0; int64_t sleep_time = 0; int64_t all_time = 0; @@ -751,6 +782,9 @@ __rw_wlock_hard(volatile uintptr_t *c, u if (SCHEDULER_STOPPED()) return; +#if defined(ADAPTIVE_RWLOCKS) || defined(KDTRACE_HOOKS) + lock_delay_arg_init(&lda, &rw_delay); +#endif rw = rwlock2rw(c); if (rw_wlocked(rw)) { @@ -775,7 +809,7 @@ __rw_wlock_hard(volatile uintptr_t *c, u if (rw->rw_lock == RW_UNLOCKED && _rw_write_lock(rw, tid)) break; #ifdef KDTRACE_HOOKS - spin_cnt++; + lda.spin_cnt++; #endif #ifdef HWPMC_HOOKS PMC_SOFT_CALL( , , lock, failed); @@ -798,12 +832,8 @@ __rw_wlock_hard(volatile uintptr_t *c, u "spinning", "lockname:\"%s\"", rw->lock_object.lo_name); while ((struct thread*)RW_OWNER(rw->rw_lock) == owner && - TD_IS_RUNNING(owner)) { - cpu_spinwait(); -#ifdef KDTRACE_HOOKS - spin_cnt++; -#endif - } + TD_IS_RUNNING(owner)) + lock_delay(&lda); KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread), "running"); continue; @@ -828,7 +858,7 @@ __rw_wlock_hard(volatile uintptr_t *c, u KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread), "running"); #ifdef KDTRACE_HOOKS - spin_cnt += rowner_loops - i; + lda.spin_cnt += rowner_loops - i; #endif if (i != rowner_loops) continue; @@ -918,7 +948,7 @@ __rw_wlock_hard(volatile uintptr_t *c, u (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); /* Record only the loops spinning and not sleeping. */ - if (spin_cnt > sleep_cnt) + if (lda.spin_cnt > sleep_cnt) LOCKSTAT_RECORD4(rw__spin, rw, all_time - sleep_time, LOCKSTAT_WRITER, (state & RW_LOCK_READ) == 0, (state & RW_LOCK_READ) == 0 ? 0 : RW_READERS(state)); Modified: user/alc/PQ_LAUNDRY/sys/kern/kern_sx.c ============================================================================== --- user/alc/PQ_LAUNDRY/sys/kern/kern_sx.c Tue Aug 2 02:29:48 2016 (r303653) +++ user/alc/PQ_LAUNDRY/sys/kern/kern_sx.c Tue Aug 2 02:32:00 2016 (r303654) @@ -53,6 +53,7 @@ __FBSDID("$FreeBSD$"); #include <sys/sched.h> #include <sys/sleepqueue.h> #include <sys/sx.h> +#include <sys/smp.h> #include <sys/sysctl.h> #if defined(SMP) && !defined(NO_ADAPTIVE_SX) @@ -145,6 +146,33 @@ static u_int asx_loops = 10000; static SYSCTL_NODE(_debug, OID_AUTO, sx, CTLFLAG_RD, NULL, "sxlock debugging"); SYSCTL_UINT(_debug_sx, OID_AUTO, retries, CTLFLAG_RW, &asx_retries, 0, ""); SYSCTL_UINT(_debug_sx, OID_AUTO, loops, CTLFLAG_RW, &asx_loops, 0, ""); + +static struct lock_delay_config sx_delay = { + .initial = 1000, + .step = 500, + .min = 100, + .max = 5000, +}; + +SYSCTL_INT(_debug_sx, OID_AUTO, delay_initial, CTLFLAG_RW, &sx_delay.initial, + 0, ""); +SYSCTL_INT(_debug_sx, OID_AUTO, delay_step, CTLFLAG_RW, &sx_delay.step, + 0, ""); +SYSCTL_INT(_debug_sx, OID_AUTO, delay_min, CTLFLAG_RW, &sx_delay.min, + 0, ""); +SYSCTL_INT(_debug_sx, OID_AUTO, delay_max, CTLFLAG_RW, &sx_delay.max, + 0, ""); + +static void +sx_delay_sysinit(void *dummy) +{ + + sx_delay.initial = mp_ncpus * 25; + sx_delay.step = (mp_ncpus * 25) / 2; + sx_delay.min = mp_ncpus * 5; + sx_delay.max = mp_ncpus * 25 * 10; +} +LOCK_DELAY_SYSINIT(sx_delay_sysinit); #endif void @@ -513,9 +541,11 @@ _sx_xlock_hard(struct sx *sx, uintptr_t int contested = 0; #endif int error = 0; +#if defined(ADAPTIVE_SX) || defined(KDTRACE_HOOKS) + struct lock_delay_arg lda; +#endif #ifdef KDTRACE_HOOKS uintptr_t state; - u_int spin_cnt = 0; u_int sleep_cnt = 0; int64_t sleep_time = 0; int64_t all_time = 0; @@ -524,6 +554,10 @@ _sx_xlock_hard(struct sx *sx, uintptr_t if (SCHEDULER_STOPPED()) return (0); +#if defined(ADAPTIVE_SX) || defined(KDTRACE_HOOKS) + lock_delay_arg_init(&lda, &sx_delay); +#endif + /* If we already hold an exclusive lock, then recurse. */ if (sx_xlocked(sx)) { KASSERT((sx->lock_object.lo_flags & LO_RECURSABLE) != 0, @@ -549,7 +583,7 @@ _sx_xlock_hard(struct sx *sx, uintptr_t atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) break; #ifdef KDTRACE_HOOKS - spin_cnt++; + lda.spin_cnt++; #endif #ifdef HWPMC_HOOKS PMC_SOFT_CALL( , , lock, failed); @@ -578,12 +612,8 @@ _sx_xlock_hard(struct sx *sx, uintptr_t sx->lock_object.lo_name); GIANT_SAVE(); while (SX_OWNER(sx->sx_lock) == x && - TD_IS_RUNNING(owner)) { - cpu_spinwait(); -#ifdef KDTRACE_HOOKS - spin_cnt++; -#endif - } + TD_IS_RUNNING(owner)) + lock_delay(&lda); KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread), "running"); continue; @@ -605,7 +635,7 @@ _sx_xlock_hard(struct sx *sx, uintptr_t break; cpu_spinwait(); #ifdef KDTRACE_HOOKS - spin_cnt++; + lda.spin_cnt++; #endif } KTR_STATE0(KTR_SCHED, "thread", @@ -725,7 +755,7 @@ _sx_xlock_hard(struct sx *sx, uintptr_t LOCKSTAT_RECORD4(sx__block, sx, sleep_time, LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0, (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); - if (spin_cnt > sleep_cnt) + if (lda.spin_cnt > sleep_cnt) LOCKSTAT_RECORD4(sx__spin, sx, all_time - sleep_time, LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0, (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); @@ -818,9 +848,11 @@ _sx_slock_hard(struct sx *sx, int opts, #endif uintptr_t x; int error = 0; +#if defined(ADAPTIVE_SX) || defined(KDTRACE_HOOKS) + struct lock_delay_arg lda; +#endif #ifdef KDTRACE_HOOKS uintptr_t state; - u_int spin_cnt = 0; u_int sleep_cnt = 0; int64_t sleep_time = 0; int64_t all_time = 0; @@ -829,6 +861,9 @@ _sx_slock_hard(struct sx *sx, int opts, if (SCHEDULER_STOPPED()) return (0); +#if defined(ADAPTIVE_SX) || defined(KDTRACE_HOOKS) + lock_delay_arg_init(&lda, &sx_delay); +#endif #ifdef KDTRACE_HOOKS state = sx->sx_lock; all_time -= lockstat_nsecs(&sx->lock_object); @@ -840,7 +875,7 @@ _sx_slock_hard(struct sx *sx, int opts, */ for (;;) { #ifdef KDTRACE_HOOKS - spin_cnt++; + lda.spin_cnt++; #endif x = sx->sx_lock; @@ -888,12 +923,8 @@ _sx_slock_hard(struct sx *sx, int opts, "lockname:\"%s\"", sx->lock_object.lo_name); GIANT_SAVE(); while (SX_OWNER(sx->sx_lock) == x && - TD_IS_RUNNING(owner)) { - cpu_spinwait(); -#ifdef KDTRACE_HOOKS - spin_cnt++; -#endif - } + TD_IS_RUNNING(owner)) + lock_delay(&lda); KTR_STATE0(KTR_SCHED, "thread", sched_tdname(curthread), "running"); continue; @@ -989,7 +1020,7 @@ _sx_slock_hard(struct sx *sx, int opts, LOCKSTAT_RECORD4(sx__block, sx, sleep_time, LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0, (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); - if (spin_cnt > sleep_cnt) + if (lda.spin_cnt > sleep_cnt) LOCKSTAT_RECORD4(sx__spin, sx, all_time - sleep_time, LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0, (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); Modified: user/alc/PQ_LAUNDRY/sys/kern/subr_lock.c ============================================================================== --- user/alc/PQ_LAUNDRY/sys/kern/subr_lock.c Tue Aug 2 02:29:48 2016 (r303653) +++ user/alc/PQ_LAUNDRY/sys/kern/subr_lock.c Tue Aug 2 02:32:00 2016 (r303654) @@ -103,6 +103,34 @@ lock_destroy(struct lock_object *lock) lock->lo_flags &= ~LO_INITIALIZED; } +void +lock_delay(struct lock_delay_arg *la) +{ + u_int i, delay, backoff, min, max; + struct lock_delay_config *lc = la->config; + + delay = la->delay; + + if (delay == 0) + delay = lc->initial; + else { + delay += lc->step; + max = lc->max; + if (delay > max) + delay = max; + } + + backoff = cpu_ticks() % delay; + min = lc->min; + if (backoff < min) + backoff = min; + for (i = 0; i < backoff; i++) + cpu_spinwait(); + + la->delay = delay; + la->spin_cnt += backoff; +} + #ifdef DDB DB_SHOW_COMMAND(lock, db_show_lock) { Modified: user/alc/PQ_LAUNDRY/sys/net/route.h ============================================================================== --- user/alc/PQ_LAUNDRY/sys/net/route.h Tue Aug 2 02:29:48 2016 (r303653) +++ user/alc/PQ_LAUNDRY/sys/net/route.h Tue Aug 2 02:32:00 2016 (r303654) @@ -360,7 +360,7 @@ struct rt_addrinfo { || (ifp)->if_link_state == LINK_STATE_UP) #define RT_LOCK_INIT(_rt) \ - mtx_init(&(_rt)->rt_mtx, "rtentry", NULL, MTX_DEF | MTX_DUPOK) + mtx_init(&(_rt)->rt_mtx, "rtentry", NULL, MTX_DEF | MTX_DUPOK | MTX_NEW) #define RT_LOCK(_rt) mtx_lock(&(_rt)->rt_mtx) #define RT_UNLOCK(_rt) mtx_unlock(&(_rt)->rt_mtx) #define RT_LOCK_DESTROY(_rt) mtx_destroy(&(_rt)->rt_mtx) Modified: user/alc/PQ_LAUNDRY/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c ============================================================================== --- user/alc/PQ_LAUNDRY/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c Tue Aug 2 02:29:48 2016 (r303653) +++ user/alc/PQ_LAUNDRY/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c Tue Aug 2 02:32:00 2016 (r303654) @@ -660,7 +660,13 @@ ipoib_unicast_send(struct mbuf *mb, stru new_path = 1; } if (path) { - _IF_ENQUEUE(&path->queue, mb); + if (_IF_QLEN(&path->queue) < IPOIB_MAX_PATH_REC_QUEUE) + _IF_ENQUEUE(&path->queue, mb); + else { + if_inc_counter(priv->dev, IFCOUNTER_OERRORS, 1); + m_freem(mb); + } + if (!path->query && path_rec_start(priv, path)) { spin_unlock_irqrestore(&priv->lock, flags); if (new_path) Modified: user/alc/PQ_LAUNDRY/sys/opencrypto/xform_aes_icm.c ============================================================================== --- user/alc/PQ_LAUNDRY/sys/opencrypto/xform_aes_icm.c Tue Aug 2 02:29:48 2016 (r303653) +++ user/alc/PQ_LAUNDRY/sys/opencrypto/xform_aes_icm.c Tue Aug 2 02:32:00 2016 (r303654) @@ -65,7 +65,7 @@ struct enc_xform enc_xform_aes_icm = { aes_icm_crypt, aes_icm_crypt, aes_icm_setkey, - rijndael128_zerokey, + aes_icm_zerokey, aes_icm_reinit, }; Modified: user/alc/PQ_LAUNDRY/sys/sys/lock.h ============================================================================== --- user/alc/PQ_LAUNDRY/sys/sys/lock.h Tue Aug 2 02:29:48 2016 (r303653) +++ user/alc/PQ_LAUNDRY/sys/sys/lock.h Tue Aug 2 02:32:00 2016 (r303654) @@ -201,9 +201,33 @@ extern struct lock_class lock_class_lock extern struct lock_class *lock_classes[]; +struct lock_delay_config { + u_int initial; + u_int step; + u_int min; + u_int max; +}; + +struct lock_delay_arg { + struct lock_delay_config *config; + u_int delay; + u_int spin_cnt; +}; + +static inline void +lock_delay_arg_init(struct lock_delay_arg *la, struct lock_delay_config *lc) { + la->config = lc; + la->delay = 0; + la->spin_cnt = 0; +} + +#define LOCK_DELAY_SYSINIT(func) \ + SYSINIT(func##_ld, SI_SUB_LOCK, SI_ORDER_ANY, func, NULL) + void lock_init(struct lock_object *, struct lock_class *, const char *, const char *, int); void lock_destroy(struct lock_object *); +void lock_delay(struct lock_delay_arg *); void spinlock_enter(void); void spinlock_exit(void); void witness_init(struct lock_object *, const char *);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201608020232.u722W09E009649>