Date: Mon, 22 Apr 2019 17:48:10 +0000 (UTC) From: Navdeep Parhar <np@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r346570 - in head/sys/dev/cxgbe: common tom Message-ID: <201904221748.x3MHmAxu048437@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: np Date: Mon Apr 22 17:48:10 2019 New Revision: 346570 URL: https://svnweb.freebsd.org/changeset/base/346570 Log: cxgbe/t4_tom: Add a "TCB history" feature that samples hardware state for a tid and maintains a running history of some interesting events. Service TCP_INFO queries from the history when the tid is being tracked there. Modified: head/sys/dev/cxgbe/common/t4_msg.h head/sys/dev/cxgbe/tom/t4_tom.c head/sys/dev/cxgbe/tom/t4_tom.h Modified: head/sys/dev/cxgbe/common/t4_msg.h ============================================================================== --- head/sys/dev/cxgbe/common/t4_msg.h Mon Apr 22 17:29:26 2019 (r346569) +++ head/sys/dev/cxgbe/common/t4_msg.h Mon Apr 22 17:48:10 2019 (r346570) @@ -923,7 +923,8 @@ struct cpl_get_tcb { WR_HDR; union opcode_tid ot; __be16 reply_ctrl; - __be16 cookie; + __u8 rsvd; + __u8 cookie; }; /* cpl_get_tcb.reply_ctrl fields */ Modified: head/sys/dev/cxgbe/tom/t4_tom.c ============================================================================== --- head/sys/dev/cxgbe/tom/t4_tom.c Mon Apr 22 17:29:26 2019 (r346569) +++ head/sys/dev/cxgbe/tom/t4_tom.c Mon Apr 22 17:48:10 2019 (r346570) @@ -386,55 +386,352 @@ t4_ctloutput(struct toedev *tod, struct tcpcb *tp, int } } -static inline int -get_tcb_bit(u_char *tcb, int bit) +static inline uint64_t +get_tcb_tflags(const uint64_t *tcb) { - int ix, shift; - ix = 127 - (bit >> 3); - shift = bit & 0x7; - - return ((tcb[ix] >> shift) & 1); + return ((be64toh(tcb[14]) << 32) | (be64toh(tcb[15]) >> 32)); } -static inline uint64_t -get_tcb_bits(u_char *tcb, int hi, int lo) +static inline uint32_t +get_tcb_field(const uint64_t *tcb, u_int word, uint32_t mask, u_int shift) { - uint64_t rc = 0; +#define LAST_WORD ((TCB_SIZE / 4) - 1) + uint64_t t1, t2; + int flit_idx; - while (hi >= lo) { - rc = (rc << 1) | get_tcb_bit(tcb, hi); - --hi; + MPASS(mask != 0); + MPASS(word <= LAST_WORD); + MPASS(shift < 32); + + flit_idx = (LAST_WORD - word) / 2; + if (word & 0x1) + shift += 32; + t1 = be64toh(tcb[flit_idx]) >> shift; + t2 = 0; + if (fls(mask) > 64 - shift) { + /* + * Will spill over into the next logical flit, which is the flit + * before this one. The flit_idx before this one must be valid. + */ + MPASS(flit_idx > 0); + t2 = be64toh(tcb[flit_idx - 1]) << (64 - shift); } + return ((t2 | t1) & mask); +#undef LAST_WORD +} +#define GET_TCB_FIELD(tcb, F) \ + get_tcb_field(tcb, W_TCB_##F, M_TCB_##F, S_TCB_##F) +/* + * Issues a CPL_GET_TCB to read the entire TCB for the tid. + */ +static int +send_get_tcb(struct adapter *sc, u_int tid) +{ + struct cpl_get_tcb *cpl; + struct wrq_cookie cookie; + + MPASS(tid < sc->tids.ntids); + + cpl = start_wrq_wr(&sc->sge.ctrlq[0], howmany(sizeof(*cpl), 16), + &cookie); + if (__predict_false(cpl == NULL)) + return (ENOMEM); + bzero(cpl, sizeof(*cpl)); + INIT_TP_WR(cpl, tid); + OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_GET_TCB, tid)); + cpl->reply_ctrl = htobe16(V_REPLY_CHAN(0) | + V_QUEUENO(sc->sge.ofld_rxq[0].iq.cntxt_id)); + cpl->cookie = 0xff; + commit_wrq_wr(&sc->sge.ctrlq[0], cpl, &cookie); + + return (0); +} + +static struct tcb_histent * +alloc_tcb_histent(struct adapter *sc, u_int tid, int flags) +{ + struct tcb_histent *te; + + MPASS(flags == M_NOWAIT || flags == M_WAITOK); + + te = malloc(sizeof(*te), M_CXGBE, M_ZERO | flags); + if (te == NULL) + return (NULL); + mtx_init(&te->te_lock, "TCB entry", NULL, MTX_DEF); + callout_init_mtx(&te->te_callout, &te->te_lock, 0); + te->te_adapter = sc; + te->te_tid = tid; + + return (te); +} + +static void +free_tcb_histent(struct tcb_histent *te) +{ + + mtx_destroy(&te->te_lock); + free(te, M_CXGBE); +} + +/* + * Start tracking the tid in the TCB history. + */ +int +add_tid_to_history(struct adapter *sc, u_int tid) +{ + struct tcb_histent *te = NULL; + struct tom_data *td = sc->tom_softc; + int rc; + + MPASS(tid < sc->tids.ntids); + + if (td->tcb_history == NULL) + return (ENXIO); + + rw_wlock(&td->tcb_history_lock); + if (td->tcb_history[tid] != NULL) { + rc = EEXIST; + goto done; + } + te = alloc_tcb_histent(sc, tid, M_NOWAIT); + if (te == NULL) { + rc = ENOMEM; + goto done; + } + mtx_lock(&te->te_lock); + rc = send_get_tcb(sc, tid); + if (rc == 0) { + te->te_flags |= TE_RPL_PENDING; + td->tcb_history[tid] = te; + } else { + free(te, M_CXGBE); + } + mtx_unlock(&te->te_lock); +done: + rw_wunlock(&td->tcb_history_lock); return (rc); } +static void +remove_tcb_histent(struct tcb_histent *te) +{ + struct adapter *sc = te->te_adapter; + struct tom_data *td = sc->tom_softc; + + rw_assert(&td->tcb_history_lock, RA_WLOCKED); + mtx_assert(&te->te_lock, MA_OWNED); + MPASS(td->tcb_history[te->te_tid] == te); + + td->tcb_history[te->te_tid] = NULL; + free_tcb_histent(te); + rw_wunlock(&td->tcb_history_lock); +} + +static inline struct tcb_histent * +lookup_tcb_histent(struct adapter *sc, u_int tid, bool addrem) +{ + struct tcb_histent *te; + struct tom_data *td = sc->tom_softc; + + MPASS(tid < sc->tids.ntids); + + if (addrem) + rw_wlock(&td->tcb_history_lock); + else + rw_rlock(&td->tcb_history_lock); + te = td->tcb_history[tid]; + if (te != NULL) { + mtx_lock(&te->te_lock); + return (te); /* with both locks held */ + } + if (addrem) + rw_wunlock(&td->tcb_history_lock); + else + rw_runlock(&td->tcb_history_lock); + + return (te); +} + +static inline void +release_tcb_histent(struct tcb_histent *te) +{ + struct adapter *sc = te->te_adapter; + struct tom_data *td = sc->tom_softc; + + mtx_assert(&te->te_lock, MA_OWNED); + mtx_unlock(&te->te_lock); + rw_assert(&td->tcb_history_lock, RA_RLOCKED); + rw_runlock(&td->tcb_history_lock); +} + +static void +request_tcb(void *arg) +{ + struct tcb_histent *te = arg; + + mtx_assert(&te->te_lock, MA_OWNED); + + /* Noone else is supposed to update the histent. */ + MPASS(!(te->te_flags & TE_RPL_PENDING)); + if (send_get_tcb(te->te_adapter, te->te_tid) == 0) + te->te_flags |= TE_RPL_PENDING; + else + callout_schedule(&te->te_callout, hz / 100); +} + +static void +update_tcb_histent(struct tcb_histent *te, const uint64_t *tcb) +{ + struct tom_data *td = te->te_adapter->tom_softc; + uint64_t tflags = get_tcb_tflags(tcb); + uint8_t sample = 0; + + if (GET_TCB_FIELD(tcb, SND_MAX_RAW) != GET_TCB_FIELD(tcb, SND_UNA_RAW)) { + if (GET_TCB_FIELD(tcb, T_RXTSHIFT) != 0) + sample |= TS_RTO; + if (GET_TCB_FIELD(tcb, T_DUPACKS) != 0) + sample |= TS_DUPACKS; + if (GET_TCB_FIELD(tcb, T_DUPACKS) >= td->dupack_threshold) + sample |= TS_FASTREXMT; + } + + if (GET_TCB_FIELD(tcb, SND_MAX_RAW) != 0) { + uint32_t snd_wnd; + + sample |= TS_SND_BACKLOGGED; /* for whatever reason. */ + + snd_wnd = GET_TCB_FIELD(tcb, RCV_ADV); + if (tflags & V_TF_RECV_SCALE(1)) + snd_wnd <<= GET_TCB_FIELD(tcb, RCV_SCALE); + if (GET_TCB_FIELD(tcb, SND_CWND) < snd_wnd) + sample |= TS_CWND_LIMITED; /* maybe due to CWND */ + } + + if (tflags & V_TF_CCTRL_ECN(1)) { + + /* + * CE marker on incoming IP hdr, echoing ECE back in the TCP + * hdr. Indicates congestion somewhere on the way from the peer + * to this node. + */ + if (tflags & V_TF_CCTRL_ECE(1)) + sample |= TS_ECN_ECE; + + /* + * ECE seen and CWR sent (or about to be sent). Might indicate + * congestion on the way to the peer. This node is reducing its + * congestion window in response. + */ + if (tflags & (V_TF_CCTRL_CWR(1) | V_TF_CCTRL_RFR(1))) + sample |= TS_ECN_CWR; + } + + te->te_sample[te->te_pidx] = sample; + if (++te->te_pidx == nitems(te->te_sample)) + te->te_pidx = 0; + memcpy(te->te_tcb, tcb, TCB_SIZE); + te->te_flags |= TE_ACTIVE; +} + +static int +do_get_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) +{ + struct adapter *sc = iq->adapter; + const struct cpl_get_tcb_rpl *cpl = mtod(m, const void *); + const uint64_t *tcb = (const uint64_t *)(const void *)(cpl + 1); + struct tcb_histent *te; + const u_int tid = GET_TID(cpl); + bool remove; + + remove = GET_TCB_FIELD(tcb, T_STATE) == TCPS_CLOSED; + te = lookup_tcb_histent(sc, tid, remove); + if (te == NULL) { + /* Not in the history. Who issued the GET_TCB for this? */ + device_printf(sc->dev, "tcb %u: flags 0x%016jx, state %u, " + "srtt %u, sscale %u, rscale %u, cookie 0x%x\n", tid, + (uintmax_t)get_tcb_tflags(tcb), GET_TCB_FIELD(tcb, T_STATE), + GET_TCB_FIELD(tcb, T_SRTT), GET_TCB_FIELD(tcb, SND_SCALE), + GET_TCB_FIELD(tcb, RCV_SCALE), cpl->cookie); + goto done; + } + + MPASS(te->te_flags & TE_RPL_PENDING); + te->te_flags &= ~TE_RPL_PENDING; + if (remove) { + remove_tcb_histent(te); + } else { + update_tcb_histent(te, tcb); + callout_reset(&te->te_callout, hz / 10, request_tcb, te); + release_tcb_histent(te); + } +done: + m_freem(m); + return (0); +} + +static void +fill_tcp_info_from_tcb(struct adapter *sc, uint64_t *tcb, struct tcp_info *ti) +{ + uint32_t v; + + ti->tcpi_state = GET_TCB_FIELD(tcb, T_STATE); + + v = GET_TCB_FIELD(tcb, T_SRTT); + ti->tcpi_rtt = tcp_ticks_to_us(sc, v); + + v = GET_TCB_FIELD(tcb, T_RTTVAR); + ti->tcpi_rttvar = tcp_ticks_to_us(sc, v); + + ti->tcpi_snd_ssthresh = GET_TCB_FIELD(tcb, SND_SSTHRESH); + ti->tcpi_snd_cwnd = GET_TCB_FIELD(tcb, SND_CWND); + ti->tcpi_rcv_nxt = GET_TCB_FIELD(tcb, RCV_NXT); + + v = GET_TCB_FIELD(tcb, TX_MAX); + ti->tcpi_snd_nxt = v - GET_TCB_FIELD(tcb, SND_NXT_RAW); + + /* Receive window being advertised by us. */ + ti->tcpi_rcv_wscale = GET_TCB_FIELD(tcb, SND_SCALE); /* Yes, SND. */ + ti->tcpi_rcv_space = GET_TCB_FIELD(tcb, RCV_WND); + + /* Send window */ + ti->tcpi_snd_wscale = GET_TCB_FIELD(tcb, RCV_SCALE); /* Yes, RCV. */ + ti->tcpi_snd_wnd = GET_TCB_FIELD(tcb, RCV_ADV); + if (get_tcb_tflags(tcb) & V_TF_RECV_SCALE(1)) + ti->tcpi_snd_wnd <<= ti->tcpi_snd_wscale; + else + ti->tcpi_snd_wscale = 0; + +} + +static void +fill_tcp_info_from_history(struct adapter *sc, struct tcb_histent *te, + struct tcp_info *ti) +{ + + fill_tcp_info_from_tcb(sc, te->te_tcb, ti); +} + /* - * Called by the kernel to allow the TOE driver to "refine" values filled up in - * the tcp_info for an offloaded connection. + * Reads the TCB for the given tid using a memory window and copies it to 'buf' + * in the same format as CPL_GET_TCB_RPL. */ static void -t4_tcp_info(struct toedev *tod, struct tcpcb *tp, struct tcp_info *ti) +read_tcb_using_memwin(struct adapter *sc, u_int tid, uint64_t *buf) { int i, j, k, rc; - struct adapter *sc = tod->tod_softc; - struct toepcb *toep = tp->t_toe; - uint32_t addr, v; - uint32_t buf[TCB_SIZE / sizeof(uint32_t)]; + uint32_t addr; u_char *tcb, tmp; - INP_WLOCK_ASSERT(tp->t_inpcb); - MPASS(ti != NULL); + MPASS(tid < sc->tids.ntids); - ti->tcpi_toe_tid = toep->tid; - - addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE) + toep->tid * TCB_SIZE; - rc = read_via_memwin(sc, 2, addr, &buf[0], TCB_SIZE); + addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE) + tid * TCB_SIZE; + rc = read_via_memwin(sc, 2, addr, (uint32_t *)buf, TCB_SIZE); if (rc != 0) return; - tcb = (u_char *)&buf[0]; + tcb = (u_char *)buf; for (i = 0, j = TCB_SIZE - 16; i < j; i += 16, j -= 16) { for (k = 0; k < 16; k++) { tmp = tcb[i + k]; @@ -442,28 +739,42 @@ t4_tcp_info(struct toedev *tod, struct tcpcb *tp, stru tcb[j + k] = tmp; } } +} - ti->tcpi_state = get_tcb_bits(tcb, 115, 112); +static void +fill_tcp_info(struct adapter *sc, u_int tid, struct tcp_info *ti) +{ + uint64_t tcb[TCB_SIZE / sizeof(uint64_t)]; + struct tcb_histent *te; - v = get_tcb_bits(tcb, 271, 256); - ti->tcpi_rtt = tcp_ticks_to_us(sc, v); + ti->tcpi_toe_tid = tid; + te = lookup_tcb_histent(sc, tid, false); + if (te != NULL) { + fill_tcp_info_from_history(sc, te, ti); + release_tcb_histent(te); + } else { + if (!(sc->debug_flags & DF_DISABLE_TCB_CACHE)) { + /* XXX: tell firmware to flush TCB cache. */ + } + read_tcb_using_memwin(sc, tid, tcb); + fill_tcp_info_from_tcb(sc, tcb, ti); + } +} - v = get_tcb_bits(tcb, 287, 272); - ti->tcpi_rttvar = tcp_ticks_to_us(sc, v); +/* + * Called by the kernel to allow the TOE driver to "refine" values filled up in + * the tcp_info for an offloaded connection. + */ +static void +t4_tcp_info(struct toedev *tod, struct tcpcb *tp, struct tcp_info *ti) +{ + struct adapter *sc = tod->tod_softc; + struct toepcb *toep = tp->t_toe; - ti->tcpi_snd_ssthresh = get_tcb_bits(tcb, 487, 460); - ti->tcpi_snd_cwnd = get_tcb_bits(tcb, 459, 432); - ti->tcpi_rcv_nxt = get_tcb_bits(tcb, 553, 522); + INP_WLOCK_ASSERT(tp->t_inpcb); + MPASS(ti != NULL); - ti->tcpi_snd_nxt = get_tcb_bits(tcb, 319, 288) - - get_tcb_bits(tcb, 375, 348); - - /* Receive window being advertised by us. */ - ti->tcpi_rcv_space = get_tcb_bits(tcb, 581, 554); - - /* Send window ceiling. */ - v = get_tcb_bits(tcb, 159, 144) << get_tcb_bits(tcb, 131, 128); - ti->tcpi_snd_wnd = min(v, ti->tcpi_snd_cwnd); + fill_tcp_info(sc, toep->tid, ti); } /* @@ -807,6 +1118,35 @@ failed: return (rc); } +static inline void +alloc_tcb_history(struct adapter *sc, struct tom_data *td) +{ + + if (sc->tids.ntids == 0 || sc->tids.ntids > 1024) + return; + rw_init(&td->tcb_history_lock, "TCB history"); + td->tcb_history = malloc(sc->tids.ntids * sizeof(*td->tcb_history), + M_CXGBE, M_ZERO | M_NOWAIT); + td->dupack_threshold = G_DUPACKTHRESH(t4_read_reg(sc, A_TP_PARA_REG0)); +} + +static inline void +free_tcb_history(struct adapter *sc, struct tom_data *td) +{ +#ifdef INVARIANTS + int i; + + if (td->tcb_history != NULL) { + for (i = 0; i < sc->tids.ntids; i++) { + MPASS(td->tcb_history[i] == NULL); + } + } +#endif + free(td->tcb_history, M_CXGBE); + if (rw_initialized(&td->tcb_history_lock)) + rw_destroy(&td->tcb_history_lock); +} + static void free_tom_data(struct adapter *sc, struct tom_data *td) { @@ -830,6 +1170,7 @@ free_tom_data(struct adapter *sc, struct tom_data *td) if (mtx_initialized(&td->toep_list_lock)) mtx_destroy(&td->toep_list_lock); + free_tcb_history(sc, td); free_tid_tabs(&sc->tids); free(td, M_CXGBE); } @@ -1097,6 +1438,8 @@ t4_tom_activate(struct adapter *sc) t4_set_reg_field(sc, A_ULP_RX_TDDP_TAGMASK, V_TDDPTAGMASK(M_TDDPTAGMASK), td->pr.pr_tag_mask); + alloc_tcb_history(sc, td); + /* toedev ops */ tod = &td->tod; init_toedev(tod); @@ -1214,6 +1557,7 @@ t4_tom_mod_load(void) struct protosw *tcp_protosw, *tcp6_protosw; /* CPL handlers */ + t4_register_cpl_handler(CPL_GET_TCB_RPL, do_get_tcb_rpl); t4_register_shared_cpl_handler(CPL_L2T_WRITE_RPL, do_l2t_write_rpl2, CPL_COOKIE_TOM); t4_init_connect_cpl_handlers(); Modified: head/sys/dev/cxgbe/tom/t4_tom.h ============================================================================== --- head/sys/dev/cxgbe/tom/t4_tom.h Mon Apr 22 17:29:26 2019 (r346569) +++ head/sys/dev/cxgbe/tom/t4_tom.h Mon Apr 22 17:48:10 2019 (r346570) @@ -33,6 +33,7 @@ #ifndef __T4_TOM_H__ #define __T4_TOM_H__ #include <sys/vmem.h> +#include "common/t4_hw.h" #include "tom/t4_tls.h" #define LISTEN_HASH_SIZE 32 @@ -254,6 +255,31 @@ struct listen_ctx { struct clip_entry *ce; }; +/* tcb_histent flags */ +#define TE_RPL_PENDING 1 +#define TE_ACTIVE 2 + +/* bits in one 8b tcb_histent sample. */ +#define TS_RTO (1 << 0) +#define TS_DUPACKS (1 << 1) +#define TS_FASTREXMT (1 << 2) +#define TS_SND_BACKLOGGED (1 << 3) +#define TS_CWND_LIMITED (1 << 4) +#define TS_ECN_ECE (1 << 5) +#define TS_ECN_CWR (1 << 6) +#define TS_RESERVED (1 << 7) /* Unused. */ + +struct tcb_histent { + struct mtx te_lock; + struct callout te_callout; + uint64_t te_tcb[TCB_SIZE / sizeof(uint64_t)]; + struct adapter *te_adapter; + u_int te_flags; + u_int te_tid; + uint8_t te_pidx; + uint8_t te_sample[100]; +}; + struct tom_data { struct toedev tod; @@ -268,6 +294,10 @@ struct tom_data { struct ppod_region pr; + struct rwlock tcb_history_lock __aligned(CACHE_LINE_SIZE); + struct tcb_histent **tcb_history; + int dupack_threshold; + /* WRs that will not be sent to the chip because L2 resolution failed */ struct mtx unsent_wr_lock; STAILQ_HEAD(, wrqe) unsent_wr_list; @@ -326,6 +356,7 @@ int select_ulp_mode(struct socket *, struct adapter *, struct offload_settings *); void set_ulp_mode(struct toepcb *, int); int negative_advice(int); +int add_tid_to_history(struct adapter *, u_int); /* t4_connect.c */ void t4_init_connect_cpl_handlers(void);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201904221748.x3MHmAxu048437>