Skip site navigation (1)Skip section navigation (2)
Date:      Tue, 03 Sep 2019 14:05:48 -0000
From:      Navdeep Parhar <np@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org
Subject:   svn commit: r345605 - stable/12/sys/dev/cxgbe/tom
Message-ID:  <201903272129.x2RLTkd1040527@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: np
Date: Wed Mar 27 21:29:45 2019
New Revision: 345605
URL: https://svnweb.freebsd.org/changeset/base/345605

Log:
  MFC r342208:
  
  cxgbe/t4_tom: fixes for issues on the passive open side.
  
  - Fix PR 227760 by getting the TOE to respond to the SYN after the call
    to toe_syncache_add, not during it.  The kernel syncache code calls
    syncache_respond just before syncache_insert.  If the ACK to the
    syncache_respond is processed in another thread it may run before the
    syncache_insert and won't find the entry.  Note that this affects only
    t4_tom because it's the only driver trying to insert and expand
    syncache entries from different threads.
  
  - Do not leak resources if an embryonic connection terminates at
    SYN_RCVD because of L2 lookup failures.
  
  - Retire lctx->synq and associated code because there is never a need to
    walk the list of embryonic connections associated with a listener.
    The per-tid state is still called a synq entry in the driver even
    though the synq itself is now gone.
  
  PR:		227760
  Sponsored by:	Chelsio Communications

Modified:
  stable/12/sys/dev/cxgbe/tom/t4_connect.c
  stable/12/sys/dev/cxgbe/tom/t4_cpl_io.c
  stable/12/sys/dev/cxgbe/tom/t4_listen.c
  stable/12/sys/dev/cxgbe/tom/t4_tom.c
  stable/12/sys/dev/cxgbe/tom/t4_tom.h
Directory Properties:
  stable/12/   (props changed)

Modified: stable/12/sys/dev/cxgbe/tom/t4_connect.c
==============================================================================
--- stable/12/sys/dev/cxgbe/tom/t4_connect.c	Wed Mar 27 21:22:59 2019	(r345604)
+++ stable/12/sys/dev/cxgbe/tom/t4_connect.c	Wed Mar 27 21:29:45 2019	(r345605)
@@ -99,7 +99,8 @@ do_act_establish(struct sge_iq *iq, const struct rss_h
 		goto done;
 	}
 
-	make_established(toep, cpl->snd_isn, cpl->rcv_isn, cpl->tcp_opt);
+	make_established(toep, be32toh(cpl->snd_isn) - 1,
+	    be32toh(cpl->rcv_isn) - 1, cpl->tcp_opt);
 
 	if (toep->ulp_mode == ULP_MODE_TLS)
 		tls_establish(toep);

Modified: stable/12/sys/dev/cxgbe/tom/t4_cpl_io.c
==============================================================================
--- stable/12/sys/dev/cxgbe/tom/t4_cpl_io.c	Wed Mar 27 21:22:59 2019	(r345604)
+++ stable/12/sys/dev/cxgbe/tom/t4_cpl_io.c	Wed Mar 27 21:29:45 2019	(r345605)
@@ -373,18 +373,15 @@ assign_rxopt(struct tcpcb *tp, unsigned int opt)
  * Completes some final bits of initialization for just established connections
  * and changes their state to TCPS_ESTABLISHED.
  *
- * The ISNs are from after the exchange of SYNs.  i.e., the true ISN + 1.
+ * The ISNs are from the exchange of SYNs.
  */
 void
-make_established(struct toepcb *toep, uint32_t snd_isn, uint32_t rcv_isn,
-    uint16_t opt)
+make_established(struct toepcb *toep, uint32_t iss, uint32_t irs, uint16_t opt)
 {
 	struct inpcb *inp = toep->inp;
 	struct socket *so = inp->inp_socket;
 	struct tcpcb *tp = intotcpcb(inp);
 	long bufsize;
-	uint32_t iss = be32toh(snd_isn) - 1;	/* true ISS */
-	uint32_t irs = be32toh(rcv_isn) - 1;	/* true IRS */
 	uint16_t tcpopt = be16toh(opt);
 	struct flowc_tx_params ftxp;
 
@@ -1245,22 +1242,12 @@ do_peer_close(struct sge_iq *iq, const struct rss_head
 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
 
 	if (__predict_false(toep->flags & TPF_SYNQE)) {
-#ifdef INVARIANTS
-		struct synq_entry *synqe = (void *)toep;
-
-		INP_WLOCK(synqe->lctx->inp);
-		if (synqe->flags & TPF_SYNQE_HAS_L2TE) {
-			KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN,
-			    ("%s: listen socket closed but tid %u not aborted.",
-			    __func__, tid));
-		} else {
-			/*
-			 * do_pass_accept_req is still running and will
-			 * eventually take care of this tid.
-			 */
-		}
-		INP_WUNLOCK(synqe->lctx->inp);
-#endif
+		/*
+		 * do_pass_establish must have run before do_peer_close and if
+		 * this is still a synqe instead of a toepcb then the connection
+		 * must be getting aborted.
+		 */
+		MPASS(toep->flags & TPF_ABORT_SHUTDOWN);
 		CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid,
 		    toep, toep->flags);
 		return (0);
@@ -1568,22 +1555,12 @@ do_rx_data(struct sge_iq *iq, const struct rss_header 
 	uint32_t ddp_placed = 0;
 
 	if (__predict_false(toep->flags & TPF_SYNQE)) {
-#ifdef INVARIANTS
-		struct synq_entry *synqe = (void *)toep;
-
-		INP_WLOCK(synqe->lctx->inp);
-		if (synqe->flags & TPF_SYNQE_HAS_L2TE) {
-			KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN,
-			    ("%s: listen socket closed but tid %u not aborted.",
-			    __func__, tid));
-		} else {
-			/*
-			 * do_pass_accept_req is still running and will
-			 * eventually take care of this tid.
-			 */
-		}
-		INP_WUNLOCK(synqe->lctx->inp);
-#endif
+		/*
+		 * do_pass_establish must have run before do_rx_data and if this
+		 * is still a synqe instead of a toepcb then the connection must
+		 * be getting aborted.
+		 */
+		MPASS(toep->flags & TPF_ABORT_SHUTDOWN);
 		CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid,
 		    toep, toep->flags);
 		m_freem(m);

Modified: stable/12/sys/dev/cxgbe/tom/t4_listen.c
==============================================================================
--- stable/12/sys/dev/cxgbe/tom/t4_listen.c	Wed Mar 27 21:22:59 2019	(r345604)
+++ stable/12/sys/dev/cxgbe/tom/t4_listen.c	Wed Mar 27 21:29:45 2019	(r345605)
@@ -87,9 +87,6 @@ static struct listen_ctx *listen_hash_find(struct adap
 static struct listen_ctx *listen_hash_del(struct adapter *, struct inpcb *);
 static struct inpcb *release_lctx(struct adapter *, struct listen_ctx *);
 
-static inline void save_qids_in_mbuf(struct mbuf *, struct vi_info *,
-    struct offload_settings *);
-static inline void get_qids_from_mbuf(struct mbuf *m, int *, int *);
 static void send_reset_synqe(struct toedev *, struct synq_entry *);
 
 static int
@@ -223,7 +220,6 @@ alloc_lctx(struct adapter *sc, struct inpcb *inp, stru
 	lctx->ctrlq = &sc->sge.ctrlq[vi->pi->port_id];
 	lctx->ofld_rxq = &sc->sge.ofld_rxq[vi->first_ofld_rxq];
 	refcount_init(&lctx->refcount, 1);
-	TAILQ_INIT(&lctx->synq);
 
 	lctx->inp = inp;
 	lctx->vnet = inp->inp_socket->so_vnet;
@@ -241,8 +237,6 @@ free_lctx(struct adapter *sc, struct listen_ctx *lctx)
 	INP_WLOCK_ASSERT(inp);
 	KASSERT(lctx->refcount == 0,
 	    ("%s: refcount %d", __func__, lctx->refcount));
-	KASSERT(TAILQ_EMPTY(&lctx->synq),
-	    ("%s: synq not empty.", __func__));
 	KASSERT(lctx->stid >= 0, ("%s: bad stid %d.", __func__, lctx->stid));
 
 	CTR4(KTR_CXGBE, "%s: stid %u, lctx %p, inp %p",
@@ -358,7 +352,7 @@ send_reset_synqe(struct toedev *tod, struct synq_entry
 	struct wrqe *wr;
 	struct fw_flowc_wr *flowc;
 	struct cpl_abort_req *req;
-	int txqid, rxqid, flowclen;
+	int flowclen;
 	struct sge_wrq *ofld_txq;
 	struct sge_ofld_rxq *ofld_rxq;
 	const int nparams = 6;
@@ -374,9 +368,8 @@ send_reset_synqe(struct toedev *tod, struct synq_entry
 		return;	/* abort already in progress */
 	synqe->flags |= TPF_ABORT_SHUTDOWN;
 
-	get_qids_from_mbuf(m, &txqid, &rxqid);
-	ofld_txq = &sc->sge.ofld_txq[txqid];
-	ofld_rxq = &sc->sge.ofld_rxq[rxqid];
+	ofld_txq = &sc->sge.ofld_txq[synqe->txqid];
+	ofld_rxq = &sc->sge.ofld_rxq[synqe->rxqid];
 
 	/* The wrqe will have two WRs - a flowc followed by an abort_req */
 	flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval);
@@ -606,7 +599,6 @@ t4_listen_stop(struct toedev *tod, struct tcpcb *tp)
 	struct listen_ctx *lctx;
 	struct adapter *sc = tod->tod_softc;
 	struct inpcb *inp = tp->t_inpcb;
-	struct synq_entry *synqe;
 
 	INP_WLOCK_ASSERT(inp);
 
@@ -622,25 +614,33 @@ t4_listen_stop(struct toedev *tod, struct tcpcb *tp)
 	 * arrive and clean up when it does.
 	 */
 	if (lctx->flags & LCTX_RPL_PENDING) {
-		KASSERT(TAILQ_EMPTY(&lctx->synq),
-		    ("%s: synq not empty.", __func__));
 		return (EINPROGRESS);
 	}
 
-	/*
-	 * The host stack will abort all the connections on the listening
-	 * socket's so_comp.  It doesn't know about the connections on the synq
-	 * so we need to take care of those.
-	 */
-	TAILQ_FOREACH(synqe, &lctx->synq, link) {
-		if (synqe->flags & TPF_SYNQE_HAS_L2TE)
-			send_reset_synqe(tod, synqe);
-	}
-
 	destroy_server(sc, lctx);
 	return (0);
 }
 
+static inline struct synq_entry *
+alloc_synqe(struct adapter *sc __unused, struct listen_ctx *lctx, int flags)
+{
+	struct synq_entry *synqe;
+
+	INP_WLOCK_ASSERT(lctx->inp);
+	MPASS(flags == M_WAITOK || flags == M_NOWAIT);
+
+	synqe = malloc(sizeof(*synqe), M_CXGBE, flags);
+	if (__predict_true(synqe != NULL)) {
+		synqe->flags = TPF_SYNQE;
+		refcount_init(&synqe->refcnt, 1);
+		synqe->lctx = lctx;
+		hold_lctx(lctx);	/* Every synqe has a ref on its lctx. */
+		synqe->syn = NULL;
+	}
+
+	return (synqe);
+}
+
 static inline void
 hold_synqe(struct synq_entry *synqe)
 {
@@ -648,17 +648,25 @@ hold_synqe(struct synq_entry *synqe)
 	refcount_acquire(&synqe->refcnt);
 }
 
-static inline void
-release_synqe(struct synq_entry *synqe)
+static inline struct inpcb *
+release_synqe(struct adapter *sc, struct synq_entry *synqe)
 {
+	struct inpcb *inp;
 
-	if (refcount_release(&synqe->refcnt)) {
-		int needfree = synqe->flags & TPF_SYNQE_NEEDFREE;
+	MPASS(synqe->flags & TPF_SYNQE);
+	MPASS(synqe->lctx != NULL);
 
+	inp = synqe->lctx->inp;
+	MPASS(inp != NULL);
+	INP_WLOCK_ASSERT(inp);
+
+	if (refcount_release(&synqe->refcnt)) {
+		inp = release_lctx(sc, synqe->lctx);
 		m_freem(synqe->syn);
-		if (needfree)
-			free(synqe, M_CXGBE);
+		free(synqe, M_CXGBE);
 	}
+
+	return (inp);
 }
 
 void
@@ -670,51 +678,45 @@ t4_syncache_added(struct toedev *tod __unused, void *a
 }
 
 void
-t4_syncache_removed(struct toedev *tod __unused, void *arg)
+t4_syncache_removed(struct toedev *tod, void *arg)
 {
+	struct adapter *sc = tod->tod_softc;
 	struct synq_entry *synqe = arg;
+	struct inpcb *inp = synqe->lctx->inp;
 
-	release_synqe(synqe);
+	/*
+	 * XXX: this is a LOR but harmless when running from the softclock.
+	 */
+	INP_WLOCK(inp);
+	inp = release_synqe(sc, synqe);
+	if (inp != NULL)
+		INP_WUNLOCK(inp);
 }
 
 int
 t4_syncache_respond(struct toedev *tod, void *arg, struct mbuf *m)
 {
-	struct adapter *sc = tod->tod_softc;
 	struct synq_entry *synqe = arg;
-	struct wrqe *wr;
-	struct l2t_entry *e;
-	struct tcpopt to;
-	struct ip *ip = mtod(m, struct ip *);
-	struct tcphdr *th;
 
-	wr = (struct wrqe *)atomic_readandclear_ptr(&synqe->wr);
-	if (wr == NULL) {
-		m_freem(m);
-		return (EALREADY);
-	}
+	if (atomic_fetchadd_int(&synqe->ok_to_respond, 1) == 0) {
+		struct tcpopt to;
+		struct ip *ip = mtod(m, struct ip *);
+		struct tcphdr *th;
 
-	if (ip->ip_v == IPVERSION)
-		th = (void *)(ip + 1);
-	else
-		th = (void *)((struct ip6_hdr *)ip + 1);
-	bzero(&to, sizeof(to));
-	tcp_dooptions(&to, (void *)(th + 1), (th->th_off << 2) - sizeof(*th),
-	    TO_SYN);
+		if (ip->ip_v == IPVERSION)
+			th = (void *)(ip + 1);
+		else
+			th = (void *)((struct ip6_hdr *)ip + 1);
+		bzero(&to, sizeof(to));
+		tcp_dooptions(&to, (void *)(th + 1),
+		    (th->th_off << 2) - sizeof(*th), TO_SYN);
 
-	/* save these for later */
-	synqe->iss = be32toh(th->th_seq);
-	synqe->ts = to.to_tsval;
-
-	if (chip_id(sc) >= CHELSIO_T5) {
-		struct cpl_t5_pass_accept_rpl *rpl5 = wrtod(wr);
-
-		rpl5->iss = th->th_seq;
+		/* save these for later */
+		synqe->iss = be32toh(th->th_seq);
+		synqe->irs = be32toh(th->th_ack) - 1;
+		synqe->ts = to.to_tsval;
 	}
 
-	e = &sc->l2t->l2tab[synqe->l2e_idx];
-	t4_l2t_send(sc, wr, e);
-
 	m_freem(m);	/* don't need this any more */
 	return (0);
 }
@@ -834,23 +836,29 @@ done_with_synqe(struct adapter *sc, struct synq_entry 
 {
 	struct listen_ctx *lctx = synqe->lctx;
 	struct inpcb *inp = lctx->inp;
-	struct vi_info *vi = synqe->syn->m_pkthdr.rcvif->if_softc;
 	struct l2t_entry *e = &sc->l2t->l2tab[synqe->l2e_idx];
 	int ntids;
 
 	INP_WLOCK_ASSERT(inp);
 	ntids = inp->inp_vflag & INP_IPV6 ? 2 : 1;
 
-	TAILQ_REMOVE(&lctx->synq, synqe, link);
-	inp = release_lctx(sc, lctx);
-	if (inp)
-		INP_WUNLOCK(inp);
 	remove_tid(sc, synqe->tid, ntids);
-	release_tid(sc, synqe->tid, &sc->sge.ctrlq[vi->pi->port_id]);
+	release_tid(sc, synqe->tid, lctx->ctrlq);
 	t4_l2t_release(e);
-	release_synqe(synqe);	/* removed from synq list */
+	inp = release_synqe(sc, synqe);
+	if (inp)
+		INP_WUNLOCK(inp);
 }
 
+void
+synack_failure_cleanup(struct adapter *sc, int tid)
+{
+	struct synq_entry *synqe = lookup_tid(sc, tid);
+
+	INP_WLOCK(synqe->lctx->inp);
+	done_with_synqe(sc, synqe);
+}
+
 int
 do_abort_req_synqe(struct sge_iq *iq, const struct rss_header *rss,
     struct mbuf *m)
@@ -861,7 +869,6 @@ do_abort_req_synqe(struct sge_iq *iq, const struct rss
 	struct synq_entry *synqe = lookup_tid(sc, tid);
 	struct listen_ctx *lctx = synqe->lctx;
 	struct inpcb *inp = lctx->inp;
-	int txqid;
 	struct sge_wrq *ofld_txq;
 #ifdef INVARIANTS
 	unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
@@ -880,8 +887,7 @@ do_abort_req_synqe(struct sge_iq *iq, const struct rss
 
 	INP_WLOCK(inp);
 
-	get_qids_from_mbuf(synqe->syn, &txqid, NULL);
-	ofld_txq = &sc->sge.ofld_txq[txqid];
+	ofld_txq = &sc->sge.ofld_txq[synqe->txqid];
 
 	/*
 	 * If we'd initiated an abort earlier the reply to it is responsible for
@@ -941,23 +947,23 @@ t4_offload_socket(struct toedev *tod, void *arg, struc
 #ifdef INVARIANTS
 	struct inpcb *inp = sotoinpcb(so);
 #endif
-	struct cpl_pass_establish *cpl = mtod(synqe->syn, void *);
-	struct toepcb *toep = *(struct toepcb **)(cpl + 1);
+	struct toepcb *toep = synqe->toep;
 
 	INP_INFO_RLOCK_ASSERT(&V_tcbinfo); /* prevents bad race with accept() */
 	INP_WLOCK_ASSERT(inp);
 	KASSERT(synqe->flags & TPF_SYNQE,
 	    ("%s: %p not a synq_entry?", __func__, arg));
+	MPASS(toep->tid == synqe->tid);
 
 	offload_socket(so, toep);
-	make_established(toep, cpl->snd_isn, cpl->rcv_isn, cpl->tcp_opt);
+	make_established(toep, synqe->iss, synqe->irs, synqe->tcp_opt);
 	toep->flags |= TPF_CPL_PENDING;
 	update_tid(sc, synqe->tid, toep);
 	synqe->flags |= TPF_SYNQE_EXPANDED;
 }
 
 static inline void
-save_qids_in_mbuf(struct mbuf *m, struct vi_info *vi,
+save_qids_in_synqe(struct synq_entry *synqe, struct vi_info *vi,
     struct offload_settings *s)
 {
 	uint32_t txqid, rxqid;
@@ -974,43 +980,10 @@ save_qids_in_mbuf(struct mbuf *m, struct vi_info *vi,
 		rxqid = arc4random() % vi->nofldrxq;
 	rxqid += vi->first_ofld_rxq;
 
-	m->m_pkthdr.flowid = (txqid << 16) | (rxqid & 0xffff);
+	synqe->txqid = txqid;
+	synqe->rxqid = rxqid;
 }
 
-static inline void
-get_qids_from_mbuf(struct mbuf *m, int *txqid, int *rxqid)
-{
-
-	if (txqid)
-		*txqid = m->m_pkthdr.flowid >> 16;
-	if (rxqid)
-		*rxqid = m->m_pkthdr.flowid & 0xffff;
-}
-
-/*
- * Use the trailing space in the mbuf in which the PASS_ACCEPT_REQ arrived to
- * store some state temporarily.
- */
-static struct synq_entry *
-mbuf_to_synqe(struct mbuf *m)
-{
-	int len = roundup2(sizeof (struct synq_entry), 8);
-	int tspace = M_TRAILINGSPACE(m);
-	struct synq_entry *synqe = NULL;
-
-	if (tspace < len) {
-		synqe = malloc(sizeof(*synqe), M_CXGBE, M_NOWAIT);
-		if (synqe == NULL)
-			return (NULL);
-		synqe->flags = TPF_SYNQE | TPF_SYNQE_NEEDFREE;
-	} else {
-		synqe = (void *)(m->m_data + m->m_len + tspace - len);
-		synqe->flags = TPF_SYNQE;
-	}
-
-	return (synqe);
-}
-
 static void
 t4opt_to_tcpopt(const struct tcp_options *t4opt, struct tcpopt *to)
 {
@@ -1210,7 +1183,39 @@ get_l2te_for_nexthop(struct port_info *pi, struct ifne
 	return (e);
 }
 
-#define REJECT_PASS_ACCEPT()	do { \
+static int
+send_synack(struct adapter *sc, struct synq_entry *synqe, uint64_t opt0,
+    uint32_t opt2, int tid)
+{
+	struct wrqe *wr;
+	struct cpl_pass_accept_rpl *rpl;
+	struct l2t_entry *e = &sc->l2t->l2tab[synqe->l2e_idx];
+
+	wr = alloc_wrqe(is_t4(sc) ? sizeof(struct cpl_pass_accept_rpl) :
+	    sizeof(struct cpl_t5_pass_accept_rpl), &sc->sge.ctrlq[0]);
+	if (wr == NULL)
+		return (ENOMEM);
+	rpl = wrtod(wr);
+
+	if (is_t4(sc))
+		INIT_TP_WR_MIT_CPL(rpl, CPL_PASS_ACCEPT_RPL, tid);
+	else {
+		struct cpl_t5_pass_accept_rpl *rpl5 = (void *)rpl;
+
+		INIT_TP_WR_MIT_CPL(rpl5, CPL_PASS_ACCEPT_RPL, tid);
+		rpl5->iss = htobe32(synqe->iss);
+	}
+	rpl->opt0 = opt0;
+	rpl->opt2 = opt2;
+
+	return (t4_l2t_send(sc, wr, e));
+}
+
+#define REJECT_PASS_ACCEPT_REQ(tunnel)	do { \
+	if (!tunnel) { \
+		m_freem(m); \
+		m = NULL; \
+	} \
 	reject_reason = __LINE__; \
 	goto reject; \
 } while (0)
@@ -1234,8 +1239,6 @@ do_pass_accept_req(struct sge_iq *iq, const struct rss
 	struct adapter *sc = iq->adapter;
 	struct toedev *tod;
 	const struct cpl_pass_accept_req *cpl = mtod(m, const void *);
-	struct cpl_pass_accept_rpl *rpl;
-	struct wrqe *wr;
 	unsigned int stid = G_PASS_OPEN_TID(be32toh(cpl->tos_stid));
 	unsigned int tid = GET_TID(cpl);
 	struct listen_ctx *lctx = lookup_stid(sc, stid);
@@ -1248,11 +1251,9 @@ do_pass_accept_req(struct sge_iq *iq, const struct rss
 	struct vi_info *vi;
 	struct ifnet *hw_ifp, *ifp;
 	struct l2t_entry *e = NULL;
-	int rscale, mtu_idx, rx_credits, rxqid, ulp_mode;
 	struct synq_entry *synqe = NULL;
 	int reject_reason, v, ntids;
-	uint16_t vid;
-	u_int wnd;
+	uint16_t vid, l2info;
 	struct epoch_tracker et;
 #ifdef INVARIANTS
 	unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
@@ -1266,36 +1267,35 @@ do_pass_accept_req(struct sge_iq *iq, const struct rss
 	CTR4(KTR_CXGBE, "%s: stid %u, tid %u, lctx %p", __func__, stid, tid,
 	    lctx);
 
-	pass_accept_req_to_protohdrs(sc, m, &inc, &th);
-	t4opt_to_tcpopt(&cpl->tcpopt, &to);
+	CURVNET_SET(lctx->vnet);	/* before any potential REJECT */
 
-	pi = sc->port[G_SYN_INTF(be16toh(cpl->l2info))];
-
-	CURVNET_SET(lctx->vnet);
-
 	/*
-	 * Use the MAC index to lookup the associated VI.  If this SYN
-	 * didn't match a perfect MAC filter, punt.
+	 * Use the MAC index to lookup the associated VI.  If this SYN didn't
+	 * match a perfect MAC filter, punt.
 	 */
-	if (!(be16toh(cpl->l2info) & F_SYN_XACT_MATCH)) {
-		m_freem(m);
-		m = NULL;
-		REJECT_PASS_ACCEPT();
+	l2info = be16toh(cpl->l2info);
+	pi = sc->port[G_SYN_INTF(l2info)];
+	if (!(l2info & F_SYN_XACT_MATCH)) {
+		REJECT_PASS_ACCEPT_REQ(false);
 	}
 	for_each_vi(pi, v, vi) {
-		if (vi->xact_addr_filt == G_SYN_MAC_IDX(be16toh(cpl->l2info)))
+		if (vi->xact_addr_filt == G_SYN_MAC_IDX(l2info))
 			goto found;
 	}
-	m_freem(m);
-	m = NULL;
-	REJECT_PASS_ACCEPT();
-
+	REJECT_PASS_ACCEPT_REQ(false);
 found:
-	hw_ifp = vi->ifp;	/* the (v)cxgbeX ifnet */
+	hw_ifp = vi->ifp;	/* the cxgbe ifnet */
 	m->m_pkthdr.rcvif = hw_ifp;
 	tod = TOEDEV(hw_ifp);
 
 	/*
+	 * Don't offload if the peer requested a TCP option that's not known to
+	 * the silicon.  Send the SYN to the kernel instead.
+	 */
+	if (__predict_false(cpl->tcpopt.unknown))
+		REJECT_PASS_ACCEPT_REQ(true);
+
+	/*
 	 * Figure out if there is a pseudo interface (vlan, lagg, etc.)
 	 * involved.  Don't offload if the SYN had a VLAN tag and the vid
 	 * doesn't match anything on this interface.
@@ -1306,75 +1306,57 @@ found:
 	if (vid != 0xfff && vid != 0) {
 		ifp = VLAN_DEVAT(hw_ifp, vid);
 		if (ifp == NULL)
-			REJECT_PASS_ACCEPT();
+			REJECT_PASS_ACCEPT_REQ(true);
 	} else
 		ifp = hw_ifp;
 
 	/*
-	 * Don't offload if the peer requested a TCP option that's not known to
-	 * the silicon.
+	 * Don't offload if the ifnet that the SYN came in on is not in the same
+	 * vnet as the listening socket.
 	 */
-	if (cpl->tcpopt.unknown)
-		REJECT_PASS_ACCEPT();
+	if (lctx->vnet != ifp->if_vnet)
+		REJECT_PASS_ACCEPT_REQ(true);
 
+	pass_accept_req_to_protohdrs(sc, m, &inc, &th);
 	if (inc.inc_flags & INC_ISIPV6) {
 
 		/* Don't offload if the ifcap isn't enabled */
 		if ((ifp->if_capenable & IFCAP_TOE6) == 0)
-			REJECT_PASS_ACCEPT();
+			REJECT_PASS_ACCEPT_REQ(true);
 
 		/*
 		 * SYN must be directed to an IP6 address on this ifnet.  This
 		 * is more restrictive than in6_localip.
 		 */
 		if (!in6_ifhasaddr(ifp, &inc.inc6_laddr))
-			REJECT_PASS_ACCEPT();
+			REJECT_PASS_ACCEPT_REQ(true);
 
 		ntids = 2;
 	} else {
 
 		/* Don't offload if the ifcap isn't enabled */
 		if ((ifp->if_capenable & IFCAP_TOE4) == 0)
-			REJECT_PASS_ACCEPT();
+			REJECT_PASS_ACCEPT_REQ(true);
 
 		/*
 		 * SYN must be directed to an IP address on this ifnet.  This
 		 * is more restrictive than in_localip.
 		 */
 		if (!in_ifhasaddr(ifp, inc.inc_laddr))
-			REJECT_PASS_ACCEPT();
+			REJECT_PASS_ACCEPT_REQ(true);
 
 		ntids = 1;
 	}
 
-	/*
-	 * Don't offload if the ifnet that the SYN came in on is not in the same
-	 * vnet as the listening socket.
-	 */
-	if (lctx->vnet != ifp->if_vnet)
-		REJECT_PASS_ACCEPT();
-
 	e = get_l2te_for_nexthop(pi, ifp, &inc);
 	if (e == NULL)
-		REJECT_PASS_ACCEPT();
+		REJECT_PASS_ACCEPT_REQ(true);
 
-	synqe = mbuf_to_synqe(m);
-	if (synqe == NULL)
-		REJECT_PASS_ACCEPT();
-
-	wr = alloc_wrqe(is_t4(sc) ? sizeof(struct cpl_pass_accept_rpl) :
-	    sizeof(struct cpl_t5_pass_accept_rpl), &sc->sge.ctrlq[pi->port_id]);
-	if (wr == NULL)
-		REJECT_PASS_ACCEPT();
-	rpl = wrtod(wr);
-
-	INP_INFO_RLOCK_ET(&V_tcbinfo, et);	/* for 4-tuple check */
-
 	/* Don't offload if the 4-tuple is already in use */
+	INP_INFO_RLOCK_ET(&V_tcbinfo, et);	/* for 4-tuple check */
 	if (toe_4tuple_check(&inc, &th, ifp) != 0) {
 		INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
-		free(wr, M_CXGBE);
-		REJECT_PASS_ACCEPT();
+		REJECT_PASS_ACCEPT_REQ(false);
 	}
 	INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
 
@@ -1383,14 +1365,8 @@ found:
 
 	/* Don't offload if the listening socket has closed */
 	if (__predict_false(inp->inp_flags & INP_DROPPED)) {
-		/*
-		 * The listening socket has closed.  The reply from the TOE to
-		 * our CPL_CLOSE_LISTSRV_REQ will ultimately release all
-		 * resources tied to this listen context.
-		 */
 		INP_WUNLOCK(inp);
-		free(wr, M_CXGBE);
-		REJECT_PASS_ACCEPT();
+		REJECT_PASS_ACCEPT_REQ(false);
 	}
 	so = inp->inp_socket;
 	rw_rlock(&sc->policy_lock);
@@ -1399,119 +1375,65 @@ found:
 	rw_runlock(&sc->policy_lock);
 	if (!settings.offload) {
 		INP_WUNLOCK(inp);
-		free(wr, M_CXGBE);
-		REJECT_PASS_ACCEPT();
+		REJECT_PASS_ACCEPT_REQ(true);	/* Rejected by COP. */
 	}
 
-	mtu_idx = find_best_mtu_idx(sc, &inc, &settings);
-	rscale = cpl->tcpopt.wsf && V_tcp_do_rfc1323 ? select_rcv_wscale() : 0;
-	/* opt0 rcv_bufsiz initially, assumes its normal meaning later */
-	wnd = max(so->sol_sbrcv_hiwat, MIN_RCV_WND);
-	wnd = min(wnd, MAX_RCV_WND);
-	rx_credits = min(wnd >> 10, M_RCV_BUFSIZ);
-
-	save_qids_in_mbuf(m, vi, &settings);
-	get_qids_from_mbuf(m, NULL, &rxqid);
-
-	if (is_t4(sc))
-		INIT_TP_WR_MIT_CPL(rpl, CPL_PASS_ACCEPT_RPL, tid);
-	else {
-		struct cpl_t5_pass_accept_rpl *rpl5 = (void *)rpl;
-
-		INIT_TP_WR_MIT_CPL(rpl5, CPL_PASS_ACCEPT_RPL, tid);
+	synqe = alloc_synqe(sc, lctx, M_NOWAIT);
+	if (synqe == NULL) {
+		INP_WUNLOCK(inp);
+		REJECT_PASS_ACCEPT_REQ(true);
 	}
-	ulp_mode = select_ulp_mode(so, sc, &settings);
-	switch (ulp_mode) {
-	case ULP_MODE_TCPDDP:
-		synqe->flags |= TPF_SYNQE_TCPDDP;
-		break;
-	case ULP_MODE_TLS:
-		synqe->flags |= TPF_SYNQE_TLS;
-		break;
-	}
-	rpl->opt0 = calc_opt0(so, vi, e, mtu_idx, rscale, rx_credits, ulp_mode,
-	    &settings);
-	rpl->opt2 = calc_opt2p(sc, pi, rxqid, &cpl->tcpopt, &th, ulp_mode,
-	    CC_ALGO(intotcpcb(inp)), &settings);
+	atomic_store_int(&synqe->ok_to_respond, 0);
 
-	synqe->tid = tid;
-	synqe->lctx = lctx;
-	synqe->syn = m;
-	m = NULL;
-	refcount_init(&synqe->refcnt, 1);	/* 1 means extra hold */
-	synqe->l2e_idx = e->idx;
-	synqe->rcv_bufsize = rx_credits;
-	atomic_store_rel_ptr(&synqe->wr, (uintptr_t)wr);
-
-	insert_tid(sc, tid, synqe, ntids);
-	TAILQ_INSERT_TAIL(&lctx->synq, synqe, link);
-	hold_synqe(synqe);	/* hold for the duration it's in the synq */
-	hold_lctx(lctx);	/* A synqe on the list has a ref on its lctx */
-
 	/*
 	 * If all goes well t4_syncache_respond will get called during
 	 * syncache_add.  Note that syncache_add releases the pcb lock.
 	 */
+	t4opt_to_tcpopt(&cpl->tcpopt, &to);
 	toe_syncache_add(&inc, &to, &th, inp, tod, synqe);
-	INP_UNLOCK_ASSERT(inp);	/* ok to assert, we have a ref on the inp */
 
-	/*
-	 * If we replied during syncache_add (synqe->wr has been consumed),
-	 * good.  Otherwise, set it to 0 so that further syncache_respond
-	 * attempts by the kernel will be ignored.
-	 */
-	if (atomic_cmpset_ptr(&synqe->wr, (uintptr_t)wr, 0)) {
+	if (atomic_load_int(&synqe->ok_to_respond) > 0) {
+		uint64_t opt0;
+		uint32_t opt2;
+		u_int wnd;
+		int rscale, mtu_idx, rx_credits;
 
-		/*
-		 * syncache may or may not have a hold on the synqe, which may
-		 * or may not be stashed in the original SYN mbuf passed to us.
-		 * Just copy it over instead of dealing with all possibilities.
-		 */
-		m = m_dup(synqe->syn, M_NOWAIT);
-		if (m)
-			m->m_pkthdr.rcvif = hw_ifp;
+		mtu_idx = find_best_mtu_idx(sc, &inc, &settings);
+		rscale = cpl->tcpopt.wsf && V_tcp_do_rfc1323 ?  select_rcv_wscale() : 0;
+		/* opt0 rcv_bufsiz initially, assumes its normal meaning later */
+		wnd = max(so->sol_sbrcv_hiwat, MIN_RCV_WND);
+		wnd = min(wnd, MAX_RCV_WND);
+		rx_credits = min(wnd >> 10, M_RCV_BUFSIZ);
 
-		remove_tid(sc, synqe->tid, ntids);
-		free(wr, M_CXGBE);
+		save_qids_in_synqe(synqe, vi, &settings);
+		synqe->ulp_mode = select_ulp_mode(so, sc, &settings);
 
-		/* Yank the synqe out of the lctx synq. */
-		INP_WLOCK(inp);
-		TAILQ_REMOVE(&lctx->synq, synqe, link);
-		release_synqe(synqe);	/* removed from synq list */
-		inp = release_lctx(sc, lctx);
-		if (inp)
-			INP_WUNLOCK(inp);
+		opt0 = calc_opt0(so, vi, e, mtu_idx, rscale, rx_credits,
+		    synqe->ulp_mode, &settings);
+		opt2 = calc_opt2p(sc, pi, synqe->rxqid, &cpl->tcpopt, &th,
+		    synqe->ulp_mode, CC_ALGO(intotcpcb(inp)), &settings);
 
-		release_synqe(synqe);	/* extra hold */
-		REJECT_PASS_ACCEPT();
-	}
+		insert_tid(sc, tid, synqe, ntids);
+		synqe->tid = tid;
+		synqe->l2e_idx = e->idx;
+		synqe->rcv_bufsize = rx_credits;
+		synqe->syn = m;
+		m = NULL;
 
-	CTR6(KTR_CXGBE, "%s: stid %u, tid %u, lctx %p, synqe %p, SYNACK mode %d",
-	    __func__, stid, tid, lctx, synqe, ulp_mode);
+		if (send_synack(sc, synqe, opt0, opt2, tid) != 0) {
+			remove_tid(sc, tid, ntids);
+			m = synqe->syn;
+			synqe->syn = NULL;
+			REJECT_PASS_ACCEPT_REQ(true);
+		}
 
-	INP_WLOCK(inp);
-	synqe->flags |= TPF_SYNQE_HAS_L2TE;
-	if (__predict_false(inp->inp_flags & INP_DROPPED)) {
-		/*
-		 * Listening socket closed but tod_listen_stop did not abort
-		 * this tid because there was no L2T entry for the tid at that
-		 * time.  Abort it now.  The reply to the abort will clean up.
-		 */
 		CTR6(KTR_CXGBE,
-		    "%s: stid %u, tid %u, lctx %p, synqe %p (0x%x), ABORT",
-		    __func__, stid, tid, lctx, synqe, synqe->flags);
-		if (!(synqe->flags & TPF_SYNQE_EXPANDED))
-			send_reset_synqe(tod, synqe);
-		INP_WUNLOCK(inp);
-		CURVNET_RESTORE();
+		    "%s: stid %u, tid %u, lctx %p, synqe %p, mode %d, SYNACK",
+		    __func__, stid, tid, lctx, synqe, synqe->ulp_mode);
+	} else
+		REJECT_PASS_ACCEPT_REQ(false);
 
-		release_synqe(synqe);	/* extra hold */
-		return (__LINE__);
-	}
-	INP_WUNLOCK(inp);
 	CURVNET_RESTORE();
-
-	release_synqe(synqe);	/* extra hold */
 	return (0);
 reject:
 	CURVNET_RESTORE();
@@ -1521,8 +1443,19 @@ reject:
 	if (e)
 		t4_l2t_release(e);
 	release_tid(sc, tid, lctx->ctrlq);
+	if (synqe) {
+		inp = synqe->lctx->inp;
+		INP_WLOCK(inp);
+		inp = release_synqe(sc, synqe);
+		if (inp)
+			INP_WUNLOCK(inp);
+	}
 
-	if (__predict_true(m != NULL)) {
+	if (m) {
+		/*
+		 * The connection request hit a TOE listener but is being passed
+		 * on to the kernel sw stack instead of getting offloaded.
+		 */
 		m_adj(m, sizeof(*cpl));
 		m->m_pkthdr.csum_flags |= (CSUM_IP_CHECKED | CSUM_IP_VALID |
 		    CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
@@ -1575,7 +1508,6 @@ do_pass_establish(struct sge_iq *iq, const struct rss_
 	struct in_conninfo inc;
 	struct toepcb *toep;
 	struct epoch_tracker et;
-	u_int txqid, rxqid;
 #ifdef INVARIANTS
 	unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
 #endif
@@ -1595,73 +1527,46 @@ do_pass_establish(struct sge_iq *iq, const struct rss_
 	    "%s: stid %u, tid %u, synqe %p (0x%x), inp_flags 0x%x",
 	    __func__, stid, tid, synqe, synqe->flags, inp->inp_flags);
 
-	if (__predict_false(inp->inp_flags & INP_DROPPED)) {
-
-		if (synqe->flags & TPF_SYNQE_HAS_L2TE) {
-			KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN,
-			    ("%s: listen socket closed but tid %u not aborted.",
-			    __func__, tid));
-		}
-
-		INP_WUNLOCK(inp);
-		INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
-		CURVNET_RESTORE();
-		return (0);
-	}
-
 	ifp = synqe->syn->m_pkthdr.rcvif;
 	vi = ifp->if_softc;
 	KASSERT(vi->pi->adapter == sc,
 	    ("%s: vi %p, sc %p mismatch", __func__, vi, sc));
 
-	get_qids_from_mbuf(synqe->syn, &txqid, &rxqid);
-	KASSERT(rxqid == iq_to_ofld_rxq(iq) - &sc->sge.ofld_rxq[0],
-	    ("%s: CPL arrived on unexpected rxq.  %d %d", __func__, rxqid,
-	    (int)(iq_to_ofld_rxq(iq) - &sc->sge.ofld_rxq[0])));
-
-	toep = alloc_toepcb(vi, txqid, rxqid, M_NOWAIT);
-	if (toep == NULL) {
+	if (__predict_false(inp->inp_flags & INP_DROPPED)) {
 reset:
-		/*
-		 * The reply to this abort will perform final cleanup.  There is
-		 * no need to check for HAS_L2TE here.  We can be here only if
-		 * we responded to the PASS_ACCEPT_REQ, and our response had the
-		 * L2T idx.
-		 */
 		send_reset_synqe(TOEDEV(ifp), synqe);
 		INP_WUNLOCK(inp);
 		INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
 		CURVNET_RESTORE();
 		return (0);
 	}
+
+	KASSERT(synqe->rxqid == iq_to_ofld_rxq(iq) - &sc->sge.ofld_rxq[0],
+	    ("%s: CPL arrived on unexpected rxq.  %d %d", __func__,
+	    synqe->rxqid, (int)(iq_to_ofld_rxq(iq) - &sc->sge.ofld_rxq[0])));
+
+	toep = alloc_toepcb(vi, synqe->txqid, synqe->rxqid, M_NOWAIT);
+	if (toep == NULL)
+		goto reset;
 	toep->tid = tid;
 	toep->l2te = &sc->l2t->l2tab[synqe->l2e_idx];
-	if (synqe->flags & TPF_SYNQE_TCPDDP)
-		set_ulp_mode(toep, ULP_MODE_TCPDDP);
-	else if (synqe->flags & TPF_SYNQE_TLS)
-		set_ulp_mode(toep, ULP_MODE_TLS);
-	else
-		set_ulp_mode(toep, ULP_MODE_NONE);
+	toep->vnet = lctx->vnet;
+	set_ulp_mode(toep, synqe->ulp_mode);
 	/* opt0 rcv_bufsiz initially, assumes its normal meaning later */
 	toep->rx_credits = synqe->rcv_bufsize;
 
-	so = inp->inp_socket;
-	KASSERT(so != NULL, ("%s: socket is NULL", __func__));
+	MPASS(be32toh(cpl->snd_isn) - 1 == synqe->iss);
+	MPASS(be32toh(cpl->rcv_isn) - 1 == synqe->irs);
+	synqe->tcp_opt = cpl->tcp_opt;
+	synqe->toep = toep;
 
 	/* Come up with something that syncache_expand should be ok with. */
 	synqe_to_protohdrs(sc, synqe, cpl, &inc, &th, &to);
+	if (inc.inc_flags & INC_ISIPV6)
+		toep->ce = t4_hold_lip(sc, &inc.inc6_laddr, lctx->ce);
+	so = inp->inp_socket;
+	KASSERT(so != NULL, ("%s: socket is NULL", __func__));
 
-	/*
-	 * No more need for anything in the mbuf that carried the
-	 * CPL_PASS_ACCEPT_REQ.  Drop the CPL_PASS_ESTABLISH and toep pointer
-	 * there.  XXX: bad form but I don't want to increase the size of synqe.
-	 */
-	m = synqe->syn;
-	KASSERT(sizeof(*cpl) + sizeof(toep) <= m->m_len,
-	    ("%s: no room in mbuf %p (m_len %d)", __func__, m, m->m_len));
-	bcopy(cpl, mtod(m, void *), sizeof(*cpl));
-	*(struct toepcb **)(mtod(m, struct cpl_pass_establish *) + 1) = toep;
-
 	if (!toe_syncache_expand(&inc, &to, &th, &so) || so == NULL) {
 		free_toepcb(toep);
 		goto reset;
@@ -1671,14 +1576,9 @@ reset:
 	new_inp = sotoinpcb(so);
 	INP_WLOCK_ASSERT(new_inp);
 	MPASS(so->so_vnet == lctx->vnet);
-	toep->vnet = lctx->vnet;
-	if (inc.inc_flags & INC_ISIPV6)
-		toep->ce = t4_hold_lip(sc, &inc.inc6_laddr, lctx->ce);
 
 	/*
-	 * This is for the unlikely case where the syncache entry that we added
-	 * has been evicted from the syncache, but the syncache_expand above
-	 * works because of syncookies.
+	 * This is for expansion from syncookies.
 	 *
 	 * XXX: we've held the tcbinfo lock throughout so there's no risk of
 	 * anyone accept'ing a connection before we've installed our hooks, but
@@ -1692,13 +1592,11 @@ reset:
 	INP_WUNLOCK(new_inp);
 
 	/* Done with the synqe */
-	TAILQ_REMOVE(&lctx->synq, synqe, link);
-	inp = release_lctx(sc, lctx);
+	inp = release_synqe(sc, synqe);
 	if (inp != NULL)
 		INP_WUNLOCK(inp);
 	INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
 	CURVNET_RESTORE();
-	release_synqe(synqe);
 
 	return (0);
 }

Modified: stable/12/sys/dev/cxgbe/tom/t4_tom.c
==============================================================================
--- stable/12/sys/dev/cxgbe/tom/t4_tom.c	Wed Mar 27 21:22:59 2019	(r345604)
+++ stable/12/sys/dev/cxgbe/tom/t4_tom.c	Wed Mar 27 21:29:45 2019	(r345605)
@@ -1020,9 +1020,9 @@ reclaim_wr_resources(void *arg, int count)
 	struct tom_data *td = arg;
 	STAILQ_HEAD(, wrqe) twr_list = STAILQ_HEAD_INITIALIZER(twr_list);
 	struct cpl_act_open_req *cpl;
-	u_int opcode, atid;
+	u_int opcode, atid, tid;
 	struct wrqe *wr;
-	struct adapter *sc;
+	struct adapter *sc = td_adapter(td);
 
 	mtx_lock(&td->unsent_wr_lock);
 	STAILQ_SWAP(&td->unsent_wr_list, &twr_list, wrqe);
@@ -1038,10 +1038,14 @@ reclaim_wr_resources(void *arg, int count)
 		case CPL_ACT_OPEN_REQ:
 		case CPL_ACT_OPEN_REQ6:
 			atid = G_TID_TID(be32toh(OPCODE_TID(cpl)));
-			sc = td_adapter(td);
-
 			CTR2(KTR_CXGBE, "%s: atid %u ", __func__, atid);
 			act_open_failure_cleanup(sc, atid, EHOSTUNREACH);
+			free(wr, M_CXGBE);
+			break;
+		case CPL_PASS_ACCEPT_RPL:
+			tid = GET_TID(cpl);
+			CTR2(KTR_CXGBE, "%s: tid %u ", __func__, tid);
+			synack_failure_cleanup(sc, tid);
 			free(wr, M_CXGBE);
 			break;

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***





Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201903272129.x2RLTkd1040527>