Date: Mon, 27 Dec 2010 00:12:57 +0000 (UTC) From: Jeff Roberson <jeff@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r216727 - projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp Message-ID: <201012270012.oBR0CvuH004401@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jeff Date: Mon Dec 27 00:12:57 2010 New Revision: 216727 URL: http://svn.freebsd.org/changeset/base/216727 Log: - Implement missing OOB support modeled after TCP. - Document and resolve some potential socket teardown races with rx. The rx path proceeds under a separate lock and ssk->socket may become NULL while it's operating. Sponsored by: Isilon Systems, iX Systems, and Panasas. Modified: projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp.h projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_rx.c Modified: projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp.h ============================================================================== --- projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp.h Sun Dec 26 23:19:16 2010 (r216726) +++ projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp.h Mon Dec 27 00:12:57 2010 (r216727) @@ -357,6 +357,7 @@ struct sdp_moderation { int moder_time; }; +/* These are flags fields. */ #define SDP_TIMEWAIT 0x0001 /* In ssk timewait state. */ #define SDP_DROPPED 0x0002 /* Socket has been dropped. */ #define SDP_SOCKREF 0x0004 /* Holding a sockref for close. */ @@ -364,8 +365,10 @@ struct sdp_moderation { #define SDP_NEEDFIN 0x0010 /* Send a fin on the next tx. */ #define SDP_DREQWAIT 0x0020 /* Waiting on DREQ. */ #define SDP_HAVEOOB 0x0040 /* Have OOB data. */ -#define SDP_HADOOB 0x0080 /* Had OOB data. */ -#define SDP_DESTROY 0x0100 /* Being destroyed. */ + +/* These are oobflags */ +#define SDP_HADOOB 0x0001 /* Had OOB data. */ +#define SDP_DESTROY 0x0002 /* Being destroyed. */ struct sdp_sock { LIST_ENTRY(sdp_sock) list; @@ -383,6 +386,7 @@ struct sdp_sock { in_port_t fport; in_addr_t faddr; int flags; + int oobflags; /* protected by rx lock. */ int state; int softerror; int recv_bytes; /* Bytes per recv. buf including header */ Modified: projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c ============================================================================== --- projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c Sun Dec 26 23:19:16 2010 (r216726) +++ projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_main.c Mon Dec 27 00:12:57 2010 (r216727) @@ -1391,6 +1391,10 @@ sdp_rcvoob(struct socket *so, struct mbu ssk = sdp_sk(so); SDP_WLOCK(ssk); + if (!rx_ring_trylock(&ssk->rx_ring)) { + SDP_WUNLOCK(ssk); + return (ECONNRESET); + } if (ssk->flags & (SDP_TIMEWAIT | SDP_DROPPED)) { error = ECONNRESET; goto out; @@ -1398,20 +1402,20 @@ sdp_rcvoob(struct socket *so, struct mbu if ((so->so_oobmark == 0 && (so->so_rcv.sb_state & SBS_RCVATMARK) == 0) || so->so_options & SO_OOBINLINE || - ssk->flags & SDP_HADOOB) { + ssk->oobflags & SDP_HADOOB) { error = EINVAL; goto out; } - if ((ssk->flags & SDP_HAVEOOB) == 0) { + if ((ssk->oobflags & SDP_HAVEOOB) == 0) { error = EWOULDBLOCK; goto out; } m->m_len = 1; *mtod(m, caddr_t) = ssk->iobc; if ((flags & MSG_PEEK) == 0) - ssk->flags ^= (SDP_HAVEOOB | SDP_HADOOB); - + ssk->oobflags ^= (SDP_HAVEOOB | SDP_HADOOB); out: + rx_ring_unlock(&ssk->rx_ring); SDP_WUNLOCK(ssk); return (error); } @@ -1425,6 +1429,28 @@ sdp_sock_init(void *mem, int size, int f return (0); } +void +sdp_urg(struct sdp_sock *ssk, struct mbuf *mb) +{ + struct mbuf *m; + struct socket *so; + + so = ssk->socket; + if (so == NULL) + return; + + so->so_oobmark = so->so_rcv.sb_cc + mb->m_pkthdr.len - 1; + sohasoutofband(so); + ssk->oobflags &= ~(SDP_HAVEOOB | SDP_HADOOB); + if (!(so->so_options & SO_OOBINLINE)) { + for (m = mb; m->m_next != NULL; m = m->m_next); + ssk->iobc = *(mtod(m, char *) + m->m_len - 1); + ssk->oobflags |= SDP_HAVEOOB; + m->m_len--; + mb->m_pkthdr.len--; + } +} + /* * Notify a sdp socket of an asynchronous error. * Modified: projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_rx.c ============================================================================== --- projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_rx.c Sun Dec 26 23:19:16 2010 (r216726) +++ projects/ofed/head/sys/ofed/drivers/infiniband/ulp/sdp/sdp_rx.c Mon Dec 27 00:12:57 2010 (r216727) @@ -166,7 +166,7 @@ sdp_post_recvs_needed(struct sdp_sock *s int buffer_size = ssk->recv_bytes; unsigned long max_bytes; - if (!ssk->qp_active) + if (!ssk->qp_active || !ssk->socket) return 0; max_bytes = ssk->socket->so_snd.sb_mbmax * scale; @@ -243,8 +243,8 @@ sdp_sock_queue_rcv_mb(struct socket *sk, m_adj(mb, SDP_HEAD_SIZE); SOCKBUF_LOCK(&sk->so_rcv); -/* if (unlikely(h->flags & SDP_OOB_PRES)) - sdp_urg(ssk, mb); XXX */ + if (unlikely(h->flags & SDP_OOB_PRES)) + sdp_urg(ssk, mb); sbappend_locked(&sk->so_rcv, mb); sorwakeup_locked(sk); return mb; @@ -327,9 +327,10 @@ static int sdp_process_rx_ctl_mb(struct sdp_sock *ssk, struct mbuf *mb) { struct sdp_bsdh *h; - struct socket *sk = ssk->socket; + struct socket *sk; SDP_WLOCK_ASSERT(ssk); + sk = ssk->socket; h = mtod(mb, struct sdp_bsdh *); switch (h->mid) { case SDP_MID_DATA: @@ -404,12 +405,23 @@ sdp_process_rx_ctl_mb(struct sdp_sock *s static int sdp_process_rx_mb(struct sdp_sock *ssk, struct mbuf *mb) { - struct socket *sk = ssk->socket; + struct socket *sk; struct sdp_bsdh *h; unsigned long mseq_ack; int credits_before; h = mtod(mb, struct sdp_bsdh *); + sk = ssk->socket; + /* + * If another thread is in so_pcbfree this may be partially torn + * down but no further synchronization is required as the destroying + * thread will wait for receive to shutdown before discarding the + * socket. + */ + if (sk == NULL) { + m_freem(mb); + return 0; + } SDPSTATS_HIST_LINEAR(credits_before_update, tx_credits(ssk)); @@ -481,7 +493,7 @@ sdp_process_rx_wc(struct sdp_sock *ssk, return NULL; if (unlikely(wc->status)) { - if (ssk->qp_active) { + if (ssk->qp_active && sk) { sdp_dbg(sk, "Recv completion with error. " "Status %d, vendor: %d\n", wc->status, wc->vendor_err); @@ -524,7 +536,7 @@ sdp_bzcopy_write_space(struct sdp_sock * { struct socket *sk = ssk->socket; - if (tx_credits(ssk) >= ssk->min_bufs) + if (tx_credits(ssk) >= ssk->min_bufs && sk) sowwakeup(sk); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201012270012.oBR0CvuH004401>