Date: Wed, 20 Feb 2008 22:11:57 GMT From: Steve Wise <swise@FreeBSD.org> To: Perforce Change Reviews <perforce@FreeBSD.org> Subject: PERFORCE change 135841 for review Message-ID: <200802202211.m1KMBvtN008006@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=135841 Change 135841 by swise@swise:vic10:iwarp on 2008/02/20 22:11:49 Move socket creation into the rdma core: - rdma-cma owns socket creation and destruction for users of the rdma-cma API. If an app wants toe create its own socket and do streaming IO before migrating to rdma mode, then it must use the IWCM API directly. (this is TBD) - rdma-cma binds socket when appropriate via sobind() to unify the TCP port space. - iwcm interface changed to allow passing in the socket ptr when the iw_cm_id is created. - chelsio rdma driver changed to take the socket from the iw_cm_id and use it for rdma connection setup. Affected files ... .. //depot/projects/iwarp/sys/contrib/rdma/iw_cm.h#2 edit .. //depot/projects/iwarp/sys/contrib/rdma/rdma_cma.c#10 edit .. //depot/projects/iwarp/sys/contrib/rdma/rdma_iwcm.c#8 edit .. //depot/projects/iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c#5 edit Differences ... ==== //depot/projects/iwarp/sys/contrib/rdma/iw_cm.h#2 (text+ko) ==== @@ -35,6 +35,9 @@ #include <contrib/rdma/ib_cm.h> +#include <sys/socket.h> +#include <sys/socketvar.h> + struct iw_cm_id; enum iw_cm_event_type { @@ -62,6 +65,7 @@ void *private_data; u8 private_data_len; void* provider_data; + struct socket *so; }; /** @@ -97,6 +101,7 @@ /* Used by provider to add and remove refs on IW cm_id */ void (*add_ref)(struct iw_cm_id *); void (*rem_ref)(struct iw_cm_id *); + struct socket *so; }; struct iw_cm_conn_param { @@ -134,11 +139,13 @@ * iw_create_cm_id - Create an IW CM identifier. * * @device: The IB device on which to create the IW CM identier. + * @so: The socket to be used for establishing the rdma connection. * @event_handler: User callback invoked to report events associated with the * returned IW CM identifier. * @context: User specified context associated with the id. */ struct iw_cm_id *iw_create_cm_id(struct ib_device *device, + struct socket *so, iw_cm_handler cm_handler, void *context); /** ==== //depot/projects/iwarp/sys/contrib/rdma/rdma_cma.c#10 (text+ko) ==== @@ -125,6 +125,7 @@ struct rdma_cm_id id; struct rdma_bind_list *bind_list; + struct socket *so; TAILQ_ENTRY(rdma_id_private) node; LIST_ENTRY(rdma_id_private) list; /* listen_any_list or cma_dev.list */ LIST_HEAD(, rdma_id_private) listen_list; /* per-device listens */ @@ -408,7 +409,7 @@ id_priv = malloc(sizeof *id_priv, M_DEVBUF, M_WAITOK); if (!id_priv) - return ERR_PTR(ENOMEM); + return ERR_PTR(-ENOMEM); bzero(id_priv, sizeof *id_priv); id_priv->state = CMA_IDLE; @@ -841,6 +842,8 @@ free(bind_list, M_DEVBUF); } mtx_unlock(&lock); + if (id_priv->so) + soclose(id_priv->so); } #ifdef IB_SUPPORTED @@ -1362,6 +1365,7 @@ *sin = iw_event->local_addr; sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr; *sin = iw_event->remote_addr; + conn_id->so = cm_id->so; memset(&event, 0, sizeof event); event.event = RDMA_CM_EVENT_CONNECT_REQUEST; @@ -1417,9 +1421,8 @@ int ret; struct sockaddr_in *sin; - id_priv->cm_id.iw = iw_create_cm_id(id_priv->id.device, - iw_conn_req_handler, - id_priv); + id_priv->cm_id.iw = iw_create_cm_id(id_priv->id.device, id_priv->so, + iw_conn_req_handler, id_priv); if (IS_ERR(id_priv->cm_id.iw)) return PTR_ERR(id_priv->cm_id.iw); @@ -1462,6 +1465,7 @@ dev_id_priv->state = CMA_ADDR_BOUND; memcpy(&id->route.addr.src_addr, &id_priv->id.route.addr.src_addr, ip_addr_size(&id_priv->id.route.addr.src_addr)); + dev_id_priv->so = id_priv->so; /* XXX */ cma_attach_to_dev(dev_id_priv, cma_dev); LIST_INSERT_HEAD(&id_priv->listen_list, dev_id_priv, listen_entry); @@ -1492,6 +1496,7 @@ memset(&addr_in, 0, sizeof addr_in); addr_in.sin_family = af; + addr_in.sin_len = sizeof addr_in; return rdma_bind_addr(id, (struct sockaddr *) &addr_in); } @@ -2023,7 +2028,7 @@ if (cma_any_addr(&cur_id->id.route.addr.src_addr)) return (EADDRNOTAVAIL); - cur_sin = (struct sockaddr_in *) &cur_id->id.route.addr.src_addr; + cur_sin = (struct sockaddr_in *)&cur_id->id.route.addr.src_addr; if (sin->sin_addr.s_addr == cur_sin->sin_addr.s_addr) return (EADDRINUSE); } @@ -2032,6 +2037,31 @@ return 0; } +static int cma_get_tcp_port(struct rdma_id_private *id_priv) +{ + int ret; + struct socket *so; + + ret = socreate(AF_INET, &so, SOCK_STREAM, IPPROTO_TCP, + curthread->td_ucred, curthread); + if (ret) { + printf("%s socreate err %d\n", __FUNCTION__, ret); + return ret; + } + + ret = sobind(so, (struct sockaddr *)&id_priv->id.route.addr.src_addr, + curthread); + KASSERT(ret == 0, ("sobind failed err %d", ret)); + if (ret) { + printf("%s sobind err %d\n", __FUNCTION__, ret); + soclose(so); + return ret; + } + printf("%s created/bound id_priv %p socket %p\n", __FUNCTION__, id_priv, so); + id_priv->so = so; + return 0; +} + static int cma_get_port(struct rdma_id_private *id_priv) { struct idr *ps; @@ -2043,6 +2073,9 @@ break; case RDMA_PS_TCP: ps = &tcp_ps; + ret = cma_get_tcp_port(id_priv); /* Synch with native stack */ + if (ret) + return ret; break; case RDMA_PS_UDP: ps = &udp_ps; @@ -2315,7 +2348,8 @@ int ret; struct iw_cm_conn_param iw_param; - cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv); + cm_id = iw_create_cm_id(id_priv->id.device, id_priv->so, + cma_iw_handler, id_priv); if (IS_ERR(cm_id)) { ret = PTR_ERR(cm_id); goto out; ==== //depot/projects/iwarp/sys/contrib/rdma/rdma_iwcm.c#8 (text+ko) ==== @@ -44,6 +44,7 @@ #include <sys/kernel.h> #include <sys/libkern.h> #include <sys/socket.h> +#include <sys/socketvar.h> #include <sys/module.h> #include <sys/linux_compat.h> #include <sys/lock.h> @@ -230,11 +231,13 @@ static int cm_event_handler(struct iw_cm_id *cm_id, struct iw_cm_event *event); struct iw_cm_id *iw_create_cm_id(struct ib_device *device, + struct socket *so, iw_cm_handler cm_handler, void *context) { struct iwcm_id_private *cm_id_priv; + KASSERT(so, ("iw_create_cm_id called with NULL socket!")); cm_id_priv = malloc(sizeof(*cm_id_priv), M_DEVBUF, M_WAITOK); if (!cm_id_priv) return ERR_PTR(ENOMEM); @@ -247,6 +250,7 @@ cm_id_priv->id.event_handler = cm_event_handler; cm_id_priv->id.add_ref = add_ref; cm_id_priv->id.rem_ref = rem_ref; + cm_id_priv->id.so = so; mtx_init(&cm_id_priv->lock, "cm_id_priv", NULL, MTX_DUPOK|MTX_SPIN); atomic_set(&cm_id_priv->refcount, 1); TAILQ_INIT(&cm_id_priv->work_list); @@ -667,6 +671,7 @@ mtx_unlock_spin(&listen_id_priv->lock); cm_id = iw_create_cm_id(listen_id_priv->id.device, + iw_event->so, listen_id_priv->id.cm_handler, listen_id_priv->id.context); /* If the cm_id could not be created, ignore the request */ ==== //depot/projects/iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c#5 (text+ko) ==== @@ -345,7 +345,7 @@ epc->so->so_upcallarg = NULL; epc->so->so_rcv.sb_flags &= ~SB_UPCALL; SOCK_UNLOCK(epc->so); - soclose(epc->so); + soshutdown(epc->so, SHUT_WR|SHUT_RD); epc->so = NULL; } @@ -589,6 +589,7 @@ event.private_data_len = ep->plen; event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); event.provider_data = ep; + event.so = ep->com.so; if (state_read(&ep->parent_ep->com) != DEAD) ep->parent_ep->com.cm_id->event_handler( ep->parent_ep->com.cm_id, @@ -1046,6 +1047,7 @@ } if (state != ABORTING) { + close_socket(&ep->com); state_set(&ep->com, DEAD); put_ep(&ep->com); } @@ -1284,23 +1286,16 @@ return err; } -static int create_sock(struct iwch_ep_common *epc) +static int init_sock(struct iwch_ep_common *epc) { int err; struct sockopt sopt; int on=1; - err = socreate(AF_INET, &epc->so, SOCK_STREAM, IPPROTO_TCP, - epc->thread->td_ucred, epc->thread); - if (err) - return err; epc->so->so_upcall = iwch_so_upcall; epc->so->so_upcallarg = epc; epc->so->so_rcv.sb_flags |= SB_UPCALL; epc->so->so_state |= SS_NBIO; - - PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); - sopt.sopt_dir = SOPT_SET; sopt.sopt_level = SOL_SOCKET; sopt.sopt_name = SO_NO_DDP; @@ -1345,7 +1340,8 @@ PDBG("%s qpn 0x%x qp %p cm_id %p\n", __FUNCTION__, conn_param->qpn, ep->com.qp, cm_id); - err = create_sock(&ep->com); + ep->com.so = cm_id->so; + err = init_sock(&ep->com); if (err) goto fail2; @@ -1378,14 +1374,6 @@ state_set(&ep->com, CONNECTING); ep->com.local_addr = cm_id->local_addr; ep->com.remote_addr = cm_id->remote_addr; - - if (!in_nullhost(ep->com.local_addr.sin_addr)) { - err = sobind(ep->com.so, (struct sockaddr *)&ep->com.local_addr, - ep->com.thread); - if (err) - goto fail3; - } - err = soconnect(ep->com.so, (struct sockaddr *)&ep->com.remote_addr, ep->com.thread); if (!err) @@ -1418,25 +1406,18 @@ ep->com.thread = curthread; state_set(&ep->com, LISTEN); - err = create_sock(&ep->com); + ep->com.so = cm_id->so; + err = init_sock(&ep->com); if (err) - goto fail1; + goto fail; - err = sobind(ep->com.so, (struct sockaddr *)&ep->com.local_addr, - ep->com.thread); - if (err) { - printf("sobind failed with errno %d\n", err); - goto fail2; - } - err = solisten(ep->com.so, ep->backlog, ep->com.thread); if (!err) { cm_id->provider_data = ep; goto out; } -fail2: close_socket(&ep->com); -fail1: +fail: cm_id->rem_ref(cm_id); put_ep(&ep->com); out:
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200802202211.m1KMBvtN008006>