Date: Wed, 28 Nov 2007 14:58:36 GMT From: Steve Wise <swise@FreeBSD.org> To: Perforce Change Reviews <perforce@FreeBSD.org> Subject: PERFORCE change 129705 for review Message-ID: <200711281458.lASEwaDa001978@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=129705 Change 129705 by swise@swise:vic10:iwarp on 2007/11/28 14:57:39 krping fixes. - printk/debug -> uprintf - msleep_spin/wakeup Affected files ... .. //depot/projects/iwarp/sys/contrib/rdma/krping/krping.c#2 edit .. //depot/projects/iwarp/sys/contrib/rdma/krping/krping.h#2 edit .. //depot/projects/iwarp/sys/contrib/rdma/krping/krping_dev.c#2 edit Differences ... ==== //depot/projects/iwarp/sys/contrib/rdma/krping/krping.c#2 (text+ko) ==== @@ -41,6 +41,8 @@ #include <sys/module.h> #include <sys/endian.h> #include <sys/limits.h> +#include <sys/proc.h> +#include <sys/signalvar.h> #include <sys/linux_compat.h> #include <sys/lock.h> @@ -57,7 +59,7 @@ #define PFX "krping: " static int debug = 1; -#define DEBUG_LOG if (debug) printf +#define DEBUG_LOG if (debug) uprintf MODULE_AUTHOR("Steve Wise"); MODULE_DESCRIPTION("RDMA ping server"); @@ -206,10 +208,15 @@ static void krping_wait(struct krping_cb *cb, int state) { - mtx_lock(&cb->lock); - if (cb->state < state) - cv_wait(&cb->sem, &cb->lock); - mtx_unlock(&cb->lock); + mtx_lock_spin(&cb->lock); + while (cb->state < state) { + msleep_spin(cb, &cb->lock, "krping", hz); + if (SIGPENDING(curthread)) { + DEBUG_LOG(PFX "interrupt!\n"); + break; + } + } + mtx_unlock_spin(&cb->lock); } static int krping_cma_event_handler(struct rdma_cm_id *cma_id, @@ -221,35 +228,35 @@ DEBUG_LOG("cma_event type %d cma_id %p (%s)\n", event->event, cma_id, (cma_id == cb->cm_id) ? "parent" : "child"); - mtx_lock(&cb->lock); + mtx_lock_spin(&cb->lock); switch (event->event) { case RDMA_CM_EVENT_ADDR_RESOLVED: cb->state = ADDR_RESOLVED; ret = rdma_resolve_route(cma_id, 2000); if (ret) { - printf(PFX "rdma_resolve_route error %d\n", + uprintf(PFX "rdma_resolve_route error %d\n", ret); - cv_signal(&cb->sem); + wakeup(cb); } break; case RDMA_CM_EVENT_ROUTE_RESOLVED: cb->state = ROUTE_RESOLVED; - cv_signal(&cb->sem); + wakeup(cb); break; case RDMA_CM_EVENT_CONNECT_REQUEST: cb->state = CONNECT_REQUEST; cb->child_cm_id = cma_id; DEBUG_LOG("child cma %p\n", cb->child_cm_id); - cv_signal(&cb->sem); + wakeup(cb); break; case RDMA_CM_EVENT_ESTABLISHED: DEBUG_LOG("ESTABLISHED\n"); if (!cb->server) { cb->state = CONNECTED; - cv_signal(&cb->sem); + wakeup(cb); } break; @@ -258,35 +265,35 @@ case RDMA_CM_EVENT_CONNECT_ERROR: case RDMA_CM_EVENT_UNREACHABLE: case RDMA_CM_EVENT_REJECTED: - printf(PFX "cma event %d, error %d\n", event->event, + uprintf(PFX "cma event %d, error %d\n", event->event, event->status); cb->state = ERROR; - cv_signal(&cb->sem); + wakeup(cb); break; case RDMA_CM_EVENT_DISCONNECTED: - printf(PFX "DISCONNECT EVENT...\n"); + DEBUG_LOG(PFX "DISCONNECT EVENT...\n"); cb->state = ERROR; - cv_signal(&cb->sem); + wakeup(cb); break; case RDMA_CM_EVENT_DEVICE_REMOVAL: - printf(PFX "cma detected device removal!!!!\n"); + DEBUG_LOG(PFX "cma detected device removal!!!!\n"); break; default: - printf(PFX "oof bad type!\n"); - cv_signal(&cb->sem); + uprintf(PFX "oof bad type!\n"); + wakeup(cb); break; } - mtx_unlock(&cb->lock); + mtx_unlock_spin(&cb->lock); return 0; } static int server_recv(struct krping_cb *cb, struct ib_wc *wc) { if (wc->byte_len != sizeof(cb->recv_buf)) { - printf(PFX "Received bogus data, size %d\n", + uprintf(PFX "Received bogus data, size %d\n", wc->byte_len); return -1; } @@ -298,12 +305,12 @@ cb->remote_rkey, (unsigned long long)cb->remote_addr, cb->remote_len); - mtx_lock(&cb->lock); + mtx_lock_spin(&cb->lock); if (cb->state <= CONNECTED || cb->state == RDMA_WRITE_COMPLETE) cb->state = RDMA_READ_ADV; else cb->state = RDMA_WRITE_ADV; - mtx_unlock(&cb->lock); + mtx_unlock_spin(&cb->lock); return 0; } @@ -311,17 +318,17 @@ static int client_recv(struct krping_cb *cb, struct ib_wc *wc) { if (wc->byte_len != sizeof(cb->recv_buf)) { - printf(PFX "Received bogus data, size %d\n", + uprintf(PFX "Received bogus data, size %d\n", wc->byte_len); return -1; } - mtx_lock(&cb->lock); + mtx_lock_spin(&cb->lock); if (cb->state == RDMA_READ_ADV) cb->state = RDMA_WRITE_ADV; else cb->state = RDMA_WRITE_COMPLETE; - mtx_unlock(&cb->lock); + mtx_unlock_spin(&cb->lock); return 0; } @@ -333,17 +340,17 @@ struct ib_recv_wr *bad_wr; int ret; - mtx_lock(&cb->lock); + mtx_lock_spin(&cb->lock); BUG_ON(cb->cq != cq); if (cb->state == ERROR) { - printf(PFX "cq completion in ERROR state\n"); - mtx_unlock(&cb->lock); + uprintf(PFX "cq completion in ERROR state\n"); + mtx_unlock_spin(&cb->lock); return; } ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); while ((ret = ib_poll_cq(cb->cq, 1, &wc)) == 1) { if (wc.status) { - printf(PFX "cq completion failed status %d\n", + uprintf(PFX "cq completion failed status %d\n", wc.status); goto error; } @@ -360,7 +367,7 @@ cb->stats.write_bytes += cb->rdma_sq_wr.sg_list->length; cb->stats.write_msgs++; cb->state = RDMA_WRITE_COMPLETE; - cv_signal(&cb->sem); + wakeup(cb); break; case IB_WC_RDMA_READ: @@ -368,7 +375,7 @@ cb->stats.read_bytes += cb->rdma_sq_wr.sg_list->length; cb->stats.read_msgs++; cb->state = RDMA_READ_COMPLETE; - cv_signal(&cb->sem); + wakeup(cb); break; case IB_WC_RECV: @@ -378,17 +385,17 @@ ret = cb->server ? server_recv(cb, &wc) : client_recv(cb, &wc); if (ret) { - printf(PFX "recv wc error: %d\n", ret); + uprintf(PFX "recv wc error: %d\n", ret); goto error; } ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); if (ret) { - printf(PFX "post recv error: %d\n", + uprintf(PFX "post recv error: %d\n", ret); goto error; } - cv_signal(&cb->sem); + wakeup(cb); break; default: @@ -397,15 +404,15 @@ } } if (ret) { - printf(PFX "poll error %d\n", ret); + uprintf(PFX "poll error %d\n", ret); goto error; } - mtx_unlock(&cb->lock); + mtx_unlock_spin(&cb->lock); return; error: cb->state = ERROR; - cv_signal(&cb->sem); - mtx_unlock(&cb->lock); + wakeup(cb); + mtx_unlock_spin(&cb->lock); } static int krping_accept(struct krping_cb *cb) @@ -421,13 +428,13 @@ ret = rdma_accept(cb->child_cm_id, &conn_param); if (ret) { - printf(PFX "rdma_accept error: %d\n", ret); + uprintf(PFX "rdma_accept error: %d\n", ret); return ret; } krping_wait(cb, CONNECTED); if (cb->state == ERROR) { - printf(PFX "wait for CONNECTED state %d\n", cb->state); + uprintf(PFX "wait for CONNECTED state %d\n", cb->state); return -1; } return 0; @@ -473,7 +480,7 @@ IB_ACCESS_REMOTE_READ| IB_ACCESS_REMOTE_WRITE); if (IS_ERR(cb->dma_mr)) { - printf(PFX "recv_buf reg_mr failed\n"); + uprintf(PFX "recv_buf reg_mr failed\n"); return PTR_ERR(cb->dma_mr); } @@ -481,7 +488,7 @@ 4096, 1024*1024); if (!cb->rdma_buf) { - printf(PFX "rdma_buf malloc failed\n"); + uprintf(PFX "rdma_buf malloc failed\n"); ret = -ENOMEM; goto err1; } @@ -490,7 +497,7 @@ cb->start_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, 0, (1L<<22), 4096, 1024*1024); if (!cb->start_buf) { - printf(PFX "start_buf malloc failed\n"); + uprintf(PFX "start_buf malloc failed\n"); ret = -ENOMEM; goto err2; } @@ -570,7 +577,7 @@ int ret; cb->pd = ib_alloc_pd(cm_id->device); if (IS_ERR(cb->pd)) { - printf(PFX "ib_alloc_pd failed\n"); + uprintf(PFX "ib_alloc_pd failed\n"); return PTR_ERR(cb->pd); } DEBUG_LOG("created pd %p\n", cb->pd); @@ -578,7 +585,7 @@ cb->cq = ib_create_cq(cm_id->device, krping_cq_event_handler, NULL, cb, RPING_SQ_DEPTH * 2, 0); if (IS_ERR(cb->cq)) { - printf(PFX "ib_create_cq failed\n"); + uprintf(PFX "ib_create_cq failed\n"); ret = PTR_ERR(cb->cq); goto err1; } @@ -586,13 +593,13 @@ ret = ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); if (ret) { - printf(PFX "ib_create_cq failed\n"); + uprintf(PFX "ib_create_cq failed\n"); goto err2; } ret = krping_create_qp(cb); if (ret) { - printf(PFX "krping_create_qp failed: %d\n", ret); + uprintf(PFX "krping_create_qp failed: %d\n", ret); goto err2; } DEBUG_LOG("created qp %p\n", cb->qp); @@ -626,7 +633,7 @@ /* Wait for client's Start STAG/TO/Len */ krping_wait(cb, RDMA_READ_ADV); if (cb->state != RDMA_READ_ADV) { - printf(PFX "wait for RDMA_READ_ADV state %d\n", + uprintf(PFX "wait for RDMA_READ_ADV state %d\n", cb->state); break; } @@ -641,7 +648,7 @@ ret = ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr); if (ret) { - printf(PFX "post send error %d\n", ret); + uprintf(PFX "post send error %d\n", ret); break; } DEBUG_LOG("server posted rdma read req \n"); @@ -649,7 +656,7 @@ /* Wait for read completion */ krping_wait(cb, RDMA_READ_COMPLETE); if (cb->state != RDMA_READ_COMPLETE) { - printf(PFX + uprintf(PFX "wait for RDMA_READ_COMPLETE state %d\n", cb->state); break; @@ -658,12 +665,12 @@ /* Display data in recv buf */ if (cb->verbose) - printf("server ping data: %s\n", cb->rdma_buf); + uprintf("server ping data: %s\n", cb->rdma_buf); /* Tell client to continue */ ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); if (ret) { - printf(PFX "post send error %d\n", ret); + uprintf(PFX "post send error %d\n", ret); break; } DEBUG_LOG("server posted go ahead\n"); @@ -671,7 +678,7 @@ /* Wait for client's RDMA STAG/TO/Len */ krping_wait(cb, RDMA_WRITE_ADV); if (cb->state != RDMA_WRITE_ADV) { - printf(PFX + uprintf(PFX "wait for RDMA_WRITE_ADV state %d\n", cb->state); break; @@ -690,14 +697,14 @@ ret = ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr); if (ret) { - printf(PFX "post send error %d\n", ret); + uprintf(PFX "post send error %d\n", ret); break; } /* Wait for completion */ krping_wait(cb, RDMA_WRITE_COMPLETE); if (cb->state != RDMA_WRITE_COMPLETE) { - printf(PFX + uprintf(PFX "wait for RDMA_WRITE_COMPLETE state %d\n", cb->state); break; @@ -709,7 +716,7 @@ /* Tell client to begin again */ ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); if (ret) { - printf(PFX "post send error %d\n", ret); + uprintf(PFX "post send error %d\n", ret); break; } DEBUG_LOG("server posted go ahead\n"); @@ -728,7 +735,7 @@ ret = rdma_bind_addr(cb->cm_id, (struct sockaddr *) &sin); if (ret) { - printf(PFX "rdma_bind_addr error %d\n", ret); + uprintf(PFX "rdma_bind_addr error %d\n", ret); return ret; } DEBUG_LOG("rdma_bind_addr successful\n"); @@ -736,13 +743,13 @@ DEBUG_LOG("rdma_listen\n"); ret = rdma_listen(cb->cm_id, 3); if (ret) { - printf(PFX "rdma_listen failed: %d\n", ret); + uprintf(PFX "rdma_listen failed: %d\n", ret); return ret; } krping_wait(cb, CONNECT_REQUEST); if (cb->state != CONNECT_REQUEST) { - printf(PFX "wait for CONNECT_REQUEST state %d\n", + uprintf(PFX "wait for CONNECT_REQUEST state %d\n", cb->state); return -1; } @@ -761,25 +768,25 @@ ret = krping_setup_qp(cb, cb->child_cm_id); if (ret) { - printf(PFX "setup_qp failed: %d\n", ret); + uprintf(PFX "setup_qp failed: %d\n", ret); return; } ret = krping_setup_buffers(cb); if (ret) { - printf(PFX "krping_setup_buffers failed: %d\n", ret); + uprintf(PFX "krping_setup_buffers failed: %d\n", ret); goto err1; } ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); if (ret) { - printf(PFX "ib_post_recv failed: %d\n", ret); + uprintf(PFX "ib_post_recv failed: %d\n", ret); goto err2; } ret = krping_accept(cb); if (ret) { - printf(PFX "connect error %d\n", ret); + uprintf(PFX "connect error %d\n", ret); goto err2; } @@ -818,14 +825,14 @@ krping_format_send(cb, cb->start_addr, cb->dma_mr); ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); if (ret) { - printf(PFX "post send error %d\n", ret); + uprintf(PFX "post send error %d\n", ret); break; } /* Wait for server to ACK */ krping_wait(cb, RDMA_WRITE_ADV); if (cb->state != RDMA_WRITE_ADV) { - printf(PFX + uprintf(PFX "wait for RDMA_WRITE_ADV state %d\n", cb->state); break; @@ -834,14 +841,14 @@ krping_format_send(cb, cb->rdma_addr, cb->dma_mr); ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); if (ret) { - printf(PFX "post send error %d\n", ret); + uprintf(PFX "post send error %d\n", ret); break; } /* Wait for the server to say the RDMA Write is complete. */ krping_wait(cb, RDMA_WRITE_COMPLETE); if (cb->state != RDMA_WRITE_COMPLETE) { - printf(PFX + uprintf(PFX "wait for RDMA_WRITE_COMPLETE state %d\n", cb->state); break; @@ -849,12 +856,12 @@ if (cb->validate) if (memcmp(cb->start_buf, cb->rdma_buf, cb->size)) { - printf(PFX "data mismatch!\n"); + uprintf(PFX "data mismatch!\n"); break; } if (cb->verbose) - printf("ping data: %s\n", cb->rdma_buf); + uprintf("ping data: %s\n", cb->rdma_buf); } } @@ -870,13 +877,13 @@ ret = rdma_connect(cb->cm_id, &conn_param); if (ret) { - printf(PFX "rdma_connect error %d\n", ret); + uprintf(PFX "rdma_connect error %d\n", ret); return ret; } krping_wait(cb, CONNECTED); if (cb->state == ERROR) { - printf(PFX "wait for CONNECTED state %d\n", cb->state); + uprintf(PFX "wait for CONNECTED state %d\n", cb->state); return -1; } @@ -897,13 +904,13 @@ ret = rdma_resolve_addr(cb->cm_id, NULL, (struct sockaddr *) &sin, 2000); if (ret) { - printf(PFX "rdma_resolve_addr error %d\n", ret); + uprintf(PFX "rdma_resolve_addr error %d\n", ret); return ret; } krping_wait(cb, ROUTE_RESOLVED); if (cb->state != ROUTE_RESOLVED) { - printf(PFX + uprintf(PFX "addr/route resolution did not resolve: state %d\n", cb->state); return -EINTR; @@ -924,25 +931,25 @@ ret = krping_setup_qp(cb, cb->cm_id); if (ret) { - printf(PFX "setup_qp failed: %d\n", ret); + uprintf(PFX "setup_qp failed: %d\n", ret); return; } ret = krping_setup_buffers(cb); if (ret) { - printf(PFX "krping_setup_buffers failed: %d\n", ret); + uprintf(PFX "krping_setup_buffers failed: %d\n", ret); goto err1; } ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); if (ret) { - printf(PFX "ib_post_recv failed: %d\n", ret); + uprintf(PFX "ib_post_recv failed: %d\n", ret); goto err2; } ret = krping_connect_client(cb); if (ret) { - printf(PFX "connect error %d\n", ret); + uprintf(PFX "connect error %d\n", ret); goto err2; } @@ -974,7 +981,6 @@ cb->server = -1; cb->state = IDLE; cb->size = 64; - cv_init(&cb->sem, "krping cv"); mtx_init(&cb->lock, "krping mtx", NULL, MTX_DUPOK|MTX_SPIN); while ((op = krping_getopt("krping", &cmd, krping_opts, NULL, &optarg, @@ -984,7 +990,7 @@ cb->addr_str = optarg; DEBUG_LOG("ipaddr (%s)\n", optarg); if (!inet_aton(optarg, &cb->addr)) { - printf(PFX "bad addr string %s\n", optarg); + uprintf(PFX "bad addr string %s\n", optarg); ret = -EINVAL; } break; @@ -1004,7 +1010,7 @@ cb->size = optint; if ((cb->size < 1) || (cb->size > RPING_BUFSIZE)) { - printf(PFX "Invalid size %d " + uprintf(PFX "Invalid size %d " "(valid range is 1 to %d)\n", cb->size, RPING_BUFSIZE); ret = EINVAL; @@ -1014,7 +1020,7 @@ case 'C': cb->count = optint; if (cb->count < 0) { - printf(PFX "Invalid count %d\n", + uprintf(PFX "Invalid count %d\n", cb->count); ret = EINVAL; } else @@ -1032,7 +1038,7 @@ debug++; break; default: - printf(PFX "unknown opt %s\n", optarg); + uprintf(PFX "unknown opt %s\n", optarg); ret = -EINVAL; break; } @@ -1041,7 +1047,7 @@ goto out; if (cb->server == -1) { - printf(PFX "must be either client or server\n"); + uprintf(PFX "must be either client or server\n"); ret = EINVAL; goto out; } @@ -1049,16 +1055,14 @@ cb->cm_id = rdma_create_id(krping_cma_event_handler, cb, RDMA_PS_TCP); if (IS_ERR(cb->cm_id)) { ret = PTR_ERR(cb->cm_id); - printf(PFX "rdma_create_id error %d\n", ret); + uprintf(PFX "rdma_create_id error %d\n", ret); goto out; } DEBUG_LOG("created cm_id %p\n", cb->cm_id); - if (cb->server) krping_run_server(cb); else krping_run_client(cb); - DEBUG_LOG("destroy cm_id %p\n", cb->cm_id); rdma_destroy_id(cb->cm_id); out: @@ -1068,3 +1072,9 @@ free(cb, M_DEVBUF); return ret; } + +void krping_init(void) +{ + mtx_init(&krping_mutex, "krping lock", NULL, MTX_DEF); + TAILQ_INIT(&krping_cbs); +} ==== //depot/projects/iwarp/sys/contrib/rdma/krping/krping.h#2 (text+ko) ==== @@ -1,4 +1,3 @@ -#include <sys/condvar.h> #include <contrib/rdma/ib_verbs.h> #include <netinet/in.h> @@ -82,7 +81,6 @@ enum test_state state; /* used for cond/signalling */ struct mtx lock; - struct cv sem; struct krping_stats stats; uint16_t port; /* dst port in NBO */ @@ -101,7 +99,8 @@ }; extern struct mtx krping_mutex; +TAILQ_HEAD(krping_cb_list, krping_cb); +extern struct krping_cb_list krping_cbs; int krping_doit(char *cmd); -TAILQ_HEAD(krping_cb_list, krping_cb); -extern struct krping_cb_list krping_cbs; +void krping_init(void); ==== //depot/projects/iwarp/sys/contrib/rdma/krping/krping_dev.c#2 (text+ko) ==== @@ -57,7 +57,7 @@ switch (what) { case MOD_LOAD: /* kldload */ - mtx_init(&krping_mutex, "krping lock", NULL, MTX_DEF); + krping_init(); krping_dev = make_dev(&krping_cdevsw, 0, UID_ROOT, @@ -105,7 +105,7 @@ mtx_lock(&krping_mutex); TAILQ_FOREACH(cb, &krping_cbs, list) { if (cb->pd) { - printf("krping: %d-%s %lld %lld %lld %lld %lld %lld %lld %lld\n", + uprintf("krping: %d-%s %lld %lld %lld %lld %lld %lld %lld %lld\n", num++, cb->pd->device->name, cb->stats.send_bytes, cb->stats.send_msgs, cb->stats.recv_bytes, cb->stats.recv_msgs, cb->stats.write_bytes, @@ -113,7 +113,7 @@ cb->stats.read_bytes, cb->stats.read_msgs); } else { - printf("krping: %d listen\n", num++); + uprintf("krping: %d listen\n", num++); } } mtx_unlock(&krping_mutex); @@ -129,16 +129,18 @@ err = copyin(uio->uio_iov->iov_base, krpingmsg->msg, MIN(uio->uio_iov->iov_len, BUFFERSIZE - 1)); - /* Now we need to null terminate, then record the length */ - *(krpingmsg->msg + MIN(uio->uio_iov->iov_len, BUFFERSIZE - 1)) = 0; - krpingmsg->len = MIN(uio->uio_iov->iov_len, BUFFERSIZE); + /* null terminate and remove the \n */ + *(krpingmsg->msg + MIN(uio->uio_iov->iov_len-1, BUFFERSIZE - 1)) = 0; + krpingmsg->len = MIN(uio->uio_iov->iov_len-1, BUFFERSIZE); if (err != 0) { uprintf("Write failed: bad address!\n"); return err; } + uio->uio_resid = 0; + uprintf("krping: write string = |%s|\n", krpingmsg->msg); krping_doit(krpingmsg->msg); - return(err); + return(0); } MODULE_DEPEND(krping, rdma_core, 1, 1, 1);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200711281458.lASEwaDa001978>