Date: Tue, 15 Jan 2008 02:09:10 GMT From: Steve Wise <swise@FreeBSD.org> To: Perforce Change Reviews <perforce@FreeBSD.org> Subject: PERFORCE change 133298 for review Message-ID: <200801150209.m0F29AJr011283@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=133298 Change 133298 by swise@swise:vic10:iwarp on 2008/01/15 02:08:46 Enhance krping to allow using but dma-mrs and regular mrs. Affected files ... .. //depot/projects/iwarp/sys/contrib/rdma/krping/krping.c#6 edit .. //depot/projects/iwarp/sys/contrib/rdma/krping/krping.h#4 edit Differences ... ==== //depot/projects/iwarp/sys/contrib/rdma/krping/krping.c#6 (text+ko) ==== @@ -74,6 +74,8 @@ {"validate", OPT_NOPARAM, 'V'}, {"server", OPT_NOPARAM, 's'}, {"client", OPT_NOPARAM, 'c'}, + {"dmamr", OPT_NOPARAM, 'D'}, + {"debug", OPT_NOPARAM, 'd'}, {NULL, 0, 0} }; @@ -446,13 +448,19 @@ /* XXX X86 only here... not mapping for dma! */ cb->recv_sgl.addr = virt_to_phys(&cb->recv_buf); cb->recv_sgl.length = sizeof cb->recv_buf; - cb->recv_sgl.lkey = cb->dma_mr->lkey; + if (cb->use_dmamr) + cb->recv_sgl.lkey = cb->dma_mr->lkey; + else + cb->recv_sgl.lkey = cb->recv_mr->lkey; cb->rq_wr.sg_list = &cb->recv_sgl; cb->rq_wr.num_sge = 1; cb->send_sgl.addr = virt_to_phys(&cb->send_buf); cb->send_sgl.length = sizeof cb->send_buf; - cb->send_sgl.lkey = cb->dma_mr->lkey; + if (cb->use_dmamr) + cb->send_sgl.lkey = cb->dma_mr->lkey; + else + cb->send_sgl.lkey = cb->send_mr->lkey; cb->sq_wr.opcode = IB_WR_SEND; cb->sq_wr.send_flags = IB_SEND_SIGNALED; @@ -461,7 +469,10 @@ cb->rdma_addr = virt_to_phys(cb->rdma_buf); cb->rdma_sgl.addr = cb->rdma_addr; - cb->rdma_sgl.lkey = cb->dma_mr->lkey; + if (cb->use_dmamr) + cb->rdma_sgl.lkey = cb->dma_mr->lkey; + else + cb->rdma_sgl.lkey = cb->rdma_mr->lkey; cb->rdma_sq_wr.send_flags = IB_SEND_SIGNALED; cb->rdma_sq_wr.sg_list = &cb->rdma_sgl; cb->rdma_sq_wr.num_sge = 1; @@ -474,15 +485,44 @@ static int krping_setup_buffers(struct krping_cb *cb) { int ret; + struct ib_phys_buf buf; + u64 iovbase; DEBUG_LOG(PFX "krping_setup_buffers called on cb %p\n", cb); - cb->dma_mr = ib_get_dma_mr(cb->pd, IB_ACCESS_LOCAL_WRITE| + if (cb->use_dmamr) { + cb->dma_mr = ib_get_dma_mr(cb->pd, IB_ACCESS_LOCAL_WRITE| IB_ACCESS_REMOTE_READ| IB_ACCESS_REMOTE_WRITE); - if (IS_ERR(cb->dma_mr)) { - DEBUG_LOG(PFX "recv_buf reg_mr failed\n"); - return PTR_ERR(cb->dma_mr); + if (IS_ERR(cb->dma_mr)) { + DEBUG_LOG(PFX "reg_dmamr failed\n"); + return PTR_ERR(cb->dma_mr); + } + } else { + + buf.addr = virt_to_phys(&cb->recv_buf); + buf.size = sizeof cb->recv_buf; + iovbase = virt_to_phys(&cb->recv_buf); + cb->recv_mr = ib_reg_phys_mr(cb->pd, &buf, 1, + IB_ACCESS_LOCAL_WRITE, + &iovbase); + + if (IS_ERR(cb->recv_mr)) { + DEBUG_LOG(PFX "recv_buf reg_mr failed\n"); + return PTR_ERR(cb->recv_mr); + } + + buf.addr = virt_to_phys(&cb->send_buf); + buf.size = sizeof cb->send_buf; + iovbase = virt_to_phys(&cb->send_buf); + cb->send_mr = ib_reg_phys_mr(cb->pd, &buf, 1, + 0, &iovbase); + + if (IS_ERR(cb->send_mr)) { + DEBUG_LOG(PFX "send_buf reg_mr failed\n"); + ib_dereg_mr(cb->recv_mr); + return PTR_ERR(cb->send_mr); + } } cb->rdma_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, 0, (1L<<22), @@ -493,7 +533,23 @@ ret = -ENOMEM; goto err1; } + if (!cb->use_dmamr) { + buf.addr = virt_to_phys(cb->rdma_buf); + buf.size = cb->size; + iovbase = virt_to_phys(cb->rdma_buf); + cb->rdma_mr = ib_reg_phys_mr(cb->pd, &buf, 1, + IB_ACCESS_REMOTE_READ| + IB_ACCESS_REMOTE_WRITE, + &iovbase); + + if (IS_ERR(cb->rdma_buf)) { + DEBUG_LOG(PFX "rdma_buf reg_mr failed\n"); + ret = PTR_ERR(cb->rdma_mr); + goto err2; + } + } + if (!cb->server) { cb->start_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, 0, (1L<<22), 4096, 1024*1024); @@ -502,15 +558,40 @@ ret = -ENOMEM; goto err2; } + if (!cb->use_dmamr) { + + buf.addr = virt_to_phys(cb->start_buf); + buf.size = cb->size; + iovbase = virt_to_phys(cb->start_buf); + cb->start_mr = ib_reg_phys_mr(cb->pd, &buf, 1, + IB_ACCESS_REMOTE_READ, + &iovbase); + + if (IS_ERR(cb->start_buf)) { + DEBUG_LOG(PFX "start_buf reg_mr failed\n"); + ret = PTR_ERR(cb->start_mr); + goto err3; + } + } } krping_setup_wr(cb); DEBUG_LOG(PFX "allocated & registered buffers...\n"); return 0; +err3: + contigfree(cb->start_buf, cb->size, M_DEVBUF); + + if (!cb->use_dmamr) + ib_dereg_mr(cb->rdma_mr); err2: contigfree(cb->rdma_buf, cb->size, M_DEVBUF); err1: - ib_dereg_mr(cb->dma_mr); + if (cb->use_dmamr) + ib_dereg_mr(cb->dma_mr); + else { + ib_dereg_mr(cb->recv_mr); + ib_dereg_mr(cb->send_mr); + } return ret; } @@ -536,7 +617,15 @@ cb->size, DMA_BIDIRECTIONAL); contigfree(cb->start_buf, cb->size, M_DEVBUF); } - ib_dereg_mr(cb->dma_mr); + if (cb->use_dmamr) + ib_dereg_mr(cb->dma_mr); + else { + ib_dereg_mr(cb->send_mr); + ib_dereg_mr(cb->recv_mr); + ib_dereg_mr(cb->rdma_mr); + if (!cb->server) + ib_dereg_mr(cb->start_mr); + } } static int krping_create_qp(struct krping_cb *cb) @@ -823,7 +912,11 @@ start = 65; cb->start_buf[cb->size - 1] = 0; - krping_format_send(cb, cb->start_addr, cb->dma_mr); + if (cb->dma_mr) + krping_format_send(cb, cb->start_addr, cb->dma_mr); + else + krping_format_send(cb, cb->start_addr, cb->start_mr); + ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); if (ret) { DEBUG_LOG(PFX "post send error %d\n", ret); @@ -839,7 +932,11 @@ break; } - krping_format_send(cb, cb->rdma_addr, cb->dma_mr); + if (cb->dma_mr) + krping_format_send(cb, cb->rdma_addr, cb->dma_mr); + else + krping_format_send(cb, cb->rdma_addr, cb->rdma_mr); + ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); if (ret) { DEBUG_LOG(PFX "post send error %d\n", ret); @@ -995,6 +1092,10 @@ ret = -EINVAL; } break; + case 'D': + cb->use_dmamr = 1; + DEBUG_LOG(PFX "using dma mr\n"); + break; case 'p': cb->port = htons(optint); DEBUG_LOG(PFX "port %d\n", (int)optint); ==== //depot/projects/iwarp/sys/contrib/rdma/krping/krping.h#4 (text+ko) ==== @@ -54,22 +54,26 @@ struct ib_pd *pd; struct ib_qp *qp; struct ib_mr *dma_mr; + int use_dmamr; struct ib_recv_wr rq_wr; /* recv work request record */ struct ib_sge recv_sgl; /* recv single SGE */ struct krping_rdma_info recv_buf;/* malloc'd buffer */ DECLARE_PCI_UNMAP_ADDR(recv_mapping) + struct ib_mr *recv_mr; struct ib_send_wr sq_wr; /* send work requrest record */ struct ib_sge send_sgl; struct krping_rdma_info send_buf;/* single send buf */ DECLARE_PCI_UNMAP_ADDR(send_mapping) + struct ib_mr *send_mr; struct ib_send_wr rdma_sq_wr; /* rdma work request record */ struct ib_sge rdma_sgl; /* rdma single SGE */ char *rdma_buf; /* used as rdma sink */ u64 rdma_addr; DECLARE_PCI_UNMAP_ADDR(rdma_mapping) + struct ib_mr *rdma_mr; uint32_t remote_rkey; /* remote guys RKEY */ uint64_t remote_addr; /* remote guys TO */ @@ -78,6 +82,7 @@ char *start_buf; /* rdma read src */ u64 start_addr; DECLARE_PCI_UNMAP_ADDR(start_mapping) + struct ib_mr *start_mr; enum test_state state; /* used for cond/signalling */ struct mtx lock;
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200801150209.m0F29AJr011283>