Skip site navigation (1)Skip section navigation (2)
Date:      Thu, 26 May 2016 09:49:29 +0000 (UTC)
From:      Edward Tomasz Napierala <trasz@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r300723 - in head/sys: dev/iser modules/iser
Message-ID:  <201605260949.u4Q9nTx1069928@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: trasz
Date: Thu May 26 09:49:29 2016
New Revision: 300723
URL: https://svnweb.freebsd.org/changeset/base/300723

Log:
  Bring in the Mellanox implementation of iSER (iSCSI over RDMA) initiator,
  written by Sagi Grimberg <sagig at mellanox.com> and Max Gurtovoy
  <maxg at mellanox.com>.
  
  This code comes from https://github.com/sagigrimberg/iser-freebsd, branch
  iser-rebase-11-current-r291993.  It's not connected to the build just yet;
  it still needs some tweaks to adapt to my changes to iSCSI infrastructure.
  
  Big thanks to Mellanox for their support for FreeBSD!
  
  Obtained from:	Mellanox Technologies
  MFC after:	1 month
  Relnotes:	yes

Added:
  head/sys/dev/iser/
  head/sys/dev/iser/icl_iser.c   (contents, props changed)
  head/sys/dev/iser/icl_iser.h   (contents, props changed)
  head/sys/dev/iser/iser_initiator.c   (contents, props changed)
  head/sys/dev/iser/iser_memory.c   (contents, props changed)
  head/sys/dev/iser/iser_verbs.c   (contents, props changed)
  head/sys/modules/iser/
  head/sys/modules/iser/Makefile   (contents, props changed)

Added: head/sys/dev/iser/icl_iser.c
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ head/sys/dev/iser/icl_iser.c	Thu May 26 09:49:29 2016	(r300723)
@@ -0,0 +1,582 @@
+/* $FreeBSD$ */
+/*-
+ * Copyright (c) 2015, Mellanox Technologies, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "icl_iser.h"
+
+SYSCTL_NODE(_kern, OID_AUTO, iser, CTLFLAG_RW, 0, "iSER module");
+int iser_debug = 0;
+SYSCTL_INT(_kern_iser, OID_AUTO, debug, CTLFLAG_RWTUN,
+    &iser_debug, 0, "Enable iser debug messages");
+
+static MALLOC_DEFINE(M_ICL_ISER, "icl_iser", "iSCSI iser backend");
+static uma_zone_t icl_pdu_zone;
+
+static volatile u_int	icl_iser_ncons;
+struct iser_global ig;
+
+static icl_conn_new_pdu_t	iser_conn_new_pdu;
+static icl_conn_pdu_free_t	iser_conn_pdu_free;
+static icl_conn_pdu_data_segment_length_t iser_conn_pdu_data_segment_length;
+static icl_conn_pdu_append_data_t	iser_conn_pdu_append_data;
+static icl_conn_pdu_queue_t	iser_conn_pdu_queue;
+static icl_conn_handoff_t	iser_conn_handoff;
+static icl_conn_free_t		iser_conn_free;
+static icl_conn_close_t		iser_conn_close;
+static icl_conn_release_t	iser_conn_release;
+static icl_conn_connect_t	iser_conn_connect;
+static icl_conn_connected_t	iser_conn_connected;
+static icl_conn_task_setup_t	iser_conn_task_setup;
+static icl_conn_task_done_t	iser_conn_task_done;
+static icl_conn_pdu_get_data_t	iser_conn_pdu_get_data;
+
+static kobj_method_t icl_iser_methods[] = {
+	KOBJMETHOD(icl_conn_new_pdu, iser_conn_new_pdu),
+	KOBJMETHOD(icl_conn_pdu_free, iser_conn_pdu_free),
+	KOBJMETHOD(icl_conn_pdu_data_segment_length, iser_conn_pdu_data_segment_length),
+	KOBJMETHOD(icl_conn_pdu_append_data, iser_conn_pdu_append_data),
+	KOBJMETHOD(icl_conn_pdu_queue, iser_conn_pdu_queue),
+	KOBJMETHOD(icl_conn_handoff, iser_conn_handoff),
+	KOBJMETHOD(icl_conn_free, iser_conn_free),
+	KOBJMETHOD(icl_conn_close, iser_conn_close),
+	KOBJMETHOD(icl_conn_release, iser_conn_release),
+	KOBJMETHOD(icl_conn_connect, iser_conn_connect),
+	KOBJMETHOD(icl_conn_connected, iser_conn_connected),
+	KOBJMETHOD(icl_conn_task_setup, iser_conn_task_setup),
+	KOBJMETHOD(icl_conn_task_done, iser_conn_task_done),
+	KOBJMETHOD(icl_conn_pdu_get_data, iser_conn_pdu_get_data),
+	{ 0, 0 }
+};
+
+DEFINE_CLASS(icl_iser, icl_iser_methods, sizeof(struct iser_conn));
+
+/**
+ * iser_initialize_headers() - Initialize task headers
+ * @pdu:       iser pdu
+ * @iser_conn:    iser connection
+ *
+ * Notes:
+ * This routine may race with iser teardown flow for scsi
+ * error handling TMFs. So for TMF we should acquire the
+ * state mutex to avoid dereferencing the IB device which
+ * may have already been terminated (racing teardown sequence).
+ */
+int
+iser_initialize_headers(struct icl_iser_pdu *pdu, struct iser_conn *iser_conn)
+{
+	struct iser_tx_desc *tx_desc = &pdu->desc;
+	struct iser_device *device = iser_conn->ib_conn.device;
+	u64 dma_addr;
+	int ret = 0;
+
+	dma_addr = ib_dma_map_single(device->ib_device, (void *)tx_desc,
+				ISER_HEADERS_LEN, DMA_TO_DEVICE);
+	if (ib_dma_mapping_error(device->ib_device, dma_addr)) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	tx_desc->mapped = true;
+	tx_desc->dma_addr = dma_addr;
+	tx_desc->tx_sg[0].addr   = tx_desc->dma_addr;
+	tx_desc->tx_sg[0].length = ISER_HEADERS_LEN;
+	tx_desc->tx_sg[0].lkey   = device->mr->lkey;
+
+out:
+
+	return (ret);
+}
+
+int
+iser_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *request,
+			  const void *addr, size_t len, int flags)
+{
+	struct iser_conn *iser_conn = icl_to_iser_conn(ic);
+
+	if (request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_LOGIN_REQUEST ||
+	    request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_TEXT_REQUEST) {
+		ISER_DBG("copy to login buff");
+		memcpy(iser_conn->login_req_buf, addr, len);
+		request->ip_data_len = len;
+	}
+
+	return (0);
+}
+
+void
+iser_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip,
+		       size_t off, void *addr, size_t len)
+{
+	/* If we have a receive data, copy it to upper layer buffer */
+	if (ip->ip_data_mbuf)
+		memcpy(addr, ip->ip_data_mbuf + off, len);
+}
+
+/*
+ * Allocate icl_pdu with empty BHS to fill up by the caller.
+ */
+struct icl_pdu *
+iser_new_pdu(struct icl_conn *ic, int flags)
+{
+	struct icl_iser_pdu *iser_pdu;
+	struct icl_pdu *ip;
+	struct iser_conn *iser_conn = icl_to_iser_conn(ic);
+
+	iser_pdu = uma_zalloc(icl_pdu_zone, flags | M_ZERO);
+	if (iser_pdu == NULL) {
+		ISER_WARN("failed to allocate %zd bytes", sizeof(*iser_pdu));
+		return (NULL);
+	}
+
+	iser_pdu->iser_conn = iser_conn;
+	ip = &iser_pdu->icl_pdu;
+	ip->ip_conn = ic;
+	ip->ip_bhs = &iser_pdu->desc.iscsi_header;
+
+	return (ip);
+}
+
+struct icl_pdu *
+iser_conn_new_pdu(struct icl_conn *ic, int flags)
+{
+	return (iser_new_pdu(ic, flags));
+}
+
+void
+iser_pdu_free(struct icl_conn *ic, struct icl_pdu *ip)
+{
+	struct icl_iser_pdu *iser_pdu = icl_to_iser_pdu(ip);
+
+	uma_zfree(icl_pdu_zone, iser_pdu);
+}
+
+size_t
+iser_conn_pdu_data_segment_length(struct icl_conn *ic,
+				  const struct icl_pdu *request)
+{
+	uint32_t len = 0;
+
+	len += request->ip_bhs->bhs_data_segment_len[0];
+	len <<= 8;
+	len += request->ip_bhs->bhs_data_segment_len[1];
+	len <<= 8;
+	len += request->ip_bhs->bhs_data_segment_len[2];
+
+	return (len);
+}
+
+void
+iser_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip)
+{
+	iser_pdu_free(ic, ip);
+}
+
+static bool
+is_control_opcode(uint8_t opcode)
+{
+	bool is_control = false;
+
+	switch (opcode & ISCSI_OPCODE_MASK) {
+		case ISCSI_BHS_OPCODE_NOP_OUT:
+		case ISCSI_BHS_OPCODE_LOGIN_REQUEST:
+		case ISCSI_BHS_OPCODE_LOGOUT_REQUEST:
+		case ISCSI_BHS_OPCODE_TEXT_REQUEST:
+			is_control = true;
+			break;
+		case ISCSI_BHS_OPCODE_SCSI_COMMAND:
+			is_control = false;
+			break;
+		default:
+			ISER_ERR("unknown opcode %d", opcode);
+	}
+
+	return (is_control);
+}
+
+void
+iser_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip)
+{
+	struct iser_conn *iser_conn = icl_to_iser_conn(ic);
+	struct icl_iser_pdu *iser_pdu = icl_to_iser_pdu(ip);
+	int ret;
+
+	ret = iser_initialize_headers(iser_pdu, iser_conn);
+	if (ret) {
+		ISER_ERR("Failed to map TX descriptor pdu %p", iser_pdu);
+		return;
+	}
+
+	if (is_control_opcode(ip->ip_bhs->bhs_opcode)) {
+		ret = iser_send_control(iser_conn, iser_pdu);
+		if (unlikely(ret))
+			ISER_ERR("Failed to send control pdu %p", iser_pdu);
+	} else {
+		ret = iser_send_command(iser_conn, iser_pdu);
+		if (unlikely(ret))
+			ISER_ERR("Failed to send command pdu %p", iser_pdu);
+	}
+}
+
+static struct icl_conn *
+iser_new_conn(const char *name, struct mtx *lock)
+{
+	struct iser_conn *iser_conn;
+	struct icl_conn *ic;
+
+	refcount_acquire(&icl_iser_ncons);
+
+	iser_conn = (struct iser_conn *)kobj_create(&icl_iser_class, M_ICL_ISER, M_WAITOK | M_ZERO);
+	if (!iser_conn) {
+		ISER_ERR("failed to allocate iser conn");
+		refcount_release(&icl_iser_ncons);
+		return (NULL);
+	}
+
+	cv_init(&iser_conn->up_cv, "iser_cv");
+	sx_init(&iser_conn->state_mutex, "iser_conn_state_mutex");
+	mtx_init(&iser_conn->ib_conn.beacon.flush_lock, "flush_lock", NULL, MTX_DEF);
+	cv_init(&iser_conn->ib_conn.beacon.flush_cv, "flush_cv");
+	mtx_init(&iser_conn->ib_conn.lock, "lock", NULL, MTX_DEF);
+
+	ic = &iser_conn->icl_conn;
+	ic->ic_lock = lock;
+	ic->ic_name = name;
+	ic->ic_driver = strdup("iser", M_TEMP);
+	ic->ic_iser = true;
+
+	return (ic);
+}
+
+void
+iser_conn_free(struct icl_conn *ic)
+{
+	struct iser_conn *iser_conn = icl_to_iser_conn(ic);
+
+	cv_destroy(&iser_conn->ib_conn.beacon.flush_cv);
+	mtx_destroy(&iser_conn->ib_conn.beacon.flush_lock);
+	sx_destroy(&iser_conn->state_mutex);
+	cv_destroy(&iser_conn->up_cv);
+	kobj_delete((struct kobj *)iser_conn, M_ICL_ISER);
+	refcount_release(&icl_iser_ncons);
+}
+
+int
+iser_conn_handoff(struct icl_conn *ic, int cmds_max)
+{
+	struct iser_conn *iser_conn = icl_to_iser_conn(ic);
+	int error = 0;
+
+	sx_xlock(&iser_conn->state_mutex);
+	if (iser_conn->state != ISER_CONN_UP) {
+		error = EINVAL;
+		ISER_ERR("iser_conn %p state is %d, teardown started\n",
+			 iser_conn, iser_conn->state);
+		goto out;
+	}
+
+	/*
+	 * In discovery session no need to allocate rx desc and posting recv
+	 * work request
+	 */
+	if (ic->ic_session_type_discovery(ic))
+		goto out;
+
+	error = iser_alloc_rx_descriptors(iser_conn, cmds_max);
+	if (error)
+		goto out;
+
+	error = iser_post_recvm(iser_conn, iser_conn->min_posted_rx);
+	if (error)
+		goto post_error;
+
+	sx_xunlock(&iser_conn->state_mutex);
+	return (error);
+
+post_error:
+	iser_free_rx_descriptors(iser_conn);
+out:
+	sx_xunlock(&iser_conn->state_mutex);
+	return (error);
+
+}
+
+/**
+ * Frees all conn objects
+ */
+void
+iser_conn_release(struct icl_conn *ic)
+{
+	struct iser_conn *iser_conn = icl_to_iser_conn(ic);
+	struct ib_conn *ib_conn = &iser_conn->ib_conn;
+	struct iser_conn *curr, *tmp;
+
+	mtx_lock(&ig.connlist_mutex);
+	/*
+	 * Search for iser connection in global list.
+	 * It may not be there in case of failure in connection establishment
+	 * stage.
+	 */
+	list_for_each_entry_safe(curr, tmp, &ig.connlist, conn_list) {
+		if (iser_conn == curr) {
+			ISER_WARN("found iser_conn %p", iser_conn);
+			list_del(&iser_conn->conn_list);
+		}
+	}
+	mtx_unlock(&ig.connlist_mutex);
+
+	/*
+	 * In case we reconnecting or removing session, we need to
+	 * release IB resources (which is safe to call more than once).
+	 */
+	sx_xlock(&iser_conn->state_mutex);
+	iser_free_ib_conn_res(iser_conn, true);
+	sx_xunlock(&iser_conn->state_mutex);
+
+	if (ib_conn->cma_id != NULL) {
+		rdma_destroy_id(ib_conn->cma_id);
+		ib_conn->cma_id = NULL;
+	}
+
+}
+
+void
+iser_conn_close(struct icl_conn *ic)
+{
+	struct iser_conn *iser_conn = icl_to_iser_conn(ic);
+
+	ISER_INFO("closing conn %p", iser_conn);
+
+	sx_xlock(&iser_conn->state_mutex);
+	/*
+	 * In case iser connection is waiting on conditional variable
+	 * (state PENDING) and we try to close it before connection establishment,
+	 * we need to signal it to continue releasing connection properly.
+	 */
+	if (!iser_conn_terminate(iser_conn) && iser_conn->state == ISER_CONN_PENDING)
+		cv_signal(&iser_conn->up_cv);
+	sx_xunlock(&iser_conn->state_mutex);
+
+}
+
+int
+iser_conn_connect(struct icl_conn *ic, int domain, int socktype,
+		int protocol, struct sockaddr *from_sa, struct sockaddr *to_sa)
+{
+	struct iser_conn *iser_conn = icl_to_iser_conn(ic);
+	struct ib_conn *ib_conn = &iser_conn->ib_conn;
+	int err = 0;
+
+	sx_xlock(&iser_conn->state_mutex);
+	 /* the device is known only --after-- address resolution */
+	ib_conn->device = NULL;
+
+	iser_conn->state = ISER_CONN_PENDING;
+
+	ib_conn->cma_id = rdma_create_id(iser_cma_handler, (void *)iser_conn,
+			RDMA_PS_TCP, IB_QPT_RC);
+	if (IS_ERR(ib_conn->cma_id)) {
+		err = -PTR_ERR(ib_conn->cma_id);
+		ISER_ERR("rdma_create_id failed: %d", err);
+		goto id_failure;
+	}
+
+	err = rdma_resolve_addr(ib_conn->cma_id, from_sa, to_sa, 1000);
+	if (err) {
+		ISER_ERR("rdma_resolve_addr failed: %d", err);
+		if (err < 0)
+			err = -err;
+		goto addr_failure;
+	}
+
+	ISER_DBG("before cv_wait: %p", iser_conn);
+	cv_wait(&iser_conn->up_cv, &iser_conn->state_mutex);
+	ISER_DBG("after cv_wait: %p", iser_conn);
+
+	if (iser_conn->state != ISER_CONN_UP) {
+		err = EIO;
+		goto addr_failure;
+	}
+
+	err = iser_alloc_login_buf(iser_conn);
+	if (err)
+		goto addr_failure;
+	sx_xunlock(&iser_conn->state_mutex);
+
+	mtx_lock(&ig.connlist_mutex);
+	list_add(&iser_conn->conn_list, &ig.connlist);
+	mtx_unlock(&ig.connlist_mutex);
+
+	return (0);
+
+id_failure:
+	ib_conn->cma_id = NULL;
+addr_failure:
+	sx_xunlock(&iser_conn->state_mutex);
+	return (err);
+}
+
+/**
+ * Called with session spinlock held.
+ * No need to lock state mutex on an advisory check.
+ **/
+bool
+iser_conn_connected(struct icl_conn *ic)
+{
+	struct iser_conn *iser_conn = icl_to_iser_conn(ic);
+
+	return (iser_conn->state == ISER_CONN_UP);
+}
+
+int
+iser_conn_task_setup(struct icl_conn *ic, struct ccb_scsiio *csio,
+		     uint32_t *task_tagp, void **prvp, struct icl_pdu *ip)
+{
+	struct icl_iser_pdu *iser_pdu = icl_to_iser_pdu(ip);
+
+	*prvp = ip;
+	iser_pdu->csio = csio;
+
+	return (0);
+}
+
+void
+iser_conn_task_done(struct icl_conn *ic, void *prv)
+{
+	struct icl_pdu *ip = prv;
+	struct icl_iser_pdu *iser_pdu = icl_to_iser_pdu(ip);
+	struct iser_device *device = iser_pdu->iser_conn->ib_conn.device;
+	struct iser_tx_desc *tx_desc = &iser_pdu->desc;
+
+	if (iser_pdu->dir[ISER_DIR_IN]) {
+		iser_unreg_rdma_mem(iser_pdu, ISER_DIR_IN);
+		iser_dma_unmap_task_data(iser_pdu,
+					 &iser_pdu->data[ISER_DIR_IN],
+					 DMA_FROM_DEVICE);
+	}
+
+	if (iser_pdu->dir[ISER_DIR_OUT]) {
+		iser_unreg_rdma_mem(iser_pdu, ISER_DIR_OUT);
+		iser_dma_unmap_task_data(iser_pdu,
+					 &iser_pdu->data[ISER_DIR_OUT],
+					 DMA_TO_DEVICE);
+	}
+
+	if (likely(tx_desc->mapped)) {
+		ib_dma_unmap_single(device->ib_device, tx_desc->dma_addr,
+				    ISER_HEADERS_LEN, DMA_TO_DEVICE);
+		tx_desc->mapped = false;
+	}
+
+	iser_pdu_free(ic, ip);
+}
+
+static u_int32_t
+iser_hba_misc()
+{
+	return (PIM_UNMAPPED);
+}
+
+static int
+iser_limits(size_t *limitp)
+{
+	*limitp = 128 * 1024;
+
+	return (0);
+}
+
+static int
+icl_iser_load(void)
+{
+	int error;
+
+	ISER_DBG("Starting iSER datamover...");
+
+	icl_pdu_zone = uma_zcreate("icl_iser_pdu", sizeof(struct icl_iser_pdu),
+				   NULL, NULL, NULL, NULL,
+				   UMA_ALIGN_PTR, 0);
+	/* FIXME: Check rc */
+
+	refcount_init(&icl_iser_ncons, 0);
+
+	error = icl_register("iser", 0, iser_limits, iser_new_conn, iser_hba_misc);
+	KASSERT(error == 0, ("failed to register iser"));
+
+	memset(&ig, 0, sizeof(struct iser_global));
+
+	/* device init is called only after the first addr resolution */
+	sx_init(&ig.device_list_mutex,  "global_device_lock");
+	INIT_LIST_HEAD(&ig.device_list);
+	mtx_init(&ig.connlist_mutex, "global_conn_lock", NULL, MTX_DEF);
+	INIT_LIST_HEAD(&ig.connlist);
+	sx_init(&ig.close_conns_mutex,  "global_close_conns_lock");
+
+	return (error);
+}
+
+static int
+icl_iser_unload(void)
+{
+	ISER_DBG("Removing iSER datamover...");
+
+	if (icl_iser_ncons != 0)
+		return (EBUSY);
+
+	sx_destroy(&ig.close_conns_mutex);
+	mtx_destroy(&ig.connlist_mutex);
+	sx_destroy(&ig.device_list_mutex);
+
+	icl_unregister("iser");
+
+	uma_zdestroy(icl_pdu_zone);
+
+	return (0);
+}
+
+static int
+icl_iser_modevent(module_t mod, int what, void *arg)
+{
+	switch (what) {
+	case MOD_LOAD:
+		return (icl_iser_load());
+	case MOD_UNLOAD:
+		return (icl_iser_unload());
+	default:
+		return (EINVAL);
+	}
+}
+
+moduledata_t icl_iser_data = {
+	.name = "icl_iser",
+	.evhand = icl_iser_modevent,
+	.priv = 0
+};
+
+DECLARE_MODULE(icl_iser, icl_iser_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
+MODULE_DEPEND(icl_iser, icl, 1, 1, 1);
+MODULE_DEPEND(icl_iser, iscsi, 1, 1, 1);
+MODULE_DEPEND(icl_iser, ibcore, 1, 1, 1);
+MODULE_DEPEND(icl_iser, linuxkpi, 1, 1, 1);
+MODULE_VERSION(icl_iser, 1);
+

Added: head/sys/dev/iser/icl_iser.h
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ head/sys/dev/iser/icl_iser.h	Thu May 26 09:49:29 2016	(r300723)
@@ -0,0 +1,547 @@
+/* $FreeBSD$ */
+/*-
+ * Copyright (c) 2015, Mellanox Technologies, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef ICL_ISER_H
+#define ICL_ISER_H
+
+/*
+ * iSCSI Common Layer for RDMA.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/capsicum.h>
+#include <sys/condvar.h>
+#include <sys/conf.h>
+#include <sys/file.h>
+#include <sys/kernel.h>
+#include <sys/kthread.h>
+#include <sys/lock.h>
+#include <sys/mbuf.h>
+#include <sys/mutex.h>
+#include <sys/module.h>
+#include <sys/protosw.h>
+#include <sys/socket.h>
+#include <sys/socketvar.h>
+#include <sys/sysctl.h>
+#include <sys/systm.h>
+#include <sys/sx.h>
+#include <sys/uio.h>
+#include <sys/taskqueue.h>
+#include <sys/bio.h>
+#include <vm/uma.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <dev/iscsi/icl.h>
+#include <dev/iscsi/iscsi_proto.h>
+#include <icl_conn_if.h>
+#include <cam/cam.h>
+#include <cam/cam_ccb.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_fmr_pool.h>
+#include <rdma/rdma_cm.h>
+
+
+#define	ISER_DBG(X, ...)						\
+	do {								\
+		if (unlikely(iser_debug > 2))				\
+			printf("DEBUG: %s: " X "\n",			\
+				__func__, ## __VA_ARGS__);		\
+	} while (0)
+
+#define	ISER_INFO(X, ...)						\
+	do {								\
+		if (unlikely(iser_debug > 1))				\
+			printf("INFO: %s: " X "\n",			\
+				__func__, ## __VA_ARGS__);		\
+	} while (0)
+
+#define	ISER_WARN(X, ...)						\
+	do {								\
+		if (unlikely(iser_debug > 0)) {				\
+			printf("WARNING: %s: " X "\n",			\
+				__func__, ## __VA_ARGS__);		\
+		}							\
+	} while (0)
+
+#define	ISER_ERR(X, ...) 						\
+	printf("ERROR: %s: " X "\n", __func__, ## __VA_ARGS__)
+
+#define ISER_VER			0x10
+#define ISER_WSV			0x08
+#define ISER_RSV			0x04
+
+#define ISER_FASTREG_LI_WRID		0xffffffffffffffffULL
+#define ISER_BEACON_WRID		0xfffffffffffffffeULL
+
+#define SHIFT_4K	12
+#define SIZE_4K	(1ULL << SHIFT_4K)
+#define MASK_4K	(~(SIZE_4K-1))
+
+/* support up to 512KB in one RDMA */
+#define ISCSI_ISER_SG_TABLESIZE         (0x80000 >> SHIFT_4K)
+#define ISER_DEF_XMIT_CMDS_MAX 256
+
+/* the max RX (recv) WR supported by the iSER QP is defined by                 *
+ * max_recv_wr = commands_max + recv_beacon                                    */
+#define ISER_QP_MAX_RECV_DTOS  (ISER_DEF_XMIT_CMDS_MAX + 1)
+#define ISER_MIN_POSTED_RX		(ISER_DEF_XMIT_CMDS_MAX >> 2)
+
+/* QP settings */
+/* Maximal bounds on received asynchronous PDUs */
+#define ISER_MAX_RX_MISC_PDUS           4 /* NOOP_IN(2) , ASYNC_EVENT(2)   */
+#define ISER_MAX_TX_MISC_PDUS           6 /* NOOP_OUT(2), TEXT(1), SCSI_TMFUNC(2), LOGOUT(1) */
+
+/* the max TX (send) WR supported by the iSER QP is defined by                 *
+ * max_send_wr = T * (1 + D) + C ; D is how many inflight dataouts we expect   *
+ * to have at max for SCSI command. The tx posting & completion handling code  *
+ * supports -EAGAIN scheme where tx is suspended till the QP has room for more *
+ * send WR. D=8 comes from 64K/8K                                              */
+
+#define ISER_INFLIGHT_DATAOUTS		8
+
+/* the send_beacon increase the max_send_wr by 1  */
+#define ISER_QP_MAX_REQ_DTOS		(ISER_DEF_XMIT_CMDS_MAX *    \
+					(1 + ISER_INFLIGHT_DATAOUTS) + \
+					ISER_MAX_TX_MISC_PDUS        + \
+					ISER_MAX_RX_MISC_PDUS + 1)
+
+#define ISER_GET_MAX_XMIT_CMDS(send_wr) ((send_wr			\
+					 - ISER_MAX_TX_MISC_PDUS	\
+					 - ISER_MAX_RX_MISC_PDUS - 1) /	\
+					 (1 + ISER_INFLIGHT_DATAOUTS))
+
+#define ISER_WC_BATCH_COUNT   16
+#define ISER_SIGNAL_CMD_COUNT 32
+
+/* Maximal QP's recommended per CQ. In case we use more QP's per CQ we might   *
+ * encounter a CQ overrun state.                                               */
+#define ISCSI_ISER_MAX_CONN	8
+#define ISER_MAX_RX_LEN		(ISER_QP_MAX_RECV_DTOS * ISCSI_ISER_MAX_CONN)
+#define ISER_MAX_TX_LEN		(ISER_QP_MAX_REQ_DTOS  * ISCSI_ISER_MAX_CONN)
+#define ISER_MAX_CQ_LEN		(ISER_MAX_RX_LEN + ISER_MAX_TX_LEN + \
+				 ISCSI_ISER_MAX_CONN)
+
+#define ISER_ZBVA_NOT_SUPPORTED                0x80
+#define ISER_SEND_W_INV_NOT_SUPPORTED	0x40
+
+#define icl_to_iser_conn(ic) \
+	container_of(ic, struct iser_conn, icl_conn)
+#define icl_to_iser_pdu(ip) \
+	container_of(ip, struct icl_iser_pdu, icl_pdu)
+
+/**
+ * struct iser_hdr - iSER header
+ *
+ * @flags:        flags support (zbva, remote_inv)
+ * @rsvd:         reserved
+ * @write_stag:   write rkey
+ * @write_va:     write virtual address
+ * @reaf_stag:    read rkey
+ * @read_va:      read virtual address
+ */
+struct iser_hdr {
+	u8      flags;
+	u8      rsvd[3];
+	__be32  write_stag;
+	__be64  write_va;
+	__be32  read_stag;
+	__be64  read_va;
+} __attribute__((packed));
+
+struct iser_cm_hdr {
+	u8      flags;
+	u8      rsvd[3];
+} __packed;
+
+/* Constant PDU lengths calculations */
+#define ISER_HEADERS_LEN  (sizeof(struct iser_hdr) + ISCSI_BHS_SIZE)
+
+#define ISER_RECV_DATA_SEG_LEN	128
+#define ISER_RX_PAYLOAD_SIZE	(ISER_HEADERS_LEN + ISER_RECV_DATA_SEG_LEN)
+
+#define ISER_RX_LOGIN_SIZE	(ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN)
+
+enum iser_conn_state {
+	ISER_CONN_INIT,		   /* descriptor allocd, no conn          */
+	ISER_CONN_PENDING,	   /* in the process of being established */
+	ISER_CONN_UP,		   /* up and running                      */
+	ISER_CONN_TERMINATING,	   /* in the process of being terminated  */
+	ISER_CONN_DOWN,		   /* shut down                           */
+	ISER_CONN_STATES_NUM
+};
+
+enum iser_task_status {
+	ISER_TASK_STATUS_INIT = 0,
+	ISER_TASK_STATUS_STARTED,
+	ISER_TASK_STATUS_COMPLETED
+};
+
+enum iser_data_dir {
+	ISER_DIR_IN = 0,	   /* to initiator */
+	ISER_DIR_OUT,		   /* from initiator */
+	ISER_DIRS_NUM
+};
+
+/**
+ * struct iser_mem_reg - iSER memory registration info
+ *
+ * @sge:          memory region sg element
+ * @rkey:         memory region remote key
+ * @mem_h:        pointer to registration context (FMR/Fastreg)
+ */
+struct iser_mem_reg {
+	struct ib_sge	 sge;
+	u32		 rkey;
+	void		*mem_h;
+};
+
+enum iser_desc_type {
+	ISCSI_TX_CONTROL ,
+	ISCSI_TX_SCSI_COMMAND,
+	ISCSI_TX_DATAOUT
+};
+
+/**
+ * struct iser_data_buf - iSER data buffer
+ *
+ * @sg:           pointer to the sg list
+ * @size:         num entries of this sg
+ * @data_len:     total beffer byte len
+ * @dma_nents:    returned by dma_map_sg
+ * @copy_buf:     allocated copy buf for SGs unaligned
+ *                for rdma which are copied
+ * @orig_sg:      pointer to the original sg list (in case
+ *                we used a copy)
+ * @sg_single:    SG-ified clone of a non SG SC or
+ *                unaligned SG
+ */
+struct iser_data_buf {
+	struct scatterlist sgl[ISCSI_ISER_SG_TABLESIZE];
+	void               *sg;
+	unsigned int       size;
+	unsigned long      data_len;
+	unsigned int       dma_nents;
+	char               *copy_buf;
+	struct scatterlist *orig_sg;
+	struct scatterlist sg_single;
+  };
+
+/* fwd declarations */
+struct iser_conn;
+struct ib_conn;
+struct iser_device;
+
+/**
+ * struct iser_tx_desc - iSER TX descriptor (for send wr_id)
+ *
+ * @iser_header:   iser header
+ * @iscsi_header:  iscsi header (bhs)
+ * @type:          command/control/dataout
+ * @dma_addr:      header buffer dma_address
+ * @tx_sg:         sg[0] points to iser/iscsi headers
+ *                 sg[1] optionally points to either of immediate data
+ *                 unsolicited data-out or control
+ * @num_sge:       number sges used on this TX task
+ * @mapped:        indicates if the descriptor is dma mapped
+ */
+struct iser_tx_desc {
+	struct iser_hdr              iser_header;
+	struct iscsi_bhs             iscsi_header __attribute__((packed));
+	enum   iser_desc_type        type;
+	u64		             dma_addr;
+	struct ib_sge		     tx_sg[2];
+	int                          num_sge;
+	bool                         mapped;
+};
+
+#define ISER_RX_PAD_SIZE	(256 - (ISER_RX_PAYLOAD_SIZE + \
+					sizeof(u64) + sizeof(struct ib_sge)))
+/**
+ * struct iser_rx_desc - iSER RX descriptor (for recv wr_id)
+ *
+ * @iser_header:   iser header
+ * @iscsi_header:  iscsi header
+ * @data:          received data segment
+ * @dma_addr:      receive buffer dma address
+ * @rx_sg:         ib_sge of receive buffer
+ * @pad:           for sense data TODO: Modify to maximum sense length supported
+ */
+struct iser_rx_desc {
+	struct iser_hdr              iser_header;
+	struct iscsi_bhs             iscsi_header;
+	char		             data[ISER_RECV_DATA_SEG_LEN];
+	u64		             dma_addr;
+	struct ib_sge		     rx_sg;
+	char		             pad[ISER_RX_PAD_SIZE];
+} __attribute__((packed));
+
+struct icl_iser_pdu {
+	struct icl_pdu               icl_pdu;
+	struct iser_tx_desc          desc;
+	struct iser_conn             *iser_conn;
+	enum iser_task_status        status;
+	struct ccb_scsiio 			 *csio;
+	int                          command_sent;
+	int                          dir[ISER_DIRS_NUM];
+	struct iser_mem_reg          rdma_reg[ISER_DIRS_NUM];
+	struct iser_data_buf         data[ISER_DIRS_NUM];
+};
+
+/**
+ * struct iser_comp - iSER completion context
+ *
+ * @device:     pointer to device handle
+ * @cq:         completion queue
+ * @wcs:        work completion array
+ * @tq:    	taskqueue handle
+ * @task:    	task to run task_fn
+ * @active_qps: Number of active QPs attached
+ *              to completion context
+ */
+struct iser_comp {
+	struct iser_device      *device;
+	struct ib_cq		*cq;
+	struct ib_wc		 wcs[ISER_WC_BATCH_COUNT];
+	struct taskqueue        *tq;
+	struct task             task;
+	int                      active_qps;
+};
+
+/**
+ * struct iser_device - iSER device handle
+ *
+ * @ib_device:     RDMA device
+ * @pd:            Protection Domain for this device
+ * @dev_attr:      Device attributes container
+ * @mr:            Global DMA memory region
+ * @event_handler: IB events handle routine
+ * @ig_list:	   entry in devices list
+ * @refcount:      Reference counter, dominated by open iser connections
+ * @comps_used:    Number of completion contexts used, Min between online
+ *                 cpus and device max completion vectors
+ * @comps:         Dinamically allocated array of completion handlers
+ */
+struct iser_device {
+	struct ib_device             *ib_device;
+	struct ib_pd	             *pd;
+	struct ib_device_attr	     dev_attr;
+	struct ib_mr	             *mr;
+	struct ib_event_handler      event_handler;
+	struct list_head             ig_list;
+	int                          refcount;
+	int			     comps_used;
+	struct iser_comp	     *comps;
+};
+
+/**
+ * struct iser_reg_resources - Fast registration recources
+ *
+ * @mr:         memory region
+ * @frpl:       fast reg page list
+ * @mr_valid:   is mr valid indicator
+ */
+struct iser_reg_resources {
+	struct ib_mr                     *mr;
+	struct ib_fast_reg_page_list     *frpl;
+	u8                                mr_valid:1;
+};
+
+/**
+ * struct fast_reg_descriptor - Fast registration descriptor
+ *
+ * @list:           entry in connection fastreg pool
+ * @rsc:            data buffer registration resources
+ */
+struct fast_reg_descriptor {
+	struct list_head		  list;
+	struct iser_reg_resources	  rsc;
+};
+
+
+/**
+ * struct iser_beacon - beacon to signal all flush errors were drained
+ *
+ * @send:           send wr
+ * @recv:           recv wr
+ * @flush_lock:     protects flush_cv
+ * @flush_cv:       condition variable for beacon flush
+ */
+struct iser_beacon {
+	union {
+		struct ib_send_wr	send;
+		struct ib_recv_wr	recv;
+	};
+	struct mtx		     flush_lock;
+	struct cv		     flush_cv;
+};
+
+/**
+ * struct ib_conn - Infiniband related objects
+ *
+ * @cma_id:              rdma_cm connection maneger handle
+ * @qp:                  Connection Queue-pair
+ * @device:              reference to iser device
+ * @comp:                iser completion context
+  */
+struct ib_conn {
+	struct rdma_cm_id           *cma_id;
+	struct ib_qp	            *qp;

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201605260949.u4Q9nTx1069928>