librdmacm: provide OFED compatibility library.

Provide a port of librdmacm.

Because of the use of overlapped structures and events, the library
has a scalability limitation of about 60 connections. The scalability limit will be addressed in a subsequent version, once the needs of a real application are determined.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>
git-svn-id: svn://openib.tc.cornell.edu/gen1/trunk@1879 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

diff --git a/ulp/dirs b/ulp/dirs
index 9de2e84..3cf61e2 100644 (file)
--- a/ulp/dirs
+++ b/ulp/dirs
@@ -1,5 +1,4 @@
 DIRS =                 \\r
 DIRS =                 \\r

-       opensm          \\r

        dapl            \\r
        dapl2           \\r
        ipoib           \\r
        dapl            \\r
        dapl2           \\r
        ipoib           \\r


@@ -9,4 +8,5 @@ DIRS =                  \
        libibverbs      \\r
        libibumad       \\r
        libibmad        \\r
        libibverbs      \\r
        libibumad       \\r
        libibmad        \\r

+       librdmacm       \\r

        nd\r
        nd\r

diff --git a/ulp/librdmacm/AUTHORS b/ulp/librdmacm/AUTHORS
new file mode 100644 (file)
index 0000000..589c93d
--- /dev/null
+++ b/ulp/librdmacm/AUTHORS
@@ -0,0 +1 @@
+Sean Hefty             <sean.hefty@intel.com>\r

diff --git a/ulp/librdmacm/COPYING b/ulp/librdmacm/COPYING
new file mode 100644 (file)
index 0000000..4136760
--- /dev/null
+++ b/ulp/librdmacm/COPYING
@@ -0,0 +1,26 @@
+Copyright (c) 2008 Intel Corporation.  All rights reserved.\r
+\r
+This software is available to you under the OpenFabrics.org BSD license\r
+below:\r
+\r
+     Redistribution and use in source and binary forms, with or\r
+     without modification, are permitted provided that the following\r
+     conditions are met:\r
+\r
+      - Redistributions of source code must retain the above\r
+        copyright notice, this list of conditions and the following\r
+        disclaimer.\r
+\r
+      - Redistributions in binary form must reproduce the above\r
+        copyright notice, this list of conditions and the following\r
+        disclaimer in the documentation and/or other materials\r
+        provided with the distribution.\r
+\r
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV\r
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+SOFTWARE.\r

diff --git a/ulp/librdmacm/dirs b/ulp/librdmacm/dirs
new file mode 100644 (file)
index 0000000..b105190
--- /dev/null
+++ b/ulp/librdmacm/dirs
@@ -0,0 +1,3 @@
+DIRS =                 \\r
+       src                     \\r
+       examples\r

diff --git a/ulp/librdmacm/examples/cmatose/SOURCES b/ulp/librdmacm/examples/cmatose/SOURCES
new file mode 100644 (file)
index 0000000..cfe59ff
--- /dev/null
+++ b/ulp/librdmacm/examples/cmatose/SOURCES
@@ -0,0 +1,30 @@
+TARGETNAME = rdma_cmatose\r
+TARGETPATH = ..\..\..\..\bin\user\obj$(BUILD_ALT_DIR)\r
+TARGETTYPE = PROGRAM\r
+\r
+UMTYPE = console\r
+UMENTRY = main\r
+\r
+USE_MSVCRT = 1\r
+USE_STL = 1\r
+USE_NATIVE_EH = 1\r
+USE_IOSTREAM = 1\r
+\r
+SOURCES = cmatose.c\r
+       \r
+INCLUDES = ..;..\..\include;..\..\..\..\inc;..\..\..\..\inc\user;..\..\..\libibverbs\include;\r
+\r
+TARGETLIBS =                                           \\r
+       $(SDK_LIB_PATH)\kernel32.lib    \\r
+       $(SDK_LIB_PATH)\advapi32.lib    \\r
+       $(SDK_LIB_PATH)\user32.lib              \\r
+       $(SDK_LIB_PATH)\ole32.lib               \\r
+       $(SDK_LIB_PATH)\ws2_32.lib              \\r
+!if $(FREEBUILD)\r
+       $(TARGETPATH)\*\libibverbs.lib  \\r
+       $(TARGETPATH)\*\librdmacm.lib\r
+!else\r
+       $(TARGETPATH)\*\libibverbsd.lib \\r
+       $(TARGETPATH)\*\librdmacmd.lib\r
+!endif\r
+\r

diff --git a/ulp/librdmacm/examples/cmatose/cmatose.c b/ulp/librdmacm/examples/cmatose/cmatose.c
new file mode 100644 (file)
index 0000000..7f32f27
--- /dev/null
+++ b/ulp/librdmacm/examples/cmatose/cmatose.c
@@ -0,0 +1,729 @@
+/*\r
+ * Copyright (c) 2005-2009 Intel Corporation.  All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ *     Redistribution and use in source and binary forms, with or\r
+ *     without modification, are permitted provided that the following\r
+ *     conditions are met:\r
+ *\r
+ *      - Redistributions of source code must retain the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer.\r
+ *\r
+ *      - Redistributions in binary form must reproduce the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer in the documentation and/or other materials\r
+ *        provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ */\r
+\r
+#include <stdio.h>\r
+#include <stdlib.h>\r
+#include <string.h>\r
+#include <ws2tcpip.h>\r
+#include <winsock2.h>\r
+#include <time.h>\r
+\r
+#include "..\..\..\..\etc\user\getopt.c"\r
+#include <rdma/rdma_cma.h>\r
+\r
+struct cmatest_node {\r
+       int                                     id;\r
+       struct rdma_cm_id       *cma_id;\r
+       int                                     connected;\r
+       struct ibv_pd           *pd;\r
+       struct ibv_cq           *cq[2];\r
+       struct ibv_mr           *mr;\r
+       void                            *mem;\r
+};\r
+\r
+enum cq_index {\r
+       SEND_CQ_INDEX,\r
+       RECV_CQ_INDEX\r
+};\r
+\r
+struct cmatest {\r
+       struct rdma_event_channel       *channel;\r
+       struct cmatest_node                     *nodes;\r
+       int                                                     conn_index;\r
+       int                                                     connects_left;\r
+       int                                                     disconnects_left;\r
+\r
+       struct sockaddr_in                      dst_in;\r
+       struct sockaddr                         *dst_addr;\r
+       struct sockaddr_in                      src_in;\r
+       struct sockaddr                         *src_addr;\r
+};\r
+\r
+static struct cmatest test;\r
+static int connections = 1;\r
+static int message_size = 100;\r
+static int message_count = 10;\r
+static uint16_t port = 7471;\r
+static uint8_t set_tos = 0;\r
+static uint8_t tos;\r
+static uint8_t migrate = 0;\r
+static char *dst_addr;\r
+static char *src_addr;\r
+\r
+static int create_message(struct cmatest_node *node)\r
+{\r
+       if (!message_size)\r
+               message_count = 0;\r
+\r
+       if (!message_count)\r
+               return 0;\r
+\r
+       node->mem = malloc(message_size);\r
+       if (!node->mem) {\r
+               printf("failed message allocation\n");\r
+               return -1;\r
+       }\r
+       node->mr = ibv_reg_mr(node->pd, node->mem, message_size,\r
+                            IBV_ACCESS_LOCAL_WRITE);\r
+       if (!node->mr) {\r
+               printf("failed to reg MR\n");\r
+               goto err;\r
+       }\r
+       return 0;\r
+err:\r
+       free(node->mem);\r
+       return -1;\r
+}\r
+\r
+static int init_node(struct cmatest_node *node)\r
+{\r
+       struct ibv_qp_init_attr init_qp_attr;\r
+       int cqe, ret;\r
+\r
+       node->pd = ibv_alloc_pd(node->cma_id->verbs);\r
+       if (!node->pd) {\r
+               ret = -1;\r
+               printf("cmatose: unable to allocate PD\n");\r
+               goto out;\r
+       }\r
+\r
+       cqe = message_count ? message_count : 1;\r
+       node->cq[SEND_CQ_INDEX] = ibv_create_cq(node->cma_id->verbs, cqe, node, 0, 0);\r
+       node->cq[RECV_CQ_INDEX] = ibv_create_cq(node->cma_id->verbs, cqe, node, 0, 0);\r
+       if (!node->cq[SEND_CQ_INDEX] || !node->cq[RECV_CQ_INDEX]) {\r
+               ret = -1;\r
+               printf("cmatose: unable to create CQs\n");\r
+               goto out;\r
+       }\r
+\r
+       memset(&init_qp_attr, 0, sizeof init_qp_attr);\r
+       init_qp_attr.cap.max_send_wr = cqe;\r
+       init_qp_attr.cap.max_recv_wr = cqe;\r
+       init_qp_attr.cap.max_send_sge = 1;\r
+       init_qp_attr.cap.max_recv_sge = 1;\r
+       init_qp_attr.qp_context = node;\r
+       init_qp_attr.sq_sig_all = 1;\r
+       init_qp_attr.qp_type = IBV_QPT_RC;\r
+       init_qp_attr.send_cq = node->cq[SEND_CQ_INDEX];\r
+       init_qp_attr.recv_cq = node->cq[RECV_CQ_INDEX];\r
+       ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr);\r
+       if (ret) {\r
+               printf("cmatose: unable to create QP: 0x%x\n", ret);\r
+               goto out;\r
+       }\r
+\r
+       ret = create_message(node);\r
+       if (ret) {\r
+               printf("cmatose: failed to create messages: 0x%x\n", ret);\r
+               goto out;\r
+       }\r
+out:\r
+       return ret;\r
+}\r
+\r
+static int post_recvs(struct cmatest_node *node)\r
+{\r
+       struct ibv_recv_wr recv_wr, *recv_failure;\r
+       struct ibv_sge sge;\r
+       int i, ret = 0;\r
+\r
+       if (!message_count)\r
+               return 0;\r
+\r
+       recv_wr.next = NULL;\r
+       recv_wr.sg_list = &sge;\r
+       recv_wr.num_sge = 1;\r
+       recv_wr.wr_id = (uintptr_t) node;\r
+\r
+       sge.length = message_size;\r
+       sge.lkey = node->mr->lkey;\r
+       sge.addr = (uintptr_t) node->mem;\r
+\r
+       for (i = 0; i < message_count && !ret; i++ ) {\r
+               ret = ibv_post_recv(node->cma_id->qp, &recv_wr, &recv_failure);\r
+               if (ret) {\r
+                       printf("failed to post receives: 0x%x\n", ret);\r
+                       break;\r
+               }\r
+       }\r
+       return ret;\r
+}\r
+\r
+static int post_sends(struct cmatest_node *node)\r
+{\r
+       struct ibv_send_wr send_wr, *bad_send_wr;\r
+       struct ibv_sge sge;\r
+       int i, ret = 0;\r
+\r
+       if (!node->connected || !message_count)\r
+               return 0;\r
+\r
+       send_wr.next = NULL;\r
+       send_wr.sg_list = &sge;\r
+       send_wr.num_sge = 1;\r
+       send_wr.opcode = IBV_WR_SEND;\r
+       send_wr.send_flags = 0;\r
+       send_wr.wr_id = (ULONG_PTR) node;\r
+\r
+       sge.length = message_size;\r
+       sge.lkey = node->mr->lkey;\r
+       sge.addr = (uintptr_t) node->mem;\r
+\r
+       for (i = 0; i < message_count && !ret; i++) {\r
+               ret = ibv_post_send(node->cma_id->qp, &send_wr, &bad_send_wr);\r
+               if (ret) \r
+                       printf("failed to post sends: 0x%x\n", ret);\r
+       }\r
+       return ret;\r
+}\r
+\r
+static void connect_error(void)\r
+{\r
+       test.disconnects_left--;\r
+       test.connects_left--;\r
+}\r
+\r
+static int addr_handler(struct cmatest_node *node)\r
+{\r
+       int ret;\r
+\r
+       if (set_tos) {\r
+               ret = rdma_set_option(node->cma_id, RDMA_OPTION_ID,\r
+                                     RDMA_OPTION_ID_TOS, &tos, sizeof tos);\r
+               if (ret)\r
+                       printf("cmatose: set TOS option failed: 0x%x\n", ret);\r
+       }\r
+\r
+       ret = rdma_resolve_route(node->cma_id, 2000);\r
+       if (ret) {\r
+               printf("cmatose: resolve route failed: 0x%x\n", ret);\r
+               connect_error();\r
+       }\r
+       return ret;\r
+}\r
+\r
+static int route_handler(struct cmatest_node *node)\r
+{\r
+       struct rdma_conn_param conn_param;\r
+       int ret;\r
+\r
+       ret = init_node(node);\r
+       if (ret)\r
+               goto err;\r
+\r
+       ret = post_recvs(node);\r
+       if (ret)\r
+               goto err;\r
+\r
+       memset(&conn_param, 0, sizeof conn_param);\r
+       conn_param.responder_resources = 1;\r
+       conn_param.initiator_depth = 1;\r
+       conn_param.retry_count = 5;\r
+       ret = rdma_connect(node->cma_id, &conn_param);\r
+       if (ret) {\r
+               printf("cmatose: failure connecting: 0x%x\n", ret);\r
+               goto err;\r
+       }\r
+       return 0;\r
+err:\r
+       connect_error();\r
+       return ret;\r
+}\r
+\r
+static int connect_handler(struct rdma_cm_id *cma_id)\r
+{\r
+       struct cmatest_node *node;\r
+       struct rdma_conn_param conn_param;\r
+       int ret;\r
+\r
+       if (test.conn_index == connections) {\r
+               ret = -1;\r
+               goto err1;\r
+       }\r
+       node = &test.nodes[test.conn_index++];\r
+\r
+       node->cma_id = cma_id;\r
+       cma_id->context = node;\r
+\r
+       ret = init_node(node);\r
+       if (ret)\r
+               goto err2;\r
+\r
+       ret = post_recvs(node);\r
+       if (ret)\r
+               goto err2;\r
+\r
+       memset(&conn_param, 0, sizeof conn_param);\r
+       conn_param.responder_resources = 1;\r
+       conn_param.initiator_depth = 1;\r
+       ret = rdma_accept(node->cma_id, &conn_param);\r
+       if (ret) {\r
+               printf("cmatose: failure accepting: 0x%x\n", ret);\r
+               goto err2;\r
+       }\r
+       return 0;\r
+\r
+err2:\r
+       node->cma_id = NULL;\r
+       connect_error();\r
+err1:\r
+       printf("cmatose: failing connection request\n");\r
+       rdma_reject(cma_id, NULL, 0);\r
+       return ret;\r
+}\r
+\r
+static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)\r
+{\r
+       int ret = 0;\r
+\r
+       switch (event->event) {\r
+       case RDMA_CM_EVENT_ADDR_RESOLVED:\r
+               ret = addr_handler(cma_id->context);\r
+               break;\r
+       case RDMA_CM_EVENT_ROUTE_RESOLVED:\r
+               ret = route_handler(cma_id->context);\r
+               break;\r
+       case RDMA_CM_EVENT_CONNECT_REQUEST:\r
+               ret = connect_handler(cma_id);\r
+               break;\r
+       case RDMA_CM_EVENT_ESTABLISHED:\r
+               ((struct cmatest_node *) cma_id->context)->connected = 1;\r
+               test.connects_left--;\r
+               break;\r
+       case RDMA_CM_EVENT_ADDR_ERROR:\r
+       case RDMA_CM_EVENT_ROUTE_ERROR:\r
+       case RDMA_CM_EVENT_CONNECT_ERROR:\r
+       case RDMA_CM_EVENT_UNREACHABLE:\r
+       case RDMA_CM_EVENT_REJECTED:\r
+               printf("cmatose: event: %s, error: 0x%x\n",\r
+                      rdma_event_str(event->event), event->status);\r
+               connect_error();\r
+               break;\r
+       case RDMA_CM_EVENT_DISCONNECTED:\r
+               rdma_disconnect(cma_id);\r
+               test.disconnects_left--;\r
+               break;\r
+       case RDMA_CM_EVENT_DEVICE_REMOVAL:\r
+               /* Cleanup will occur after test completes. */\r
+               break;\r
+       default:\r
+               break;\r
+       }\r
+       return ret;\r
+}\r
+\r
+static void destroy_node(struct cmatest_node *node)\r
+{\r
+       if (!node->cma_id)\r
+               return;\r
+\r
+       if (node->cma_id->qp)\r
+               rdma_destroy_qp(node->cma_id);\r
+\r
+       if (node->cq[SEND_CQ_INDEX])\r
+               ibv_destroy_cq(node->cq[SEND_CQ_INDEX]);\r
+\r
+       if (node->cq[RECV_CQ_INDEX])\r
+               ibv_destroy_cq(node->cq[RECV_CQ_INDEX]);\r
+\r
+       if (node->mem) {\r
+               ibv_dereg_mr(node->mr);\r
+               free(node->mem);\r
+       }\r
+\r
+       if (node->pd)\r
+               ibv_dealloc_pd(node->pd);\r
+\r
+       /* Destroy the RDMA ID after all device resources */\r
+       rdma_destroy_id(node->cma_id);\r
+}\r
+\r
+static int alloc_nodes(void)\r
+{\r
+       int ret, i;\r
+\r
+       test.nodes = malloc(sizeof *test.nodes * connections);\r
+       if (!test.nodes) {\r
+               printf("cmatose: unable to allocate memory for test nodes\n");\r
+               return -1;\r
+       }\r
+       memset(test.nodes, 0, sizeof *test.nodes * connections);\r
+\r
+       for (i = 0; i < connections; i++) {\r
+               test.nodes[i].id = i;\r
+               if (dst_addr) {\r
+                       ret = rdma_create_id(test.channel,\r
+                                            &test.nodes[i].cma_id,\r
+                                            &test.nodes[i], RDMA_PS_TCP);\r
+                       if (ret)\r
+                               goto err;\r
+               }\r
+       }\r
+       return 0;\r
+err:\r
+       while (--i >= 0)\r
+               rdma_destroy_id(test.nodes[i].cma_id);\r
+       free(test.nodes);\r
+       return ret;\r
+}\r
+\r
+static void destroy_nodes(void)\r
+{\r
+       int i;\r
+\r
+       for (i = 0; i < connections; i++)\r
+               destroy_node(&test.nodes[i]);\r
+       free(test.nodes);\r
+}\r
+\r
+static int poll_cqs(enum CQ_INDEX index)\r
+{\r
+       struct ibv_wc wc[8];\r
+       int done, i, ret;\r
+\r
+       for (i = 0; i < connections; i++) {\r
+               if (!test.nodes[i].connected)\r
+                       continue;\r
+\r
+               for (done = 0; done < message_count; done += ret) {\r
+                       ret = ibv_poll_cq(test.nodes[i].cq[index], 8, wc);\r
+                       if (ret < 0) {\r
+                               printf("cmatose: failed polling CQ: 0x%x\n", ret);\r
+                               return ret;\r
+                       }\r
+               }\r
+       }\r
+       return 0;\r
+}\r
+\r
+static int connect_events(void)\r
+{\r
+       struct rdma_cm_event *event;\r
+       int err = 0, ret = 0;\r
+\r
+       while (test.connects_left && !err) {\r
+               err = rdma_get_cm_event(test.channel, &event);\r
+               if (!err) {\r
+                       cma_handler(event->id, event);\r
+                       rdma_ack_cm_event(event);\r
+               } else {\r
+                       printf("cmatose: failure in rdma_get_cm_event in connect events\n");\r
+                       ret = err;\r
+               }\r
+       }\r
+\r
+       return ret;\r
+}\r
+\r
+static int disconnect_events(void)\r
+{\r
+       struct rdma_cm_event *event;\r
+       int err = 0, ret = 0;\r
+\r
+       while (test.disconnects_left && !err) {\r
+               err = rdma_get_cm_event(test.channel, &event);\r
+               if (!err) {\r
+                       cma_handler(event->id, event);\r
+                       rdma_ack_cm_event(event);\r
+               } else {\r
+                       printf("cmatose: failure in rdma_get_cm_event in disconnect events\n");\r
+                       ret = err;\r
+               }\r
+       }\r
+\r
+       return ret;\r
+}\r
+\r
+static int migrate_channel(struct rdma_cm_id *listen_id)\r
+{\r
+       struct rdma_event_channel *channel;\r
+       int i, ret;\r
+\r
+       printf("migrating to new event channel\n");\r
+\r
+       channel = rdma_create_event_channel();\r
+       if (!channel) {\r
+               printf("cmatose: failed to create event channel\n");\r
+               return -1;\r
+       }\r
+\r
+       ret = 0;\r
+       if (listen_id)\r
+               ret = rdma_migrate_id(listen_id, channel);\r
+\r
+       for (i = 0; i < connections && !ret; i++)\r
+               ret = rdma_migrate_id(test.nodes[i].cma_id, channel);\r
+\r
+       if (!ret) {\r
+               rdma_destroy_event_channel(test.channel);\r
+               test.channel = channel;\r
+       } else\r
+               printf("cmatose: failure migrating to channel: 0x%x\n", ret);\r
+\r
+       return ret;\r
+}\r
+\r
+static int get_addr(char *dst, struct sockaddr_in *addr)\r
+{\r
+       struct addrinfo *res;\r
+       int ret;\r
+\r
+       ret = getaddrinfo(dst, NULL, NULL, &res);\r
+       if (ret) {\r
+               printf("getaddrinfo failed - invalid hostname or IP address\n");\r
+               return ret;\r
+       }\r
+\r
+       if (res->ai_family != PF_INET) {\r
+               ret = -1;\r
+               goto out;\r
+       }\r
+\r
+       *addr = *(struct sockaddr_in *) res->ai_addr;\r
+out:\r
+       freeaddrinfo(res);\r
+       return ret;\r
+}\r
+\r
+static int run_server(void)\r
+{\r
+       struct rdma_cm_id *listen_id;\r
+       int i, ret;\r
+\r
+       printf("cmatose: starting server\n");\r
+       ret = rdma_create_id(test.channel, &listen_id, &test, RDMA_PS_TCP);\r
+       if (ret) {\r
+               printf("cmatose: listen request failed\n");\r
+               return ret;\r
+       }\r
+\r
+       if (src_addr) {\r
+               ret = get_addr(src_addr, &test.src_in);\r
+               if (ret)\r
+                       goto out;\r
+       } else\r
+               test.src_in.sin_family = PF_INET;\r
+\r
+       test.src_in.sin_port = port;\r
+       ret = rdma_bind_addr(listen_id, test.src_addr);\r
+       if (ret) {\r
+               printf("cmatose: bind address failed: 0x%x\n", ret);\r
+               goto out;\r
+       }\r
+\r
+       ret = rdma_listen(listen_id, 0);\r
+       if (ret) {\r
+               printf("cmatose: failure trying to listen: 0x%x\n", ret);\r
+               goto out;\r
+       }\r
+\r
+       ret = connect_events();\r
+       if (ret)\r
+               goto out;\r
+\r
+       if (message_count) {\r
+               printf("initiating data transfers\n");\r
+               for (i = 0; i < connections; i++) {\r
+                       ret = post_sends(&test.nodes[i]);\r
+                       if (ret)\r
+                               goto out;\r
+               }\r
+\r
+               printf("completing sends\n");\r
+               ret = poll_cqs(SEND_CQ_INDEX);\r
+               if (ret)\r
+                       goto out;\r
+\r
+               printf("receiving data transfers\n");\r
+               ret = poll_cqs(RECV_CQ_INDEX);\r
+               if (ret)\r
+                       goto out;\r
+               printf("data transfers complete\n");\r
+\r
+       }\r
+\r
+       if (migrate) {\r
+               ret = migrate_channel(listen_id);\r
+               if (ret)\r
+                       goto out;\r
+       }\r
+\r
+       printf("cmatose: disconnecting\n");\r
+       for (i = 0; i < connections; i++) {\r
+               if (!test.nodes[i].connected)\r
+                       continue;\r
+\r
+               test.nodes[i].connected = 0;\r
+               rdma_disconnect(test.nodes[i].cma_id);\r
+       }\r
+\r
+       ret = disconnect_events();\r
+\r
+       printf("disconnected\n");\r
+\r
+out:\r
+       rdma_destroy_id(listen_id);\r
+       return ret;\r
+}\r
+\r
+static int run_client(void)\r
+{\r
+       int i, ret, ret2;\r
+\r
+       printf("cmatose: starting client\n");\r
+       if (src_addr) {\r
+               ret = get_addr(src_addr, &test.src_in);\r
+               if (ret)\r
+                       return ret;\r
+       }\r
+\r
+       ret = get_addr(dst_addr, &test.dst_in);\r
+       if (ret)\r
+               return ret;\r
+\r
+       test.dst_in.sin_port = port;\r
+\r
+       printf("cmatose: connecting\n");\r
+       for (i = 0; i < connections; i++) {\r
+               ret = rdma_resolve_addr(test.nodes[i].cma_id,\r
+                                       src_addr ? test.src_addr : NULL,\r
+                                       test.dst_addr, 2000);\r
+               if (ret) {\r
+                       printf("cmatose: failure getting addr: 0x%x\n", ret);\r
+                       connect_error();\r
+                       return ret;\r
+               }\r
+       }\r
+\r
+       ret = connect_events();\r
+       if (ret)\r
+               goto disc;\r
+\r
+       if (message_count) {\r
+               printf("receiving data transfers\n");\r
+               ret = poll_cqs(RECV_CQ_INDEX);\r
+               if (ret)\r
+                       goto disc;\r
+\r
+               printf("sending replies\n");\r
+               for (i = 0; i < connections; i++) {\r
+                       ret = post_sends(&test.nodes[i]);\r
+                       if (ret)\r
+                               goto disc;\r
+               }\r
+\r
+               printf("data transfers complete\n");\r
+       }\r
+\r
+       ret = 0;\r
+\r
+       if (migrate) {\r
+               ret = migrate_channel(NULL);\r
+               if (ret)\r
+                       goto out;\r
+       }\r
+disc:\r
+       ret2 = disconnect_events();\r
+       if (ret2)\r
+               ret = ret2;\r
+out:\r
+       return ret;\r
+}\r
+\r
+int __cdecl main(int argc, char **argv)\r
+{\r
+       int op, ret;\r
+\r
+       while ((op = getopt(argc, argv, "s:b:c:C:S:t:p:m")) != -1) {\r
+               switch (op) {\r
+               case 's':\r
+                       dst_addr = optarg;\r
+                       break;\r
+               case 'b':\r
+                       src_addr = optarg;\r
+                       break;\r
+               case 'c':\r
+                       connections = atoi(optarg);\r
+                       break;\r
+               case 'C':\r
+                       message_count = atoi(optarg);\r
+                       break;\r
+               case 'S':\r
+                       message_size = atoi(optarg);\r
+                       break;\r
+               case 't':\r
+                       set_tos = 1;\r
+                       tos = (uint8_t) atoi(optarg);\r
+                       break;\r
+               case 'p':\r
+                       port = (uint16_t) atoi(optarg);\r
+                       break;\r
+               case 'm':\r
+                       migrate = 1;\r
+                       break;\r
+               default:\r
+                       printf("usage: %s\n", argv[0]);\r
+                       printf("\t[-s server_address]\n");\r
+                       printf("\t[-b bind_address]\n");\r
+                       printf("\t[-c connections]\n");\r
+                       printf("\t[-C message_count]\n");\r
+                       printf("\t[-S message_size]\n");\r
+                       printf("\t[-t type_of_service]\n");\r
+                       printf("\t[-p port_number]\n");\r
+                       printf("\t[-m(igrate)]\n");\r
+                       exit(1);\r
+               }\r
+       }\r
+\r
+       test.dst_addr = (struct sockaddr *) &test.dst_in;\r
+       test.src_addr = (struct sockaddr *) &test.src_in;\r
+       test.connects_left = connections;\r
+       test.disconnects_left = connections;\r
+\r
+       test.channel = rdma_create_event_channel();\r
+       if (!test.channel) {\r
+               printf("failed to create event channel\n");\r
+               exit(1);\r
+       }\r
+\r
+       if (alloc_nodes())\r
+               exit(1);\r
+\r
+       if (dst_addr)\r
+               ret = run_client();\r
+       else\r
+               ret = run_server();\r
+\r
+       printf("test complete\n");\r
+       destroy_nodes();\r
+       rdma_destroy_event_channel(test.channel);\r
+\r
+       printf("return status 0x%x\n", ret);\r
+       return ret;\r
+}\r

diff --git a/ulp/librdmacm/examples/cmatose/makefile b/ulp/librdmacm/examples/cmatose/makefile
new file mode 100644 (file)
index 0000000..a0c0627
--- /dev/null
+++ b/ulp/librdmacm/examples/cmatose/makefile
@@ -0,0 +1,7 @@
+#\r
+# DO NOT EDIT THIS FILE!!!  Edit .\sources. if you want to add a new source\r
+# file to this component.  This file merely indirects to the real make file\r
+# that is shared by all the driver components of the OpenIB Windows project.\r
+#\r
+\r
+!INCLUDE ..\..\..\..\inc\openib.def\r

diff --git a/ulp/librdmacm/examples/dirs b/ulp/librdmacm/examples/dirs
new file mode 100644 (file)
index 0000000..2bdf906
--- /dev/null
+++ b/ulp/librdmacm/examples/dirs
@@ -0,0 +1,2 @@
+DIRS =                 \\r
+       cmatose\r

diff --git a/ulp/librdmacm/examples/mckey/mckey.c b/ulp/librdmacm/examples/mckey/mckey.c
new file mode 100644 (file)
index 0000000..a858ad6
--- /dev/null
+++ b/ulp/librdmacm/examples/mckey/mckey.c
@@ -0,0 +1,579 @@
+/*\r
+ * Copyright (c) 2005-2007 Intel Corporation.  All rights reserved.\r
+ *\r
+ * This software is available to you under a choice of one of two\r
+ * licenses.  You may choose to be licensed under the terms of the GNU\r
+ * General Public License (GPL) Version 2, available from the file\r
+ * COPYING in the main directory of this source tree, or the\r
+ * OpenIB.org BSD license below:\r
+ *\r
+ *     Redistribution and use in source and binary forms, with or\r
+ *     without modification, are permitted provided that the following\r
+ *     conditions are met:\r
+ *\r
+ *      - Redistributions of source code must retain the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer.\r
+ *\r
+ *      - Redistributions in binary form must reproduce the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer in the documentation and/or other materials\r
+ *        provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ *\r
+ * $Id$\r
+ */\r
+\r
+#include <stdlib.h>\r
+#include <string.h>\r
+#include <stdio.h>\r
+#include <errno.h>\r
+#include <sys/types.h>\r
+#include <netinet/in.h>\r
+#include <arpa/inet.h>\r
+#include <sys/socket.h>\r
+#include <netdb.h>\r
+#include <byteswap.h>\r
+#include <unistd.h>\r
+#include <getopt.h>\r
+\r
+#include <rdma/rdma_cma.h>\r
+\r
+struct cmatest_node {\r
+       int                     id;\r
+       struct rdma_cm_id       *cma_id;\r
+       int                     connected;\r
+       struct ibv_pd           *pd;\r
+       struct ibv_cq           *cq;\r
+       struct ibv_mr           *mr;\r
+       struct ibv_ah           *ah;\r
+       uint32_t                remote_qpn;\r
+       uint32_t                remote_qkey;\r
+       void                    *mem;\r
+};\r
+\r
+struct cmatest {\r
+       struct rdma_event_channel *channel;\r
+       struct cmatest_node     *nodes;\r
+       int                     conn_index;\r
+       int                     connects_left;\r
+\r
+       struct sockaddr_in6     dst_in;\r
+       struct sockaddr         *dst_addr;\r
+       struct sockaddr_in6     src_in;\r
+       struct sockaddr         *src_addr;\r
+};\r
+\r
+static struct cmatest test;\r
+static int connections = 1;\r
+static int message_size = 100;\r
+static int message_count = 10;\r
+static int is_sender;\r
+static int unmapped_addr;\r
+static char *dst_addr;\r
+static char *src_addr;\r
+static enum rdma_port_space port_space = RDMA_PS_UDP;\r
+\r
+static int create_message(struct cmatest_node *node)\r
+{\r
+       if (!message_size)\r
+               message_count = 0;\r
+\r
+       if (!message_count)\r
+               return 0;\r
+\r
+       node->mem = malloc(message_size + sizeof(struct ibv_grh));\r
+       if (!node->mem) {\r
+               printf("failed message allocation\n");\r
+               return -1;\r
+       }\r
+       node->mr = ibv_reg_mr(node->pd, node->mem,\r
+                             message_size + sizeof(struct ibv_grh),\r
+                             IBV_ACCESS_LOCAL_WRITE);\r
+       if (!node->mr) {\r
+               printf("failed to reg MR\n");\r
+               goto err;\r
+       }\r
+       return 0;\r
+err:\r
+       free(node->mem);\r
+       return -1;\r
+}\r
+\r
+static int verify_test_params(struct cmatest_node *node)\r
+{\r
+       struct ibv_port_attr port_attr;\r
+       int ret;\r
+\r
+       ret = ibv_query_port(node->cma_id->verbs, node->cma_id->port_num,\r
+                            &port_attr);\r
+       if (ret)\r
+               return ret;\r
+\r
+       if (message_count && message_size > (1 << (port_attr.active_mtu + 7))) {\r
+               printf("mckey: message_size %d is larger than active mtu %d\n",\r
+                      message_size, 1 << (port_attr.active_mtu + 7));\r
+               return -EINVAL;\r
+       }\r
+\r
+       return 0;\r
+}\r
+\r
+static int init_node(struct cmatest_node *node)\r
+{\r
+       struct ibv_qp_init_attr init_qp_attr;\r
+       int cqe, ret;\r
+\r
+       node->pd = ibv_alloc_pd(node->cma_id->verbs);\r
+       if (!node->pd) {\r
+               ret = -ENOMEM;\r
+               printf("mckey: unable to allocate PD\n");\r
+               goto out;\r
+       }\r
+\r
+       cqe = message_count ? message_count * 2 : 2;\r
+       node->cq = ibv_create_cq(node->cma_id->verbs, cqe, node, 0, 0);\r
+       if (!node->cq) {\r
+               ret = -ENOMEM;\r
+               printf("mckey: unable to create CQ\n");\r
+               goto out;\r
+       }\r
+\r
+       memset(&init_qp_attr, 0, sizeof init_qp_attr);\r
+       init_qp_attr.cap.max_send_wr = message_count ? message_count : 1;\r
+       init_qp_attr.cap.max_recv_wr = message_count ? message_count : 1;\r
+       init_qp_attr.cap.max_send_sge = 1;\r
+       init_qp_attr.cap.max_recv_sge = 1;\r
+       init_qp_attr.qp_context = node;\r
+       init_qp_attr.sq_sig_all = 0;\r
+       init_qp_attr.qp_type = IBV_QPT_UD;\r
+       init_qp_attr.send_cq = node->cq;\r
+       init_qp_attr.recv_cq = node->cq;\r
+       ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr);\r
+       if (ret) {\r
+               printf("mckey: unable to create QP: %d\n", ret);\r
+               goto out;\r
+       }\r
+\r
+       ret = create_message(node);\r
+       if (ret) {\r
+               printf("mckey: failed to create messages: %d\n", ret);\r
+               goto out;\r
+       }\r
+out:\r
+       return ret;\r
+}\r
+\r
+static int post_recvs(struct cmatest_node *node)\r
+{\r
+       struct ibv_recv_wr recv_wr, *recv_failure;\r
+       struct ibv_sge sge;\r
+       int i, ret = 0;\r
+\r
+       if (!message_count)\r
+               return 0;\r
+\r
+       recv_wr.next = NULL;\r
+       recv_wr.sg_list = &sge;\r
+       recv_wr.num_sge = 1;\r
+       recv_wr.wr_id = (uintptr_t) node;\r
+\r
+       sge.length = message_size + sizeof(struct ibv_grh);\r
+       sge.lkey = node->mr->lkey;\r
+       sge.addr = (uintptr_t) node->mem;\r
+\r
+       for (i = 0; i < message_count && !ret; i++ ) {\r
+               ret = ibv_post_recv(node->cma_id->qp, &recv_wr, &recv_failure);\r
+               if (ret) {\r
+                       printf("failed to post receives: %d\n", ret);\r
+                       break;\r
+               }\r
+       }\r
+       return ret;\r
+}\r
+\r
+static int post_sends(struct cmatest_node *node, int signal_flag)\r
+{\r
+       struct ibv_send_wr send_wr, *bad_send_wr;\r
+       struct ibv_sge sge;\r
+       int i, ret = 0;\r
+\r
+       if (!node->connected || !message_count)\r
+               return 0;\r
+\r
+       send_wr.next = NULL;\r
+       send_wr.sg_list = &sge;\r
+       send_wr.num_sge = 1;\r
+       send_wr.opcode = IBV_WR_SEND_WITH_IMM;\r
+       send_wr.send_flags = signal_flag;\r
+       send_wr.wr_id = (unsigned long)node;\r
+       send_wr.imm_data = htonl(node->cma_id->qp->qp_num);\r
+\r
+       send_wr.wr.ud.ah = node->ah;\r
+       send_wr.wr.ud.remote_qpn = node->remote_qpn;\r
+       send_wr.wr.ud.remote_qkey = node->remote_qkey;\r
+\r
+       sge.length = message_size;\r
+       sge.lkey = node->mr->lkey;\r
+       sge.addr = (uintptr_t) node->mem;\r
+\r
+       for (i = 0; i < message_count && !ret; i++) {\r
+               ret = ibv_post_send(node->cma_id->qp, &send_wr, &bad_send_wr);\r
+               if (ret)\r
+                       printf("failed to post sends: %d\n", ret);\r
+       }\r
+       return ret;\r
+}\r
+\r
+static void connect_error(void)\r
+{\r
+       test.connects_left--;\r
+}\r
+\r
+static int addr_handler(struct cmatest_node *node)\r
+{\r
+       int ret;\r
+\r
+       ret = verify_test_params(node);\r
+       if (ret)\r
+               goto err;\r
+\r
+       ret = init_node(node);\r
+       if (ret)\r
+               goto err;\r
+\r
+       if (!is_sender) {\r
+               ret = post_recvs(node);\r
+               if (ret)\r
+                       goto err;\r
+       }\r
+\r
+       ret = rdma_join_multicast(node->cma_id, test.dst_addr, node);\r
+       if (ret) {\r
+               printf("mckey: failure joining: %d\n", ret);\r
+               goto err;\r
+       }\r
+       return 0;\r
+err:\r
+       connect_error();\r
+       return ret;\r
+}\r
+\r
+static int join_handler(struct cmatest_node *node,\r
+                       struct rdma_ud_param *param)\r
+{\r
+       char buf[40];\r
+\r
+       inet_ntop(AF_INET6, param->ah_attr.grh.dgid.raw, buf, 40);\r
+       printf("mckey: joined dgid: %s\n", buf);\r
+\r
+       node->remote_qpn = param->qp_num;\r
+       node->remote_qkey = param->qkey;\r
+       node->ah = ibv_create_ah(node->pd, &param->ah_attr);\r
+       if (!node->ah) {\r
+               printf("mckey: failure creating address handle\n");\r
+               goto err;\r
+       }\r
+\r
+       node->connected = 1;\r
+       test.connects_left--;\r
+       return 0;\r
+err:\r
+       connect_error();\r
+       return -1;\r
+}\r
+\r
+static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)\r
+{\r
+       int ret = 0;\r
+\r
+       switch (event->event) {\r
+       case RDMA_CM_EVENT_ADDR_RESOLVED:\r
+               ret = addr_handler(cma_id->context);\r
+               break;\r
+       case RDMA_CM_EVENT_MULTICAST_JOIN:\r
+               ret = join_handler(cma_id->context, &event->param.ud);\r
+               break;\r
+       case RDMA_CM_EVENT_ADDR_ERROR:\r
+       case RDMA_CM_EVENT_ROUTE_ERROR:\r
+       case RDMA_CM_EVENT_MULTICAST_ERROR:\r
+               printf("mckey: event: %s, error: %d\n",\r
+                      rdma_event_str(event->event), event->status);\r
+               connect_error();\r
+               ret = event->status;\r
+               break;\r
+       case RDMA_CM_EVENT_DEVICE_REMOVAL:\r
+               /* Cleanup will occur after test completes. */\r
+               break;\r
+       default:\r
+               break;\r
+       }\r
+       return ret;\r
+}\r
+\r
+static void destroy_node(struct cmatest_node *node)\r
+{\r
+       if (!node->cma_id)\r
+               return;\r
+\r
+       if (node->ah)\r
+               ibv_destroy_ah(node->ah);\r
+\r
+       if (node->cma_id->qp)\r
+               rdma_destroy_qp(node->cma_id);\r
+\r
+       if (node->cq)\r
+               ibv_destroy_cq(node->cq);\r
+\r
+       if (node->mem) {\r
+               ibv_dereg_mr(node->mr);\r
+               free(node->mem);\r
+       }\r
+\r
+       if (node->pd)\r
+               ibv_dealloc_pd(node->pd);\r
+\r
+       /* Destroy the RDMA ID after all device resources */\r
+       rdma_destroy_id(node->cma_id);\r
+}\r
+\r
+static int alloc_nodes(void)\r
+{\r
+       int ret, i;\r
+\r
+       test.nodes = malloc(sizeof *test.nodes * connections);\r
+       if (!test.nodes) {\r
+               printf("mckey: unable to allocate memory for test nodes\n");\r
+               return -ENOMEM;\r
+       }\r
+       memset(test.nodes, 0, sizeof *test.nodes * connections);\r
+\r
+       for (i = 0; i < connections; i++) {\r
+               test.nodes[i].id = i;\r
+               ret = rdma_create_id(test.channel, &test.nodes[i].cma_id,\r
+                                    &test.nodes[i], port_space);\r
+               if (ret)\r
+                       goto err;\r
+       }\r
+       return 0;\r
+err:\r
+       while (--i >= 0)\r
+               rdma_destroy_id(test.nodes[i].cma_id);\r
+       free(test.nodes);\r
+       return ret;\r
+}\r
+\r
+static void destroy_nodes(void)\r
+{\r
+       int i;\r
+\r
+       for (i = 0; i < connections; i++)\r
+               destroy_node(&test.nodes[i]);\r
+       free(test.nodes);\r
+}\r
+\r
+static int poll_cqs(void)\r
+{\r
+       struct ibv_wc wc[8];\r
+       int done, i, ret;\r
+\r
+       for (i = 0; i < connections; i++) {\r
+               if (!test.nodes[i].connected)\r
+                       continue;\r
+\r
+               for (done = 0; done < message_count; done += ret) {\r
+                       ret = ibv_poll_cq(test.nodes[i].cq, 8, wc);\r
+                       if (ret < 0) {\r
+                               printf("mckey: failed polling CQ: %d\n", ret);\r
+                               return ret;\r
+                       }\r
+               }\r
+       }\r
+       return 0;\r
+}\r
+\r
+static int connect_events(void)\r
+{\r
+       struct rdma_cm_event *event;\r
+       int ret = 0;\r
+\r
+       while (test.connects_left && !ret) {\r
+               ret = rdma_get_cm_event(test.channel, &event);\r
+               if (!ret) {\r
+                       ret = cma_handler(event->id, event);\r
+                       rdma_ack_cm_event(event);\r
+               }\r
+       }\r
+       return ret;\r
+}\r
+\r
+static int get_addr(char *dst, struct sockaddr *addr)\r
+{\r
+       struct addrinfo *res;\r
+       int ret;\r
+\r
+       ret = getaddrinfo(dst, NULL, NULL, &res);\r
+       if (ret) {\r
+               printf("getaddrinfo failed - invalid hostname or IP address\n");\r
+               return ret;\r
+       }\r
+\r
+       memcpy(addr, res->ai_addr, res->ai_addrlen);\r
+       freeaddrinfo(res);\r
+       return ret;\r
+}\r
+\r
+static int run(void)\r
+{\r
+       int i, ret;\r
+\r
+       printf("mckey: starting %s\n", is_sender ? "client" : "server");\r
+       if (src_addr) {\r
+               ret = get_addr(src_addr, (struct sockaddr *) &test.src_in);\r
+               if (ret)\r
+                       return ret;\r
+       }\r
+\r
+       ret = get_addr(dst_addr, (struct sockaddr *) &test.dst_in);\r
+       if (ret)\r
+               return ret;\r
+\r
+       printf("mckey: joining\n");\r
+       for (i = 0; i < connections; i++) {\r
+               if (src_addr) {\r
+                       ret = rdma_bind_addr(test.nodes[i].cma_id,\r
+                                            test.src_addr);\r
+                       if (ret) {\r
+                               printf("mckey: addr bind failure: %d\n", ret);\r
+                               connect_error();\r
+                               return ret;\r
+                       }\r
+               }\r
+\r
+               if (unmapped_addr)\r
+                       ret = addr_handler(&test.nodes[i]);\r
+               else\r
+                       ret = rdma_resolve_addr(test.nodes[i].cma_id,\r
+                                               test.src_addr, test.dst_addr,\r
+                                               2000);\r
+               if (ret) {\r
+                       printf("mckey: resolve addr failure: %d\n", ret);\r
+                       connect_error();\r
+                       return ret;\r
+               }\r
+       }\r
+\r
+       ret = connect_events();\r
+       if (ret)\r
+               goto out;\r
+\r
+       /*\r
+        * Pause to give SM chance to configure switches.  We don't want to\r
+        * handle reliability issue in this simple test program.\r
+        */\r
+       sleep(3);\r
+\r
+       if (message_count) {\r
+               if (is_sender) {\r
+                       printf("initiating data transfers\n");\r
+                       for (i = 0; i < connections; i++) {\r
+                               ret = post_sends(&test.nodes[i], 0);\r
+                               if (ret)\r
+                                       goto out;\r
+                       }\r
+               } else {\r
+                       printf("receiving data transfers\n");\r
+                       ret = poll_cqs();\r
+                       if (ret)\r
+                               goto out;\r
+               }\r
+               printf("data transfers complete\n");\r
+       }\r
+out:\r
+       for (i = 0; i < connections; i++) {\r
+               ret = rdma_leave_multicast(test.nodes[i].cma_id,\r
+                                          test.dst_addr);\r
+               if (ret)\r
+                       printf("mckey: failure leaving: %d\n", ret);\r
+       }\r
+       return ret;\r
+}\r
+\r
+int main(int argc, char **argv)\r
+{\r
+       int op, ret;\r
+\r
+\r
+       while ((op = getopt(argc, argv, "m:M:sb:c:C:S:p:")) != -1) {\r
+               switch (op) {\r
+               case 'm':\r
+                       dst_addr = optarg;\r
+                       break;\r
+               case 'M':\r
+                       unmapped_addr = 1;\r
+                       dst_addr = optarg;\r
+                       break;\r
+               case 's':\r
+                       is_sender = 1;\r
+                       break;\r
+               case 'b':\r
+                       src_addr = optarg;\r
+                       test.src_addr = (struct sockaddr *) &test.src_in;\r
+                       break;\r
+               case 'c':\r
+                       connections = atoi(optarg);\r
+                       break;\r
+               case 'C':\r
+                       message_count = atoi(optarg);\r
+                       break;\r
+               case 'S':\r
+                       message_size = atoi(optarg);\r
+                       break;\r
+               case 'p':\r
+                       port_space = strtol(optarg, NULL, 0);\r
+                       break;\r
+               default:\r
+                       printf("usage: %s\n", argv[0]);\r
+                       printf("\t-m multicast_address\n");\r
+                       printf("\t[-M unmapped_multicast_address]\n"\r
+                              "\t replaces -m and requires -b\n");\r
+                       printf("\t[-s(ender)]\n");\r
+                       printf("\t[-b bind_address]\n");\r
+                       printf("\t[-c connections]\n");\r
+                       printf("\t[-C message_count]\n");\r
+                       printf("\t[-S message_size]\n");\r
+                       printf("\t[-p port_space - %#x for UDP (default), "\r
+                              "%#x for IPOIB]\n", RDMA_PS_UDP, RDMA_PS_IPOIB);\r
+                       exit(1);\r
+               }\r
+       }\r
+\r
+       test.dst_addr = (struct sockaddr *) &test.dst_in;\r
+       test.connects_left = connections;\r
+\r
+       test.channel = rdma_create_event_channel();\r
+       if (!test.channel) {\r
+               printf("failed to create event channel\n");\r
+               exit(1);\r
+       }\r
+\r
+       if (alloc_nodes())\r
+               exit(1);\r
+\r
+       ret = run();\r
+\r
+       printf("test complete\n");\r
+       destroy_nodes();\r
+       rdma_destroy_event_channel(test.channel);\r
+\r
+       printf("return status %d\n", ret);\r
+       return ret;\r
+}\r

diff --git a/ulp/librdmacm/examples/rping/rping.c b/ulp/librdmacm/examples/rping/rping.c
new file mode 100644 (file)
index 0000000..1e264fe
--- /dev/null
+++ b/ulp/librdmacm/examples/rping/rping.c
@@ -0,0 +1,1122 @@
+/*\r
+ * Copyright (c) 2005 Ammasso, Inc. All rights reserved.\r
+ * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.\r
+ * Copyright (c) 2009 Intel Corp.  All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ *     Redistribution and use in source and binary forms, with or\r
+ *     without modification, are permitted provided that the following\r
+ *     conditions are met:\r
+ *\r
+ *      - Redistributions of source code must retain the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer.\r
+ *\r
+ *      - Redistributions in binary form must reproduce the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer in the documentation and/or other materials\r
+ *        provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ */\r
+\r
+#include <stdio.h>\r
+#include <stdlib.h>\r
+#include <string.h>\r
+#include <ws2tcpip.h>\r
+#include <winsock2.h>\r
+#include <time.h>\r
+\r
+#include "..\..\..\..\etc\user\getopt.c"\r
+#include <rdma/rdma_cma.h>\r
+\r
+#include <rdma/rdma_cma.h>\r
+\r
+static int debug = 0;\r
+#define DEBUG_LOG if (debug) printf\r
+\r
+/*\r
+ * rping "ping/pong" loop:\r
+ *     client sends source rkey/addr/len\r
+ *     server receives source rkey/add/len\r
+ *     server rdma reads "ping" data from source\r
+ *     server sends "go ahead" on rdma read completion\r
+ *     client sends sink rkey/addr/len\r
+ *     server receives sink rkey/addr/len\r
+ *     server rdma writes "pong" data to sink\r
+ *     server sends "go ahead" on rdma write completion\r
+ *     <repeat loop>\r
+ */\r
+\r
+/*\r
+ * These states are used to signal events between the completion handler\r
+ * and the main client or server thread.\r
+ *\r
+ * Once CONNECTED, they cycle through RDMA_READ_ADV, RDMA_WRITE_ADV, \r
+ * and RDMA_WRITE_COMPLETE for each ping.\r
+ */\r
+enum test_state {\r
+       IDLE = 1,\r
+       CONNECT_REQUEST,\r
+       ADDR_RESOLVED,\r
+       ROUTE_RESOLVED,\r
+       CONNECTED,\r
+       RDMA_READ_ADV,\r
+       RDMA_READ_COMPLETE,\r
+       RDMA_WRITE_ADV,\r
+       RDMA_WRITE_COMPLETE,\r
+       RDMA_ERROR\r
+};\r
+\r
+struct rping_rdma_info {\r
+       uint64_t buf;\r
+       uint32_t rkey;\r
+       uint32_t size;\r
+};\r
+\r
+#define RPING_SQ_DEPTH 16\r
+\r
+#define RPING_MSG_FMT           "rdma-ping-%d: "\r
+#define RPING_MIN_BUFSIZE       16\r
+\r
+/*\r
+ * Control block struct.\r
+ */\r
+struct rping_cb {\r
+       int server;                     /* 0 iff client */\r
+       pthread_t cqthread;\r
+       struct ibv_comp_channel *channel;\r
+       struct ibv_cq *cq;\r
+       struct ibv_pd *pd;\r
+       struct ibv_qp *qp;\r
+\r
+       struct ibv_recv_wr rq_wr;       /* recv work request record */\r
+       struct ibv_sge recv_sgl;        /* recv single SGE */\r
+       struct rping_rdma_info recv_buf;/* malloc'd buffer */\r
+       struct ibv_mr *recv_mr;         /* MR associated with this buffer */\r
+\r
+       struct ibv_send_wr sq_wr;       /* send work request record */\r
+       struct ibv_sge send_sgl;\r
+       struct rping_rdma_info send_buf;/* single send buf */\r
+       struct ibv_mr *send_mr;\r
+\r
+       struct ibv_send_wr rdma_sq_wr;  /* rdma work request record */\r
+       struct ibv_sge rdma_sgl;        /* rdma single SGE */\r
+       char *rdma_buf;                 /* used as rdma sink */\r
+       struct ibv_mr *rdma_mr;\r
+\r
+       uint32_t remote_rkey;           /* remote guys RKEY */\r
+       uint64_t remote_addr;           /* remote guys TO */\r
+       uint32_t remote_len;            /* remote guys LEN */\r
+\r
+       char *start_buf;                /* rdma read src */\r
+       struct ibv_mr *start_mr;\r
+\r
+       enum test_state state;          /* used for cond/signalling */\r
+//     sem_t sem;\r
+\r
+       struct sockaddr_in sin;\r
+       uint16_t port;                  /* dst port in NBO */\r
+       int verbose;                    /* verbose logging */\r
+       int count;                      /* ping count */\r
+       int size;                       /* ping data size */\r
+       int validate;                   /* validate ping data */\r
+\r
+       /* CM stuff */\r
+//     pthread_t cmthread;\r
+       struct rdma_event_channel *cm_channel;\r
+       struct rdma_cm_id *cm_id;       /* connection on client side,*/\r
+                                       /* listener on service side. */\r
+       struct rdma_cm_id *child_cm_id; /* connection on server side */\r
+};\r
+\r
+struct rping_cb *cb;\r
+static void *cm_thread(void *arg);\r
+static void *cq_thread(void *arg);\r
+\r
+static int rping_cma_event_handler(struct rdma_cm_id *cma_id,\r
+                                   struct rdma_cm_event *event)\r
+{\r
+       int ret = 0;\r
+       struct rping_cb *cb = cma_id->context;\r
+\r
+       DEBUG_LOG("cma_event type %s cma_id %p (%s)\n",\r
+                 rdma_event_str(event->event), cma_id,\r
+                 (cma_id == cb->cm_id) ? "parent" : "child");\r
+\r
+       switch (event->event) {\r
+       case RDMA_CM_EVENT_ADDR_RESOLVED:\r
+               cb->state = ADDR_RESOLVED;\r
+               ret = rdma_resolve_route(cma_id, 2000);\r
+               if (ret) {\r
+                       cb->state = RDMA_ERROR;\r
+                       fprintf(stderr, "rdma_resolve_route error %d\n", ret);\r
+//                     sem_post(&cb->sem);\r
+               }\r
+               break;\r
+\r
+       case RDMA_CM_EVENT_ROUTE_RESOLVED:\r
+               cb->state = ROUTE_RESOLVED;\r
+//             sem_post(&cb->sem);\r
+               break;\r
+\r
+       case RDMA_CM_EVENT_CONNECT_REQUEST:\r
+               cb->state = CONNECT_REQUEST;\r
+               cb->child_cm_id = cma_id;\r
+               DEBUG_LOG("child cma %p\n", cb->child_cm_id);\r
+//             sem_post(&cb->sem);\r
+               break;\r
+\r
+       case RDMA_CM_EVENT_ESTABLISHED:\r
+               DEBUG_LOG("ESTABLISHED\n");\r
+\r
+               /*\r
+                * Server will wake up when first RECV completes.\r
+                */\r
+               if (!cb->server) {\r
+                       cb->state = CONNECTED;\r
+               }\r
+//             sem_post(&cb->sem);\r
+               break;\r
+\r
+       case RDMA_CM_EVENT_ADDR_ERROR:\r
+       case RDMA_CM_EVENT_ROUTE_ERROR:\r
+       case RDMA_CM_EVENT_CONNECT_ERROR:\r
+       case RDMA_CM_EVENT_UNREACHABLE:\r
+       case RDMA_CM_EVENT_REJECTED:\r
+               fprintf(stderr, "cma event %s, error %d\n",\r
+                       rdma_event_str(event->event), event->status);\r
+//             sem_post(&cb->sem);\r
+               ret = -1;\r
+               break;\r
+\r
+       case RDMA_CM_EVENT_DISCONNECTED:\r
+               fprintf(stderr, "%s DISCONNECT EVENT...\n",\r
+                       cb->server ? "server" : "client");\r
+//             sem_post(&cb->sem);\r
+               break;\r
+\r
+       case RDMA_CM_EVENT_DEVICE_REMOVAL:\r
+               fprintf(stderr, "cma detected device removal!!!!\n");\r
+               ret = -1;\r
+               break;\r
+\r
+       default:\r
+               fprintf(stderr, "unhandled event: %s, ignoring\n",\r
+                       rdma_event_str(event->event));\r
+               break;\r
+       }\r
+\r
+       return ret;\r
+}\r
+\r
+static int server_recv(struct rping_cb *cb, struct ibv_wc *wc)\r
+{\r
+       if (wc->byte_len != sizeof(cb->recv_buf)) {\r
+               fprintf(stderr, "Received bogus data, size %d\n", wc->byte_len);\r
+               return -1;\r
+       }\r
+\r
+       cb->remote_rkey = ntohl(cb->recv_buf.rkey);\r
+       cb->remote_addr = ntohll(cb->recv_buf.buf);\r
+       cb->remote_len  = ntohl(cb->recv_buf.size);\r
+       DEBUG_LOG("Received rkey %x addr %" PRIx64 " len %d from peer\n",\r
+                 cb->remote_rkey, cb->remote_addr, cb->remote_len);\r
+\r
+       if (cb->state <= CONNECTED || cb->state == RDMA_WRITE_COMPLETE)\r
+               cb->state = RDMA_READ_ADV;\r
+       else\r
+               cb->state = RDMA_WRITE_ADV;\r
+\r
+       return 0;\r
+}\r
+\r
+static int client_recv(struct rping_cb *cb, struct ibv_wc *wc)\r
+{\r
+       if (wc->byte_len != sizeof(cb->recv_buf)) {\r
+               fprintf(stderr, "Received bogus data, size %d\n", wc->byte_len);\r
+               return -1;\r
+       }\r
+\r
+       if (cb->state == RDMA_READ_ADV)\r
+               cb->state = RDMA_WRITE_ADV;\r
+       else\r
+               cb->state = RDMA_WRITE_COMPLETE;\r
+\r
+       return 0;\r
+}\r
+\r
+static int rping_cq_event_handler(struct rping_cb *cb)\r
+{\r
+       struct ibv_wc wc;\r
+       struct ibv_recv_wr *bad_wr;\r
+       int ret;\r
+\r
+       while ((ret = ibv_poll_cq(cb->cq, 1, &wc)) == 1) {\r
+               ret = 0;\r
+\r
+               if (wc.status) {\r
+                       fprintf(stderr, "cq completion failed status %d\n",\r
+                               wc.status);\r
+                       if (wc.status != IBV_WC_WR_FLUSH_ERR)\r
+                               ret = -1;\r
+                       goto error;\r
+               }\r
+\r
+               switch (wc.opcode) {\r
+               case IBV_WC_SEND:\r
+                       DEBUG_LOG("send completion\n");\r
+                       break;\r
+\r
+               case IBV_WC_RDMA_WRITE:\r
+                       DEBUG_LOG("rdma write completion\n");\r
+                       cb->state = RDMA_WRITE_COMPLETE;\r
+//                     sem_post(&cb->sem);\r
+                       break;\r
+\r
+               case IBV_WC_RDMA_READ:\r
+                       DEBUG_LOG("rdma read completion\n");\r
+                       cb->state = RDMA_READ_COMPLETE;\r
+//                     sem_post(&cb->sem);\r
+                       break;\r
+\r
+               case IBV_WC_RECV:\r
+                       DEBUG_LOG("recv completion\n");\r
+                       ret = cb->server ? server_recv(cb, &wc) :\r
+                                          client_recv(cb, &wc);\r
+                       if (ret) {\r
+                               fprintf(stderr, "recv wc error: %d\n", ret);\r
+                               goto error;\r
+                       }\r
+\r
+                       ret = ibv_post_recv(cb->qp, &cb->rq_wr, &bad_wr);\r
+                       if (ret) {\r
+                               fprintf(stderr, "post recv error: %d\n", ret);\r
+                               goto error;\r
+                       }\r
+//                     sem_post(&cb->sem);\r
+                       break;\r
+\r
+               default:\r
+                       DEBUG_LOG("unknown!!!!! completion\n");\r
+                       ret = -1;\r
+                       goto error;\r
+               }\r
+       }\r
+       if (ret) {\r
+               fprintf(stderr, "poll error %d\n", ret);\r
+               goto error;\r
+       }\r
+       return 0;\r
+\r
+error:\r
+       cb->state = RDMA_ERROR;\r
+//     sem_post(&cb->sem);\r
+       return ret;\r
+}\r
+\r
+static int rping_accept(struct rping_cb *cb)\r
+{\r
+       struct rdma_conn_param conn_param;\r
+       int ret;\r
+\r
+       DEBUG_LOG("accepting client connection request\n");\r
+\r
+       memset(&conn_param, 0, sizeof conn_param);\r
+       conn_param.responder_resources = 1;\r
+       conn_param.initiator_depth = 1;\r
+\r
+       ret = rdma_accept(cb->child_cm_id, &conn_param);\r
+       if (ret) {\r
+               fprintf(stderr, "rdma_accept error: %d\n", ret);\r
+               return ret;\r
+       }\r
+\r
+       cm_thread(cb);\r
+//     sem_wait(&cb->sem);\r
+       if (cb->state == RDMA_ERROR) {\r
+               fprintf(stderr, "wait for CONNECTED state %d\n", cb->state);\r
+               return -1;\r
+       }\r
+       return 0;\r
+}\r
+\r
+static void rping_setup_wr(struct rping_cb *cb)\r
+{\r
+       cb->recv_sgl.addr = (uint64_t) (unsigned long) &cb->recv_buf;\r
+       cb->recv_sgl.length = sizeof cb->recv_buf;\r
+       cb->recv_sgl.lkey = cb->recv_mr->lkey;\r
+       cb->rq_wr.sg_list = &cb->recv_sgl;\r
+       cb->rq_wr.num_sge = 1;\r
+\r
+       cb->send_sgl.addr = (uint64_t) (unsigned long) &cb->send_buf;\r
+       cb->send_sgl.length = sizeof cb->send_buf;\r
+       cb->send_sgl.lkey = cb->send_mr->lkey;\r
+\r
+       cb->sq_wr.opcode = IBV_WR_SEND;\r
+       cb->sq_wr.send_flags = IBV_SEND_SIGNALED;\r
+       cb->sq_wr.sg_list = &cb->send_sgl;\r
+       cb->sq_wr.num_sge = 1;\r
+\r
+       cb->rdma_sgl.addr = (uint64_t) (unsigned long) cb->rdma_buf;\r
+       cb->rdma_sgl.lkey = cb->rdma_mr->lkey;\r
+       cb->rdma_sq_wr.send_flags = IBV_SEND_SIGNALED;\r
+       cb->rdma_sq_wr.sg_list = &cb->rdma_sgl;\r
+       cb->rdma_sq_wr.num_sge = 1;\r
+}\r
+\r
+static int rping_setup_buffers(struct rping_cb *cb)\r
+{\r
+       int ret;\r
+\r
+       DEBUG_LOG("rping_setup_buffers called on cb %p\n", cb);\r
+\r
+       cb->recv_mr = ibv_reg_mr(cb->pd, &cb->recv_buf, sizeof cb->recv_buf,\r
+                                IBV_ACCESS_LOCAL_WRITE);\r
+       if (!cb->recv_mr) {\r
+               fprintf(stderr, "recv_buf reg_mr failed\n");\r
+               return errno;\r
+       }\r
+\r
+       cb->send_mr = ibv_reg_mr(cb->pd, &cb->send_buf, sizeof cb->send_buf, 0);\r
+       if (!cb->send_mr) {\r
+               fprintf(stderr, "send_buf reg_mr failed\n");\r
+               ret = errno;\r
+               goto err1;\r
+       }\r
+\r
+       cb->rdma_buf = malloc(cb->size);\r
+       if (!cb->rdma_buf) {\r
+               fprintf(stderr, "rdma_buf malloc failed\n");\r
+               ret = -ENOMEM;\r
+               goto err2;\r
+       }\r
+\r
+       cb->rdma_mr = ibv_reg_mr(cb->pd, cb->rdma_buf, cb->size,\r
+                                IBV_ACCESS_LOCAL_WRITE |\r
+                                IBV_ACCESS_REMOTE_READ |\r
+                                IBV_ACCESS_REMOTE_WRITE);\r
+       if (!cb->rdma_mr) {\r
+               fprintf(stderr, "rdma_buf reg_mr failed\n");\r
+               ret = errno;\r
+               goto err3;\r
+       }\r
+\r
+       if (!cb->server) {\r
+               cb->start_buf = malloc(cb->size);\r
+               if (!cb->start_buf) {\r
+                       fprintf(stderr, "start_buf malloc failed\n");\r
+                       ret = -ENOMEM;\r
+                       goto err4;\r
+               }\r
+\r
+               cb->start_mr = ibv_reg_mr(cb->pd, cb->start_buf, cb->size,\r
+                                         IBV_ACCESS_LOCAL_WRITE | \r
+                                         IBV_ACCESS_REMOTE_READ |\r
+                                         IBV_ACCESS_REMOTE_WRITE);\r
+               if (!cb->start_mr) {\r
+                       fprintf(stderr, "start_buf reg_mr failed\n");\r
+                       ret = errno;\r
+                       goto err5;\r
+               }\r
+       }\r
+\r
+       rping_setup_wr(cb);\r
+       DEBUG_LOG("allocated & registered buffers...\n");\r
+       return 0;\r
+\r
+err5:\r
+       free(cb->start_buf);\r
+err4:\r
+       ibv_dereg_mr(cb->rdma_mr);\r
+err3:\r
+       free(cb->rdma_buf);\r
+err2:\r
+       ibv_dereg_mr(cb->send_mr);\r
+err1:\r
+       ibv_dereg_mr(cb->recv_mr);\r
+       return ret;\r
+}\r
+\r
+static void rping_free_buffers(struct rping_cb *cb)\r
+{\r
+       DEBUG_LOG("rping_free_buffers called on cb %p\n", cb);\r
+       ibv_dereg_mr(cb->recv_mr);\r
+       ibv_dereg_mr(cb->send_mr);\r
+       ibv_dereg_mr(cb->rdma_mr);\r
+       free(cb->rdma_buf);\r
+       if (!cb->server) {\r
+               ibv_dereg_mr(cb->start_mr);\r
+               free(cb->start_buf);\r
+       }\r
+}\r
+\r
+static int rping_create_qp(struct rping_cb *cb)\r
+{\r
+       struct ibv_qp_init_attr init_attr;\r
+       int ret;\r
+\r
+       memset(&init_attr, 0, sizeof(init_attr));\r
+       init_attr.cap.max_send_wr = RPING_SQ_DEPTH;\r
+       init_attr.cap.max_recv_wr = 2;\r
+       init_attr.cap.max_recv_sge = 1;\r
+       init_attr.cap.max_send_sge = 1;\r
+       init_attr.qp_type = IBV_QPT_RC;\r
+       init_attr.send_cq = cb->cq;\r
+       init_attr.recv_cq = cb->cq;\r
+\r
+       if (cb->server) {\r
+               ret = rdma_create_qp(cb->child_cm_id, cb->pd, &init_attr);\r
+               if (!ret)\r
+                       cb->qp = cb->child_cm_id->qp;\r
+       } else {\r
+               ret = rdma_create_qp(cb->cm_id, cb->pd, &init_attr);\r
+               if (!ret)\r
+                       cb->qp = cb->cm_id->qp;\r
+       }\r
+\r
+       return ret;\r
+}\r
+\r
+static void rping_free_qp(struct rping_cb *cb)\r
+{\r
+       ibv_destroy_qp(cb->qp);\r
+       ibv_destroy_cq(cb->cq);\r
+       ibv_destroy_comp_channel(cb->channel);\r
+       ibv_dealloc_pd(cb->pd);\r
+}\r
+\r
+static int rping_setup_qp(struct rping_cb *cb, struct rdma_cm_id *cm_id)\r
+{\r
+       int ret;\r
+\r
+       cb->pd = ibv_alloc_pd(cm_id->verbs);\r
+       if (!cb->pd) {\r
+               fprintf(stderr, "ibv_alloc_pd failed\n");\r
+               return errno;\r
+       }\r
+       DEBUG_LOG("created pd %p\n", cb->pd);\r
+\r
+       cb->channel = ibv_create_comp_channel(cm_id->verbs);\r
+       if (!cb->channel) {\r
+               fprintf(stderr, "ibv_create_comp_channel failed\n");\r
+               ret = errno;\r
+               goto err1;\r
+       }\r
+       DEBUG_LOG("created channel %p\n", cb->channel);\r
+\r
+       cb->cq = ibv_create_cq(cm_id->verbs, RPING_SQ_DEPTH * 2, cb,\r
+                               cb->channel, 0);\r
+       if (!cb->cq) {\r
+               fprintf(stderr, "ibv_create_cq failed\n");\r
+               ret = errno;\r
+               goto err2;\r
+       }\r
+       DEBUG_LOG("created cq %p\n", cb->cq);\r
+\r
+       ret = ibv_req_notify_cq(cb->cq, 0);\r
+       if (ret) {\r
+               fprintf(stderr, "ibv_create_cq failed\n");\r
+               ret = errno;\r
+               goto err3;\r
+       }\r
+\r
+       ret = rping_create_qp(cb);\r
+       if (ret) {\r
+               fprintf(stderr, "rping_create_qp failed: %d\n", ret);\r
+               goto err3;\r
+       }\r
+       DEBUG_LOG("created qp %p\n", cb->qp);\r
+       return 0;\r
+\r
+err3:\r
+       ibv_destroy_cq(cb->cq);\r
+err2:\r
+       ibv_destroy_comp_channel(cb->channel);\r
+err1:\r
+       ibv_dealloc_pd(cb->pd);\r
+       return ret;\r
+}\r
+\r
+static void cm_thread(void *arg)\r
+{\r
+       struct rping_cb *cb = arg;\r
+       struct rdma_cm_event *event;\r
+       int ret;\r
+\r
+//     while (1) {\r
+       ret = rdma_get_cm_event(cb->cm_channel, &event);\r
+       if (ret) {\r
+               fprintf(stderr, "rdma_get_cm_event err %d\n", ret);\r
+               return;\r
+//                     exit(ret);\r
+       }\r
+       ret = rping_cma_event_handler(event->id, event);\r
+       rdma_ack_cm_event(event);\r
+//             if (ret)\r
+//                     exit(ret);\r
+//     }\r
+}\r
+\r
+static void cq_thread(void *arg)\r
+{\r
+       struct rping_cb *cb = arg;\r
+       struct ibv_cq *ev_cq;\r
+       void *ev_ctx;\r
+       int ret;\r
+       \r
+//     DEBUG_LOG("cq_thread started.\n");\r
+\r
+//     while (1) {     \r
+//             pthread_testcancel();\r
+\r
+       ret = ibv_get_cq_event(cb->channel, &ev_cq, &ev_ctx);\r
+       if (ret) {\r
+               fprintf(stderr, "Failed to get cq event!\n");\r
+               return;\r
+//             pthread_exit(NULL);\r
+       }\r
+       if (ev_cq != cb->cq) {\r
+               fprintf(stderr, "Unknown CQ!\n");\r
+               return;\r
+//             pthread_exit(NULL);\r
+       }\r
+       ret = ibv_req_notify_cq(cb->cq, 0);\r
+       if (ret) {\r
+               fprintf(stderr, "Failed to set notify!\n");\r
+               return;\r
+//             pthread_exit(NULL);\r
+       }\r
+       ret = rping_cq_event_handler(cb);\r
+       ibv_ack_cq_events(cb->cq, 1);\r
+//             pthread_exit(NULL);\r
+//     }\r
+}\r
+\r
+static void rping_format_send(struct rping_cb *cb, char *buf, struct ibv_mr *mr)\r
+{\r
+       struct rping_rdma_info *info = &cb->send_buf;\r
+\r
+       info->buf = htonll((uint64_t) (unsigned long) buf);\r
+       info->rkey = htonl(mr->rkey);\r
+       info->size = htonl(cb->size);\r
+\r
+       DEBUG_LOG("RDMA addr %" PRIx64" rkey %x len %d\n",\r
+                 ntohll(info->buf), ntohl(info->rkey), ntohl(info->size));\r
+}\r
+\r
+static int rping_test_server(struct rping_cb *cb)\r
+{\r
+       struct ibv_send_wr *bad_wr;\r
+       int ret;\r
+\r
+       while (1) {\r
+               /* Wait for client's Start STAG/TO/Len */\r
+               cq_thread(cb);\r
+//             sem_wait(&cb->sem);\r
+               if (cb->state != RDMA_READ_ADV) {\r
+                       fprintf(stderr, "wait for RDMA_READ_ADV state %d\n",\r
+                               cb->state);\r
+                       ret = -1;\r
+                       break;\r
+               }\r
+\r
+               DEBUG_LOG("server received sink adv\n");\r
+\r
+               /* Issue RDMA Read. */\r
+               cb->rdma_sq_wr.opcode = IBV_WR_RDMA_READ;\r
+               cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey;\r
+               cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr;\r
+               cb->rdma_sq_wr.sg_list->length = cb->remote_len;\r
+\r
+               ret = ibv_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr);\r
+               if (ret) {\r
+                       fprintf(stderr, "post send error %d\n", ret);\r
+                       break;\r
+               }\r
+               DEBUG_LOG("server posted rdma read req \n");\r
+\r
+               /* Wait for read completion */\r
+               cq_thread(cb);\r
+//             sem_wait(&cb->sem);\r
+               if (cb->state != RDMA_READ_COMPLETE) {\r
+                       fprintf(stderr, "wait for RDMA_READ_COMPLETE state %d\n",\r
+                               cb->state);\r
+                       ret = -1;\r
+                       break;\r
+               }\r
+               DEBUG_LOG("server received read complete\n");\r
+\r
+               /* Display data in recv buf */\r
+               if (cb->verbose)\r
+                       printf("server ping data: %s\n", cb->rdma_buf);\r
+\r
+               /* Tell client to continue */\r
+               ret = ibv_post_send(cb->qp, &cb->sq_wr, &bad_wr);\r
+               if (ret) {\r
+                       fprintf(stderr, "post send error %d\n", ret);\r
+                       break;\r
+               }\r
+               DEBUG_LOG("server posted go ahead\n");\r
+\r
+               /* Wait for client's RDMA STAG/TO/Len */\r
+               cq_thread(cb);\r
+//             sem_wait(&cb->sem);\r
+               if (cb->state != RDMA_WRITE_ADV) {\r
+                       fprintf(stderr, "wait for RDMA_WRITE_ADV state %d\n",\r
+                               cb->state);\r
+                       ret = -1;\r
+                       break;\r
+               }\r
+               DEBUG_LOG("server received sink adv\n");\r
+\r
+               /* RDMA Write echo data */\r
+               cb->rdma_sq_wr.opcode = IBV_WR_RDMA_WRITE;\r
+               cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey;\r
+               cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr;\r
+               cb->rdma_sq_wr.sg_list->length = strlen(cb->rdma_buf) + 1;\r
+               DEBUG_LOG("rdma write from lkey %x laddr %" PRIx64 " len %d\n",\r
+                         cb->rdma_sq_wr.sg_list->lkey,\r
+                         cb->rdma_sq_wr.sg_list->addr,\r
+                         cb->rdma_sq_wr.sg_list->length);\r
+\r
+               ret = ibv_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr);\r
+               if (ret) {\r
+                       fprintf(stderr, "post send error %d\n", ret);\r
+                       break;\r
+               }\r
+\r
+               /* Wait for completion */\r
+               cq_thread(cb);\r
+//             ret = sem_wait(&cb->sem);\r
+               if (cb->state != RDMA_WRITE_COMPLETE) {\r
+                       fprintf(stderr, "wait for RDMA_WRITE_COMPLETE state %d\n",\r
+                               cb->state);\r
+                       ret = -1;\r
+                       break;\r
+               }\r
+               DEBUG_LOG("server rdma write complete \n");\r
+\r
+               /* Tell client to begin again */\r
+               ret = ibv_post_send(cb->qp, &cb->sq_wr, &bad_wr);\r
+               if (ret) {\r
+                       fprintf(stderr, "post send error %d\n", ret);\r
+                       break;\r
+               }\r
+               DEBUG_LOG("server posted go ahead\n");\r
+       }\r
+\r
+       return ret;\r
+}\r
+\r
+static int rping_bind_server(struct rping_cb *cb)\r
+{\r
+       int ret;\r
+\r
+       cb->sin.sin_port = cb->port;\r
+       ret = rdma_bind_addr(cb->cm_id, (struct sockaddr *) &cb->sin);\r
+       if (ret) {\r
+               fprintf(stderr, "rdma_bind_addr error %d\n", ret);\r
+               return ret;\r
+       }\r
+       DEBUG_LOG("rdma_bind_addr successful\n");\r
+\r
+       DEBUG_LOG("rdma_listen\n");\r
+       ret = rdma_listen(cb->cm_id, 3);\r
+       if (ret) {\r
+               fprintf(stderr, "rdma_listen failed: %d\n", ret);\r
+               return ret;\r
+       }\r
+\r
+       return 0;\r
+}\r
+\r
+static struct rping_cb *clone_cb(struct rping_cb *listening_cb)\r
+{\r
+       struct rping_cb *cb = malloc(sizeof *cb);\r
+       if (!cb)\r
+               return NULL;\r
+       *cb = *listening_cb;\r
+       cb->child_cm_id->context = cb;\r
+       return cb;\r
+}\r
+\r
+static void free_cb(struct rping_cb *cb)\r
+{\r
+       free(cb);\r
+}\r
+\r
+static int rping_run_server(struct rping_cb *cb)\r
+{\r
+       struct ibv_recv_wr *bad_wr;\r
+       int ret;\r
+\r
+       ret = rping_bind_server(cb);\r
+       if (ret)\r
+               return ret;\r
+\r
+       cm_thread(cb);\r
+//     sem_wait(&cb->sem);\r
+       if (cb->state != CONNECT_REQUEST) {\r
+               fprintf(stderr, "wait for CONNECT_REQUEST state %d\n",\r
+                       cb->state);\r
+               return -1;\r
+       }\r
+\r
+       ret = rping_setup_qp(cb, cb->child_cm_id);\r
+       if (ret) {\r
+               fprintf(stderr, "setup_qp failed: %d\n", ret);\r
+               return ret;\r
+       }\r
+\r
+       ret = rping_setup_buffers(cb);\r
+       if (ret) {\r
+               fprintf(stderr, "rping_setup_buffers failed: %d\n", ret);\r
+               goto err1;\r
+       }\r
+\r
+       ret = ibv_post_recv(cb->qp, &cb->rq_wr, &bad_wr);\r
+       if (ret) {\r
+               fprintf(stderr, "ibv_post_recv failed: %d\n", ret);\r
+               goto err2;\r
+       }\r
+\r
+//     pthread_create(&cb->cqthread, NULL, cq_thread, cb);\r
+\r
+       ret = rping_accept(cb);\r
+       if (ret) {\r
+               fprintf(stderr, "connect error %d\n", ret);\r
+               goto err2;\r
+       }\r
+\r
+       rping_test_server(cb);\r
+       rdma_disconnect(cb->child_cm_id);\r
+       rdma_destroy_id(cb->child_cm_id);\r
+err2:\r
+       rping_free_buffers(cb);\r
+err1:\r
+       rping_free_qp(cb);\r
+\r
+       return ret;\r
+}\r
+\r
+static int rping_test_client(struct rping_cb *cb)\r
+{\r
+       int ping, start, cc, i, ret = 0;\r
+       struct ibv_send_wr *bad_wr;\r
+       unsigned char c;\r
+\r
+       start = 65;\r
+       for (ping = 0; !cb->count || ping < cb->count; ping++) {\r
+               cb->state = RDMA_READ_ADV;\r
+\r
+               /* Put some ascii text in the buffer. */\r
+               cc = sprintf(cb->start_buf, RPING_MSG_FMT, ping);\r
+               for (i = cc, c = start; i < cb->size; i++) {\r
+                       cb->start_buf[i] = c;\r
+                       c++;\r
+                       if (c > 122)\r
+                               c = 65;\r
+               }\r
+               start++;\r
+               if (start > 122)\r
+                       start = 65;\r
+               cb->start_buf[cb->size - 1] = 0;\r
+\r
+               rping_format_send(cb, cb->start_buf, cb->start_mr);\r
+               ret = ibv_post_send(cb->qp, &cb->sq_wr, &bad_wr);\r
+               if (ret) {\r
+                       fprintf(stderr, "post send error %d\n", ret);\r
+                       break;\r
+               }\r
+\r
+               /* Wait for server to ACK */\r
+               cq_thread(cb);\r
+//             sem_wait(&cb->sem);\r
+               if (cb->state != RDMA_WRITE_ADV) {\r
+                       fprintf(stderr, "wait for RDMA_WRITE_ADV state %d\n",\r
+                               cb->state);\r
+                       ret = -1;\r
+                       break;\r
+               }\r
+\r
+               rping_format_send(cb, cb->rdma_buf, cb->rdma_mr);\r
+               ret = ibv_post_send(cb->qp, &cb->sq_wr, &bad_wr);\r
+               if (ret) {\r
+                       fprintf(stderr, "post send error %d\n", ret);\r
+                       break;\r
+               }\r
+\r
+               /* Wait for the server to say the RDMA Write is complete. */\r
+               cq_thread(cb);\r
+//             sem_wait(&cb->sem);\r
+               if (cb->state != RDMA_WRITE_COMPLETE) {\r
+                       fprintf(stderr, "wait for RDMA_WRITE_COMPLETE state %d\n",\r
+                               cb->state);\r
+                       ret = -1;\r
+                       break;\r
+               }\r
+\r
+               if (cb->validate)\r
+                       if (memcmp(cb->start_buf, cb->rdma_buf, cb->size)) {\r
+                               fprintf(stderr, "data mismatch!\n");\r
+                               ret = -1;\r
+                               break;\r
+                       }\r
+\r
+               if (cb->verbose)\r
+                       printf("ping data: %s\n", cb->rdma_buf);\r
+       }\r
+\r
+       return ret;\r
+}\r
+\r
+static int rping_connect_client(struct rping_cb *cb)\r
+{\r
+       struct rdma_conn_param conn_param;\r
+       int ret;\r
+\r
+       memset(&conn_param, 0, sizeof conn_param);\r
+       conn_param.responder_resources = 1;\r
+       conn_param.initiator_depth = 1;\r
+       conn_param.retry_count = 10;\r
+\r
+       ret = rdma_connect(cb->cm_id, &conn_param);\r
+       if (ret) {\r
+               fprintf(stderr, "rdma_connect error %d\n", ret);\r
+               return ret;\r
+       }\r
+\r
+       cm_thread(cb);\r
+//     sem_wait(&cb->sem);\r
+       if (cb->state != CONNECTED) {\r
+               fprintf(stderr, "wait for CONNECTED state %d\n", cb->state);\r
+               return -1;\r
+       }\r
+\r
+       DEBUG_LOG("rmda_connect successful\n");\r
+       return 0;\r
+}\r
+\r
+static int rping_bind_client(struct rping_cb *cb)\r
+{\r
+       int ret;\r
+\r
+       cb->sin.sin_port = cb->port;\r
+       ret = rdma_resolve_addr(cb->cm_id, NULL, (struct sockaddr *) &cb->sin, 2000);\r
+       if (ret) {\r
+               fprintf(stderr, "rdma_resolve_addr error %d\n", ret);\r
+               return ret;\r
+       }\r
+\r
+       cm_thread(cb);\r
+//     sem_wait(&cb->sem);\r
+       if (cb->state != ROUTE_RESOLVED) {\r
+               fprintf(stderr, "waiting for addr/route resolution state %d\n",\r
+                       cb->state);\r
+               return -1;\r
+       }\r
+\r
+       DEBUG_LOG("rdma_resolve_addr - rdma_resolve_route successful\n");\r
+       return 0;\r
+}\r
+\r
+static int rping_run_client(struct rping_cb *cb)\r
+{\r
+       struct ibv_recv_wr *bad_wr;\r
+       int ret;\r
+\r
+       ret = rping_bind_client(cb);\r
+       if (ret)\r
+               return ret;\r
+\r
+       ret = rping_setup_qp(cb, cb->cm_id);\r
+       if (ret) {\r
+               fprintf(stderr, "setup_qp failed: %d\n", ret);\r
+               return ret;\r
+       }\r
+\r
+       ret = rping_setup_buffers(cb);\r
+       if (ret) {\r
+               fprintf(stderr, "rping_setup_buffers failed: %d\n", ret);\r
+               goto err1;\r
+       }\r
+\r
+       ret = ibv_post_recv(cb->qp, &cb->rq_wr, &bad_wr);\r
+       if (ret) {\r
+               fprintf(stderr, "ibv_post_recv failed: %d\n", ret);\r
+               goto err2;\r
+       }\r
+\r
+//     pthread_create(&cb->cqthread, NULL, cq_thread, cb);\r
+\r
+       ret = rping_connect_client(cb);\r
+       if (ret) {\r
+               fprintf(stderr, "connect error %d\n", ret);\r
+               goto err2;\r
+       }\r
+\r
+       rping_test_client(cb);\r
+       rdma_disconnect(cb->cm_id);\r
+err2:\r
+       rping_free_buffers(cb);\r
+err1:\r
+       rping_free_qp(cb);\r
+\r
+       return ret;\r
+}\r
+\r
+static int get_addr(char *dst, struct sockaddr_in *addr)\r
+{\r
+       struct addrinfo *res;\r
+       int ret;\r
+\r
+       ret = getaddrinfo(dst, NULL, NULL, &res);\r
+       if (ret) {\r
+               printf("getaddrinfo failed - invalid hostname or IP address\n");\r
+               return ret;\r
+       }\r
+\r
+       if (res->ai_family != PF_INET) {\r
+               ret = -1;\r
+               goto out;\r
+       }\r
+\r
+       *addr = *(struct sockaddr_in *) res->ai_addr;\r
+out:\r
+       freeaddrinfo(res);\r
+       return ret;\r
+}\r
+\r
+static void usage()\r
+{\r
+       printf("rdma_rping -s [-vVd] [-S size] [-C count] [-a addr] [-p port]\n");\r
+       printf("rdma_rping -c [-vVd] [-S size] [-C count] -a addr [-p port]\n");\r
+       printf("\t-c\t\tclient side\n");\r
+       printf("\t-s\t\tserver side\n");\r
+       printf("\t-v\t\tdisplay ping data to stdout\n");\r
+       printf("\t-V\t\tvalidate ping data\n");\r
+       printf("\t-d\t\tdebug printfs\n");\r
+       printf("\t-S size \tping data size\n");\r
+       printf("\t-C count\tping count times\n");\r
+       printf("\t-a addr\t\taddress\n");\r
+       printf("\t-p port\t\tport\n");\r
+}\r
+\r
+int main(int argc, char *argv[])\r
+{\r
+//     struct rping_cb *cb;\r
+       int op;\r
+       int ret = 0;\r
+\r
+       cb = malloc(sizeof(*cb));\r
+       if (!cb)\r
+               return -ENOMEM;\r
+\r
+       memset(cb, 0, sizeof(*cb));\r
+       cb->server = -1;\r
+       cb->state = IDLE;\r
+       cb->size = 64;\r
+       cb->sin.sin_family = PF_INET;\r
+       cb->port = htons(7174);\r
+//     sem_init(&cb->sem, 0, 0);\r
+\r
+       opterr = 0;\r
+       while ((op=getopt(argc, argv, "a:Pp:C:S:t:scvVd")) != -1) {\r
+               switch (op) {\r
+               case 'a':\r
+                       ret = get_addr(optarg, &cb->sin);\r
+                       break;\r
+               case 'p':\r
+                       cb->port = htons(atoi(optarg));\r
+                       DEBUG_LOG("port %d\n", (int) atoi(optarg));\r
+                       break;\r
+               case 's':\r
+                       cb->server = 1;\r
+                       DEBUG_LOG("server\n");\r
+                       break;\r
+               case 'c':\r
+                       cb->server = 0;\r
+                       DEBUG_LOG("client\n");\r
+                       break;\r
+               case 'S':\r
+                       cb->size = atoi(optarg);\r
+                       if (cb->size < RPING_MIN_BUFSIZE) {\r
+                               fprintf(stderr, "Invalid size (minimum is %d) " RPING_MIN_BUFSIZE);\r
+                               ret = EINVAL;\r
+                       } else\r
+                               DEBUG_LOG("size %d\n", (int) atoi(optarg));\r
+                       break;\r
+               case 'C':\r
+                       cb->count = atoi(optarg);\r
+                       if (cb->count < 0) {\r
+                               fprintf(stderr, "Invalid count %d\n", cb->count);\r
+                               ret = EINVAL;\r
+                       } else\r
+                               DEBUG_LOG("count %d\n", (int) cb->count);\r
+                       break;\r
+               case 'v':\r
+                       cb->verbose++;\r
+                       DEBUG_LOG("verbose\n");\r
+                       break;\r
+               case 'V':\r
+                       cb->validate++;\r
+                       DEBUG_LOG("validate data\n");\r
+                       break;\r
+               case 'd':\r
+                       debug++;\r
+                       break;\r
+               default:\r
+                       usage();\r
+                       ret = EINVAL;\r
+                       goto out;\r
+               }\r
+       }\r
+       if (ret)\r
+               goto out;\r
+\r
+       if (cb->server == -1) {\r
+               usage();\r
+               ret = EINVAL;\r
+               goto out;\r
+       }\r
+\r
+       cb->cm_channel = rdma_create_event_channel();\r
+       if (!cb->cm_channel) {\r
+               ret = errno;\r
+               fprintf(stderr, "rdma_create_event_channel error %d\n", ret);\r
+               goto out;\r
+       }\r
+\r
+       ret = rdma_create_id(cb->cm_channel, &cb->cm_id, cb, RDMA_PS_TCP);\r
+       if (ret) {\r
+               ret = errno;\r
+               fprintf(stderr, "rdma_create_id error %d\n", ret);\r
+               goto out2;\r
+       }\r
+       DEBUG_LOG("created cm_id %p\n", cb->cm_id);\r
+\r
+//     pthread_create(&cb->cmthread, NULL, cm_thread, cb);\r
+\r
+       if (cb->server)\r
+               ret = rping_run_server(cb);\r
+       else\r
+               ret = rping_run_client(cb);\r
+\r
+       DEBUG_LOG("destroy cm_id %p\n", cb->cm_id);\r
+       rdma_destroy_id(cb->cm_id);\r
+out2:\r
+       rdma_destroy_event_channel(cb->cm_channel);\r
+out:\r
+       free(cb);\r
+       return ret;\r
+}\r

diff --git a/ulp/librdmacm/examples/udaddy/udaddy.c b/ulp/librdmacm/examples/udaddy/udaddy.c
new file mode 100644 (file)
index 0000000..9e74a22
--- /dev/null
+++ b/ulp/librdmacm/examples/udaddy/udaddy.c
@@ -0,0 +1,701 @@
+/*\r
+ * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.\r
+ *\r
+ * This software is available to you under a choice of one of two\r
+ * licenses.  You may choose to be licensed under the terms of the GNU\r
+ * General Public License (GPL) Version 2, available from the file\r
+ * COPYING in the main directory of this source tree, or the\r
+ * OpenIB.org BSD license below:\r
+ *\r
+ *     Redistribution and use in source and binary forms, with or\r
+ *     without modification, are permitted provided that the following\r
+ *     conditions are met:\r
+ *\r
+ *      - Redistributions of source code must retain the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer.\r
+ *\r
+ *      - Redistributions in binary form must reproduce the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer in the documentation and/or other materials\r
+ *        provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ *\r
+ * $Id$\r
+ */\r
+\r
+#include <stdlib.h>\r
+#include <string.h>\r
+#include <stdio.h>\r
+#include <errno.h>\r
+#include <sys/types.h>\r
+#include <netinet/in.h>\r
+#include <sys/socket.h>\r
+#include <netdb.h>\r
+#include <byteswap.h>\r
+#include <getopt.h>\r
+\r
+#include <rdma/rdma_cma.h>\r
+\r
+struct cmatest_node {\r
+       int                     id;\r
+       struct rdma_cm_id       *cma_id;\r
+       int                     connected;\r
+       struct ibv_pd           *pd;\r
+       struct ibv_cq           *cq;\r
+       struct ibv_mr           *mr;\r
+       struct ibv_ah           *ah;\r
+       uint32_t                remote_qpn;\r
+       uint32_t                remote_qkey;\r
+       void                    *mem;\r
+};\r
+\r
+struct cmatest {\r
+       struct rdma_event_channel *channel;\r
+       struct cmatest_node     *nodes;\r
+       int                     conn_index;\r
+       int                     connects_left;\r
+\r
+       struct sockaddr_in      dst_in;\r
+       struct sockaddr         *dst_addr;\r
+       struct sockaddr_in      src_in;\r
+       struct sockaddr         *src_addr;\r
+};\r
+\r
+static struct cmatest test;\r
+static int connections = 1;\r
+static int message_size = 100;\r
+static int message_count = 10;\r
+static uint16_t port = 7174;\r
+static uint8_t set_tos = 0;\r
+static uint8_t tos;\r
+static char *dst_addr;\r
+static char *src_addr;\r
+static enum rdma_port_space port_space = RDMA_PS_UDP;\r
+\r
+static int create_message(struct cmatest_node *node)\r
+{\r
+       if (!message_size)\r
+               message_count = 0;\r
+\r
+       if (!message_count)\r
+               return 0;\r
+\r
+       node->mem = malloc(message_size + sizeof(struct ibv_grh));\r
+       if (!node->mem) {\r
+               printf("failed message allocation\n");\r
+               return -1;\r
+       }\r
+       node->mr = ibv_reg_mr(node->pd, node->mem,\r
+                             message_size + sizeof(struct ibv_grh),\r
+                             IBV_ACCESS_LOCAL_WRITE);\r
+       if (!node->mr) {\r
+               printf("failed to reg MR\n");\r
+               goto err;\r
+       }\r
+       return 0;\r
+err:\r
+       free(node->mem);\r
+       return -1;\r
+}\r
+\r
+static int verify_test_params(struct cmatest_node *node)\r
+{\r
+       struct ibv_port_attr port_attr;\r
+       int ret;\r
+\r
+       ret = ibv_query_port(node->cma_id->verbs, node->cma_id->port_num,\r
+                            &port_attr);\r
+       if (ret)\r
+               return ret;\r
+\r
+       if (message_count && message_size > (1 << (port_attr.active_mtu + 7))) {\r
+               printf("udaddy: message_size %d is larger than active mtu %d\n",\r
+                      message_size, 1 << (port_attr.active_mtu + 7));\r
+               return -EINVAL;\r
+       }\r
+\r
+       return 0;\r
+}\r
+\r
+static int init_node(struct cmatest_node *node)\r
+{\r
+       struct ibv_qp_init_attr init_qp_attr;\r
+       int cqe, ret;\r
+\r
+       node->pd = ibv_alloc_pd(node->cma_id->verbs);\r
+       if (!node->pd) {\r
+               ret = -ENOMEM;\r
+               printf("udaddy: unable to allocate PD\n");\r
+               goto out;\r
+       }\r
+\r
+       cqe = message_count ? message_count * 2 : 2;\r
+       node->cq = ibv_create_cq(node->cma_id->verbs, cqe, node, 0, 0);\r
+       if (!node->cq) {\r
+               ret = -ENOMEM;\r
+               printf("udaddy: unable to create CQ\n");\r
+               goto out;\r
+       }\r
+\r
+       memset(&init_qp_attr, 0, sizeof init_qp_attr);\r
+       init_qp_attr.cap.max_send_wr = message_count ? message_count : 1;\r
+       init_qp_attr.cap.max_recv_wr = message_count ? message_count : 1;\r
+       init_qp_attr.cap.max_send_sge = 1;\r
+       init_qp_attr.cap.max_recv_sge = 1;\r
+       init_qp_attr.qp_context = node;\r
+       init_qp_attr.sq_sig_all = 0;\r
+       init_qp_attr.qp_type = IBV_QPT_UD;\r
+       init_qp_attr.send_cq = node->cq;\r
+       init_qp_attr.recv_cq = node->cq;\r
+       ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr);\r
+       if (ret) {\r
+               printf("udaddy: unable to create QP: %d\n", ret);\r
+               goto out;\r
+       }\r
+\r
+       ret = create_message(node);\r
+       if (ret) {\r
+               printf("udaddy: failed to create messages: %d\n", ret);\r
+               goto out;\r
+       }\r
+out:\r
+       return ret;\r
+}\r
+\r
+static int post_recvs(struct cmatest_node *node)\r
+{\r
+       struct ibv_recv_wr recv_wr, *recv_failure;\r
+       struct ibv_sge sge;\r
+       int i, ret = 0;\r
+\r
+       if (!message_count)\r
+               return 0;\r
+\r
+       recv_wr.next = NULL;\r
+       recv_wr.sg_list = &sge;\r
+       recv_wr.num_sge = 1;\r
+       recv_wr.wr_id = (uintptr_t) node;\r
+\r
+       sge.length = message_size + sizeof(struct ibv_grh);\r
+       sge.lkey = node->mr->lkey;\r
+       sge.addr = (uintptr_t) node->mem;\r
+\r
+       for (i = 0; i < message_count && !ret; i++ ) {\r
+               ret = ibv_post_recv(node->cma_id->qp, &recv_wr, &recv_failure);\r
+               if (ret) {\r
+                       printf("failed to post receives: %d\n", ret);\r
+                       break;\r
+               }\r
+       }\r
+       return ret;\r
+}\r
+\r
+static int post_sends(struct cmatest_node *node, int signal_flag)\r
+{\r
+       struct ibv_send_wr send_wr, *bad_send_wr;\r
+       struct ibv_sge sge;\r
+       int i, ret = 0;\r
+\r
+       if (!node->connected || !message_count)\r
+               return 0;\r
+\r
+       send_wr.next = NULL;\r
+       send_wr.sg_list = &sge;\r
+       send_wr.num_sge = 1;\r
+       send_wr.opcode = IBV_WR_SEND_WITH_IMM;\r
+       send_wr.send_flags = signal_flag;\r
+       send_wr.wr_id = (unsigned long)node;\r
+       send_wr.imm_data = htonl(node->cma_id->qp->qp_num);\r
+\r
+       send_wr.wr.ud.ah = node->ah;\r
+       send_wr.wr.ud.remote_qpn = node->remote_qpn;\r
+       send_wr.wr.ud.remote_qkey = node->remote_qkey;\r
+\r
+       sge.length = message_size;\r
+       sge.lkey = node->mr->lkey;\r
+       sge.addr = (uintptr_t) node->mem;\r
+\r
+       for (i = 0; i < message_count && !ret; i++) {\r
+               ret = ibv_post_send(node->cma_id->qp, &send_wr, &bad_send_wr);\r
+               if (ret) \r
+                       printf("failed to post sends: %d\n", ret);\r
+       }\r
+       return ret;\r
+}\r
+\r
+static void connect_error(void)\r
+{\r
+       test.connects_left--;\r
+}\r
+\r
+static int addr_handler(struct cmatest_node *node)\r
+{\r
+       int ret;\r
+\r
+       if (set_tos) {\r
+               ret = rdma_set_option(node->cma_id, RDMA_OPTION_ID,\r
+                                     RDMA_OPTION_ID_TOS, &tos, sizeof tos);\r
+               if (ret)\r
+                       printf("udaddy: set TOS option failed: %d\n", ret);\r
+       }\r
+\r
+       ret = rdma_resolve_route(node->cma_id, 2000);\r
+       if (ret) {\r
+               printf("udaddy: resolve route failed: %d\n", ret);\r
+               connect_error();\r
+       }\r
+       return ret;\r
+}\r
+\r
+static int route_handler(struct cmatest_node *node)\r
+{\r
+       struct rdma_conn_param conn_param;\r
+       int ret;\r
+\r
+       ret = verify_test_params(node);\r
+       if (ret)\r
+               goto err;\r
+\r
+       ret = init_node(node);\r
+       if (ret)\r
+               goto err;\r
+\r
+       ret = post_recvs(node);\r
+       if (ret)\r
+               goto err;\r
+\r
+       memset(&conn_param, 0, sizeof conn_param);\r
+       ret = rdma_connect(node->cma_id, &conn_param);\r
+       if (ret) {\r
+               printf("udaddy: failure connecting: %d\n", ret);\r
+               goto err;\r
+       }\r
+       return 0;\r
+err:\r
+       connect_error();\r
+       return ret;\r
+}\r
+\r
+static int connect_handler(struct rdma_cm_id *cma_id)\r
+{\r
+       struct cmatest_node *node;\r
+       struct rdma_conn_param conn_param;\r
+       int ret;\r
+\r
+       if (test.conn_index == connections) {\r
+               ret = -ENOMEM;\r
+               goto err1;\r
+       }\r
+       node = &test.nodes[test.conn_index++];\r
+\r
+       node->cma_id = cma_id;\r
+       cma_id->context = node;\r
+\r
+       ret = verify_test_params(node);\r
+       if (ret)\r
+               goto err2;\r
+\r
+       ret = init_node(node);\r
+       if (ret)\r
+               goto err2;\r
+\r
+       ret = post_recvs(node);\r
+       if (ret)\r
+               goto err2;\r
+\r
+       memset(&conn_param, 0, sizeof conn_param);\r
+       conn_param.qp_num = node->cma_id->qp->qp_num;\r
+       ret = rdma_accept(node->cma_id, &conn_param);\r
+       if (ret) {\r
+               printf("udaddy: failure accepting: %d\n", ret);\r
+               goto err2;\r
+       }\r
+       node->connected = 1;\r
+       test.connects_left--;\r
+       return 0;\r
+\r
+err2:\r
+       node->cma_id = NULL;\r
+       connect_error();\r
+err1:\r
+       printf("udaddy: failing connection request\n");\r
+       rdma_reject(cma_id, NULL, 0);\r
+       return ret;\r
+}\r
+\r
+static int resolved_handler(struct cmatest_node *node,\r
+                           struct rdma_cm_event *event)\r
+{\r
+       node->remote_qpn = event->param.ud.qp_num;\r
+       node->remote_qkey = event->param.ud.qkey;\r
+       node->ah = ibv_create_ah(node->pd, &event->param.ud.ah_attr);\r
+       if (!node->ah) {\r
+               printf("udaddy: failure creating address handle\n");\r
+               goto err;\r
+       }\r
+\r
+       node->connected = 1;\r
+       test.connects_left--;\r
+       return 0;\r
+err:\r
+       connect_error();\r
+       return -1;\r
+}\r
+\r
+static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)\r
+{\r
+       int ret = 0;\r
+\r
+       switch (event->event) {\r
+       case RDMA_CM_EVENT_ADDR_RESOLVED:\r
+               ret = addr_handler(cma_id->context);\r
+               break;\r
+       case RDMA_CM_EVENT_ROUTE_RESOLVED:\r
+               ret = route_handler(cma_id->context);\r
+               break;\r
+       case RDMA_CM_EVENT_CONNECT_REQUEST:\r
+               ret = connect_handler(cma_id);\r
+               break;\r
+       case RDMA_CM_EVENT_ESTABLISHED:\r
+               ret = resolved_handler(cma_id->context, event);\r
+               break;\r
+       case RDMA_CM_EVENT_ADDR_ERROR:\r
+       case RDMA_CM_EVENT_ROUTE_ERROR:\r
+       case RDMA_CM_EVENT_CONNECT_ERROR:\r
+       case RDMA_CM_EVENT_UNREACHABLE:\r
+       case RDMA_CM_EVENT_REJECTED:\r
+               printf("udaddy: event: %s, error: %d\n",\r
+                      rdma_event_str(event->event), event->status);\r
+               connect_error();\r
+               ret = event->status;\r
+               break;\r
+       case RDMA_CM_EVENT_DEVICE_REMOVAL:\r
+               /* Cleanup will occur after test completes. */\r
+               break;\r
+       default:\r
+               break;\r
+       }\r
+       return ret;\r
+}\r
+\r
+static void destroy_node(struct cmatest_node *node)\r
+{\r
+       if (!node->cma_id)\r
+               return;\r
+\r
+       if (node->ah)\r
+               ibv_destroy_ah(node->ah);\r
+\r
+       if (node->cma_id->qp)\r
+               rdma_destroy_qp(node->cma_id);\r
+\r
+       if (node->cq)\r
+               ibv_destroy_cq(node->cq);\r
+\r
+       if (node->mem) {\r
+               ibv_dereg_mr(node->mr);\r
+               free(node->mem);\r
+       }\r
+\r
+       if (node->pd)\r
+               ibv_dealloc_pd(node->pd);\r
+\r
+       /* Destroy the RDMA ID after all device resources */\r
+       rdma_destroy_id(node->cma_id);\r
+}\r
+\r
+static int alloc_nodes(void)\r
+{\r
+       int ret, i;\r
+\r
+       test.nodes = malloc(sizeof *test.nodes * connections);\r
+       if (!test.nodes) {\r
+               printf("udaddy: unable to allocate memory for test nodes\n");\r
+               return -ENOMEM;\r
+       }\r
+       memset(test.nodes, 0, sizeof *test.nodes * connections);\r
+\r
+       for (i = 0; i < connections; i++) {\r
+               test.nodes[i].id = i;\r
+               if (dst_addr) {\r
+                       ret = rdma_create_id(test.channel,\r
+                                            &test.nodes[i].cma_id,\r
+                                            &test.nodes[i], port_space);\r
+                       if (ret)\r
+                               goto err;\r
+               }\r
+       }\r
+       return 0;\r
+err:\r
+       while (--i >= 0)\r
+               rdma_destroy_id(test.nodes[i].cma_id);\r
+       free(test.nodes);\r
+       return ret;\r
+}\r
+\r
+static void destroy_nodes(void)\r
+{\r
+       int i;\r
+\r
+       for (i = 0; i < connections; i++)\r
+               destroy_node(&test.nodes[i]);\r
+       free(test.nodes);\r
+}\r
+\r
+static void create_reply_ah(struct cmatest_node *node, struct ibv_wc *wc)\r
+{\r
+       struct ibv_qp_attr attr;\r
+       struct ibv_qp_init_attr init_attr;\r
+\r
+       node->ah = ibv_create_ah_from_wc(node->pd, wc, node->mem,\r
+                                        node->cma_id->port_num);\r
+       node->remote_qpn = ntohl(wc->imm_data);\r
+\r
+       ibv_query_qp(node->cma_id->qp, &attr, IBV_QP_QKEY, &init_attr);\r
+       node->remote_qkey = attr.qkey;\r
+}\r
+\r
+static int poll_cqs(void)\r
+{\r
+       struct ibv_wc wc[8];\r
+       int done, i, ret;\r
+\r
+       for (i = 0; i < connections; i++) {\r
+               if (!test.nodes[i].connected)\r
+                       continue;\r
+\r
+               for (done = 0; done < message_count; done += ret) {\r
+                       ret = ibv_poll_cq(test.nodes[i].cq, 8, wc);\r
+                       if (ret < 0) {\r
+                               printf("udaddy: failed polling CQ: %d\n", ret);\r
+                               return ret;\r
+                       }\r
+\r
+                       if (ret && !test.nodes[i].ah)\r
+                               create_reply_ah(&test.nodes[i], wc);\r
+               }\r
+       }\r
+       return 0;\r
+}\r
+\r
+static int connect_events(void)\r
+{\r
+       struct rdma_cm_event *event;\r
+       int ret = 0;\r
+\r
+       while (test.connects_left && !ret) {\r
+               ret = rdma_get_cm_event(test.channel, &event);\r
+               if (!ret) {\r
+                       ret = cma_handler(event->id, event);\r
+                       rdma_ack_cm_event(event);\r
+               }\r
+       }\r
+       return ret;\r
+}\r
+\r
+static int get_addr(char *dst, struct sockaddr_in *addr)\r
+{\r
+       struct addrinfo *res;\r
+       int ret;\r
+\r
+       ret = getaddrinfo(dst, NULL, NULL, &res);\r
+       if (ret) {\r
+               printf("getaddrinfo failed - invalid hostname or IP address\n");\r
+               return ret;\r
+       }\r
+\r
+       if (res->ai_family != PF_INET) {\r
+               ret = -1;\r
+               goto out;\r
+       }\r
+\r
+       *addr = *(struct sockaddr_in *) res->ai_addr;\r
+out:\r
+       freeaddrinfo(res);\r
+       return ret;\r
+}\r
+\r
+static int run_server(void)\r
+{\r
+       struct rdma_cm_id *listen_id;\r
+       int i, ret;\r
+\r
+       printf("udaddy: starting server\n");\r
+       ret = rdma_create_id(test.channel, &listen_id, &test, port_space);\r
+       if (ret) {\r
+               printf("udaddy: listen request failed\n");\r
+               return ret;\r
+       }\r
+\r
+       if (src_addr) {\r
+               ret = get_addr(src_addr, &test.src_in);\r
+               if (ret)\r
+                       goto out;\r
+       } else\r
+               test.src_in.sin_family = PF_INET;\r
+\r
+       test.src_in.sin_port = port;\r
+       ret = rdma_bind_addr(listen_id, test.src_addr);\r
+       if (ret) {\r
+               printf("udaddy: bind address failed: %d\n", ret);\r
+               return ret;\r
+       }\r
+\r
+       ret = rdma_listen(listen_id, 0);\r
+       if (ret) {\r
+               printf("udaddy: failure trying to listen: %d\n", ret);\r
+               goto out;\r
+       }\r
+\r
+       connect_events();\r
+\r
+       if (message_count) {\r
+               printf("receiving data transfers\n");\r
+               ret = poll_cqs();\r
+               if (ret)\r
+                       goto out;\r
+\r
+               printf("sending replies\n");\r
+               for (i = 0; i < connections; i++) {\r
+                       ret = post_sends(&test.nodes[i], IBV_SEND_SIGNALED);\r
+                       if (ret)\r
+                               goto out;\r
+               }\r
+\r
+               ret = poll_cqs();\r
+               if (ret)\r
+                       goto out;\r
+               printf("data transfers complete\n");\r
+       }\r
+out:\r
+       rdma_destroy_id(listen_id);\r
+       return ret;\r
+}\r
+\r
+static int run_client(void)\r
+{\r
+       int i, ret;\r
+\r
+       printf("udaddy: starting client\n");\r
+       if (src_addr) {\r
+               ret = get_addr(src_addr, &test.src_in);\r
+               if (ret)\r
+                       return ret;\r
+       }\r
+\r
+       ret = get_addr(dst_addr, &test.dst_in);\r
+       if (ret)\r
+               return ret;\r
+\r
+       test.dst_in.sin_port = port;\r
+\r
+       printf("udaddy: connecting\n");\r
+       for (i = 0; i < connections; i++) {\r
+               ret = rdma_resolve_addr(test.nodes[i].cma_id,\r
+                                       src_addr ? test.src_addr : NULL,\r
+                                       test.dst_addr, 2000);\r
+               if (ret) {\r
+                       printf("udaddy: failure getting addr: %d\n", ret);\r
+                       connect_error();\r
+                       return ret;\r
+               }\r
+       }\r
+\r
+       ret = connect_events();\r
+       if (ret)\r
+               goto out;\r
+\r
+       if (message_count) {\r
+               printf("initiating data transfers\n");\r
+               for (i = 0; i < connections; i++) {\r
+                       ret = post_sends(&test.nodes[i], 0);\r
+                       if (ret)\r
+                               goto out;\r
+               }\r
+               printf("receiving data transfers\n");\r
+               ret = poll_cqs();\r
+               if (ret)\r
+                       goto out;\r
+\r
+               printf("data transfers complete\n");\r
+       }\r
+out:\r
+       return ret;\r
+}\r
+\r
+int main(int argc, char **argv)\r
+{\r
+       int op, ret;\r
+\r
+       while ((op = getopt(argc, argv, "s:b:c:C:S:t:p:")) != -1) {\r
+               switch (op) {\r
+               case 's':\r
+                       dst_addr = optarg;\r
+                       break;\r
+               case 'b':\r
+                       src_addr = optarg;\r
+                       break;\r
+               case 'c':\r
+                       connections = atoi(optarg);\r
+                       break;\r
+               case 'C':\r
+                       message_count = atoi(optarg);\r
+                       break;\r
+               case 'S':\r
+                       message_size = atoi(optarg);\r
+                       break;\r
+               case 't':\r
+                       set_tos = 1;\r
+                       tos = (uint8_t) atoi(optarg);\r
+                       break;\r
+               case 'p':\r
+                       port_space = strtol(optarg, NULL, 0);\r
+                       break;\r
+               default:\r
+                       printf("usage: %s\n", argv[0]);\r
+                       printf("\t[-s server_address]\n");\r
+                       printf("\t[-b bind_address]\n");\r
+                       printf("\t[-c connections]\n");\r
+                       printf("\t[-C message_count]\n");\r
+                       printf("\t[-S message_size]\n");\r
+                       printf("\t[-t type_of_service]\n");\r
+                       printf("\t[-p port_space - %#x for UDP (default), "\r
+                              "%#x for IPOIB]\n", RDMA_PS_UDP, RDMA_PS_IPOIB);\r
+                       exit(1);\r
+               }\r
+       }\r
+\r
+       test.dst_addr = (struct sockaddr *) &test.dst_in;\r
+       test.src_addr = (struct sockaddr *) &test.src_in;\r
+       test.connects_left = connections;\r
+\r
+       test.channel = rdma_create_event_channel();\r
+       if (!test.channel) {\r
+               printf("failed to create event channel\n");\r
+               exit(1);\r
+       }\r
+\r
+       if (alloc_nodes())\r
+               exit(1);\r
+\r
+       if (dst_addr)\r
+               ret = run_client();\r
+       else\r
+               ret = run_server();\r
+\r
+       printf("test complete\n");\r
+       destroy_nodes();\r
+       rdma_destroy_event_channel(test.channel);\r
+\r
+       printf("return status %d\n", ret);\r
+       return ret;\r
+}\r

diff --git a/ulp/librdmacm/include/rdma/rdma_cma.h b/ulp/librdmacm/include/rdma/rdma_cma.h
new file mode 100644 (file)
index 0000000..3a5a4bb
--- /dev/null
+++ b/ulp/librdmacm/include/rdma/rdma_cma.h
@@ -0,0 +1,637 @@
+/*\r
+ * Copyright (c) 2005-2009 Intel Corporation.  All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenFabrics.org BSD license\r
+ * below:\r
+ *\r
+ *     Redistribution and use in source and binary forms, with or\r
+ *     without modification, are permitted provided that the following\r
+ *     conditions are met:\r
+ *\r
+ *      - Redistributions of source code must retain the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer.\r
+ *\r
+ *      - Redistributions in binary form must reproduce the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer in the documentation and/or other materials\r
+ *        provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ */\r
+\r
+#pragma once\r
+\r
+#if !defined(RDMA_CMA_H)\r
+#define RDMA_CMA_H\r
+\r
+#include <windows.h>\r
+#include <infiniband\verbs.h>\r
+#include <rdma\winverbs.h>\r
+\r
+#ifdef __cplusplus\r
+extern "C" {\r
+#endif\r
+\r
+/*\r
+ * Interfaces based on librdmacm 1.0.8.\r
+ */\r
+\r
+typedef unsigned __int8                uint8_t;\r
+typedef unsigned __int16       uint16_t;\r
+typedef unsigned __int32       uint32_t;\r
+typedef unsigned __int64       uint64_t;\r
+\r
+/*\r
+ * Upon receiving a device removal event, users must destroy the associated\r
+ * RDMA identifier and release all resources allocated with the device.\r
+ */\r
+enum rdma_cm_event_type\r
+{\r
+       RDMA_CM_EVENT_ADDR_RESOLVED,\r
+       RDMA_CM_EVENT_ADDR_ERROR,\r
+       RDMA_CM_EVENT_ROUTE_RESOLVED,\r
+       RDMA_CM_EVENT_ROUTE_ERROR,\r
+       RDMA_CM_EVENT_CONNECT_REQUEST,\r
+       RDMA_CM_EVENT_CONNECT_RESPONSE,\r
+       RDMA_CM_EVENT_CONNECT_ERROR,\r
+       RDMA_CM_EVENT_UNREACHABLE,\r
+       RDMA_CM_EVENT_REJECTED,\r
+       RDMA_CM_EVENT_ESTABLISHED,\r
+       RDMA_CM_EVENT_DISCONNECTED,\r
+       RDMA_CM_EVENT_DEVICE_REMOVAL,\r
+       RDMA_CM_EVENT_MULTICAST_JOIN,\r
+       RDMA_CM_EVENT_MULTICAST_ERROR,\r
+       RDMA_CM_EVENT_ADDR_CHANGE,\r
+       RDMA_CM_EVENT_TIMEWAIT_EXIT\r
+};\r
+\r
+enum rdma_port_space\r
+{\r
+       RDMA_PS_IPOIB   = 0x0002,\r
+       RDMA_PS_TCP             = 0x0106,\r
+       RDMA_PS_UDP             = 0x0111,\r
+};\r
+\r
+/*\r
+ * Global qkey value for UDP QPs and multicast groups created via the \r
+ * RDMA CM.\r
+ */\r
+#define RDMA_UDP_QKEY 0x01234567\r
+\r
+struct ib_addr\r
+{\r
+       union ibv_gid   sgid;\r
+       union ibv_gid   dgid;\r
+       uint16_t                pkey;\r
+};\r
+\r
+struct rdma_addr\r
+{\r
+       struct sockaddr         src_addr;\r
+       uint8_t                         src_pad[sizeof(SOCKADDR_IN6) -\r
+                                                               sizeof(struct sockaddr)];\r
+       struct sockaddr         dst_addr;\r
+       uint8_t                         dst_pad[sizeof(SOCKADDR_IN6) -\r
+                                                               sizeof(struct sockaddr)];\r
+       union\r
+       {\r
+               struct ib_addr  ibaddr;\r
+       }       addr;\r
+};\r
+\r
+struct ibv_sa_path_rec\r
+{\r
+       uint8_t                         data[64];\r
+};\r
+\r
+struct rdma_route\r
+{\r
+       struct rdma_addr                addr;\r
+       struct ibv_sa_path_rec  *path_rec;\r
+       int                                             num_paths;\r
+};\r
+\r
+struct rdma_event_channel\r
+{\r
+       uint32_t                timeout;\r
+};\r
+\r
+struct rdma_cm_id\r
+{\r
+       struct ibv_context                      *verbs;\r
+       struct rdma_event_channel       *channel;\r
+       void                                            *context;\r
+       struct ibv_qp                           *qp;\r
+       struct rdma_route                       route;\r
+       enum rdma_port_space            ps;\r
+       uint8_t                                         port_num;\r
+\r
+       union {\r
+               IWVConnectEndpoint              *connect;\r
+               IWVDatagramEndpoint             *datagram;\r
+       }       ep;\r
+       OVERLAPPED                                      overlap;\r
+       uint32_t                                        events_completed;\r
+};\r
+\r
+struct rdma_conn_param\r
+{\r
+       const void *private_data;\r
+       uint8_t private_data_len;\r
+       uint8_t responder_resources;\r
+       uint8_t initiator_depth;\r
+       uint8_t flow_control;\r
+       uint8_t retry_count;            /* ignored when accepting */\r
+       uint8_t rnr_retry_count;\r
+       /* Fields below ignored if a QP is created on the rdma_cm_id. */\r
+       uint8_t srq;\r
+       uint32_t qp_num;\r
+};\r
+\r
+struct rdma_ud_param\r
+{\r
+       const void *private_data;\r
+       uint8_t private_data_len;\r
+       struct ibv_ah_attr ah_attr;\r
+       uint32_t qp_num;\r
+       uint32_t qkey;\r
+};\r
+\r
+struct rdma_cm_event\r
+{\r
+       struct rdma_cm_id               *id;\r
+       struct rdma_cm_id               *listen_id;\r
+       enum rdma_cm_event_type  event;\r
+       int                                              status;\r
+       union\r
+       {\r
+               struct rdma_conn_param conn;\r
+               struct rdma_ud_param   ud;\r
+\r
+       }       param;\r
+};\r
+\r
+/**\r
+ * rdma_create_event_channel - Open a channel used to report communication events.\r
+ * Description:\r
+ *   Asynchronous events are reported to users through event channels.  Each\r
+ *   event channel maps to a file descriptor.\r
+ * Notes:\r
+ *   All created event channels must be destroyed by calling\r
+ *   rdma_destroy_event_channel.  Users should call rdma_get_cm_event to\r
+ *   retrieve events on an event channel.\r
+ * See also:\r
+ *   rdma_get_cm_event, rdma_destroy_event_channel\r
+ */\r
+__declspec(dllexport)\r
+struct rdma_event_channel *rdma_create_event_channel(void);\r
+\r
+/**\r
+ * rdma_destroy_event_channel - Close an event communication channel.\r
+ * @channel: The communication channel to destroy.\r
+ * Description:\r
+ *   Release all resources associated with an event channel and closes the\r
+ *   associated file descriptor.\r
+ * Notes:\r
+ *   All rdma_cm_id's associated with the event channel must be destroyed,\r
+ *   and all returned events must be acked before calling this function.\r
+ * See also:\r
+ *  rdma_create_event_channel, rdma_get_cm_event, rdma_ack_cm_event\r
+ */\r
+__declspec(dllexport)\r
+void rdma_destroy_event_channel(struct rdma_event_channel *channel);\r
+\r
+/**\r
+ * rdma_create_id - Allocate a communication identifier.\r
+ * @channel: The communication channel that events associated with the\r
+ *   allocated rdma_cm_id will be reported on.\r
+ * @id: A reference where the allocated communication identifier will be\r
+ *   returned.\r
+ * @context: User specified context associated with the rdma_cm_id.\r
+ * @ps: RDMA port space.\r
+ * Description:\r
+ *   Creates an identifier that is used to track communication information.\r
+ * Notes:\r
+ *   Rdma_cm_id's are conceptually equivalent to a socket for RDMA\r
+ *   communication.  The difference is that RDMA communication requires\r
+ *   explicitly binding to a specified RDMA device before communication\r
+ *   can occur, and most operations are asynchronous in nature.  Communication\r
+ *   events on an rdma_cm_id are reported through the associated event\r
+ *   channel.  Users must release the rdma_cm_id by calling rdma_destroy_id.\r
+ * See also:\r
+ *   rdma_create_event_channel, rdma_destroy_id, rdma_get_devices,\r
+ *   rdma_bind_addr, rdma_resolve_addr, rdma_connect, rdma_listen,\r
+ */\r
+__declspec(dllexport)\r
+int rdma_create_id(struct rdma_event_channel *channel,\r
+                                  struct rdma_cm_id **id, void *context,\r
+                                  enum rdma_port_space ps);\r
+\r
+/**\r
+ * rdma_destroy_id - Release a communication identifier.\r
+ * @id: The communication identifier to destroy.\r
+ * Description:\r
+ *   Destroys the specified rdma_cm_id and cancels any outstanding\r
+ *   asynchronous operation.\r
+ * Notes:\r
+ *   Users must free any associated QP with the rdma_cm_id before\r
+ *   calling this routine and ack an related events.\r
+ * See also:\r
+ *   rdma_create_id, rdma_destroy_qp, rdma_ack_cm_event\r
+ */\r
+__declspec(dllexport)\r
+int rdma_destroy_id(struct rdma_cm_id *id);\r
+\r
+/**\r
+ * rdma_bind_addr - Bind an RDMA identifier to a source address.\r
+ * @id: RDMA identifier.\r
+ * @addr: Local address information.  Wildcard values are permitted.\r
+ * Description:\r
+ *   Associates a source address with an rdma_cm_id.  The address may be\r
+ *   wildcarded.  If binding to a specific local address, the rdma_cm_id\r
+ *   will also be bound to a local RDMA device.\r
+ * Notes:\r
+ *   Typically, this routine is called before calling rdma_listen to bind\r
+ *   to a specific port number, but it may also be called on the active side\r
+ *   of a connection before calling rdma_resolve_addr to bind to a specific\r
+ *   address.\r
+ * See also:\r
+ *   rdma_create_id, rdma_listen, rdma_resolve_addr, rdma_create_qp\r
+ */\r
+__declspec(dllexport)\r
+int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr);\r
+\r
+/**\r
+ * rdma_resolve_addr - Resolve destination and optional source addresses.\r
+ * @id: RDMA identifier.\r
+ * @src_addr: Source address information.  This parameter may be NULL.\r
+ * @dst_addr: Destination address information.\r
+ * @timeout_ms: Time to wait for resolution to complete.\r
+ * Description:\r
+ *   Resolve destination and optional source addresses from IP addresses\r
+ *   to an RDMA address.  If successful, the specified rdma_cm_id will\r
+ *   be bound to a local device.\r
+ * Notes:\r
+ *   This call is used to map a given destination IP address to a usable RDMA\r
+ *   address.  If a source address is given, the rdma_cm_id is bound to that\r
+ *   address, the same as if rdma_bind_addr were called.  If no source\r
+ *   address is given, and the rdma_cm_id has not yet been bound to a device,\r
+ *   then the rdma_cm_id will be bound to a source address based on the\r
+ *   local routing tables.  After this call, the rdma_cm_id will be bound to\r
+ *   an RDMA device.  This call is typically made from the active side of a\r
+ *   connection before calling rdma_resolve_route and rdma_connect.\r
+ * See also:\r
+ *   rdma_create_id, rdma_resolve_route, rdma_connect, rdma_create_qp,\r
+ *   rdma_get_cm_event, rdma_bind_addr\r
+ */\r
+__declspec(dllexport)\r
+int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,\r
+                                         struct sockaddr *dst_addr, int timeout_ms);\r
+\r
+/**\r
+ * rdma_resolve_route - Resolve the route information needed to establish a connection.\r
+ * @id: RDMA identifier.\r
+ * @timeout_ms: Time to wait for resolution to complete.\r
+ * Description:\r
+ *   Resolves an RDMA route to the destination address in order to establish\r
+ *   a connection.  The destination address must have already been resolved\r
+ *   by calling rdma_resolve_addr.\r
+ * Notes:\r
+ *   This is called on the client side of a connection after calling\r
+ *   rdma_resolve_addr, but before calling rdma_connect.\r
+ * See also:\r
+ *   rdma_resolve_addr, rdma_connect, rdma_get_cm_event\r
+ */\r
+__declspec(dllexport)\r
+int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms);\r
+\r
+/**\r
+ * rdma_create_qp - Allocate a QP.\r
+ * @id: RDMA identifier.\r
+ * @pd: protection domain for the QP.\r
+ * @qp_init_attr: initial QP attributes.\r
+ * Description:\r
+ *  Allocate a QP associated with the specified rdma_cm_id and transition it\r
+ *  for sending and receiving.\r
+ * Notes:\r
+ *   The rdma_cm_id must be bound to a local RDMA device before calling this\r
+ *   function, and the protection domain must be for that same device.\r
+ *   QPs allocated to an rdma_cm_id are automatically transitioned by the\r
+ *   librdmacm through their states.  After being allocated, the QP will be\r
+ *   ready to handle posting of receives.  If the QP is unconnected, it will\r
+ *   be ready to post sends.\r
+ * See also:\r
+ *   rdma_bind_addr, rdma_resolve_addr, rdma_destroy_qp, ibv_create_qp,\r
+ *   ibv_modify_qp\r
+ */\r
+__declspec(dllexport)\r
+int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd,\r
+                                  struct ibv_qp_init_attr *qp_init_attr);\r
+\r
+/**\r
+ * rdma_destroy_qp - Deallocate a QP.\r
+ * @id: RDMA identifier.\r
+ * Description:\r
+ *   Destroy a QP allocated on the rdma_cm_id.\r
+ * Notes:\r
+ *   Users must destroy any QP associated with an rdma_cm_id before\r
+ *   destroying the ID.\r
+ * See also:\r
+ *   rdma_create_qp, rdma_destroy_id, ibv_destroy_qp\r
+ */\r
+__declspec(dllexport)\r
+void rdma_destroy_qp(struct rdma_cm_id *id);\r
+\r
+/**\r
+ * rdma_connect - Initiate an active connection request.\r
+ * @id: RDMA identifier.\r
+ * @conn_param: connection parameters.\r
+ * Description:\r
+ *   For a connected rdma_cm_id, this call initiates a connection request\r
+ *   to a remote destination.  For an unconnected rdma_cm_id, it initiates\r
+ *   a lookup of the remote QP providing the datagram service.\r
+ * Notes:\r
+ *   Users must have resolved a route to the destination address\r
+ *   by having called rdma_resolve_route before calling this routine.\r
+ * See also:\r
+ *   rdma_resolve_route, rdma_disconnect, rdma_listen, rdma_get_cm_event\r
+ */\r
+__declspec(dllexport)\r
+int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param);\r
+\r
+/**\r
+ * rdma_listen - Listen for incoming connection requests.\r
+ * @id: RDMA identifier.\r
+ * @backlog: backlog of incoming connection requests.\r
+ * Description:\r
+ *   Initiates a listen for incoming connection requests or datagram service\r
+ *   lookup.  The listen will be restricted to the locally bound source\r
+ *   address.\r
+ * Notes:\r
+ *   Users must have bound the rdma_cm_id to a local address by calling\r
+ *   rdma_bind_addr before calling this routine.  If the rdma_cm_id is\r
+ *   bound to a specific IP address, the listen will be restricted to that\r
+ *   address and the associated RDMA device.  If the rdma_cm_id is bound\r
+ *   to an RDMA port number only, the listen will occur across all RDMA\r
+ *   devices.\r
+ * See also:\r
+ *   rdma_bind_addr, rdma_connect, rdma_accept, rdma_reject, rdma_get_cm_event\r
+ */\r
+__declspec(dllexport)\r
+int rdma_listen(struct rdma_cm_id *id, int backlog);\r
+\r
+/**\r
+ * rdma_accept - Called to accept a connection request.\r
+ * @id: Connection identifier associated with the request.\r
+ * @conn_param: Information needed to establish the connection.\r
+ * Description:\r
+ *   Called from the listening side to accept a connection or datagram\r
+ *   service lookup request.\r
+ * Notes:\r
+ *   Unlike the socket accept routine, rdma_accept is not called on a\r
+ *   listening rdma_cm_id.  Instead, after calling rdma_listen, the user\r
+ *   waits for a connection request event to occur.  Connection request\r
+ *   events give the user a newly created rdma_cm_id, similar to a new\r
+ *   socket, but the rdma_cm_id is bound to a specific RDMA device.\r
+ *   rdma_accept is called on the new rdma_cm_id.\r
+ * See also:\r
+ *   rdma_listen, rdma_reject, rdma_get_cm_event\r
+ */\r
+__declspec(dllexport)\r
+int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param);\r
+\r
+/**\r
+ * rdma_reject - Called to reject a connection request.\r
+ * @id: Connection identifier associated with the request.\r
+ * @private_data: Optional private data to send with the reject message.\r
+ * @private_data_len: Size of the private_data to send, in bytes.\r
+ * Description:\r
+ *   Called from the listening side to reject a connection or datagram\r
+ *   service lookup request.\r
+ * Notes:\r
+ *   After receiving a connection request event, a user may call rdma_reject\r
+ *   to reject the request.  If the underlying RDMA transport supports\r
+ *   private data in the reject message, the specified data will be passed to\r
+ *   the remote side.\r
+ * See also:\r
+ *   rdma_listen, rdma_accept, rdma_get_cm_event\r
+ */\r
+__declspec(dllexport)\r
+int rdma_reject(struct rdma_cm_id *id, const void *private_data,\r
+                               uint8_t private_data_len);\r
+\r
+/**\r
+ * rdma_notify - Notifies the librdmacm of an asynchronous event.\r
+ * @id: RDMA identifier.\r
+ * @event: Asynchronous event.\r
+ * Description:\r
+ *   Used to notify the librdmacm of asynchronous events that have occurred\r
+ *   on a QP associated with the rdma_cm_id.\r
+ * Notes:\r
+ *   Asynchronous events that occur on a QP are reported through the user's\r
+ *   device event handler.  This routine is used to notify the librdmacm of\r
+ *   communication events.  In most cases, use of this routine is not\r
+ *   necessary, however if connection establishment is done out of band\r
+ *   (such as done through Infiniband), it's possible to receive data on a\r
+ *   QP that is not yet considered connected.  This routine forces the\r
+ *   connection into an established state in this case in order to handle\r
+ *   the rare situation where the connection never forms on its own.\r
+ *   Events that should be reported to the CM are: IB_EVENT_COMM_EST.\r
+ * See also:\r
+ *   rdma_connect, rdma_accept, rdma_listen\r
+ */\r
+__declspec(dllexport)\r
+int rdma_notify(struct rdma_cm_id *id, enum ibv_event_type event);\r
+\r
+/**\r
+ * rdma_disconnect - This function disconnects a connection.\r
+ * @id: RDMA identifier.\r
+ * Description:\r
+ *   Disconnects a connection and transitions any associated QP to the\r
+ *   error state.\r
+ * See also:\r
+ *   rdma_connect, rdma_listen, rdma_accept\r
+ */\r
+__declspec(dllexport)\r
+int rdma_disconnect(struct rdma_cm_id *id);\r
+\r
+/**\r
+ * rdma_join_multicast - Joins a multicast group.\r
+ * @id: Communication identifier associated with the request.\r
+ * @addr: Multicast address identifying the group to join.\r
+ * @context: User-defined context associated with the join request.\r
+ * Description:\r
+ *   Joins a multicast group and attaches an associated QP to the group.\r
+ * Notes:\r
+ *   Before joining a multicast group, the rdma_cm_id must be bound to\r
+ *   an RDMA device by calling rdma_bind_addr or rdma_resolve_addr.  Use of\r
+ *   rdma_resolve_addr requires the local routing tables to resolve the\r
+ *   multicast address to an RDMA device.  The user must call\r
+ *   rdma_leave_multicast to leave the multicast group and release any\r
+ *   multicast resources.  The context is returned to the user through\r
+ *   the private_data field in the rdma_cm_event.\r
+ * See also:\r
+ *   rdma_leave_multicast, rdma_bind_addr, rdma_resolve_addr, rdma_create_qp\r
+ */\r
+__declspec(dllexport)\r
+int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,\r
+                                               void *context);\r
+\r
+/**\r
+ * rdma_leave_multicast - Leaves a multicast group.\r
+ * @id: Communication identifier associated with the request.\r
+ * @addr: Multicast address identifying the group to leave.\r
+ * Description:\r
+ *   Leaves a multicast group and detaches an associated QP from the group.\r
+ * Notes:\r
+ *   Calling this function before a group has been fully joined results in\r
+ *   canceling the join operation.  Users should be aware that messages\r
+ *   received from the multicast group may stilled be queued for\r
+ *   completion processing immediately after leaving a multicast group.\r
+ *   Destroying an rdma_cm_id will automatically leave all multicast groups.\r
+ * See also:\r
+ *   rdma_join_multicast, rdma_destroy_qp\r
+ */\r
+__declspec(dllexport)\r
+int rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr);\r
+\r
+/**\r
+ * rdma_get_cm_event - Retrieves the next pending communication event.\r
+ * @channel: Event channel to check for events.\r
+ * @event: Allocated information about the next communication event.\r
+ * Description:\r
+ *   Retrieves a communication event.  If no events are pending, by default,\r
+ *   the call will block until an event is received.\r
+ * Notes:\r
+ *   The default synchronous behavior of this routine can be changed by\r
+ *   modifying the file descriptor associated with the given channel.  All\r
+ *   events that are reported must be acknowledged by calling rdma_ack_cm_event.\r
+ *   Destruction of an rdma_cm_id will block until related events have been\r
+ *   acknowledged.\r
+ * See also:\r
+ *   rdma_ack_cm_event, rdma_create_event_channel, rdma_event_str\r
+ */\r
+__declspec(dllexport)\r
+int rdma_get_cm_event(struct rdma_event_channel *channel,\r
+                                         struct rdma_cm_event **event);\r
+\r
+/**\r
+ * rdma_ack_cm_event - Free a communication event.\r
+ * @event: Event to be released.\r
+ * Description:\r
+ *   All events which are allocated by rdma_get_cm_event must be released,\r
+ *   there should be a one-to-one correspondence between successful gets\r
+ *   and acks.\r
+ * See also:\r
+ *   rdma_get_cm_event, rdma_destroy_id\r
+ */\r
+__declspec(dllexport)\r
+int rdma_ack_cm_event(struct rdma_cm_event *event);\r
+\r
+static uint16_t rdma_get_src_port(struct rdma_cm_id *id)\r
+{\r
+       return  id->route.addr.src_addr.sa_family == PF_INET6 ?\r
+               ((struct sockaddr_in6 *) &id->route.addr.src_addr)->sin6_port :\r
+               ((struct sockaddr_in *) &id->route.addr.src_addr)->sin_port;\r
+}\r
+\r
+static uint16_t rdma_get_dst_port(struct rdma_cm_id *id)\r
+{\r
+       return  id->route.addr.dst_addr.sa_family == PF_INET6 ?\r
+               ((struct sockaddr_in6 *) &id->route.addr.dst_addr)->sin6_port :\r
+               ((struct sockaddr_in *) &id->route.addr.dst_addr)->sin_port;\r
+}\r
+\r
+static struct sockaddr *rdma_get_local_addr(struct rdma_cm_id *id)\r
+{\r
+       return &id->route.addr.src_addr;\r
+}\r
+\r
+static struct sockaddr *rdma_get_peer_addr(struct rdma_cm_id *id)\r
+{\r
+       return &id->route.addr.dst_addr;\r
+}\r
+\r
+/**\r
+ * rdma_get_devices - Get list of RDMA devices currently available.\r
+ * @num_devices: If non-NULL, set to the number of devices returned.\r
+ * Description:\r
+ *   Return a NULL-terminated array of opened RDMA devices.  Callers can use\r
+ *   this routine to allocate resources on specific RDMA devices that will be\r
+ *   shared across multiple rdma_cm_id's.\r
+ * Notes:\r
+ *   The returned array must be released by calling rdma_free_devices.  Devices\r
+ *   remain opened while the librdmacm is loaded.\r
+ * See also:\r
+ *   rdma_free_devices\r
+ */\r
+__declspec(dllexport)\r
+struct ibv_context **rdma_get_devices(int *num_devices);\r
+\r
+/**\r
+ * rdma_free_devices - Frees the list of devices returned by rdma_get_devices.\r
+ * @list: List of devices returned from rdma_get_devices.\r
+ * Description:\r
+ *   Frees the device array returned by rdma_get_devices.\r
+ * See also:\r
+ *   rdma_get_devices\r
+ */\r
+__declspec(dllexport)\r
+void rdma_free_devices(struct ibv_context **list);\r
+\r
+/**\r
+ * rdma_event_str - Returns a string representation of an rdma cm event.\r
+ * @event: Asynchronous event.\r
+ * Description:\r
+ *   Returns a string representation of an asynchronous event.\r
+ * See also:\r
+ *   rdma_get_cm_event\r
+ */\r
+__declspec(dllexport)\r
+const char *rdma_event_str(enum rdma_cm_event_type event);\r
+\r
+/* Option levels */\r
+enum\r
+{\r
+       RDMA_OPTION_ID          = 0\r
+};\r
+\r
+/* Option details */\r
+enum\r
+{\r
+       RDMA_OPTION_ID_TOS      = 0     /* uint8_t: RFC 2474 */\r
+};\r
+\r
+/**\r
+ * rdma_set_option - Set options for an rdma_cm_id.\r
+ * @id: Communication identifier to set option for.\r
+ * @level: Protocol level of the option to set.\r
+ * @optname: Name of the option to set.\r
+ * @optval: Reference to the option data.\r
+ * @optlen: The size of the %optval buffer.\r
+ */\r
+__declspec(dllexport)\r
+int rdma_set_option(struct rdma_cm_id *id, int level, int optname,\r
+                                       void *optval, size_t optlen);\r
+\r
+/**\r
+ * rdma_migrate_id - Move an rdma_cm_id to a new event channel.\r
+ * @id: Communication identifier to migrate.\r
+ * @channel: New event channel for rdma_cm_id events.\r
+ */\r
+__declspec(dllexport)\r
+int rdma_migrate_id(struct rdma_cm_id *id, struct rdma_event_channel *channel);\r
+\r
+#ifdef __cplusplus\r
+}\r
+#endif\r
+\r
+#endif /* RDMA_CMA_H */\r

diff --git a/ulp/librdmacm/src/Sources b/ulp/librdmacm/src/Sources
new file mode 100644 (file)
index 0000000..3a77169
--- /dev/null
+++ b/ulp/librdmacm/src/Sources
@@ -0,0 +1,40 @@
+!if $(FREEBUILD)\r
+TARGETNAME = librdmacm\r
+!else\r
+TARGETNAME = librdmacmd\r
+!endif\r
+\r
+TARGETPATH = ..\..\..\bin\user\obj$(BUILD_ALT_DIR)\r
+TARGETTYPE = DYNLINK\r
+\r
+!if $(_NT_TOOLS_VERSION) == 0x700\r
+DLLDEF = $O\cma_exports.def\r
+!else\r
+DLLDEF = $(OBJ_PATH)\$O\cma_exports.def\r
+!endif\r
+\r
+DLLENTRY = DllMain\r
+USE_MSVCRT = 1\r
+\r
+SOURCES =                      \\r
+       cma.rc                  \\r
+       cma_main.cpp    \\r
+       cma.cpp\r
+\r
+INCLUDES = ..\include;..\..\..\inc;..\..\..\inc\user;..\..\libibverbs\include\r
+\r
+USER_C_FLAGS = $(USER_C_FLAGS) -DEXPORT_CMA_SYMBOLS\r
+\r
+TARGETLIBS = \\r
+       $(SDK_LIB_PATH)\kernel32.lib    \\r
+       $(SDK_LIB_PATH)\uuid.lib                \\r
+       $(SDK_LIB_PATH)\ws2_32.lib              \\r
+       $(SDK_LIB_PATH)\iphlpapi.lib    \\r
+       $(TARGETPATH)\*\ibat.lib                \\r
+!if $(FREEBUILD)\r
+       $(TARGETPATH)\*\libibverbs.lib  \\r
+       $(TARGETPATH)\*\winverbs.lib\r
+!else\r
+       $(TARGETPATH)\*\libibverbsd.lib \\r
+       $(TARGETPATH)\*\winverbsd.lib\r
+!endif\r

diff --git a/ulp/librdmacm/src/cma.cpp b/ulp/librdmacm/src/cma.cpp
new file mode 100644 (file)
index 0000000..2ce0cd2
--- /dev/null
+++ b/ulp/librdmacm/src/cma.cpp
@@ -0,0 +1,1084 @@
+/*\r
+ * Copyright (c) 2005-2009 Intel Corporation.  All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ *     Redistribution and use in source and binary forms, with or\r
+ *     without modification, are permitted provided that the following\r
+ *     conditions are met:\r
+ *\r
+ *      - Redistributions of source code must retain the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer.\r
+ *\r
+ *      - Redistributions in binary form must reproduce the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer in the documentation and/or other materials\r
+ *        provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ */\r
+\r
+#include <windows.h>\r
+#include <winsock2.h>\r
+#include <stdio.h>\r
+#include <iphlpapi.h>\r
+\r
+#include <rdma/rdma_cma.h>\r
+#include <infiniband/verbs.h>\r
+#include <iba/ibat.h>\r
+#include "cma.h"\r
+\r
+IWVProvider *prov;\r
+__declspec(dllexport)\r
+IWVProvider *ibv_get_winverbs(void);\r
+\r
+enum cma_state\r
+{\r
+       cma_idle,\r
+       cma_listening,\r
+       cma_get_request,\r
+       cma_addr_resolve,\r
+       cma_route_resolve,\r
+       cma_passive_connect,\r
+       cma_active_connect,\r
+       cma_active_accept,\r
+       cma_accepting,\r
+       cma_connected,\r
+       cma_active_disconnect,\r
+       cma_passive_disconnect,\r
+       cma_disconnected\r
+};\r
+\r
+#define CMA_DEFAULT_BACKLOG            16\r
+\r
+struct cma_id_private\r
+{\r
+       struct rdma_cm_id                       id;\r
+       enum cma_state                          state;\r
+       int                                                     channel_index;\r
+       struct cma_device                       *cma_dev;\r
+       int                                                     backlog;\r
+       int                                                     index;\r
+       struct rdma_cm_id                       **req_list;\r
+};\r
+\r
+struct cma_event_channel\r
+{\r
+       struct rdma_event_channel       channel;\r
+       CRITICAL_SECTION                        lock;\r
+       struct cma_id_private           *id[MAXIMUM_WAIT_OBJECTS];\r
+       HANDLE                                          event[MAXIMUM_WAIT_OBJECTS];\r
+       int                                                     count;\r
+};\r
+\r
+struct cma_device\r
+{\r
+       struct ibv_context      *verbs;\r
+       uint64_t                        guid;\r
+       int                                     port_cnt;\r
+       uint8_t                         max_initiator_depth;\r
+       uint8_t                         max_responder_resources;\r
+};\r
+\r
+struct cma_event {\r
+       struct rdma_cm_event    event;\r
+       uint8_t                                 private_data[56];\r
+       struct cma_id_private   *id_priv;\r
+};\r
+\r
+static struct cma_device *cma_dev_array;\r
+static int cma_dev_cnt;\r
+\r
+static void ucma_cleanup(void)\r
+{\r
+       if (cma_dev_cnt > 0) {\r
+               while (cma_dev_cnt > 0) {\r
+                       ibv_close_device(cma_dev_array[--cma_dev_cnt].verbs);\r
+               }\r
+               delete cma_dev_array;\r
+               cma_dev_cnt = 0;\r
+       }\r
+       if (prov != NULL) {\r
+               prov->Release();\r
+               prov = NULL;\r
+       }\r
+}\r
+\r
+static int ucma_init(void)\r
+{\r
+       struct ibv_device **dev_list = NULL;\r
+       struct cma_device *cma_dev;\r
+       struct ibv_device_attr attr;\r
+       int i, ret;\r
+\r
+       EnterCriticalSection(&lock);\r
+       if (cma_dev_cnt > 0) {\r
+               goto out;\r
+       }\r
+\r
+       prov = ibv_get_winverbs();\r
+       if (prov == NULL) {\r
+               ret = -1;\r
+               goto err;\r
+       }\r
+\r
+       dev_list = ibv_get_device_list(&cma_dev_cnt);\r
+       if (dev_list == NULL) {\r
+               ret = -1;\r
+               goto err;\r
+       }\r
+\r
+       cma_dev_array = new struct cma_device[cma_dev_cnt];\r
+       if (cma_dev_array == NULL) {\r
+               ret = -1;\r
+               goto err;\r
+       }\r
+\r
+       for (i = 0; dev_list[i]; ++i) {\r
+               cma_dev = &cma_dev_array[i];\r
+\r
+               cma_dev->guid = ibv_get_device_guid(dev_list[i]);\r
+               cma_dev->verbs = ibv_open_device(dev_list[i]);\r
+               if (cma_dev->verbs == NULL) {\r
+                       ret = -1;\r
+                       goto err;\r
+               }\r
+\r
+               ret = ibv_query_device(cma_dev->verbs, &attr);\r
+               if (ret) {\r
+                       goto err;\r
+               }\r
+\r
+               cma_dev->port_cnt = attr.phys_port_cnt;\r
+               cma_dev->max_initiator_depth = (uint8_t) attr.max_qp_init_rd_atom;\r
+               cma_dev->max_responder_resources = (uint8_t) attr.max_qp_rd_atom;\r
+       }\r
+       ibv_free_device_list(dev_list);\r
+out:\r
+       LeaveCriticalSection(&lock);\r
+       return 0;\r
+\r
+err:\r
+       ucma_cleanup();\r
+       LeaveCriticalSection(&lock);\r
+       if (dev_list) {\r
+               ibv_free_device_list(dev_list);\r
+       }\r
+       return ret;\r
+}\r
+\r
+__declspec(dllexport)\r
+struct ibv_context **rdma_get_devices(int *num_devices)\r
+{\r
+       struct ibv_context **devs = NULL;\r
+       int i;\r
+\r
+       if (!cma_dev_cnt && ucma_init()) {\r
+               goto out;\r
+       }\r
+\r
+       devs = new struct ibv_context *[cma_dev_cnt + 1];\r
+       if (devs == NULL) {\r
+               goto out;\r
+       }\r
+\r
+       for (i = 0; i < cma_dev_cnt; i++) {\r
+               devs[i] = cma_dev_array[i].verbs;\r
+       }\r
+       devs[i] = NULL;\r
+out:\r
+       if (num_devices != NULL) {\r
+               *num_devices = devs ? cma_dev_cnt : 0;\r
+       }\r
+       return devs;\r
+}\r
+\r
+__declspec(dllexport)\r
+void rdma_free_devices(struct ibv_context **list)\r
+{\r
+       delete list;\r
+}\r
+\r
+__declspec(dllexport)\r
+struct rdma_event_channel *rdma_create_event_channel(void)\r
+{\r
+       struct cma_event_channel *chan;\r
+\r
+       if (!cma_dev_cnt && ucma_init()) {\r
+               return NULL;\r
+       }\r
+\r
+       chan = new struct cma_event_channel;\r
+       if (chan == NULL) {\r
+               return NULL;\r
+       }\r
+\r
+       InitializeCriticalSection(&chan->lock);\r
+       chan->count = 0;\r
+       chan->channel.timeout = INFINITE;\r
+\r
+       return &chan->channel;\r
+}\r
+\r
+__declspec(dllexport)\r
+void rdma_destroy_event_channel(struct rdma_event_channel *channel)\r
+{\r
+       struct cma_event_channel *chan;\r
+\r
+       chan = CONTAINING_RECORD(channel, struct cma_event_channel, channel);\r
+       DeleteCriticalSection(&chan->lock);     \r
+       delete chan;\r
+}\r
+\r
+static int cma_event_channel_insert_id(struct rdma_event_channel *channel,\r
+                                                                          struct cma_id_private *id_priv)\r
+{\r
+       struct cma_event_channel *chan;\r
+       int ret = 0;\r
+\r
+       chan = CONTAINING_RECORD(channel, struct cma_event_channel, channel);\r
+\r
+       EnterCriticalSection(&chan->lock);\r
+       if (chan->count == MAXIMUM_WAIT_OBJECTS) {\r
+               ret = -1;\r
+               goto out;\r
+       }\r
+\r
+       chan->id[chan->count] = id_priv;\r
+       chan->event[chan->count] = id_priv->id.overlap.hEvent;\r
+       id_priv->channel_index = chan->count++;\r
+out:\r
+       LeaveCriticalSection(&chan->lock);\r
+       return ret;\r
+}\r
+\r
+/*\r
+ * TODO: we cannot call cma_event_channel_remove_id() while another\r
+ * thread is calling rdma_get_event().  If this is needed, then we\r
+ * need to halt the rdma_get_event() thread, modify the event list,\r
+ * then restart the rdma_get_event() thread.\r
+ */\r
+static void cma_event_channel_remove_id(struct rdma_event_channel *channel,\r
+                                                                               struct cma_id_private *id_priv)\r
+{\r
+       struct cma_event_channel *chan;\r
+\r
+       chan = CONTAINING_RECORD(channel, struct cma_event_channel, channel);\r
+\r
+       EnterCriticalSection(&chan->lock);\r
+       chan->count--;\r
+       chan->id[id_priv->channel_index] = chan->id[chan->count];\r
+       chan->event[id_priv->channel_index] = chan->event[chan->count];\r
+       chan->id[id_priv->channel_index]->channel_index = id_priv->channel_index;\r
+       LeaveCriticalSection(&chan->lock);\r
+}\r
+\r
+__declspec(dllexport)\r
+int rdma_create_id(struct rdma_event_channel *channel,\r
+                                  struct rdma_cm_id **id, void *context,\r
+                                  enum rdma_port_space ps)\r
+{\r
+       struct cma_id_private *id_priv;\r
+       HRESULT hr;\r
+\r
+       hr = cma_dev_cnt ? 0 : ucma_init();\r
+       if (hr) {\r
+               return hr;\r
+       }\r
+\r
+       id_priv = new struct cma_id_private;\r
+       if (id_priv == NULL) {\r
+               return NULL;\r
+       }\r
+\r
+       RtlZeroMemory(id_priv, sizeof(struct cma_id_private));\r
+       id_priv->id.overlap.hEvent = CreateEvent(NULL, FALSE, FALSE, NULL);\r
+       if (id_priv->id.overlap.hEvent == NULL) {\r
+               goto err1;\r
+       }\r
+\r
+       id_priv->id.context = context;\r
+       id_priv->id.channel = channel;\r
+       id_priv->id.ps = ps;\r
+\r
+       if (ps == RDMA_PS_TCP) {\r
+               hr = prov->CreateConnectEndpoint(&id_priv->id.ep.connect);\r
+       } else {\r
+               hr = prov->CreateDatagramEndpoint(&id_priv->id.ep.datagram);\r
+       }\r
+       if (FAILED(hr)) {\r
+               goto err2;\r
+       }\r
+\r
+       hr = cma_event_channel_insert_id(channel, id_priv);\r
+       if (FAILED(hr)) {\r
+               goto err3;\r
+       }\r
+\r
+       *id = &id_priv->id;\r
+       return 0;\r
+\r
+err3:\r
+       if (ps == RDMA_PS_TCP) {\r
+               id_priv->id.ep.connect->Release();\r
+       } else {\r
+               id_priv->id.ep.datagram->Release();\r
+       }\r
+err2:\r
+       CloseHandle(id_priv->id.overlap.hEvent);\r
+err1:\r
+       delete id_priv;\r
+       return -1;\r
+}\r
+\r
+static void ucma_destroy_listen(struct cma_id_private *id_priv)\r
+{\r
+       while (--id_priv->backlog >= 0) {\r
+               if (id_priv->req_list[id_priv->backlog] != NULL) {\r
+                       rdma_destroy_id(id_priv->req_list[id_priv->backlog]);\r
+               }\r
+       }\r
+\r
+       delete id_priv->req_list;\r
+}\r
+\r
+__declspec(dllexport)\r
+int rdma_destroy_id(struct rdma_cm_id *id)\r
+{\r
+       struct cma_id_private   *id_priv;\r
+\r
+       id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);\r
+       if (id->ps == RDMA_PS_TCP) {\r
+               id->ep.connect->CancelOverlappedRequests();\r
+       } else {\r
+               id->ep.datagram->CancelOverlappedRequests();\r
+       }\r
+\r
+       cma_event_channel_remove_id(id->channel, id_priv);\r
+\r
+       if (id_priv->backlog > 0) {\r
+               ucma_destroy_listen(id_priv);\r
+       }\r
+\r
+       if (id_priv->id.ps == RDMA_PS_TCP) {\r
+               id_priv->id.ep.connect->Release();\r
+       } else {\r
+               id_priv->id.ep.datagram->Release();\r
+       }\r
+\r
+       delete id_priv;\r
+       return 0;\r
+}\r
+\r
+static int ucma_addrlen(struct sockaddr *addr)\r
+{\r
+       if (addr->sa_family == PF_INET) {\r
+               return sizeof(struct sockaddr_in);\r
+       } else {\r
+               return sizeof(struct sockaddr_in6);\r
+       }\r
+}\r
+\r
+static int ucma_get_device(struct cma_id_private *id_priv, uint64_t guid)\r
+{\r
+       struct cma_device *cma_dev;\r
+       int i;\r
+\r
+       for (i = 0; i < cma_dev_cnt; i++) {\r
+               cma_dev = &cma_dev_array[i];\r
+               if (cma_dev->guid == guid) {\r
+                       id_priv->cma_dev = cma_dev;\r
+                       id_priv->id.verbs = cma_dev->verbs;\r
+                       return 0;\r
+               }\r
+       }\r
+       return -1;\r
+}\r
+\r
+static int ucma_query_connect(struct rdma_cm_id *id, struct rdma_conn_param *param)\r
+{\r
+       struct cma_id_private *id_priv;\r
+       WV_CONNECT_ATTRIBUTES attr;\r
+       HRESULT hr;\r
+\r
+       id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);\r
+       hr = id->ep.connect->Query(&attr);\r
+       if (FAILED(hr)) {\r
+               return hr;\r
+       }\r
+\r
+       RtlCopyMemory(&id->route.addr.src_addr, &attr.LocalAddress,\r
+                                 sizeof attr.LocalAddress);\r
+       RtlCopyMemory(&id->route.addr.dst_addr, &attr.PeerAddress,\r
+                                 sizeof attr.PeerAddress);\r
+\r
+       if (param != NULL) {\r
+               RtlCopyMemory((void *) param->private_data, attr.Param.Data,\r
+                                         attr.Param.DataLength);\r
+               param->private_data_len = (uint8_t) attr.Param.DataLength;\r
+               param->responder_resources = (uint8_t) attr.Param.ResponderResources;\r
+               param->initiator_depth = (uint8_t) attr.Param.InitiatorDepth;\r
+               param->flow_control = 1;\r
+               param->retry_count = attr.Param.RetryCount;\r
+               param->rnr_retry_count = attr.Param.RnrRetryCount;\r
+       }\r
+\r
+       if (id_priv->cma_dev == NULL && attr.Device.DeviceGuid != 0) {\r
+               hr = ucma_get_device(id_priv, attr.Device.DeviceGuid);\r
+               if (FAILED(hr)) {\r
+                       return hr;\r
+               }\r
+\r
+               id->route.addr.addr.ibaddr.pkey = attr.Device.Pkey;\r
+               id_priv->id.port_num = attr.Device.PortNumber;\r
+       }\r
+\r
+       return 0;\r
+}\r
+\r
+static int ucma_query_datagram(struct rdma_cm_id *id, struct rdma_ud_param *param)\r
+{\r
+       struct cma_id_private *id_priv;\r
+       WV_DATAGRAM_ATTRIBUTES attr;\r
+       HRESULT hr;\r
+\r
+       id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);\r
+       hr = id->ep.datagram->Query(&attr);\r
+       if (FAILED(hr)) {\r
+               return hr;\r
+       }\r
+\r
+       RtlCopyMemory(&id->route.addr.src_addr, &attr.LocalAddress,\r
+                                 sizeof attr.LocalAddress);\r
+       RtlCopyMemory(&id->route.addr.dst_addr, &attr.PeerAddress,\r
+                                 sizeof attr.PeerAddress);\r
+\r
+       if (param != NULL) {\r
+               RtlCopyMemory((void *) param->private_data, attr.Param.Data,\r
+                                         attr.Param.DataLength);\r
+               param->private_data_len = (uint8_t) attr.Param.DataLength;\r
+               // ucma_convert_av(&attr.Param.AddressVector, param->ah_attr)\r
+               param->qp_num = attr.Param.Qpn;\r
+               param->qkey = attr.Param.Qkey;\r
+       }\r
+\r
+       if (id_priv->cma_dev == NULL && attr.Device.DeviceGuid != 0) {\r
+               hr = ucma_get_device(id_priv, attr.Device.DeviceGuid);\r
+               if (FAILED(hr))\r
+                       return hr;\r
+               id->route.addr.addr.ibaddr.pkey = attr.Device.Pkey;\r
+               id_priv->id.port_num = attr.Device.PortNumber;\r
+       }\r
+       return 0;\r
+}\r
+\r
+__declspec(dllexport)\r
+int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)\r
+{\r
+       HRESULT hr;\r
+\r
+       if (id->ps == RDMA_PS_TCP) {\r
+               hr = id->ep.connect->BindAddress(addr);\r
+               if (SUCCEEDED(hr)) {\r
+                       hr = ucma_query_connect(id, NULL);\r
+               }\r
+       } else {\r
+               hr = id->ep.datagram->BindAddress(addr);\r
+               if (SUCCEEDED(hr)) {\r
+                       hr = ucma_query_datagram(id, NULL);\r
+               }\r
+       }\r
+\r
+       return hr;\r
+}\r
+\r
+__declspec(dllexport)\r
+int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,\r
+                                         struct sockaddr *dst_addr, int timeout_ms)\r
+{\r
+       struct cma_id_private *id_priv;\r
+       WV_SOCKADDR addr;\r
+       SOCKET s;\r
+       DWORD size;\r
+       HRESULT hr;\r
+\r
+       if (src_addr == NULL) {\r
+               if (id->ps == RDMA_PS_TCP) {\r
+                       s = socket(dst_addr->sa_family, SOCK_STREAM, IPPROTO_TCP);\r
+               } else {\r
+                       s = socket(dst_addr->sa_family, SOCK_DGRAM, IPPROTO_UDP);\r
+               }\r
+               if (s == INVALID_SOCKET) {\r
+                       return WSAGetLastError();\r
+               }\r
+\r
+               hr = WSAIoctl(s, SIO_ROUTING_INTERFACE_QUERY, dst_addr, ucma_addrlen(dst_addr),\r
+                                         &addr, sizeof addr, &size, NULL, NULL);\r
+               closesocket(s);\r
+               if (FAILED(hr)) {\r
+                       return WSAGetLastError();\r
+               }\r
+               src_addr = &addr.Sa;\r
+       }\r
+\r
+       hr = rdma_bind_addr(id, src_addr);\r
+       if (FAILED(hr)) {\r
+               return hr;\r
+       }\r
+\r
+       RtlCopyMemory(&id->route.addr.dst_addr, dst_addr, ucma_addrlen(dst_addr));\r
+       id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);\r
+       id_priv->state = cma_addr_resolve;\r
+       SetEvent(id->overlap.hEvent);\r
+       return 0;\r
+}\r
+\r
+__declspec(dllexport)\r
+int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)\r
+{\r
+       struct cma_id_private *id_priv;\r
+       IBAT_PATH_BLOB path;\r
+       HRESULT hr;\r
+\r
+       hr = IBAT::Resolve(&id->route.addr.src_addr, &id->route.addr.dst_addr, &path);\r
+       if (FAILED(hr)) {\r
+               return hr;\r
+       }\r
+\r
+       hr = (id->ps == RDMA_PS_TCP) ?\r
+                id->ep.connect->Modify(WV_EP_OPTION_ROUTE, &path, sizeof path) :\r
+                id->ep.datagram->Modify(WV_EP_OPTION_ROUTE, &path, sizeof path);\r
+       if (FAILED(hr)) {\r
+               return hr;\r
+       }\r
+\r
+       id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);\r
+       id_priv->state = cma_route_resolve;\r
+       SetEvent(id->overlap.hEvent);\r
+       return 0;\r
+}\r
+\r
+static int ucma_modify_qp_init(struct cma_id_private *id_priv, struct ibv_qp *qp)\r
+{\r
+       struct ibv_qp_attr qp_attr;\r
+       DWORD index;\r
+       HRESULT hr;\r
+\r
+       RtlZeroMemory(&qp_attr, sizeof qp_attr);\r
+       qp_attr.qp_state = IBV_QPS_INIT;\r
+       qp_attr.port_num = id_priv->id.port_num;\r
+       hr = qp->context->cmd_if->FindPkey(id_priv->id.port_num,\r
+                                                                          id_priv->id.route.addr.addr.ibaddr.pkey,\r
+                                                                          &index);\r
+       if (FAILED(hr)) {\r
+               return hr;\r
+       }\r
+\r
+       qp_attr.pkey_index = (uint16_t) index;\r
+       return ibv_modify_qp(qp, &qp_attr, (enum ibv_qp_attr_mask)\r
+                                                (IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT));\r
+}\r
+\r
+static int ucma_init_ud_qp(struct cma_id_private *id_priv, struct ibv_qp *qp)\r
+{\r
+       struct ibv_qp_attr qp_attr;\r
+       int qp_attr_mask, ret;\r
+\r
+       ret = ucma_modify_qp_init(id_priv, qp);\r
+       if (ret) {\r
+               return ret;\r
+       }\r
+\r
+       qp_attr.qp_state = IBV_QPS_RTR;\r
+       ret = ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE);\r
+       if (ret) {\r
+               return ret;\r
+       }\r
+\r
+       qp_attr.qp_state = IBV_QPS_RTS;\r
+       qp_attr.sq_psn = 0;\r
+       return ibv_modify_qp(qp, &qp_attr, (enum ibv_qp_attr_mask)\r
+                                                (IBV_QP_STATE | IBV_QP_SQ_PSN));\r
+}\r
+\r
+__declspec(dllexport)\r
+int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd,\r
+                                  struct ibv_qp_init_attr *qp_init_attr)\r
+{\r
+       struct cma_id_private *id_priv;\r
+       struct ibv_qp *qp;\r
+       int ret;\r
+\r
+       id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);\r
+       if (id->verbs != pd->context) {\r
+               return -1;\r
+       }\r
+\r
+       qp = ibv_create_qp(pd, qp_init_attr);\r
+       if (!qp) {\r
+               return -1;\r
+       }\r
+\r
+       if (id->ps == RDMA_PS_TCP) {\r
+               ret = ucma_modify_qp_init(id_priv, qp);\r
+       } else {\r
+               ret = ucma_init_ud_qp(id_priv, qp);\r
+       }\r
+       if (ret) {\r
+               goto err;\r
+       }\r
+\r
+       id->qp = qp;\r
+       return 0;\r
+err:\r
+       ibv_destroy_qp(qp);\r
+       return ret;\r
+}\r
+\r
+__declspec(dllexport)\r
+void rdma_destroy_qp(struct rdma_cm_id *id)\r
+{\r
+       ibv_destroy_qp(id->qp);\r
+}\r
+\r
+static int ucma_valid_param(struct cma_id_private *id_priv,\r
+                                                       struct rdma_conn_param *conn_param)\r
+{\r
+       if (id_priv->id.ps != RDMA_PS_TCP) {\r
+               return 0;\r
+       }\r
+\r
+       if ((conn_param->responder_resources > id_priv->cma_dev->max_responder_resources) ||\r
+               (conn_param->initiator_depth > id_priv->cma_dev->max_initiator_depth)) {\r
+               return -1;\r
+       }\r
+\r
+       return 0;\r
+}\r
+\r
+__declspec(dllexport)\r
+int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)\r
+{\r
+       struct cma_id_private *id_priv;\r
+       WV_CONNECT_PARAM attr;\r
+       HRESULT hr;\r
+       \r
+       id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);\r
+       hr = ucma_valid_param(id_priv, conn_param);\r
+       if (FAILED(hr)) {\r
+               return hr;\r
+       }\r
+\r
+       RtlZeroMemory(&attr, sizeof attr);\r
+       attr.ResponderResources = conn_param->responder_resources;\r
+       attr.InitiatorDepth = conn_param->initiator_depth;\r
+       attr.RetryCount = conn_param->retry_count;\r
+       attr.RnrRetryCount = conn_param->rnr_retry_count;\r
+       if ((attr.DataLength = conn_param->private_data_len)) {\r
+               RtlCopyMemory(attr.Data, conn_param->private_data, attr.DataLength);\r
+       }\r
+\r
+       id_priv->state = cma_active_connect;\r
+       hr = id->ep.connect->Connect(id->qp->conn_handle, &id->route.addr.dst_addr,\r
+                                                                &attr, &id->overlap);\r
+       if (FAILED(hr) && hr != WV_IO_PENDING) {\r
+               id_priv->state = cma_route_resolve;\r
+               return hr;\r
+       }\r
+\r
+       return 0;\r
+}\r
+\r
+static int ucma_get_request(struct cma_id_private *listen, int index)\r
+{\r
+       struct cma_id_private *id_priv;\r
+       HRESULT hr;\r
+\r
+       hr = rdma_create_id(listen->id.channel, &listen->req_list[index],\r
+                                               listen, listen->id.ps);\r
+       if (FAILED(hr)) {\r
+               return hr;\r
+       }\r
+\r
+       id_priv = CONTAINING_RECORD(listen->req_list[index], struct cma_id_private, id);\r
+       id_priv->index = index;\r
+       id_priv->state = cma_get_request;\r
+\r
+       if (listen->id.ps == RDMA_PS_TCP) {\r
+               hr = listen->id.ep.connect->GetRequest(id_priv->id.ep.connect,\r
+                                                                                          &id_priv->id.overlap);\r
+       } else {\r
+               hr = listen->id.ep.datagram->GetRequest(id_priv->id.ep.datagram,\r
+                                                                                               &id_priv->id.overlap);\r
+       }\r
+\r
+       return (FAILED(hr) && hr != WV_IO_PENDING) ? hr : 0;\r
+}\r
+\r
+__declspec(dllexport)\r
+int rdma_listen(struct rdma_cm_id *id, int backlog)\r
+{\r
+       struct cma_id_private *id_priv, *req_id;\r
+       HRESULT hr;\r
+       int i;\r
+\r
+       if (backlog <= 0 || backlog > CMA_DEFAULT_BACKLOG) {\r
+               backlog = CMA_DEFAULT_BACKLOG;\r
+       }\r
+\r
+       id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);\r
+       id_priv->req_list = new struct rdma_cm_id*[backlog];\r
+       if (id_priv->req_list == NULL) {\r
+               return -1;\r
+       }\r
+\r
+       RtlZeroMemory(id_priv->req_list, sizeof(struct rdma_cm_id *) * backlog);\r
+       id_priv->backlog = backlog;\r
+\r
+       id_priv->state = cma_listening;\r
+       hr = (id->ps == RDMA_PS_TCP) ?\r
+                id->ep.connect->Listen(backlog) : id->ep.datagram->Listen(backlog);\r
+       if (FAILED(hr)) {\r
+               return hr;\r
+       }\r
+\r
+       for (i = 0; i < backlog; i++) {\r
+               hr = ucma_get_request(id_priv, i);\r
+               if (FAILED(hr)) {\r
+                       return hr;\r
+               }\r
+       }\r
+\r
+       return 0;\r
+}\r
+\r
+__declspec(dllexport)\r
+int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)\r
+{\r
+       struct cma_id_private *id_priv;\r
+       WV_CONNECT_PARAM attr;\r
+       HRESULT hr;\r
+\r
+       id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);\r
+       hr = ucma_valid_param(id_priv, conn_param);\r
+       if (FAILED(hr)) {\r
+               return hr;\r
+       }\r
+\r
+       RtlZeroMemory(&attr, sizeof attr);\r
+       attr.ResponderResources = conn_param->responder_resources;\r
+       attr.InitiatorDepth = conn_param->initiator_depth;\r
+       attr.RetryCount = conn_param->retry_count;\r
+       attr.RnrRetryCount = conn_param->rnr_retry_count;\r
+       if ((attr.DataLength = conn_param->private_data_len)) {\r
+               RtlCopyMemory(attr.Data, conn_param->private_data, attr.DataLength);\r
+       }\r
+\r
+       id_priv->state = cma_accepting;\r
+       hr = id->ep.connect->Accept(id->qp->conn_handle, &attr, &id->overlap);\r
+       if (FAILED(hr) && hr != WV_IO_PENDING) {\r
+               id_priv->state = cma_disconnected;\r
+               return hr;\r
+       }\r
+\r
+       return 0;\r
+}\r
+\r
+__declspec(dllexport)\r
+int rdma_reject(struct rdma_cm_id *id, const void *private_data,\r
+                               uint8_t private_data_len)\r
+{\r
+       struct cma_id_private *id_priv;\r
+       HRESULT hr;\r
+\r
+       id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);\r
+       id_priv->state = cma_disconnected;\r
+       hr = id->ep.connect->Reject(private_data, private_data_len);\r
+       if (FAILED(hr)) {\r
+               return hr;\r
+       }\r
+       return 0;\r
+}\r
+\r
+__declspec(dllexport)\r
+int rdma_notify(struct rdma_cm_id *id, enum ibv_event_type event)\r
+{\r
+       return 0;\r
+}\r
+\r
+__declspec(dllexport)\r
+int rdma_disconnect(struct rdma_cm_id *id)\r
+{\r
+       struct cma_id_private *id_priv;\r
+       HRESULT hr;\r
+\r
+       id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);\r
+       if (id_priv->state == cma_connected) {\r
+               id_priv->state = cma_active_disconnect;\r
+       } else {\r
+               id_priv->state = cma_disconnected;\r
+       }\r
+       hr = id->ep.connect->Disconnect();\r
+       if (FAILED(hr)) {\r
+               return hr;\r
+       }\r
+\r
+       return 0;\r
+}\r
+\r
+__declspec(dllexport)\r
+int rdma_ack_cm_event(struct rdma_cm_event *event)\r
+{\r
+       struct cma_event *evt;\r
+\r
+       evt = CONTAINING_RECORD(event, struct cma_event, event);\r
+       delete evt;\r
+       return 0;\r
+}\r
+\r
+static int ucma_process_conn_req(struct cma_event *event)\r
+{\r
+       struct cma_id_private *listen;\r
+       HRESULT hr = 0;\r
+\r
+       listen = (struct cma_id_private *) event->id_priv->id.context;\r
+\r
+       if (SUCCEEDED(event->event.status)) {\r
+               event->event.status = ucma_query_connect(&event->id_priv->id,\r
+                                                                                                &event->event.param.conn);\r
+       }\r
+\r
+       if (SUCCEEDED(event->event.status)) {\r
+               event->event.event = RDMA_CM_EVENT_CONNECT_REQUEST;\r
+               event->id_priv->state = cma_passive_connect;\r
+\r
+               listen->req_list[event->id_priv->index] = NULL;\r
+               ucma_get_request(listen, event->id_priv->index);\r
+       } else {\r
+               hr = listen->id.ep.connect->GetRequest(event->id_priv->id.ep.connect,\r
+                                                                                          &event->id_priv->id.overlap);\r
+               if (hr == WV_IO_PENDING) {\r
+                       hr = 0;\r
+               }\r
+       }\r
+\r
+       return hr;\r
+}\r
+\r
+static int ucma_process_conn_resp(struct cma_event *event)\r
+{\r
+       struct rdma_cm_id       *id;\r
+       WV_CONNECT_PARAM        attr;\r
+       HRESULT                         hr;\r
+\r
+       if (FAILED(event->event.status)) {\r
+               goto err;\r
+       }\r
+\r
+       RtlZeroMemory(&attr, sizeof(attr));\r
+       event->id_priv->state = cma_accepting;\r
+\r
+       id = &event->id_priv->id;\r
+       hr = id->ep.connect->Accept(id->qp->conn_handle, &attr, &id->overlap);\r
+       if (FAILED(hr) && hr != WV_IO_PENDING) {\r
+               event->event.status = hr;\r
+               goto err;\r
+       }\r
+\r
+       return WV_IO_PENDING;\r
+\r
+err:\r
+       event->event.event = (event->event.status == WV_REJECTED) ?\r
+                                                RDMA_CM_EVENT_REJECTED :\r
+                                                RDMA_CM_EVENT_CONNECT_ERROR;\r
+       event->id_priv->state = cma_disconnected;\r
+       return 0;\r
+}\r
+\r
+static void ucma_process_establish(struct cma_event *event)\r
+{\r
+       if (SUCCEEDED(event->event.status)) {\r
+               event->event.status = ucma_query_connect(&event->id_priv->id,\r
+                                                                                                &event->event.param.conn);\r
+       }\r
+\r
+       if (SUCCEEDED(event->event.status)) {\r
+               event->event.event = RDMA_CM_EVENT_ESTABLISHED;\r
+               event->id_priv->state = cma_connected;\r
+\r
+               event->id_priv->id.ep.connect->NotifyDisconnect(&event->id_priv->id.overlap);\r
+       } else {\r
+               event->event.event = RDMA_CM_EVENT_CONNECT_ERROR;\r
+               event->id_priv->state = cma_disconnected;\r
+       }\r
+}\r
+\r
+static int ucma_process_event(struct cma_event *event)\r
+{\r
+       WV_CONNECT_ATTRIBUTES attr;\r
+       HRESULT hr = 0;\r
+\r
+       switch (event->id_priv->state) {\r
+       case cma_get_request:\r
+               hr = ucma_process_conn_req(event);\r
+               break;\r
+       case cma_addr_resolve:\r
+               event->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;\r
+               break;\r
+       case cma_route_resolve:\r
+               event->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;\r
+               break;\r
+       case cma_active_connect:\r
+               hr = ucma_process_conn_resp(event);\r
+               break;\r
+       case cma_accepting:\r
+               ucma_process_establish(event);\r
+               break;\r
+       case cma_connected:\r
+               event->event.event = RDMA_CM_EVENT_DISCONNECTED;\r
+               event->id_priv->state = cma_passive_disconnect;\r
+               break;\r
+       case cma_active_disconnect:\r
+               event->event.event = RDMA_CM_EVENT_DISCONNECTED;\r
+               event->id_priv->state = cma_disconnected;\r
+               break;\r
+       default:\r
+               return -1;\r
+       }\r
+\r
+       return hr;\r
+}\r
+\r
+__declspec(dllexport)\r
+int rdma_get_cm_event(struct rdma_event_channel *channel,\r
+                                         struct rdma_cm_event **event)\r
+{\r
+       struct cma_event_channel *chan;\r
+       struct cma_event *evt;\r
+       struct cma_id_private *id_priv;\r
+       struct rdma_cm_id *id;\r
+       DWORD bytes;\r
+       HRESULT hr;\r
+\r
+       evt = new struct cma_event;\r
+       if (evt == NULL) {\r
+               return -1;\r
+       }\r
+\r
+       do {\r
+               RtlZeroMemory(evt, sizeof(struct cma_event));\r
+\r
+               chan = CONTAINING_RECORD(channel, struct cma_event_channel, channel);\r
+               hr = WaitForMultipleObjects(chan->count, chan->event, FALSE,\r
+                                                                       chan->channel.timeout);\r
+               if (hr == WAIT_TIMEOUT) {\r
+                       return hr;\r
+               } else if (hr == WAIT_FAILED) {\r
+                       return HRESULT_FROM_WIN32(GetLastError());\r
+               }\r
+\r
+               EnterCriticalSection(&chan->lock);\r
+               evt->id_priv = chan->id[hr];\r
+               LeaveCriticalSection(&chan->lock);\r
+\r
+               id = &evt->id_priv->id;\r
+               evt->event.id = id;\r
+               evt->event.param.conn.private_data = evt->private_data;\r
+               if (id->ep.connect->GetOverlappedResult(&id->overlap, &bytes, FALSE) == 0) {\r
+                       evt->event.status = HRESULT_FROM_WIN32(GetLastError());\r
+               }\r
+\r
+               hr = ucma_process_event(evt);\r
+       } while (FAILED(hr));\r
+       \r
+       *event = &evt->event;\r
+       return 0;\r
+}\r
+\r
+\r
+__declspec(dllexport)\r
+int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,\r
+                                               void *context)\r
+{\r
+       return WV_NOT_SUPPORTED;\r
+}\r
+\r
+__declspec(dllexport)\r
+int rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)\r
+{\r
+       return WV_NOT_SUPPORTED;\r
+}\r
+\r
+__declspec(dllexport)\r
+const char *rdma_event_str(enum rdma_cm_event_type event)\r
+{\r
+       switch (event) {\r
+       case RDMA_CM_EVENT_ADDR_RESOLVED:\r
+               return "RDMA_CM_EVENT_ADDR_RESOLVED";\r
+       case RDMA_CM_EVENT_ADDR_ERROR:\r
+               return "RDMA_CM_EVENT_ADDR_ERROR";\r
+       case RDMA_CM_EVENT_ROUTE_RESOLVED:\r
+               return "RDMA_CM_EVENT_ROUTE_RESOLVED";\r
+       case RDMA_CM_EVENT_ROUTE_ERROR:\r
+               return "RDMA_CM_EVENT_ROUTE_ERROR";\r
+       case RDMA_CM_EVENT_CONNECT_REQUEST:\r
+               return "RDMA_CM_EVENT_CONNECT_REQUEST";\r
+       case RDMA_CM_EVENT_CONNECT_RESPONSE:\r
+               return "RDMA_CM_EVENT_CONNECT_RESPONSE";\r
+       case RDMA_CM_EVENT_CONNECT_ERROR:\r
+               return "RDMA_CM_EVENT_CONNECT_ERROR";\r
+       case RDMA_CM_EVENT_UNREACHABLE:\r
+               return "RDMA_CM_EVENT_UNREACHABLE";\r
+       case RDMA_CM_EVENT_REJECTED:\r
+               return "RDMA_CM_EVENT_REJECTED";\r
+       case RDMA_CM_EVENT_ESTABLISHED:\r
+               return "RDMA_CM_EVENT_ESTABLISHED";\r
+       case RDMA_CM_EVENT_DISCONNECTED:\r
+               return "RDMA_CM_EVENT_DISCONNECTED";\r
+       case RDMA_CM_EVENT_DEVICE_REMOVAL:\r
+               return "RDMA_CM_EVENT_DEVICE_REMOVAL";\r
+       case RDMA_CM_EVENT_MULTICAST_JOIN:\r
+               return "RDMA_CM_EVENT_MULTICAST_JOIN";\r
+       case RDMA_CM_EVENT_MULTICAST_ERROR:\r
+               return "RDMA_CM_EVENT_MULTICAST_ERROR";\r
+       case RDMA_CM_EVENT_ADDR_CHANGE:\r
+               return "RDMA_CM_EVENT_ADDR_CHANGE";\r
+       case RDMA_CM_EVENT_TIMEWAIT_EXIT:\r
+               return "RDMA_CM_EVENT_TIMEWAIT_EXIT";\r
+       default:\r
+               return "UNKNOWN EVENT";\r
+       }\r
+}\r
+\r
+__declspec(dllexport)\r
+int rdma_set_option(struct rdma_cm_id *id, int level, int optname,\r
+                                       void *optval, size_t optlen)\r
+{\r
+       return WV_NOT_SUPPORTED;\r
+}\r
+\r
+__declspec(dllexport)\r
+int rdma_migrate_id(struct rdma_cm_id *id, struct rdma_event_channel *channel)\r
+{\r
+       struct cma_id_private *id_priv;\r
+\r
+       id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);\r
+       cma_event_channel_remove_id(id->channel, id_priv);\r
+       /*\r
+        * TODO: To support calling this routine while processing events on the old\r
+        * channel, we need to wait for all old events to be acknowledged.\r
+        */\r
+       id->channel = channel;\r
+       cma_event_channel_insert_id(channel, id_priv);\r
+\r
+       return 0;\r
+}\r

diff --git a/ulp/librdmacm/src/cma.h b/ulp/librdmacm/src/cma.h
new file mode 100644 (file)
index 0000000..fb65026
--- /dev/null
+++ b/ulp/librdmacm/src/cma.h
@@ -0,0 +1,46 @@
+/*\r
+ * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.\r
+ * Copyright (c) 2008-2009 Intel Corp.  All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ *     Redistribution and use in source and binary forms, with or\r
+ *     without modification, are permitted provided that the following\r
+ *     conditions are met:\r
+ *\r
+ *      - Redistributions of source code must retain the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer.\r
+ *\r
+ *      - Redistributions in binary form must reproduce the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer in the documentation and/or other materials\r
+ *        provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ */\r
+\r
+#ifndef CMA_H\r
+#define CMA_H\r
+\r
+extern CRITICAL_SECTION lock;\r
+\r
+__inline void* __cdecl operator new(size_t size)\r
+{\r
+       return HeapAlloc(GetProcessHeap(), 0, size);\r
+}\r
+\r
+__inline void __cdecl operator delete(void *pObj)\r
+{\r
+       HeapFree(GetProcessHeap(), 0, pObj);\r
+}\r
+\r
+#endif /* CMA_H */\r

diff --git a/ulp/librdmacm/src/cma.rc b/ulp/librdmacm/src/cma.rc
new file mode 100644 (file)
index 0000000..fd20545
--- /dev/null
+++ b/ulp/librdmacm/src/cma.rc
@@ -0,0 +1,46 @@
+/*\r
+ * Copyright (c) 2008 Intel Corporation.  All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ *     Redistribution and use in source and binary forms, with or\r
+ *     without modification, are permitted provided that the following\r
+ *     conditions are met:\r
+ *\r
+ *      - Redistributions of source code must retain the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer.\r
+ *\r
+ *      - Redistributions in binary form must reproduce the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer in the documentation and/or other materials\r
+ *        provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ */\r
+\r
+\r
+#include <oib_ver.h>\r
+\r
+#define VER_FILETYPE                   VFT_DLL\r
+#define VER_FILESUBTYPE                        VFT2_UNKNOWN\r
+\r
+#ifdef _DEBUG_\r
+#define VER_FILEDESCRIPTION_STR                "LibRdmaCm (Debug)"\r
+#define VER_INTERNALNAME_STR           "librdmacmd.dll"\r
+#define VER_ORIGINALFILENAME_STR       "librdmacmd.dll"\r
+#else\r
+#define VER_FILEDESCRIPTION_STR                "LibRdmaCm"\r
+#define VER_INTERNALNAME_STR           "librdmacm.dll"\r
+#define VER_ORIGINALFILENAME_STR       "librdmacm.dll"\r
+#endif\r
+\r
+#include <common.ver>\r

diff --git a/ulp/librdmacm/src/cma_export.def b/ulp/librdmacm/src/cma_export.def
new file mode 100644 (file)
index 0000000..7a2b1ec
--- /dev/null
+++ b/ulp/librdmacm/src/cma_export.def
@@ -0,0 +1,34 @@
+/*\r
+ * Copyright (c) 2008 Intel Corporation. All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ *     Redistribution and use in source and binary forms, with or\r
+ *     without modification, are permitted provided that the following\r
+ *     conditions are met:\r
+ *\r
+ *      - Redistributions of source code must retain the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer.\r
+ *\r
+ *      - Redistributions in binary form must reproduce the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer in the documentation and/or other materials\r
+ *        provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ */\r
+\r
+LIBRARY        LIBRDMACM.DLL\r
+\r
+EXPORTS\r
+       DllCanUnloadNow         PRIVATE\r
+       DllGetClassObject       PRIVATE\r

diff --git a/ulp/librdmacm/src/cma_exports.src b/ulp/librdmacm/src/cma_exports.src
new file mode 100644 (file)
index 0000000..853173b
--- /dev/null
+++ b/ulp/librdmacm/src/cma_exports.src
@@ -0,0 +1,33 @@
+#if DBG\r
+LIBRARY librdmacmd.dll\r
+#else\r
+LIBRARY librdmacm.dll\r
+#endif\r
+\r
+#ifndef _WIN64\r
+EXPORTS\r
+rdma_create_event_channel\r
+rdma_destroy_event_channel\r
+rdma_create_id\r
+rdma_destroy_id\r
+rdma_bind_addr\r
+rdma_resolve_addr\r
+rdma_resolve_route\r
+rdma_create_qp\r
+rdma_destroy_qp\r
+rdma_connect\r
+rdma_listen\r
+rdma_accept\r
+rdma_reject\r
+rdma_notify\r
+rdma_disconnect\r
+rdma_join_multicast\r
+rdma_leave_multicast\r
+rdma_get_cm_event\r
+rdma_ack_cm_event\r
+rdma_get_devices\r
+rdma_free_devices\r
+rdma_event_str\r
+rdma_set_option\r
+rdma_migrate_id\r
+#endif\r

diff --git a/ulp/librdmacm/src/cma_main.cpp b/ulp/librdmacm/src/cma_main.cpp
new file mode 100644 (file)
index 0000000..2daead7
--- /dev/null
+++ b/ulp/librdmacm/src/cma_main.cpp
@@ -0,0 +1,44 @@
+/*\r
+ * Copyright (c) 2008-2009 Intel Corporation. All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ *     Redistribution and use in source and binary forms, with or\r
+ *     without modification, are permitted provided that the following\r
+ *     conditions are met:\r
+ *\r
+ *      - Redistributions of source code must retain the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer.\r
+ *\r
+ *      - Redistributions in binary form must reproduce the above\r
+ *        copyright notice, this list of conditions and the following\r
+ *        disclaimer in the documentation and/or other materials\r
+ *        provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ */\r
+\r
+#include <windows.h>\r
+#include "cma.h"\r
+\r
+CRITICAL_SECTION lock;\r
+\r
+BOOLEAN WINAPI DllMain(HINSTANCE hInstance, DWORD dwReason, LPVOID lpReserved)\r
+{\r
+       UNREFERENCED_PARAMETER(hInstance);\r
+       UNREFERENCED_PARAMETER(dwReason);\r
+       UNREFERENCED_PARAMETER(lpReserved);\r
+\r
+       InitializeCriticalSection(&lock);\r
+\r
+       return TRUE;\r
+}\r

diff --git a/ulp/librdmacm/src/makefile b/ulp/librdmacm/src/makefile
new file mode 100644 (file)
index 0000000..bffacaa
--- /dev/null
+++ b/ulp/librdmacm/src/makefile
@@ -0,0 +1,7 @@
+#\r
+# DO NOT EDIT THIS FILE!!!  Edit .\sources. if you want to add a new source\r
+# file to this component.  This file merely indirects to the real make file\r
+# that is shared by all the driver components of the OpenIB Windows project.\r
+#\r
+\r
+!INCLUDE ..\..\..\inc\openib.def\r