librdmacm: fix event reporting
authorshefty <shefty@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Thu, 23 Apr 2009 19:37:41 +0000 (19:37 +0000)
committershefty <shefty@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Thu, 23 Apr 2009 19:37:41 +0000 (19:37 +0000)
Use the latest comp_channel changes to fix event reporting and avoid
hangs when destroying resources.  We need to track when closing
devices to make sure that events are canceled, and avoid issuing
new wait calls.

Minor correction to the cmatose test app to avoid busy polling of the CQ,
which can prevent other threads from running.  This leads to connection
failures when running more clients than there are CPUs in the system.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>
git-svn-id: svn://openib.tc.cornell.edu/gen1/trunk@2130 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

ulp/librdmacm/examples/cmatose/cmatose.c
ulp/librdmacm/src/cma.cpp

index b9a8232..c7d8981 100644 (file)
@@ -417,6 +417,7 @@ static int poll_cqs(enum CQ_INDEX index)
                                printf("cmatose: failed polling CQ: 0x%x\n", ret);\r
                                return ret;\r
                        }\r
+                       Sleep(0);\r
                }\r
        }\r
        return 0;\r
index 934e2f2..1832078 100644 (file)
 #include <iba/ibat.h>\r
 #include "cma.h"\r
 #include "..\..\..\etc\user\comp_channel.cpp"\r
+#include "..\..\..\etc\user\dlist.c"\r
 \r
-static struct ibv_windata windata;\r
+static struct ibvw_windata windata;\r
 \r
 enum cma_state\r
 {\r
        cma_idle,\r
        cma_listening,\r
        cma_get_request,\r
+       cma_addr_bind,\r
        cma_addr_resolve,\r
        cma_route_resolve,\r
        cma_passive_connect,\r
@@ -98,7 +100,7 @@ static void ucma_cleanup(void)
                cma_dev_cnt = 0;\r
        }\r
        if (windata.prov != NULL) {\r
-               ibv_release_windata(&windata, IBV_WINDATA_VERSION);\r
+               ibvw_release_windata(&windata, IBVW_WINDATA_VERSION);\r
                windata.prov = NULL;\r
        }\r
 }\r
@@ -115,7 +117,7 @@ static int ucma_init(void)
                goto out;\r
        }\r
 \r
-       ret = ibv_get_windata(&windata, IBV_WINDATA_VERSION);\r
+       ret = ibvw_get_windata(&windata, IBVW_WINDATA_VERSION);\r
        if (ret) {\r
                goto err;\r
        }\r
@@ -286,7 +288,7 @@ int rdma_destroy_id(struct rdma_cm_id *id)
                id->ep.datagram->CancelOverlappedRequests();\r
        }\r
 \r
-       CompChannelRemoveEntry(&id->channel->channel, &id->comp_entry);\r
+       CompEntryCancel(&id->comp_entry);\r
 \r
        if (id_priv->backlog > 0) {\r
                ucma_destroy_listen(id_priv);\r
@@ -407,6 +409,7 @@ static int ucma_query_datagram(struct rdma_cm_id *id, struct rdma_ud_param *para
 __declspec(dllexport)\r
 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)\r
 {\r
+       struct cma_id_private *id_priv;\r
        HRESULT hr;\r
 \r
        if (id->ps == RDMA_PS_TCP) {\r
@@ -421,6 +424,10 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
                }\r
        }\r
 \r
+       if (SUCCEEDED(hr)) {\r
+               id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);\r
+               id_priv->state = cma_addr_bind;\r
+       }\r
        return hr;\r
 }\r
 \r
@@ -434,32 +441,34 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
        DWORD size;\r
        HRESULT hr;\r
 \r
-       if (src_addr == NULL) {\r
-               if (id->ps == RDMA_PS_TCP) {\r
-                       s = socket(dst_addr->sa_family, SOCK_STREAM, IPPROTO_TCP);\r
-               } else {\r
-                       s = socket(dst_addr->sa_family, SOCK_DGRAM, IPPROTO_UDP);\r
-               }\r
-               if (s == INVALID_SOCKET) {\r
-                       return WSAGetLastError();\r
+       id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);\r
+       if (id_priv->state == cma_idle) {\r
+               if (src_addr == NULL) {\r
+                       if (id->ps == RDMA_PS_TCP) {\r
+                               s = socket(dst_addr->sa_family, SOCK_STREAM, IPPROTO_TCP);\r
+                       } else {\r
+                               s = socket(dst_addr->sa_family, SOCK_DGRAM, IPPROTO_UDP);\r
+                       }\r
+                       if (s == INVALID_SOCKET) {\r
+                               return WSAGetLastError();\r
+                       }\r
+\r
+                       hr = WSAIoctl(s, SIO_ROUTING_INTERFACE_QUERY, dst_addr, ucma_addrlen(dst_addr),\r
+                                                 &addr, sizeof addr, &size, NULL, NULL);\r
+                       closesocket(s);\r
+                       if (FAILED(hr)) {\r
+                               return WSAGetLastError();\r
+                       }\r
+                       src_addr = &addr.Sa;\r
                }\r
 \r
-               hr = WSAIoctl(s, SIO_ROUTING_INTERFACE_QUERY, dst_addr, ucma_addrlen(dst_addr),\r
-                                         &addr, sizeof addr, &size, NULL, NULL);\r
-               closesocket(s);\r
+               hr = rdma_bind_addr(id, src_addr);\r
                if (FAILED(hr)) {\r
-                       return WSAGetLastError();\r
+                       return hr;\r
                }\r
-               src_addr = &addr.Sa;\r
-       }\r
-\r
-       hr = rdma_bind_addr(id, src_addr);\r
-       if (FAILED(hr)) {\r
-               return hr;\r
        }\r
 \r
        RtlCopyMemory(&id->route.addr.dst_addr, dst_addr, ucma_addrlen(dst_addr));\r
-       id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);\r
        id_priv->state = cma_addr_resolve;\r
 \r
        CompEntryPost(&id->comp_entry);\r
@@ -613,9 +622,11 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
        }\r
 \r
        id_priv->state = cma_active_connect;\r
+       id->comp_entry.Busy = 1;\r
        hr = id->ep.connect->Connect(id->qp->conn_handle, &id->route.addr.dst_addr,\r
                                                                 &attr, &id->comp_entry.Overlap);\r
        if (FAILED(hr) && hr != WV_IO_PENDING) {\r
+               id->comp_entry.Busy = 0;\r
                id_priv->state = cma_route_resolve;\r
                return hr;\r
        }\r
@@ -638,6 +649,7 @@ static int ucma_get_request(struct cma_id_private *listen, int index)
        id_priv->index = index;\r
        id_priv->state = cma_get_request;\r
 \r
+       id_priv->id.comp_entry.Busy = 1;\r
        if (listen->id.ps == RDMA_PS_TCP) {\r
                hr = listen->id.ep.connect->GetRequest(id_priv->id.ep.connect,\r
                                                                                           &id_priv->id.comp_entry.Overlap);\r
@@ -646,6 +658,7 @@ static int ucma_get_request(struct cma_id_private *listen, int index)
                                                                                                &id_priv->id.comp_entry.Overlap);\r
        }\r
        if (FAILED(hr) && hr != WV_IO_PENDING) {\r
+               id_priv->id.comp_entry.Busy = 0;\r
                return hr;\r
        }\r
 \r
@@ -712,9 +725,11 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
        }\r
 \r
        id_priv->state = cma_accepting;\r
+       id->comp_entry.Busy = 1;\r
        hr = id->ep.connect->Accept(id->qp->conn_handle, &attr,\r
                                                                &id->comp_entry.Overlap);\r
        if (FAILED(hr) && hr != WV_IO_PENDING) {\r
+               id->comp_entry.Busy = 0;\r
                id_priv->state = cma_disconnected;\r
                return hr;\r
        }\r
@@ -790,6 +805,7 @@ static int ucma_process_conn_req(struct cma_event *event)
        if (SUCCEEDED(event->event.status)) {\r
                event->event.event = RDMA_CM_EVENT_CONNECT_REQUEST;\r
                event->id_priv->state = cma_passive_connect;\r
+               event->event.listen_id = &listen->id;\r
        } else {\r
                rdma_destroy_id(&event->id_priv->id);\r
        }\r
@@ -811,9 +827,11 @@ static int ucma_process_conn_resp(struct cma_event *event)
        event->id_priv->state = cma_accepting;\r
 \r
        id = &event->id_priv->id;\r
+       id->comp_entry.Busy = 1;\r
        hr = id->ep.connect->Accept(id->qp->conn_handle, &attr,\r
                                                                &id->comp_entry.Overlap);\r
        if (FAILED(hr) && hr != WV_IO_PENDING) {\r
+               id->comp_entry.Busy = 0;\r
                event->event.status = hr;\r
                goto err;\r
        }\r
@@ -841,6 +859,7 @@ static void ucma_process_establish(struct cma_event *event)
                event->event.event = RDMA_CM_EVENT_ESTABLISHED;\r
 \r
                id_priv->state = cma_connected;\r
+               id_priv->id.comp_entry.Busy = 1;\r
                id_priv->id.ep.connect->NotifyDisconnect(&id_priv->id.comp_entry.Overlap);\r
        } else {\r
                event->event.event = RDMA_CM_EVENT_CONNECT_ERROR;\r