librdmacm: fix event reporting when destroying listen

[mirror/winof/.git] / ulp / librdmacm / src / cma.cpp

diff --git a/ulp/librdmacm/src/cma.cpp b/ulp/librdmacm/src/cma.cpp
index 1832078..d1f7168 100644 (file)
--- a/ulp/librdmacm/src/cma.cpp
+++ b/ulp/librdmacm/src/cma.cpp
@@ -57,7 +57,8 @@ enum cma_state
        cma_connected,\r
        cma_active_disconnect,\r
        cma_passive_disconnect,\r
        cma_connected,\r
        cma_active_disconnect,\r
        cma_passive_disconnect,\r

-       cma_disconnected\r
+       cma_disconnected,\r
+       cma_destroying\r

 };\r
 \r
 #define CMA_DEFAULT_BACKLOG            16\r
 };\r
 \r
 #define CMA_DEFAULT_BACKLOG            16\r


@@ -69,6 +70,7 @@ struct cma_id_private
        struct cma_device                       *cma_dev;\r
        int                                                     backlog;\r
        int                                                     index;\r
        struct cma_device                       *cma_dev;\r
        int                                                     backlog;\r
        int                                                     index;\r

+       volatile LONG                           refcnt;\r

        struct rdma_cm_id                       **req_list;\r
 };\r
 \r
        struct rdma_cm_id                       **req_list;\r
 };\r
 \r


@@ -243,6 +245,7 @@ int rdma_create_id(struct rdma_event_channel *channel,
        }\r
 \r
        RtlZeroMemory(id_priv, sizeof(struct cma_id_private));\r
        }\r
 \r
        RtlZeroMemory(id_priv, sizeof(struct cma_id_private));\r

+       id_priv->refcnt = 1;\r

        id_priv->id.context = context;\r
        id_priv->id.channel = channel;\r
        id_priv->id.ps = ps;\r
        id_priv->id.context = context;\r
        id_priv->id.channel = channel;\r
        id_priv->id.ps = ps;\r


@@ -269,6 +272,7 @@ static void ucma_destroy_listen(struct cma_id_private *id_priv)
 {\r
        while (--id_priv->backlog >= 0) {\r
                if (id_priv->req_list[id_priv->backlog] != NULL) {\r
 {\r
        while (--id_priv->backlog >= 0) {\r
                if (id_priv->req_list[id_priv->backlog] != NULL) {\r

+                       InterlockedDecrement(&id_priv->refcnt);\r

                        rdma_destroy_id(id_priv->req_list[id_priv->backlog]);\r
                }\r
        }\r
                        rdma_destroy_id(id_priv->req_list[id_priv->backlog]);\r
                }\r
        }\r


@@ -282,13 +286,20 @@ int rdma_destroy_id(struct rdma_cm_id *id)
        struct cma_id_private *id_priv;\r
 \r
        id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);\r
        struct cma_id_private *id_priv;\r
 \r
        id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);\r

+\r
+       EnterCriticalSection(&lock);\r
+       id_priv->state = cma_destroying;\r
+       LeaveCriticalSection(&lock);\r
+\r

        if (id->ps == RDMA_PS_TCP) {\r
                id->ep.connect->CancelOverlappedRequests();\r
        } else {\r
                id->ep.datagram->CancelOverlappedRequests();\r
        }\r
 \r
        if (id->ps == RDMA_PS_TCP) {\r
                id->ep.connect->CancelOverlappedRequests();\r
        } else {\r
                id->ep.datagram->CancelOverlappedRequests();\r
        }\r
 \r

-       CompEntryCancel(&id->comp_entry);\r
+       if (CompEntryCancel(&id->comp_entry) != NULL) {\r
+               InterlockedDecrement(&id_priv->refcnt);\r
+       }\r

 \r
        if (id_priv->backlog > 0) {\r
                ucma_destroy_listen(id_priv);\r
 \r
        if (id_priv->backlog > 0) {\r
                ucma_destroy_listen(id_priv);\r


@@ -300,6 +311,10 @@ int rdma_destroy_id(struct rdma_cm_id *id)
                id_priv->id.ep.datagram->Release();\r
        }\r
 \r
                id_priv->id.ep.datagram->Release();\r
        }\r
 \r

+       InterlockedDecrement(&id_priv->refcnt);\r
+       while (id_priv->refcnt) {\r
+               Sleep(0);\r
+       }\r

        delete id_priv;\r
        return 0;\r
 }\r
        delete id_priv;\r
        return 0;\r
 }\r


@@ -471,6 +486,7 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
        RtlCopyMemory(&id->route.addr.dst_addr, dst_addr, ucma_addrlen(dst_addr));\r
        id_priv->state = cma_addr_resolve;\r
 \r
        RtlCopyMemory(&id->route.addr.dst_addr, dst_addr, ucma_addrlen(dst_addr));\r
        id_priv->state = cma_addr_resolve;\r
 \r

+       id_priv->refcnt++;\r

        CompEntryPost(&id->comp_entry);\r
        return 0;\r
 }\r
        CompEntryPost(&id->comp_entry);\r
        return 0;\r
 }\r


@@ -497,6 +513,7 @@ int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
        id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);\r
        id_priv->state = cma_route_resolve;\r
 \r
        id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);\r
        id_priv->state = cma_route_resolve;\r
 \r

+       id_priv->refcnt++;\r

        CompEntryPost(&id->comp_entry);\r
        return 0;\r
 }\r
        CompEntryPost(&id->comp_entry);\r
        return 0;\r
 }\r


@@ -622,10 +639,12 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
        }\r
 \r
        id_priv->state = cma_active_connect;\r
        }\r
 \r
        id_priv->state = cma_active_connect;\r

+       id_priv->refcnt++;\r

        id->comp_entry.Busy = 1;\r
        hr = id->ep.connect->Connect(id->qp->conn_handle, &id->route.addr.dst_addr,\r
                                                                 &attr, &id->comp_entry.Overlap);\r
        if (FAILED(hr) && hr != WV_IO_PENDING) {\r
        id->comp_entry.Busy = 1;\r
        hr = id->ep.connect->Connect(id->qp->conn_handle, &id->route.addr.dst_addr,\r
                                                                 &attr, &id->comp_entry.Overlap);\r
        if (FAILED(hr) && hr != WV_IO_PENDING) {\r

+               id_priv->refcnt--;\r

                id->comp_entry.Busy = 0;\r
                id_priv->state = cma_route_resolve;\r
                return hr;\r
                id->comp_entry.Busy = 0;\r
                id_priv->state = cma_route_resolve;\r
                return hr;\r


@@ -636,19 +655,27 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
 \r
 static int ucma_get_request(struct cma_id_private *listen, int index)\r
 {\r
 \r
 static int ucma_get_request(struct cma_id_private *listen, int index)\r
 {\r

-       struct cma_id_private *id_priv;\r
+       struct cma_id_private *id_priv = NULL;\r

        HRESULT hr;\r
 \r
        HRESULT hr;\r
 \r

+       EnterCriticalSection(&lock);\r
+       if (listen->state != cma_listening) {\r
+               hr = WV_INVALID_PARAMETER;\r
+               goto err1;\r
+       }\r
+\r
+       InterlockedIncrement(&listen->refcnt);\r

        hr = rdma_create_id(listen->id.channel, &listen->req_list[index],\r
                                                listen, listen->id.ps);\r
        if (FAILED(hr)) {\r
        hr = rdma_create_id(listen->id.channel, &listen->req_list[index],\r
                                                listen, listen->id.ps);\r
        if (FAILED(hr)) {\r

-               return hr;\r
+               goto err2;\r

        }\r
 \r
        id_priv = CONTAINING_RECORD(listen->req_list[index], struct cma_id_private, id);\r
        id_priv->index = index;\r
        id_priv->state = cma_get_request;\r
 \r
        }\r
 \r
        id_priv = CONTAINING_RECORD(listen->req_list[index], struct cma_id_private, id);\r
        id_priv->index = index;\r
        id_priv->state = cma_get_request;\r
 \r

+       id_priv->refcnt++;\r

        id_priv->id.comp_entry.Busy = 1;\r
        if (listen->id.ps == RDMA_PS_TCP) {\r
                hr = listen->id.ep.connect->GetRequest(id_priv->id.ep.connect,\r
        id_priv->id.comp_entry.Busy = 1;\r
        if (listen->id.ps == RDMA_PS_TCP) {\r
                hr = listen->id.ep.connect->GetRequest(id_priv->id.ep.connect,\r


@@ -659,10 +686,21 @@ static int ucma_get_request(struct cma_id_private *listen, int index)
        }\r
        if (FAILED(hr) && hr != WV_IO_PENDING) {\r
                id_priv->id.comp_entry.Busy = 0;\r
        }\r
        if (FAILED(hr) && hr != WV_IO_PENDING) {\r
                id_priv->id.comp_entry.Busy = 0;\r

-               return hr;\r
+               id_priv->refcnt--;\r
+               goto err2;\r

        }\r
        }\r

+       LeaveCriticalSection(&lock);\r

 \r
        return 0;\r
 \r
        return 0;\r

+\r
+err2:\r
+       InterlockedDecrement(&listen->refcnt);\r
+err1:\r
+       LeaveCriticalSection(&lock);\r
+       if (id_priv != NULL) {\r
+               rdma_destroy_id(&id_priv->id);\r
+       }\r
+       return hr;\r

 }\r
 \r
 __declspec(dllexport)\r
 }\r
 \r
 __declspec(dllexport)\r


@@ -725,10 +763,12 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
        }\r
 \r
        id_priv->state = cma_accepting;\r
        }\r
 \r
        id_priv->state = cma_accepting;\r

+       id_priv->refcnt++;\r

        id->comp_entry.Busy = 1;\r
        hr = id->ep.connect->Accept(id->qp->conn_handle, &attr,\r
                                                                &id->comp_entry.Overlap);\r
        if (FAILED(hr) && hr != WV_IO_PENDING) {\r
        id->comp_entry.Busy = 1;\r
        hr = id->ep.connect->Accept(id->qp->conn_handle, &attr,\r
                                                                &id->comp_entry.Overlap);\r
        if (FAILED(hr) && hr != WV_IO_PENDING) {\r

+               id_priv->refcnt--;\r

                id->comp_entry.Busy = 0;\r
                id_priv->state = cma_disconnected;\r
                return hr;\r
                id->comp_entry.Busy = 0;\r
                id_priv->state = cma_disconnected;\r
                return hr;\r


@@ -783,31 +823,41 @@ __declspec(dllexport)
 int rdma_ack_cm_event(struct rdma_cm_event *event)\r
 {\r
        struct cma_event *evt;\r
 int rdma_ack_cm_event(struct rdma_cm_event *event)\r
 {\r
        struct cma_event *evt;\r

+       struct cma_id_private *listen;\r

 \r
        evt = CONTAINING_RECORD(event, struct cma_event, event);\r
 \r
        evt = CONTAINING_RECORD(event, struct cma_event, event);\r

+       InterlockedDecrement(&evt->id_priv->refcnt);\r
+       if (evt->event.listen_id) {\r
+               listen = CONTAINING_RECORD(evt->event.listen_id, struct cma_id_private, id);\r
+               InterlockedDecrement(&listen->refcnt);\r
+       }\r

        delete evt;\r
        return 0;\r
 }\r
 \r
 static int ucma_process_conn_req(struct cma_event *event)\r
 {\r
        delete evt;\r
        return 0;\r
 }\r
 \r
 static int ucma_process_conn_req(struct cma_event *event)\r
 {\r

-       struct cma_id_private *listen;\r
+       struct cma_id_private *listen, *id_priv;\r

        struct cma_event_channel *chan;\r
 \r
        listen = (struct cma_id_private *) event->id_priv->id.context;\r
        struct cma_event_channel *chan;\r
 \r
        listen = (struct cma_id_private *) event->id_priv->id.context;\r

-       ucma_get_request(listen, event->id_priv->index);\r
+       id_priv = event->id_priv;\r
+\r
+       ucma_get_request(listen, id_priv->index);\r

 \r
        if (SUCCEEDED(event->event.status)) {\r
 \r
        if (SUCCEEDED(event->event.status)) {\r

-               event->event.status = ucma_query_connect(&event->id_priv->id,\r
+               event->event.status = ucma_query_connect(&id_priv->id,\r

                                                                                                 &event->event.param.conn);\r
        }\r
 \r
        if (SUCCEEDED(event->event.status)) {\r
                event->event.event = RDMA_CM_EVENT_CONNECT_REQUEST;\r
                                                                                                 &event->event.param.conn);\r
        }\r
 \r
        if (SUCCEEDED(event->event.status)) {\r
                event->event.event = RDMA_CM_EVENT_CONNECT_REQUEST;\r

-               event->id_priv->state = cma_passive_connect;\r
+               id_priv->state = cma_passive_connect;\r

                event->event.listen_id = &listen->id;\r
        } else {\r
                event->event.listen_id = &listen->id;\r
        } else {\r

-               rdma_destroy_id(&event->id_priv->id);\r
+               InterlockedDecrement(&listen->refcnt);\r
+               InterlockedDecrement(&id_priv->refcnt);\r
+               rdma_destroy_id(&id_priv->id);\r

        }\r
 \r
        return event->event.status;\r
        }\r
 \r
        return event->event.status;\r


@@ -859,6 +909,7 @@ static void ucma_process_establish(struct cma_event *event)
                event->event.event = RDMA_CM_EVENT_ESTABLISHED;\r
 \r
                id_priv->state = cma_connected;\r
                event->event.event = RDMA_CM_EVENT_ESTABLISHED;\r
 \r
                id_priv->state = cma_connected;\r

+               InterlockedIncrement(&id_priv->refcnt);\r

                id_priv->id.comp_entry.Busy = 1;\r
                id_priv->id.ep.connect->NotifyDisconnect(&id_priv->id.comp_entry.Overlap);\r
        } else {\r
                id_priv->id.comp_entry.Busy = 1;\r
                id_priv->id.ep.connect->NotifyDisconnect(&id_priv->id.comp_entry.Overlap);\r
        } else {\r


@@ -869,13 +920,25 @@ static void ucma_process_establish(struct cma_event *event)
 \r
 static int ucma_process_event(struct cma_event *event)\r
 {\r
 \r
 static int ucma_process_event(struct cma_event *event)\r
 {\r

+       struct cma_id_private *listen, *id_priv;\r

        WV_CONNECT_ATTRIBUTES attr;\r
        HRESULT hr = 0;\r
 \r
        WV_CONNECT_ATTRIBUTES attr;\r
        HRESULT hr = 0;\r
 \r

-       switch (event->id_priv->state) {\r
+       id_priv = event->id_priv;\r
+\r
+       EnterCriticalSection(&lock);\r
+       switch (id_priv->state) {\r

        case cma_get_request:\r
        case cma_get_request:\r

-               hr = ucma_process_conn_req(event);\r
-               break;\r
+               listen = (struct cma_id_private *) id_priv->id.context;\r
+               if (listen->state != cma_listening) {\r
+                       InterlockedDecrement(&id_priv->refcnt);\r
+                       hr = WV_CANCELLED;\r
+                       break;\r
+               }\r
+\r
+               listen->req_list[id_priv->index] = NULL;\r
+               LeaveCriticalSection(&lock);\r
+               return ucma_process_conn_req(event);\r

        case cma_addr_resolve:\r
                event->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;\r
                break;\r
        case cma_addr_resolve:\r
                event->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;\r
                break;\r


@@ -890,15 +953,17 @@ static int ucma_process_event(struct cma_event *event)
                break;\r
        case cma_connected:\r
                event->event.event = RDMA_CM_EVENT_DISCONNECTED;\r
                break;\r
        case cma_connected:\r
                event->event.event = RDMA_CM_EVENT_DISCONNECTED;\r

-               event->id_priv->state = cma_passive_disconnect;\r
+               id_priv->state = cma_passive_disconnect;\r

                break;\r
        case cma_active_disconnect:\r
                event->event.event = RDMA_CM_EVENT_DISCONNECTED;\r
                break;\r
        case cma_active_disconnect:\r
                event->event.event = RDMA_CM_EVENT_DISCONNECTED;\r

-               event->id_priv->state = cma_disconnected;\r
+               id_priv->state = cma_disconnected;\r

                break;\r
        default:\r
                break;\r
        default:\r

-               return -1;\r
+               InterlockedDecrement(&id_priv->refcnt);\r
+               hr = WV_CANCELLED;\r

        }\r
        }\r

+       LeaveCriticalSection(&lock);\r

 \r
        return hr;\r
 }\r
 \r
        return hr;\r
 }\r