librdmacm: allow for graceful cleanup of verbs
[mirror/winof/.git] / ulp / librdmacm / src / cma.cpp
index 7d31c30..022d291 100644 (file)
@@ -90,8 +90,9 @@ struct cma_event {
 \r
 static struct cma_device *cma_dev_array;\r
 static int cma_dev_cnt;\r
+static DWORD ref;\r
 \r
-static int ucma_init(void)\r
+static int ucma_acquire(void)\r
 {\r
        struct ibv_device **dev_list = NULL;\r
        struct cma_device *cma_dev;\r
@@ -99,7 +100,7 @@ static int ucma_init(void)
        int i, ret, dev_cnt;\r
 \r
        EnterCriticalSection(&lock);\r
-       if (cma_dev_cnt) {\r
+       if (ref++) {\r
                goto out;\r
        }\r
 \r
@@ -141,7 +142,7 @@ static int ucma_init(void)
                cma_dev->max_responder_resources = (uint8_t) attr.max_qp_rd_atom;\r
        }\r
        ibv_free_device_list(dev_list);\r
-       cma_dev_cnt = dev_cnt;\r
+\r
 out:\r
        LeaveCriticalSection(&lock);\r
        return 0;\r
@@ -156,17 +157,34 @@ err3:
 err2:\r
        ibvw_release_windata(&windata, IBVW_WINDATA_VERSION);\r
 err1:\r
+       ref--;\r
        LeaveCriticalSection(&lock);\r
        return ret;\r
 }\r
 \r
+void ucma_release(void)\r
+{\r
+       int i;\r
+\r
+       EnterCriticalSection(&lock);\r
+       if (--ref == 0) {\r
+               for (i = 0; i < cma_dev_cnt; i++) {\r
+                       ibv_close_device(cma_dev_array[i].verbs);\r
+               }\r
+               delete cma_dev_array;\r
+               cma_dev_cnt = 0;\r
+               ibvw_release_windata(&windata, IBVW_WINDATA_VERSION);\r
+       }\r
+       LeaveCriticalSection(&lock);\r
+}\r
+\r
 __declspec(dllexport)\r
 struct ibv_context **rdma_get_devices(int *num_devices)\r
 {\r
        struct ibv_context **devs = NULL;\r
        int i;\r
 \r
-       if (!cma_dev_cnt && ucma_init()) {\r
+       if (ucma_acquire()) {\r
                goto out;\r
        }\r
 \r
@@ -190,6 +208,7 @@ __declspec(dllexport)
 void rdma_free_devices(struct ibv_context **list)\r
 {\r
        delete list;\r
+       ucma_release();\r
 }\r
 \r
 __declspec(dllexport)\r
@@ -197,7 +216,7 @@ struct rdma_event_channel *rdma_create_event_channel(void)
 {\r
        struct rdma_event_channel *channel;\r
 \r
-       if (!cma_dev_cnt && ucma_init()) {\r
+       if (ucma_acquire()) {\r
                return NULL;\r
        }\r
 \r
@@ -215,6 +234,7 @@ void rdma_destroy_event_channel(struct rdma_event_channel *channel)
 {\r
        CompChannelCleanup(&channel->channel);\r
        delete channel;\r
+       ucma_release();\r
 }\r
 \r
 __declspec(dllexport)\r
@@ -225,14 +245,15 @@ int rdma_create_id(struct rdma_event_channel *channel,
        struct cma_id_private *id_priv;\r
        HRESULT hr;\r
 \r
-       hr = cma_dev_cnt ? 0 : ucma_init();\r
+       hr = ucma_acquire();\r
        if (hr) {\r
                return hr;\r
        }\r
 \r
        id_priv = new struct cma_id_private;\r
        if (id_priv == NULL) {\r
-               return NULL;\r
+               hr = ENOMEM;\r
+               goto err1;\r
        }\r
 \r
        RtlZeroMemory(id_priv, sizeof(struct cma_id_private));\r
@@ -248,14 +269,16 @@ int rdma_create_id(struct rdma_event_channel *channel,
                hr = windata.prov->CreateDatagramEndpoint(&id_priv->id.ep.datagram);\r
        }\r
        if (FAILED(hr)) {\r
-               goto err;\r
+               goto err2;\r
        }\r
 \r
        *id = &id_priv->id;\r
        return 0;\r
 \r
-err:\r
+err2:\r
        delete id_priv;\r
+err1:\r
+       ucma_release();\r
        return hr;\r
 }\r
 \r
@@ -307,6 +330,7 @@ int rdma_destroy_id(struct rdma_cm_id *id)
                Sleep(0);\r
        }\r
        delete id_priv;\r
+       ucma_release();\r
        return 0;\r
 }\r
 \r
@@ -482,23 +506,33 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
        return 0;\r
 }\r
 \r
-__declspec(dllexport)\r
-int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)\r
+static int\r
+ucma_resolve_ibat_path(struct rdma_cm_id *id, int timeout_ms,\r
+                                          IBAT_PATH_BLOB *path)\r
 {\r
-       struct cma_id_private *id_priv;\r
-       IBAT_PATH_BLOB path;\r
        HRESULT hr;\r
 \r
        do {\r
-               hr = IBAT::Resolve(&id->route.addr.src_addr, &id->route.addr.dst_addr, &path);\r
-               if (hr != E_PENDING) {\r
+               hr = IBAT::Resolve(&id->route.addr.src_addr, &id->route.addr.dst_addr,\r
+                                                  path);\r
+               if (hr != E_PENDING || timeout_ms <= 0) {\r
                        break;\r
                }\r
                timeout_ms -= 10;\r
-               if (timeout_ms > 0)\r
-                       Sleep(10);\r
+               Sleep(10);\r
        } while (timeout_ms > 0);\r
 \r
+       return hr;\r
+}\r
+\r
+__declspec(dllexport)\r
+int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)\r
+{\r
+       struct cma_id_private *id_priv;\r
+       IBAT_PATH_BLOB path;\r
+       HRESULT hr;\r
+\r
+       hr = ucma_resolve_ibat_path(id, timeout_ms, &path);\r
        if (FAILED(hr)) {\r
                return hr;\r
        }\r