libibverbs: add reference counts to track CQ events
authorshefty <shefty@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Sat, 1 Aug 2009 01:01:57 +0000 (01:01 +0000)
committershefty <shefty@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Sat, 1 Aug 2009 01:01:57 +0000 (01:01 +0000)
We need to track CQ events to block destroying CQ objects until all outstanding events have been acknowledged.  Otherwise, the caller could reference freed memory after calling ibv_get_cq_event.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>
git-svn-id: svn://openib.tc.cornell.edu/gen1/trunk@2323 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

ulp/libibverbs/include/infiniband/verbs.h
ulp/libibverbs/src/verbs.cpp

index 700259e..236ebe4 100644 (file)
@@ -660,6 +660,8 @@ struct ibv_cq
        IWVCompletionQueue              *handle;\r
        int                                             cqe;\r
        COMP_ENTRY                              comp_entry;\r
+       LONG volatile                   notify_cnt;\r
+       LONG volatile                   ack_cnt;\r
 };\r
 \r
 struct ibv_ah\r
index e5dffc6..e577733 100644 (file)
@@ -347,6 +347,8 @@ struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe, void *cq_cont
        cq->context = context;\r
        cq->channel = channel;\r
        cq->cq_context = cq_context;\r
+       cq->notify_cnt = 0;\r
+       cq->ack_cnt = 0;\r
 \r
        entries = cqe;\r
        hr = context->cmd_if->CreateCompletionQueue(&entries, &cq->handle);\r
@@ -387,10 +389,14 @@ int ibv_req_notify_cq(struct ibv_cq *cq, int solicited_only)
        HRESULT hr;\r
 \r
        if (InterlockedCompareExchange(&cq->comp_entry.Busy, 1, 0) == 0) {\r
+               InterlockedIncrement(&cq->notify_cnt);\r
                hr = cq->handle->Notify(solicited_only ? WvCqSolicited : WvCqNextCompletion,\r
                                                                &cq->comp_entry.Overlap);\r
                if (SUCCEEDED(hr) || hr == WV_IO_PENDING) {\r
                        hr = 0;\r
+               } else {\r
+                       InterlockedExchange(&cq->comp_entry.Busy, 0);\r
+                       InterlockedDecrement(&cq->notify_cnt);\r
                }\r
        } else {\r
                hr = 0;\r
@@ -417,8 +423,13 @@ int ibv_destroy_cq(struct ibv_cq *cq)
 {\r
        cq->handle->CancelOverlappedRequests();\r
        if (cq->channel != NULL) {\r
-               CompEntryCancel(&cq->comp_entry);\r
+               if (CompEntryCancel(&cq->comp_entry)) {\r
+                       InterlockedIncrement(&cq->ack_cnt);\r
+               }\r
        }\r
+\r
+       while (cq->ack_cnt < cq->notify_cnt)\r
+               Sleep(0);\r
        cq->handle->Release();\r
        delete cq;\r
        return 0;\r
@@ -443,6 +454,7 @@ int ibv_get_cq_event(struct ibv_comp_channel *channel,
 __declspec(dllexport)\r
 void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)\r
 {\r
+       InterlockedExchangeAdd(&cq->ack_cnt, (LONG) nevents);\r
 }\r
 \r
 __declspec(dllexport)\r