etc/comp_channel: add support for channel sets
authorshefty <shefty@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Wed, 2 Sep 2009 15:30:03 +0000 (15:30 +0000)
committershefty <shefty@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Wed, 2 Sep 2009 15:30:03 +0000 (15:30 +0000)
To better mimic Linux fd support, add the concept of channel sets to completion channels.  This is needed to minimize changes in platform independent uDAPL code.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>
git-svn-id: svn://openib.tc.cornell.edu/gen1/trunk@2412 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

etc/user/comp_channel.cpp
inc/user/comp_channel.h

index ca3356e..4f7e957 100644 (file)
@@ -30,7 +30,6 @@
 #include <comp_channel.h>\r
 #include <process.h>\r
 \r
-static void CompManagerQueue(COMP_MANAGER *pMgr, COMP_ENTRY *pEntry);\r
 static void CompChannelQueue(COMP_CHANNEL *pChannel, COMP_ENTRY *pEntry);\r
 \r
 \r
@@ -50,11 +49,8 @@ static unsigned __stdcall CompThreadPoll(void *Context)
                GetQueuedCompletionStatus(mgr->CompQueue, &bytes, &key,\r
                                                                  &overlap, INFINITE);\r
                entry = CONTAINING_RECORD(overlap, COMP_ENTRY, Overlap);\r
-\r
-               if (entry->Channel) {\r
+               if (entry->Channel != NULL) {\r
                        CompChannelQueue(entry->Channel, entry);\r
-               } else {\r
-                       CompManagerQueue(mgr, entry);\r
                }\r
        }\r
 \r
@@ -66,37 +62,21 @@ DWORD CompManagerOpen(COMP_MANAGER *pMgr)
 {\r
        DWORD ret;\r
 \r
-       InitializeCriticalSection(&pMgr->Lock);\r
-       pMgr->Busy = 0;\r
-       DListInit(&pMgr->DoneList);\r
-       CompEntryInit(NULL, &pMgr->Entry);\r
-\r
        pMgr->CompQueue = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, -1);\r
        if (pMgr->CompQueue == NULL) {\r
-               ret = GetLastError();\r
-               goto err1;\r
-       }\r
-\r
-       pMgr->Event = CreateEvent(NULL, TRUE, TRUE, NULL);\r
-       if (pMgr->Event == NULL) {\r
-               ret = GetLastError();\r
-               goto err2;\r
+               return GetLastError();\r
        }\r
 \r
        pMgr->Run = TRUE;\r
        pMgr->Thread = (HANDLE) _beginthreadex(NULL, 0, CompThreadPoll, pMgr, 0, NULL);\r
        if (pMgr->Thread == NULL) {\r
                ret = GetLastError();\r
-               goto err3;\r
+               goto err;\r
        }\r
        return 0;\r
 \r
-err3:\r
-       CloseHandle(pMgr->Event);\r
-err2:\r
+err:\r
        CloseHandle(pMgr->CompQueue);\r
-err1:\r
-       DeleteCriticalSection(&pMgr->Lock);     \r
        return ret;\r
 }\r
 \r
@@ -112,8 +92,6 @@ void CompManagerClose(COMP_MANAGER *pMgr)
        CloseHandle(pMgr->Thread);\r
 \r
        CloseHandle(pMgr->CompQueue);\r
-       CloseHandle(pMgr->Event);\r
-       DeleteCriticalSection(&pMgr->Lock);     \r
 }\r
 \r
 DWORD CompManagerMonitor(COMP_MANAGER *pMgr, HANDLE hFile, ULONG_PTR Key)\r
@@ -124,58 +102,74 @@ DWORD CompManagerMonitor(COMP_MANAGER *pMgr, HANDLE hFile, ULONG_PTR Key)
        return (cq == NULL) ? GetLastError() : 0;\r
 }\r
 \r
-static void CompManagerQueue(COMP_MANAGER *pMgr, COMP_ENTRY *pEntry)\r
+\r
+/*\r
+ * Completion channel sets\r
+ */\r
+\r
+DWORD CompSetInit(COMP_SET *pSet)\r
 {\r
-       EnterCriticalSection(&pMgr->Lock);\r
-       DListInsertTail(&pEntry->MgrEntry, &pMgr->DoneList);\r
-       SetEvent(pMgr->Event);\r
-       LeaveCriticalSection(&pMgr->Lock);\r
+       pSet->Head = NULL;\r
+       pSet->TailPtr = &pSet->Head;\r
+\r
+       pSet->Event = CreateEvent(NULL, TRUE, FALSE, NULL);\r
+       if (pSet->Event == NULL) {\r
+               return GetLastError();\r
+       }\r
+\r
+       return 0;\r
 }\r
 \r
-static void CompManagerRemoveEntry(COMP_MANAGER *pMgr, COMP_ENTRY *pEntry)\r
+void CompSetCleanup(COMP_SET *pSet)\r
 {\r
-       EnterCriticalSection(&pMgr->Lock);\r
-       DListRemove(&pEntry->MgrEntry);\r
-       LeaveCriticalSection(&pMgr->Lock);\r
+       CloseHandle(pSet->Event);\r
 }\r
 \r
-DWORD CompManagerPoll(COMP_MANAGER *pMgr, DWORD Milliseconds,\r
-                                         COMP_CHANNEL **ppChannel)\r
+void CompSetZero(COMP_SET *pSet)\r
 {\r
-       COMP_ENTRY *entry;\r
-       DWORD ret = 0;\r
-\r
-       EnterCriticalSection(&pMgr->Lock);\r
-       while (DListEmpty(&pMgr->DoneList)) {\r
-               ResetEvent(pMgr->Event);\r
-               LeaveCriticalSection(&pMgr->Lock);\r
-       \r
-               ret = WaitForSingleObject(pMgr->Event, Milliseconds);\r
-               if (ret) {\r
-                       return ret;\r
-               }\r
+       pSet->Head = NULL;\r
+       pSet->TailPtr = &pSet->Head;\r
+       ResetEvent(pSet->Event);\r
+}\r
+\r
+void CompSetAdd(COMP_CHANNEL *pChannel, COMP_SET *pSet)\r
+{\r
+       *pSet->TailPtr = pChannel;\r
+       pSet->TailPtr = &pChannel->Next;\r
 \r
-               EnterCriticalSection(&pMgr->Lock);\r
+       EnterCriticalSection(&pChannel->Lock);\r
+       pChannel->Set = pSet;\r
+       if (pChannel->Head != NULL) {\r
+               SetEvent(pSet->Event);\r
        }\r
+       LeaveCriticalSection(&pChannel->Lock);\r
+}\r
 \r
-       entry = CONTAINING_RECORD(pMgr->DoneList.Next, COMP_ENTRY, MgrEntry);\r
-       *ppChannel = entry->Channel;\r
-       if (entry->Channel == NULL) {\r
-               DListRemove(&entry->MgrEntry);\r
-               InterlockedExchange(&entry->Busy, 0);\r
-               ret = ERROR_CANCELLED;\r
+DWORD CompSetPoll(COMP_SET *pSet, DWORD Milliseconds)\r
+{\r
+       DLIST_ENTRY *entry;\r
+       COMP_CHANNEL *channel;\r
+       DWORD ret, cnt = 0;\r
+\r
+       *pSet->TailPtr = NULL;\r
+       ret = WaitForSingleObject(pSet->Event, Milliseconds);\r
+       if (ret == WAIT_TIMEOUT) {\r
+               ret = 0;\r
        }\r
-       LeaveCriticalSection(&pMgr->Lock);\r
 \r
-       return ret;\r
+       for (channel = pSet->Head; channel != NULL; channel = channel->Next) {\r
+               EnterCriticalSection(&channel->Lock);\r
+               channel->Set = NULL;\r
+               cnt += (channel->Head != NULL);\r
+               LeaveCriticalSection(&channel->Lock);\r
+       }\r
+\r
+       return cnt ? cnt : ret;\r
 }\r
 \r
-void CompManagerCancel(COMP_MANAGER *pMgr)\r
+void CompSetCancel(COMP_SET *pSet)\r
 {\r
-       if (InterlockedCompareExchange(&pMgr->Entry.Busy, 1, 0) == 0) {\r
-               PostQueuedCompletionStatus(pMgr->CompQueue, 0, (ULONG_PTR) pMgr,\r
-                                                                  &pMgr->Entry.Overlap);\r
-       }\r
+       SetEvent(pSet->Event);\r
 }\r
 \r
 \r
@@ -186,6 +180,8 @@ void CompManagerCancel(COMP_MANAGER *pMgr)
 DWORD CompChannelInit(COMP_MANAGER *pMgr, COMP_CHANNEL *pChannel, DWORD Milliseconds)\r
 {\r
        pChannel->Manager = pMgr;\r
+       pChannel->Next = NULL;\r
+       pChannel->Set = NULL;\r
        pChannel->Head = NULL;\r
        pChannel->TailPtr = &pChannel->Head;\r
        pChannel->Milliseconds = Milliseconds;\r
@@ -210,6 +206,7 @@ static void CompChannelInsertTail(COMP_CHANNEL *pChannel, COMP_ENTRY *pEntry)
 {\r
        *pChannel->TailPtr = pEntry;\r
        pChannel->TailPtr = &pEntry->Next;\r
+       pEntry->Next = NULL;\r
 }\r
 \r
 static COMP_ENTRY *CompChannelRemoveHead(COMP_CHANNEL *pChannel)\r
@@ -240,7 +237,6 @@ static COMP_ENTRY *CompChannelFindRemove(COMP_CHANNEL *pChannel, COMP_ENTRY *pEn
                if (pChannel->TailPtr == &pEntry->Next) {\r
                        pChannel->TailPtr = entry_ptr;\r
                }\r
-               CompManagerRemoveEntry(pChannel->Manager, pEntry);\r
                InterlockedExchange(&pEntry->Busy, 0);\r
        }\r
        LeaveCriticalSection(&pChannel->Lock);\r
@@ -251,9 +247,11 @@ static void CompChannelQueue(COMP_CHANNEL *pChannel, COMP_ENTRY *pEntry)
 {\r
        pEntry->Next = NULL;\r
        EnterCriticalSection(&pChannel->Lock);\r
-       CompManagerQueue(pChannel->Manager, pEntry);\r
        CompChannelInsertTail(pChannel, pEntry);\r
        SetEvent(pChannel->Event);\r
+       if (pChannel->Set != NULL) {\r
+               SetEvent(pChannel->Set->Event);\r
+       }\r
        LeaveCriticalSection(&pChannel->Lock);\r
 }\r
 \r
@@ -275,7 +273,6 @@ DWORD CompChannelPoll(COMP_CHANNEL *pChannel, COMP_ENTRY **ppEntry)
                EnterCriticalSection(&pChannel->Lock);\r
        }\r
        entry = CompChannelRemoveHead(pChannel);\r
-       CompManagerRemoveEntry(pChannel->Manager, entry);\r
        LeaveCriticalSection(&pChannel->Lock);\r
 \r
        InterlockedExchange(&entry->Busy, 0);\r
index 9876871..d00d4df 100644 (file)
@@ -42,7 +42,6 @@ extern "C" {
 typedef struct _COMP_ENTRY\r
 {\r
        struct _COMP_ENTRY              *Next;\r
-       DLIST_ENTRY                             MgrEntry;\r
        OVERLAPPED                              Overlap;\r
        struct _COMP_CHANNEL    *Channel;\r
        LONG volatile                   Busy;\r
@@ -52,6 +51,8 @@ typedef struct _COMP_ENTRY
 typedef struct _COMP_CHANNEL\r
 {\r
        struct _COMP_MANAGER    *Manager;\r
+       struct _COMP_CHANNEL    *Next;\r
+       struct _COMP_SET                *Set;\r
        COMP_ENTRY                              *Head;\r
        COMP_ENTRY                              **TailPtr;\r
        COMP_ENTRY                              Entry;\r
@@ -61,25 +62,32 @@ typedef struct _COMP_CHANNEL
 \r
 }      COMP_CHANNEL;\r
 \r
+typedef struct _COMP_SET\r
+{\r
+       COMP_CHANNEL                    *Head;\r
+       COMP_CHANNEL                    **TailPtr;\r
+       HANDLE                                  Event;\r
+\r
+}      COMP_SET;\r
+\r
 typedef struct _COMP_MANAGER\r
 {\r
        HANDLE                                  CompQueue;\r
-       DLIST_ENTRY                             DoneList;\r
-       COMP_ENTRY                              Entry;\r
        HANDLE                                  Thread;\r
        BOOL                                    Run;\r
-       HANDLE                                  Event;\r
-       LONG volatile                   Busy;\r
-       CRITICAL_SECTION                Lock;\r
 \r
 }      COMP_MANAGER;\r
 \r
 DWORD          CompManagerOpen(COMP_MANAGER *pMgr);\r
 void           CompManagerClose(COMP_MANAGER *pMgr);\r
 DWORD          CompManagerMonitor(COMP_MANAGER *pMgr, HANDLE hFile, ULONG_PTR Key);\r
-DWORD          CompManagerPoll(COMP_MANAGER *pMgr, DWORD Milliseconds,\r
-                                                       COMP_CHANNEL **ppChannel);\r
-void           CompManagerCancel(COMP_MANAGER *pMgr);\r
+\r
+DWORD          CompSetInit(COMP_SET *pSet);\r
+void           CompSetCleanup(COMP_SET *pSet);\r
+void           CompSetZero(COMP_SET *pSet);\r
+void           CompSetAdd(COMP_CHANNEL *pChannel, COMP_SET *pSet);\r
+DWORD          CompSetPoll(COMP_SET *pSet, DWORD Milliseconds);\r
+void           CompSetCancel(COMP_SET *pSet);\r
 \r
 DWORD          CompChannelInit(COMP_MANAGER *pMgr, COMP_CHANNEL *pChannel,\r
                                                        DWORD Milliseconds);\r