[SDP] Fixed a race in which data was not promised to be sent on shutdown. (rev 825)
authortzachid <tzachid@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Mon, 19 Dec 2005 09:03:04 +0000 (09:03 +0000)
committertzachid <tzachid@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Mon, 19 Dec 2005 09:03:04 +0000 (09:03 +0000)
git-svn-id: svn://openib.tc.cornell.edu/gen1/trunk@207 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

ulp/sdp/kernel/SdpBufferPool.cpp
ulp/sdp/kernel/SdpBufferPool.h
ulp/sdp/kernel/SdpRecvPool.cpp
ulp/sdp/kernel/SdpRecvPool.h
ulp/sdp/kernel/SdpSocket.cpp
ulp/sdp/kernel/SdpSocket.h
ulp/sdp/kernel/SdpUserFile.cpp
ulp/sdp/tests/basic/SdpConnect.cpp

index f8ccc62..17c93df 100644 (file)
@@ -112,6 +112,10 @@ BufferPool::BufferPool()
     m_CreditdBufferDescriptor = NULL;\r
     m_pSdpSocket = NULL;\r
 \r
+\r
+    m_NumberOfBytesSent = 0;\r
+    m_NumberOfBytesSentAndAcked = 0;\r
+\r
 }\r
 \r
 NTSTATUS \r
@@ -283,7 +287,7 @@ BufferPool::ReturnBuffer(BufferDescriptor *pBufferDescriptor)
             if ((RemainingToCopy(pIrp) < m_MaxMessageSize * m_FreePackets.Size())) {\r
                 rc = m_pSdpSocket->RequestCallBack();\r
                 if (!NT_SUCCESS(rc)) {\r
-                    SDP_PRINT(SDP_ERR, SDP_BUFFER_POOL, ("PostCredits failed rc = 0x%x\n", rc ));\r
+                    SDP_PRINT(SDP_ERR, SDP_BUFFER_POOL, ("RequestCallBack failed rc = 0x%x\n", rc ));\r
                     goto Cleanup;\r
                 }        \r
                 m_CallBackPending = true;\r
@@ -295,9 +299,21 @@ BufferPool::ReturnBuffer(BufferDescriptor *pBufferDescriptor)
             m_pSdpSocket->DisconectSentEvent();\r
         }\r
     }\r
+    /*\r
+        We allow buffers to be sent here since it is possible that the\r
+        socket is already closed, and the user thread is not present.\r
+    */\r
+    rc = SendBuffersIfCan();\r
+    if (!NT_SUCCESS(rc)) {\r
+        SDP_PRINT(SDP_ERR, SDP_BUFFER_POOL, ("SendBuffersIfCan failed rc = 0x%x\n", rc ));\r
+        goto Cleanup;\r
+    }        \r
+    \r
+    \r
 Cleanup:\r
     ASSERT(m_CurrentlySentBuffers != 0);\r
     m_CurrentlySentBuffers--;\r
+    \r
 \r
     return rc;\r
 }\r
@@ -493,11 +509,19 @@ Cleanup:
 VOID \r
 BufferPool::CloseSocket()\r
 {\r
+    LIST_ENTRY *item = NULL;\r
+    IRP *pIrp = NULL;\r
+\r
     SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n"));\r
     AssertLocked();\r
-\r
-    //??? Should we do something here \r
-\r
+    // All IRP's that were not compleated, will be compleated as cancelled\r
+    while (m_UserPackets.Size() > 0 ) {\r
+        item = m_UserPackets.RemoveHeadList();\r
+        pIrp = CONTAINING_RECORD(item, IRP ,Tail.Overlay.ListEntry);\r
+        pIrp->IoStatus.Status = STATUS_CANCELLED;\r
+        pIrp->IoStatus.Information = 0;\r
+        IoCompleteRequest (pIrp, IO_NO_INCREMENT);\r
+    }\r
 }\r
 \r
 \r
@@ -592,6 +616,7 @@ BufferPool::SendBuffer(BufferDescriptor *pBufferDescriptor)
         rc = IB2Status(ib_status);\r
         goto Cleanup;\r
     }\r
+    m_NumberOfBytesSent++;\r
     //????? Should we clear the post credits here ????????\r
     m_CurrentlySentBuffers ++;\r
     m_rRecvBuf--;\r
index f7f7989..bfa8a8b 100644 (file)
@@ -256,7 +256,7 @@ public:
 \r
     VOID SetRemoteRecvBuf (uint16_t rRecvBuf) {\r
         if (m_rRecvBuf == 2) {\r
-            SDP_PRINT(SDP_ALL, SDP_BUFFER_POOL,("m_rRecvBuf = %d, it is being set to %d seqnum = %d\n", m_rRecvBuf, rRecvBuf, m_SendSeq));\r
+            SDP_PRINT(SDP_DEBUG, SDP_BUFFER_POOL,("m_rRecvBuf = %d, it is being set to %d seqnum = %d\n", m_rRecvBuf, rRecvBuf, m_SendSeq));\r
         }\r
         ASSERT(rRecvBuf < 1000);\r
         m_rRecvBuf = rRecvBuf;\r
@@ -318,6 +318,13 @@ private: //?????????
     bool m_CallBackPending; // Set to true if we have requesetd a callback from \r
                             // the users thread\r
 \r
+    // TODO: The two counters bellow are for debug only. move them to be\r
+    // declared as such\r
+public:    \r
+    uint32_t m_NumberOfBytesSent;\r
+    uint32_t m_NumberOfBytesSentAndAcked;\r
+                            \r
+\r
 VOID AssertLocked();\r
 \r
 };\r
index f0442e4..333f7f4 100644 (file)
@@ -42,6 +42,8 @@ RecvPool::RecvPool()
     m_DisConnRecieved = false;\r
     m_LocaleAdvertisedBuffers = 0;\r
     m_pSdpSocket = NULL;\r
+    m_NumberOfBuffersRecievedWithError = 0;\r
+    m_NumberOfBuffersRecievedSuccessfully = 0;\r
 }\r
 \r
 NTSTATUS \r
@@ -55,7 +57,7 @@ RecvPool::Init(
     SdpSocket *pSdpSocket\r
     )\r
 {\r
-    SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n", m_pSdpSocket));\r
+    SDP_PRINT(SDP_TRACE, SDP_BUFFER_POOL, ("this = 0x%p \n", pSdpSocket));\r
     m_MaxBuffers = MaxBuffers;\r
     m_MaxConcurrentRecieves = MaxConcurrentRecieves;\r
     m_MaxMessageSize = MaxMessageSize;        \r
@@ -117,8 +119,11 @@ RecvPool::RecievedBuffer(BufferDescriptor *pBufferDescriptor, bool error)
     if (error) {\r
         // Not much that we can do in this case (only return the packet)\r
         m_FreePackets.InsertTailList(&pBufferDescriptor->BuffersList);\r
+        m_NumberOfBuffersRecievedWithError++;\r
         goto Cleanup;\r
     }\r
+    ASSERT(m_NumberOfBuffersRecievedWithError == 0);\r
+    m_NumberOfBuffersRecievedSuccessfully++;\r
     \r
     // We have recieved a "RAW" buffer, we have to make sure that the buffer\r
     // descriptor is OK.\r
@@ -175,7 +180,7 @@ RecvPool::RecievedBuffer(BufferDescriptor *pBufferDescriptor, bool error)
         } else {\r
             // This is an empty buffer\r
             ASSERT(pHeader->size == sizeof msg_hdr_bsdh);\r
-            m_FreePackets.InsertTailList(&pBufferDescriptor->BuffersList);        \r
+            m_FreePackets.InsertTailList(&pBufferDescriptor->BuffersList);\r
         }\r
     }    \r
 \r
@@ -206,8 +211,7 @@ RecvPool::RecievedBuffer(BufferDescriptor *pBufferDescriptor, bool error)
         SDP_PRINT(SDP_ERR, SDP_BUFFER_POOL, ("RecieveIfCan failed rc = 0x%x\n", rc ));\r
         goto Cleanup;\r
     }\r
-\r
-Cleanup:    \r
+Cleanup:\r
     return rc;\r
 \r
 }\r
@@ -261,7 +265,7 @@ RecvPool::GetData(
         m_ClientBeingServed = true; \r
     }\r
 \r
-    // Can we supply data to the userd right now ?\r
+    // Can we supply data to the user right now ?\r
     while (*Copied < CopySize) {\r
         if (m_FullPackets.Size()) {\r
             // We have a buffer, we can use it to copy data to the user\r
index 3852805..40e0c44 100644 (file)
@@ -118,9 +118,16 @@ private:
     SdpSocket *m_pSdpSocket;\r
 \r
     // This signals that the remote side will not be sending any data any more\r
-    bool       m_DisConnRecieved; \r
+    bool       m_DisConnRecieved;\r
 \r
-VOID AssertLocked();\r
+    // TODO: The two counters bellow are for debug only. move them to be\r
+    // declared as such\r
+    public:\r
+    uint32_t m_NumberOfBuffersRecievedWithError;\r
+    uint32_t m_NumberOfBuffersRecievedSuccessfully;\r
+    \r
+\r
+    VOID AssertLocked();\r
 \r
 };\r
 \r
index bf0d37e..6511800 100644 (file)
@@ -1097,13 +1097,6 @@ SdpSocket::WSPCloseSocket(
     SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("this = 0x%p state = %s \n",this, SS2String(m_state)));\r
     OBJECT_ATTRIBUTES   attr;\r
     HANDLE  ThreadHandle;\r
-    bool sleep = false;\r
-restart:\r
-    \r
-    if (sleep) {\r
-        Sleep(1*1000*1000);//???????\r
-    }\r
-    sleep = true;\r
 \r
     if (!m_Lock.Lock()) {\r
         SDP_PRINT(SDP_ERR, SDP_SOCKET, ("Failed to lock this = 0x%p \n",this));\r
@@ -1121,19 +1114,13 @@ restart:
         goto Cleanup;\r
     }\r
 \r
-\r
-    //?????????\r
-    if (m_SendBufferPool.m_UserPackets.Size() > 0) {\r
-        m_Lock.Unlock();\r
-        goto restart;\r
-    }\r
-\r
     // This will force that no more calls will be allowed\r
     ASSERT(m_CloseSocketCalled == FALSE); // If this is not the case \r
     // We shouldn't be able to take the lock\r
     m_CloseSocketCalled = true;\r
 \r
     // notify to all "subclients" to free all waiting clients\r
+    // and also return cancell on all IRPs\r
     m_RecvBufferPool.CloseSocket();\r
     m_SendBufferPool.CloseSocket();\r
     m_ConnectionList.CloseSocket();\r
@@ -1231,6 +1218,7 @@ Cleanup:
 \r
 VOID SdpSocket::DisconectSentEvent()\r
 {\r
+    SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("this = 0x%p\n",this));\r
     KeSetEvent( &m_DisconectSentEvent, IO_NO_INCREMENT, FALSE );\r
 }\r
 \r
@@ -1257,7 +1245,7 @@ VOID SdpSocket::CloseSocketThread()
         );\r
 \r
     if (rc == STATUS_TIMEOUT) {\r
-        SDP_PRINT(SDP_ERR, SDP_SOCKET, ("Wait failed with time out\n"));\r
+        SDP_PRINT(SDP_ERR, SDP_SOCKET, ("Wait failed with time out this = 0x%p\n", this));\r
     }\r
 \r
     ASSERT(NT_SUCCESS(rc));\r
@@ -1857,7 +1845,7 @@ SdpSocket::__recv_cb1(
     IN  const   ib_cq_handle_t  h_cq,\r
     IN  void *cq_context )\r
 {\r
-   SDP_PRINT(SDP_ALL, SDP_SOCKET, ("__recv_cb1\n"));\r
+   SDP_PRINT(SDP_DEBUG, SDP_SOCKET, ("__recv_cb1\n"));\r
 \r
     SdpSocket *pSocket = (SdpSocket *) cq_context;\r
     pSocket->m_Lock.SignalCB(RECV_CB_CALLED);\r
@@ -2033,6 +2021,8 @@ NTSTATUS SdpSocket::send_cb()
         {\r
             ASSERT( p_wc->wc_type == IB_WC_SEND );\r
             pBufferDescriptor = (BufferDescriptor*)(uintn_t)p_wc->wr_id;\r
+            m_SendBufferPool.m_NumberOfBytesSentAndAcked++;\r
+\r
             rc2 = m_SendBufferPool.ReturnBuffer(pBufferDescriptor);\r
             UpdateRc(&rc1, rc2); // We remember the error here and continue to avoid leaks\r
             switch( p_wc->status )\r
@@ -2429,7 +2419,7 @@ VOID WaitForShutdownEvent(KEVENT *ShutdownCompleteEvent)
         );\r
 \r
     if (rc == STATUS_TIMEOUT) {\r
-        SDP_PRINT(SDP_ERR, SDP_SOCKET, ("Wait failed with time out\n"));\r
+        SDP_PRINT(SDP_ERR, SDP_SOCKET, ("Wait failed with time out \n"));\r
         ASSERT(FALSE);\r
     }\r
 \r
@@ -2439,17 +2429,46 @@ VOID WaitForShutdownEvent(KEVENT *ShutdownCompleteEvent)
     KeClearEvent(ShutdownCompleteEvent);\r
 }\r
 \r
+/* \r
+    This function is called when the usercallback thread has detected\r
+    that the user thread is dead, or when we have a IRP_MJ_CLEANUP.\r
+    In this case, if CloseSocket was called, we don't do anything\r
+    (since there is alreadya thread that is garantied to close it) and on \r
+    all other cases, we close it abortively.\r
+*/\r
+\r
+VOID SdpSocket::DyingProcessDetected()\r
+{\r
+    SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("called this = 0x%p\n", this));\r
+    m_Lock.Lock(true); //????? verify must succeed\r
+    if (m_CloseSocketCalled) {\r
+        SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("this = 0x%p m_CloseSocketCalled, letting it finish his job\n", this));\r
+        m_Lock.Unlock(); // Error is ignored since this is already\r
+                     // shutdown call\r
+        return;\r
+    }\r
+    m_Lock.Unlock(); // Error is ignored since this is already\r
+                     // shutdown call\r
+    Shutdown();\r
+}\r
+\r
 VOID SdpSocket::Shutdown()\r
 {\r
-    SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("called this = 0x%p\n", this));   \r
+    SDP_PRINT(SDP_TRACE, SDP_SOCKET, ("called this = 0x%p\n", this));\r
+    SDP_PRINT(SDP_ERR, SDP_SOCKET, ("this = 0x%p m_NumberOfBytesSent = %d m_NumberOfBytesSentAndAcked = %d "\r
+        "m_NumberOfBuffersRecievedSuccessfully = %d, m_NumberOfBuffersRecievedWithError = %d\n", \r
+        this, m_SendBufferPool.m_NumberOfBytesSent, m_SendBufferPool.m_NumberOfBytesSentAndAcked, \r
+        m_RecvBufferPool.m_NumberOfBuffersRecievedSuccessfully, m_RecvBufferPool.m_NumberOfBuffersRecievedWithError));   \r
+\r
     NTSTATUS rc = STATUS_SUCCESS;\r
     ib_api_status_t     ib_status;\r
     ib_qp_mod_t         qp_mod;\r
 \r
-\r
     if (m_ShutdownCalled) {\r
         // Since this variable is always changing from false to true\r
-        // we can check it without the lock\r
+        // we can check it without the lock. Since there is no memory\r
+        // barier, this test is not true for multiple enviorments, but it\r
+        // still makes the shutodwn reentrent\r
         return;\r
     }\r
     // now take the lock and test again\r
index 3b1dbf4..b44c36c 100644 (file)
@@ -223,6 +223,8 @@ public:
         WspSocketCloseOut   *pWspSocketCloseOut\r
         );    \r
 \r
+    VOID DyingProcessDetected();\r
+\r
     VOID Shutdown();\r
 \r
     static VOID ShutdownCB(VOID* pContext);\r
index 5a62b45..e4932cc 100644 (file)
@@ -81,7 +81,9 @@ void SdpUserFile::Shutdown()
 \r
         // Don't call release with the lock being hold\r
         Lock.Unlock();\r
-        pSdpSocket->Shutdown();\r
+        // We currently call this both when the process is dead as well as \r
+        // when the device is closed\r
+        pSdpSocket->DyingProcessDetected();\r
         pSdpSocket->Release();\r
         // It seems that we shoule be protected by the m_shutdown\r
         // flag, but will take the lock just in case\r
@@ -227,7 +229,6 @@ SdpUserFile::UsersThread()
             pSdpSocket = CONTAINING_RECORD(item, SdpSocket , m_CallBackRequestList);\r
 \r
             Lock.Unlock();\r
-\r
             // Do the call back \r
             if (!ShutdownCalled) {\r
                 pSdpSocket->UsersThreadCallBack(true);\r
index c0cdcf6..257db86 100644 (file)
-
-#ifdef _WIN32
-#include <Winsock2.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-
-#define AF_INET_FAMILY PF_INET
-
-#define sleep(x) Sleep(1000 * (x));
-#define CloseSocket closesocket
-
-#define GetTimeMs() GetTickCount()
-#else
-
-#define AF_INET_FAMILY 27
-
-#include <sys/types.h>
-#include <netinet/in.h>
-#include <sys/socket.h>
-#include <stdio.h>
-#include <errno.h>
-
-#define _stricmp strcasecmp
-#define WSAGetLastError() errno
-#define CloseSocket close
-
-
-#endif
-// Windows linux 
-
-const int NumberOfThreads = 1;
-
-HANDLE g_ComplitionPort;
-HANDLE *g_pThreads;
-
-#define ASSERT assert
-
-struct sockaddr addressFromString(char *address, int port)
-{
-       unsigned int b1,b2,b3,b4;
-
-       struct sockaddr_in socket_address;
-       struct sockaddr *socket_address_ptr = (struct sockaddr*)&socket_address;
-    
-       memset(&socket_address, 0, sizeof(socket_address));
-       socket_address.sin_family = AF_INET;
-       socket_address.sin_port = htons((u_short)port);
-
-
-       sscanf(address, "%d.%d.%d.%d",&b1, &b2, &b3, &b4);
-       socket_address.sin_addr.s_addr = b4 * 256 * 256 *256 +
-                                        b3 * 256 * 256 + 
-                                        b2 * 256 +
-                                        b1;
-                                        
-       return *socket_address_ptr;
-}
-
-
-struct OverlappedSend {
-    OVERLAPPED Overlapped;
-    SOCKET Socket;
-    void *Buffer; // The data to send
-    int  DataSize;
-    DWORD NumberOfBytesSent;
-    int  RemainingSends;
-    int  SendsInAir;     // Curently sent data
-    CRITICAL_SECTION Lock;
-    HANDLE SendDoneEvent;
-
-};
-
-
-DWORD WINAPI WorkerThreadFunc( LPVOID lpParam )
-{
-    BOOL ret;
-    DWORD NumberOfBytes;
-    ULONG_PTR CompletionKey;
-    OVERLAPPED* lpOverlapped;
-    OverlappedSend *pOverLappedSend;
-    BOOL  ContinueLoop = true;
-    int iRet;
-    
-    while (TRUE) {
-        ret = GetQueuedCompletionStatus(
-                                g_ComplitionPort,
-                                &NumberOfBytes,
-                                &CompletionKey,
-                                &lpOverlapped,
-                                INFINITE);
-        ASSERT(ret != 0);
-        pOverLappedSend = CONTAINING_RECORD(lpOverlapped, OverlappedSend, Overlapped);
-        // Work on the object itself:
-        EnterCriticalSection(&pOverLappedSend->Lock);
-        pOverLappedSend->SendsInAir--;
-        ASSERT(pOverLappedSend->SendsInAir ==0); // Only one thread
-        if (pOverLappedSend->RemainingSends > 0) {
-            // do the next send
-            WSABUF Buffers;
-            Buffers.len = pOverLappedSend->DataSize;
-            Buffers.buf = (char *) pOverLappedSend->Buffer;
-            pOverLappedSend->SendsInAir++;
-            pOverLappedSend->RemainingSends--;
-            
-            iRet = WSASend(
-                pOverLappedSend->Socket,
-                &Buffers,
-                1,
-                &pOverLappedSend->NumberOfBytesSent,
-                0,
-                &pOverLappedSend->Overlapped,
-                NULL
-                );
-            ASSERT((iRet ==0) || (iRet == SOCKET_ERROR && WSAGetLastError() == WSA_IO_PENDING ));
-
-        } else {
-            // Nothing more to send - signal compleation and exit
-            SetEvent(pOverLappedSend->SendDoneEvent);
-            ContinueLoop = false;
-        }
-        LeaveCriticalSection(&pOverLappedSend->Lock);
-        if (!ContinueLoop) {
-            break;
-        }
-
-    }
-    return 0;
-}
-
-void CreateComplitionPort(int newfd)
-{
-    g_pThreads = new HANDLE [NumberOfThreads];
-    ASSERT(g_pThreads != NULL);
-
-    g_ComplitionPort = CreateIoCompletionPort((HANDLE)newfd, NULL, NULL, 2);
-    if(g_ComplitionPort == NULL) {
-        printf("Create complition port failed err=%d", GetLastError());
-        exit(1);
-    }
-
-    // Create the threads that will work on the complitions
-    g_pThreads[0] = CreateThread(NULL, 0 ,WorkerThreadFunc ,NULL, 0,NULL );
-
-
-}
-
-void CloseComplitionPort()
-{
-    //??? Post a complition thread end message
-
-    WaitForMultipleObjects(NumberOfThreads,g_pThreads, TRUE, INFINITE);
-    CloseHandle(g_ComplitionPort);
-
-}
-
-void SendDataOverlapped(int newfd, double NumberOfBuffers, double BufferSize, BOOL SendPacketNumber)
-{
-    // We will start with one operations simultaniously
-    int i ;
-    double d;
-    VOID *buffer = NULL;
-    double elapsed, acc = 0;
-    WSAOVERLAPPED wsa;
-    OverlappedSend *pOverLappedSend = NULL;
-    BOOL ret;
-    memset(&wsa, 0, sizeof wsa);
-
-    buffer = malloc ((size_t)BufferSize);
-    if (buffer == NULL) {
-        printf("Error allocating buffer\n");
-        exit(1);
-    }
-
-    CreateComplitionPort(newfd);
-
-    pOverLappedSend = new OverlappedSend;
-
-    // Set it's fields
-    memset(&pOverLappedSend->Overlapped, 0, sizeof (pOverLappedSend->Overlapped));
-    pOverLappedSend->Socket = newfd;
-    pOverLappedSend->Buffer = buffer;
-    pOverLappedSend->DataSize = (size_t)BufferSize;
-    pOverLappedSend->RemainingSends = (size_t)NumberOfBuffers;
-    pOverLappedSend->SendsInAir = 1; // To force a simulate of the compleation
-    InitializeCriticalSection(&pOverLappedSend->Lock);
-    pOverLappedSend->SendDoneEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
-
-    double Start = GetTimeMs();
-    // Post the message that will do the first send
-    ret = PostQueuedCompletionStatus(
-                              g_ComplitionPort,
-                              (DWORD)BufferSize,
-                              NULL,
-                              &pOverLappedSend->Overlapped
-                                );
-    ASSERT(ret != 0);
-
-    // wait for the send 
-    WaitForSingleObject(pOverLappedSend->SendDoneEvent, INFINITE);
-    elapsed = (GetTimeMs() - Start) / 1000;
-    printf("Finishd sending correctly %f mbytes/sec\n", NumberOfBuffers * BufferSize /elapsed/1024/1024 );
-    CloseComplitionPort();
-//Cleanup:
-    free(buffer);
-
-
-}
-
-
-
-void SendData(int newfd, double NumberOfBuffers, double BufferSize, BOOL SendPacketNumber)
-{
-    int i ;
-    double d;
-    CHAR *buffer = NULL;
-    double elapsed, acc = 0;
-
-    buffer = new char [(size_t)BufferSize];
-    if (buffer == NULL) {
-        printf("Error allocating buffer\n");
-        exit(1);
-    }
-
-    printf("Starting to send %lf messages of size %lf\n", NumberOfBuffers, BufferSize );
-
-    i = 0;
-
-    for (i = 0; i < 10; i++) {
-        buffer[i] = 'a' + i;
-    }
-    double Start = GetTimeMs();
-
-    for (d=0; d < NumberOfBuffers; d++) {
-        int j;
-        if (SendPacketNumber) {
-            memset(buffer, (char) d, (size_t)BufferSize);
-        }
-        
-        j = send(newfd, buffer, (size_t)BufferSize, 0 );
-            acc += j;
-        if (j!=BufferSize) {
-            printf("Error send not compleated  sent %lf\n", acc);
-            goto Cleanup;
-        }
-    }
-    elapsed = (GetTimeMs() - Start) / 1000;
-    printf("Finishd sending correctly %f mbytes/sec (%f seconfs)\n", acc/elapsed/1024/1024, elapsed );
-Cleanup:
-    free(buffer);
-}
-
-
-void RecvData(int newfd, double NumberOfBuffers, double BufferSize )
-{
-    int read1 = 1;
-    unsigned char *p;
-    char buffer[380000];
-    double elapsed, acc = 0;
-    int i;
-    double  Start = GetTimeMs();
-
-    while ((read1 != -1) /*&& (acc < (double)BufferSize * NumberOfBuffers)*/) {
-        read1 = recv(newfd, buffer, sizeof buffer, 0);
-        if (read1 == -1 || read1 == 0) {
-               printf("Finsihed reading, total read %lf bytes last read =%d\n", acc, read1 );
-            if (acc != BufferSize * NumberOfBuffers) {
-                printf("Error, expected to read %lf but read %lf\n",
-                    (BufferSize * NumberOfBuffers), acc);
-            }
-            break;
-        }
-        else {
-            acc += read1;
-
-//                      printf("read returned %d \"%c%c%c%c\"\n",read1,
-//                      buffer[0],buffer[1], buffer[2], buffer[3]);
-        }
-    }
-
-    if (acc != ((double)BufferSize * NumberOfBuffers)) {
-        printf("Error, expected to read %lf but read %lf",
-            ((double)BufferSize * NumberOfBuffers), acc);
-    } else {
-        elapsed = (GetTimeMs() - Start) / 1000;
-        printf("Finishd reading correctly %f mbytes/sec (time = %f)\n", acc / elapsed/1024/1024, elapsed);
-    }
-}
-
-void PrintUsage(char *name)
-{
-    printf("The program might be used in client or server mode\n");
-       printf("usage is %s <server> <port number>\n", name);
-    printf("usage is %s <client> <ip> <port number> <option number> <WritePacketNumber> <UseOverlappedSend> <Number of packets> <PacketSize>\n", name);
-    printf("The option will be sent to the remote side\n");
-       printf("Available options are:\n");
-       printf("    1 - just recieve packets forever (the other side will send xxx bytes yyy times)\n");
-       printf("    2 - send xxx bytes yyy times (based on input from network)\n");
-    printf("    WritePacketNumber can be 0 (no) or 1 (yes)\n");
-    printf("    UseOverlappedSend can be 0 (no) or 1 (yes)\n");
-
-}
-
-int Connect(int argc, char *argv[])
-{
-       struct sockaddr server_addr;
-       int port = atoi(argv[3]);
-    int s, ret;
-    int Send = 0;
-    int BufferSize, NumberOfBuffers;
-    int result = 0;
-    BOOL SendPacketNumber;
-    int  UseOverlappedSend = 0;
-
-    if (argv[4][0] == '1') {
-        Send = 1;
-    }else if (argv[4][0] == '2') {
-        Send = 0;
-    } else {
-        printf("Error - don't know if to send or recieve\n");
-        exit(1);
-    }
-
-    if (argv[5][0] == '0') {
-        SendPacketNumber = FALSE;
-    }else if (argv[5][0] == '1') {
-        SendPacketNumber = TRUE;
-    } else {
-        printf("Error - don't know if to send the packet number\n");
-        exit(1);
-    }
-
-    if (argv[6][0] == '0') {
-        UseOverlappedSend = FALSE;
-    }else if (argv[6][0] == '1') {
-        UseOverlappedSend = TRUE;
-    } else {
-        printf("Error - don't know if to send overlapped or not\n");
-        exit(1);
-    }
-    
-
-    NumberOfBuffers = atoi(argv[7]);
-    BufferSize = atoi(argv[8]);
-    
-       printf("Using port %d %s %s %s NumberOfBuffers = %d , BufferSize = %d \n", 
-        port, 
-        Send == 1 ? "sending" : "recieving", 
-        SendPacketNumber ? "Packets Are filed with data" : "Packets are not filled with data",
-        UseOverlappedSend ? "Using overlapped send" : "not using overlapped send", 
-        NumberOfBuffers,
-        BufferSize
-        );
-
-    server_addr = addressFromString(argv[2], port);
-
-       s = socket (AF_INET_FAMILY, SOCK_STREAM, 0);
-    if (s == -1) {
-            printf("Error creating socket\n");
-    }
-
-    ret = connect(s, &server_addr, sizeof(server_addr));
-    if (ret ==0) {
-        printf("connect succeeded\n");
-    } else {
-        printf("connect failed\n");
-            result = 1;
-            goto cleanup;
-    }
-
-    // Send the other side the correct information: 
-    // send(=1), UseOverlapped ,NumberOfBuffers, BufferSize,
-    // or recieve(=0),UseOverlapped ,NumberOfBuffers, BufferSize,
-    if (Send) {
-        char message[60];
-        sprintf(message, "1,%d,%d,%d,0,", UseOverlappedSend,NumberOfBuffers, BufferSize);
-        ret = send(s, message, strlen (message), 0);
-        if (ret != strlen (message)) {
-            printf("Error in send winsock error=%d\n", WSAGetLastError());
-            result = 1;
-            goto cleanup;
-        } 
-    } else {
-        char message[60];
-        sprintf(message, "2,%d,%d,%d,%d,",UseOverlappedSend, NumberOfBuffers, BufferSize, SendPacketNumber);
-        ret = send(s, message, strlen (message), 0);
-        if (ret != strlen (message)) {
-            printf("Error in send winsock error=%d\n", WSAGetLastError());
-            result = 1;
-            goto cleanup;
-        }
-    }
-    if (Send) {
-        if (UseOverlappedSend) {
-            SendDataOverlapped(s, NumberOfBuffers, BufferSize, SendPacketNumber);
-        } else {
-            SendData(s, NumberOfBuffers, BufferSize, SendPacketNumber);
-        }
-    } else {
-        RecvData(s, NumberOfBuffers, BufferSize);
-    }
-cleanup:
-    ret = CloseSocket(s);
-    if (ret == 0) {
-        printf("CloseSocket succeeded\n");
-    } else {
-        printf("CloseSocket failed\n");
-    }
-    
-    
-
-    return result;
-
-
-
-}
-int ReadNumber(int fd)
-{
-    //???
-       char Data[20]; // Don't care about buffer overflows :-)
-       int i = 0, ret;
-//    printf("ReadNumber called\n");
-
-       while (1) {
-               ret = recv(fd,&Data[i],1,0);
-        if (ret != 1) {
-            printf("Error reading data 1\n");
-            exit(1);
-        }
-               if (Data[i] == ',') break;
-               i++;
-               if (i == 20) {
-                       printf("Error reading data 2\n");
-                       exit(1);
-               }       
-       }
-       return atoi(Data);
-}
-
-int Server(int argc, char *argv[])
-{
-
-       u_short port = (u_short)atoi(argv[2]);
-    int option;
-       int s, newfd;
-       int ret;
-       struct sockaddr_in my_addr, client_addr;
-       int addr_len = sizeof(client_addr);
-    int BufferSize, NumberOfBuffers;
-    BOOL SendPacketNumber;
-    int UseOverlappedSend;
-
-       printf("Using port %d\n", port);
-    
-       memset(&my_addr, 0, sizeof(my_addr));
-       my_addr.sin_family = AF_INET;
-       my_addr.sin_port = htons(port);
-       s = socket (AF_INET_FAMILY , SOCK_STREAM, IPPROTO_TCP);
-        if (s == -1) {
-                printf("Error creating socket\n");
-        }
-
-       ret =   bind(s, (struct sockaddr*)&my_addr, sizeof(my_addr));
-       if (ret < 0) {
-               printf("Error in bind\n" );
-       }
-
-       ret = listen(s, 5);
-       for(;;) {
-               struct sockaddr name;
-               int namelen;
-               unsigned char *p;
-               
-               newfd  = accept(s, (struct sockaddr *)&client_addr, &addr_len);
-               namelen = sizeof(name);
-               getpeername(newfd, &name, &namelen);
-               p = (unsigned char *) &client_addr;
-               printf("recieved a client %d from %d.%d.%d.%d:%d\n", newfd, 
-                               p[4], p[5],p[6], p[7], p[2] *256 +  p[3] ) ;
-
-
-       option = ReadNumber(newfd);
-        UseOverlappedSend = ReadNumber(newfd);
-        NumberOfBuffers = ReadNumber(newfd);
-        BufferSize = ReadNumber(newfd);
-        SendPacketNumber = ReadNumber(newfd);
-
-
-        printf("%s ,%s Using overlapped Send,NumberOfBuffers = %d , BufferSize =%d PacketsFilled=%s \n", 
-            option == 2 ? "sending" : "recieving",
-            UseOverlappedSend ? "" : "not",
-            NumberOfBuffers,
-            BufferSize,
-            SendPacketNumber ? "true" : "false"
-            );
-        
-        if (option == 1) RecvData(newfd, NumberOfBuffers, BufferSize);
-        if (option == 2) SendData(newfd, NumberOfBuffers, BufferSize, SendPacketNumber);
-
-        ret = CloseSocket(newfd);
-        if (ret == 0) {
-            printf("CloseSocket succeeded\n");
-        } else {
-            printf("CloseSocket failed\n");
-        }
-        //break;    
-    }
-    return 0;
-}
-
-int __cdecl main(int argc, char *argv[])
-{
-    int error; 
-
-       if (argc < 3) {
-               PrintUsage(argv[0]);
-               return 1;
-       }
-
-#ifdef _WIN32
-    {
-        WSADATA wsaData;
-
-       error = WSAStartup( MAKEWORD( 2, 2 ), &wsaData);
-       if (error) {
-               printf("Error in WSAStartup: winsock error=%d\n", WSAGetLastError());
-                       return 1;
-       }
-    }
-#endif    
-    if (!_stricmp(argv[1],"client")) {
-        Connect(argc, argv);
-        //sleep(2);
-    } else if (!_stricmp(argv[1],"server")) {
-        Server(argc, argv);
-    } else {
-        printf("Unknowion option %s", argv[1]);
-               PrintUsage(argv[0]);
-        
-    }
-       SOCKET s;
-    s = socket (AF_INET_FAMILY, SOCK_STREAM, 0);
-CloseSocket(s);        
-    //printf("calling WSACleanup\n ");
-    int ret = WSACleanup ();
-    //printf("ret = %d", ret);
-    //Sleep(10000);
-    return 0;
-}
-
+\r
+#ifdef _WIN32\r
+#include <Winsock2.h>\r
+#include <stdio.h>\r
+#include <stdlib.h>\r
+#include <assert.h>\r
+\r
+#define AF_INET_FAMILY PF_INET\r
+\r
+#define sleep(x) Sleep(1000 * (x));\r
+#define CloseSocket closesocket\r
+\r
+#define GetTimeMs() GetTickCount()\r
+#else\r
+\r
+#define AF_INET_FAMILY 27\r
+\r
+#include <sys/types.h>\r
+#include <netinet/in.h>\r
+#include <sys/socket.h>\r
+#include <stdio.h>\r
+#include <errno.h>\r
+\r
+#define _stricmp strcasecmp\r
+#define WSAGetLastError() errno\r
+#define CloseSocket close\r
+\r
+\r
+#endif\r
+// Windows linux \r
+\r
+const int NumberOfThreads = 1;\r
+\r
+HANDLE g_ComplitionPort;\r
+HANDLE *g_pThreads;\r
+\r
+#define ASSERT assert\r
+\r
+struct sockaddr addressFromString(char *address, int port)\r
+{\r
+       unsigned int b1,b2,b3,b4;\r
+\r
+       struct sockaddr_in socket_address;\r
+       struct sockaddr *socket_address_ptr = (struct sockaddr*)&socket_address;\r
+    \r
+       memset(&socket_address, 0, sizeof(socket_address));\r
+       socket_address.sin_family = AF_INET;\r
+       socket_address.sin_port = htons((u_short)port);\r
+\r
+\r
+       sscanf(address, "%d.%d.%d.%d",&b1, &b2, &b3, &b4);\r
+       socket_address.sin_addr.s_addr = b4 * 256 * 256 *256 +\r
+                                        b3 * 256 * 256 + \r
+                                        b2 * 256 +\r
+                                        b1;\r
+                                        \r
+       return *socket_address_ptr;\r
+}\r
+\r
+\r
+struct OverlappedSend {\r
+    OVERLAPPED Overlapped;\r
+    SOCKET Socket;\r
+    void *Buffer; // The data to send\r
+    int  DataSize;\r
+    DWORD NumberOfBytesSent;\r
+    int  RemainingSends;\r
+    int  SendsInAir;     // Curently sent data\r
+    CRITICAL_SECTION Lock;\r
+    HANDLE SendDoneEvent;\r
+\r
+};\r
+\r
+\r
+DWORD WINAPI WorkerThreadFunc( LPVOID lpParam )\r
+{\r
+    BOOL ret;\r
+    DWORD NumberOfBytes;\r
+    ULONG_PTR CompletionKey;\r
+    OVERLAPPED* lpOverlapped;\r
+    OverlappedSend *pOverLappedSend;\r
+    BOOL  ContinueLoop = true;\r
+    int iRet;\r
+    \r
+    while (TRUE) {\r
+        ret = GetQueuedCompletionStatus(\r
+                                g_ComplitionPort,\r
+                                &NumberOfBytes,\r
+                                &CompletionKey,\r
+                                &lpOverlapped,\r
+                                INFINITE);\r
+        ASSERT(ret != 0);\r
+        pOverLappedSend = CONTAINING_RECORD(lpOverlapped, OverlappedSend, Overlapped);\r
+        // Work on the object itself:\r
+        EnterCriticalSection(&pOverLappedSend->Lock);\r
+        pOverLappedSend->SendsInAir--;\r
+        ASSERT(pOverLappedSend->SendsInAir ==0); // Only one thread\r
+        if (pOverLappedSend->RemainingSends > 0) {\r
+            // do the next send\r
+            WSABUF Buffers;\r
+            Buffers.len = pOverLappedSend->DataSize;\r
+            Buffers.buf = (char *) pOverLappedSend->Buffer;\r
+            pOverLappedSend->SendsInAir++;\r
+            pOverLappedSend->RemainingSends--;\r
+            \r
+            iRet = WSASend(\r
+                pOverLappedSend->Socket,\r
+                &Buffers,\r
+                1,\r
+                &pOverLappedSend->NumberOfBytesSent,\r
+                0,\r
+                &pOverLappedSend->Overlapped,\r
+                NULL\r
+                );\r
+            ASSERT((iRet ==0) || (iRet == SOCKET_ERROR && WSAGetLastError() == WSA_IO_PENDING ));\r
+\r
+        } else {\r
+            // Nothing more to send - signal compleation and exit\r
+            SetEvent(pOverLappedSend->SendDoneEvent);\r
+            ContinueLoop = false;\r
+        }\r
+        LeaveCriticalSection(&pOverLappedSend->Lock);\r
+        if (!ContinueLoop) {\r
+            break;\r
+        }\r
+\r
+    }\r
+    return 0;\r
+}\r
+\r
+void CreateComplitionPort(SOCKET newfd)\r
+{\r
+    g_pThreads = new HANDLE [NumberOfThreads];\r
+    ASSERT(g_pThreads != NULL);\r
+\r
+\r
+    g_ComplitionPort = CreateIoCompletionPort((HANDLE)newfd, NULL, NULL, 2);\r
+    if(g_ComplitionPort == NULL) {\r
+        printf("Create complition port failed err=%d", GetLastError());\r
+        exit(1);\r
+    }\r
+\r
+    // Create the threads that will work on the complitions\r
+    g_pThreads[0] = CreateThread(NULL, 0 ,WorkerThreadFunc ,NULL, 0,NULL );\r
+\r
+\r
+}\r
+\r
+void CloseComplitionPort()\r
+{\r
+    //??? Post a complition thread end message\r
+\r
+    WaitForMultipleObjects(NumberOfThreads,g_pThreads, TRUE, INFINITE);\r
+    CloseHandle(g_ComplitionPort);\r
+\r
+}\r
+\r
+void SendDataOverlapped(SOCKET newfd, double NumberOfBuffers, double BufferSize, BOOL SendPacketNumber)\r
+{\r
+    // We will start with one operations simultaniously\r
+    int i ;\r
+    double d;\r
+    VOID *buffer = NULL;\r
+    double elapsed, acc = 0;\r
+    WSAOVERLAPPED wsa;\r
+    OverlappedSend *pOverLappedSend = NULL;\r
+    BOOL ret;\r
+    memset(&wsa, 0, sizeof wsa);\r
+\r
+    buffer = malloc ((size_t)BufferSize);\r
+    if (buffer == NULL) {\r
+        printf("Error allocating buffer\n");\r
+        exit(1);\r
+    }\r
+\r
+    CreateComplitionPort(newfd);\r
+\r
+    pOverLappedSend = new OverlappedSend;\r
+\r
+    // Set it's fields\r
+    memset(&pOverLappedSend->Overlapped, 0, sizeof (pOverLappedSend->Overlapped));\r
+    pOverLappedSend->Socket = newfd;\r
+    pOverLappedSend->Buffer = buffer;\r
+    pOverLappedSend->DataSize = (size_t)BufferSize;\r
+    pOverLappedSend->RemainingSends = (size_t)NumberOfBuffers;\r
+    pOverLappedSend->SendsInAir = 1; // To force a simulate of the compleation\r
+    InitializeCriticalSection(&pOverLappedSend->Lock);\r
+    pOverLappedSend->SendDoneEvent = CreateEvent(NULL, FALSE, FALSE, NULL);\r
+\r
+    double Start = GetTimeMs();\r
+    // Post the message that will do the first send\r
+    ret = PostQueuedCompletionStatus(\r
+                              g_ComplitionPort,\r
+                              (DWORD)BufferSize,\r
+                              NULL,\r
+                              &pOverLappedSend->Overlapped\r
+                                );\r
+    ASSERT(ret != 0);\r
+\r
+    // wait for the send \r
+    WaitForSingleObject(pOverLappedSend->SendDoneEvent, INFINITE);\r
+    elapsed = (GetTimeMs() - Start) / 1000;\r
+    printf("Finishd sending correctly %f mbytes/sec\n", NumberOfBuffers * BufferSize /elapsed/1024/1024 );\r
+    CloseComplitionPort();\r
+//Cleanup:\r
+    free(buffer);\r
+\r
+\r
+}\r
+\r
+\r
+\r
+void SendData(SOCKET newfd, double NumberOfBuffers, double BufferSize, BOOL SendPacketNumber)\r
+{\r
+    int i ;\r
+    double d;\r
+    CHAR *buffer = NULL;\r
+    double elapsed, acc = 0;\r
+\r
+    buffer = new char [(size_t)BufferSize];\r
+    if (buffer == NULL) {\r
+        printf("Error allocating buffer\n");\r
+        exit(1);\r
+    }\r
+\r
+    printf("Starting to send %lf messages of size %lf\n", NumberOfBuffers, BufferSize );\r
+\r
+    i = 0;\r
+\r
+    for (i = 0; i < 10; i++) {\r
+        buffer[i] = 'a' + i;\r
+    }\r
+    double Start = GetTimeMs();\r
+\r
+    for (d=0; d < NumberOfBuffers; d++) {\r
+        int j;\r
+        if (SendPacketNumber) {\r
+            memset(buffer, (char) d, (size_t)BufferSize);\r
+        }\r
+        \r
+        j = send(newfd, buffer, (size_t)BufferSize, 0 );\r
+            acc += j;\r
+        if (j!=BufferSize) {\r
+            printf("Error send not compleated  sent %lf\n", acc);\r
+            goto Cleanup;\r
+        }\r
+    }\r
+    elapsed = (GetTimeMs() - Start) / 1000;\r
+    printf("Finishd sending correctly %f mbytes/sec (%f seconfs)\n", acc/elapsed/1024/1024, elapsed );\r
+Cleanup:\r
+    free(buffer);\r
+}\r
+\r
+\r
+void RecvData(SOCKET newfd, double NumberOfBuffers, double BufferSize )\r
+{\r
+    const int ReadSize = 400000;\r
+    int read1 = 1;\r
+    unsigned char *p;\r
+    double elapsed, acc = 0;\r
+    int i;\r
+    double  Start = GetTimeMs();\r
+    char *buffer = new char [ReadSize];\r
+    if (buffer == NULL) {\r
+        printf("Error allocating memory");\r
+        exit(4);\r
+    }\r
+\r
+    while ((read1 != -1) /*&& (acc < (double)BufferSize * NumberOfBuffers)*/) {\r
+        read1 = recv(newfd, buffer, ReadSize, 0);\r
+        if (read1 == -1 || read1 == 0) {\r
+               printf("Finsihed reading, total read %lf bytes last read =%d\n", acc, read1 );\r
+            if (acc != BufferSize * NumberOfBuffers) {\r
+                printf("Error, expected to read %lf but read %lf\n",\r
+                    (BufferSize * NumberOfBuffers), acc);\r
+            }\r
+            break;\r
+        }\r
+        else {\r
+            acc += read1;\r
+\r
+//                      printf("read returned %d \"%c%c%c%c\"\n",read1,\r
+//                      buffer[0],buffer[1], buffer[2], buffer[3]);\r
+        }\r
+    }\r
+\r
+    if (acc != ((double)BufferSize * NumberOfBuffers)) {\r
+        printf("Error, expected to read %lf but read %lf",\r
+            ((double)BufferSize * NumberOfBuffers), acc);\r
+    } else {\r
+        elapsed = (GetTimeMs() - Start) / 1000;\r
+        printf("Finishd reading correctly %lf mbytes/sec (time = %lf)\n", acc / elapsed/1024/1024, elapsed);\r
+    }\r
+    delete []buffer;\r
+}\r
+\r
+void PrintUsage(char *name)\r
+{\r
+    printf("The program might be used in client or server mode\n");\r
+       printf("usage is %s <server> <port number>\n", name);\r
+    printf("usage is %s <client> <ip> <port number> <option number> <WritePacketNumber> <UseOverlappedSend> <Number of packets> <PacketSize>\n", name);\r
+    printf("The option will be sent to the remote side\n");\r
+       printf("Available options are:\n");\r
+       printf("    1 - just recieve packets forever (the other side will send xxx bytes yyy times)\n");\r
+       printf("    2 - send xxx bytes yyy times (based on input from network)\n");\r
+    printf("    WritePacketNumber can be 0 (no) or 1 (yes)\n");\r
+    printf("    UseOverlappedSend can be 0 (no) or 1 (yes)\n");\r
+\r
+}\r
+\r
+int Connect(int argc, char *argv[])\r
+{\r
+       struct sockaddr server_addr;\r
+       int port = atoi(argv[3]);\r
+    int  ret;\r
+    SOCKET s;\r
+    int Send = 0;\r
+    int BufferSize, NumberOfBuffers;\r
+    int result = 0;\r
+    BOOL SendPacketNumber;\r
+    int  UseOverlappedSend = 0;\r
+\r
+    if (argv[4][0] == '1') {\r
+        Send = 1;\r
+    }else if (argv[4][0] == '2') {\r
+        Send = 0;\r
+    } else {\r
+        printf("Error - don't know if to send or recieve\n");\r
+        exit(1);\r
+    }\r
+\r
+    if (argv[5][0] == '0') {\r
+        SendPacketNumber = FALSE;\r
+    }else if (argv[5][0] == '1') {\r
+        SendPacketNumber = TRUE;\r
+    } else {\r
+        printf("Error - don't know if to send the packet number\n");\r
+        exit(1);\r
+    }\r
+\r
+    if (argv[6][0] == '0') {\r
+        UseOverlappedSend = FALSE;\r
+    }else if (argv[6][0] == '1') {\r
+        UseOverlappedSend = TRUE;\r
+    } else {\r
+        printf("Error - don't know if to send overlapped or not\n");\r
+        exit(1);\r
+    }\r
+    \r
+\r
+    NumberOfBuffers = atoi(argv[7]);\r
+    BufferSize = atoi(argv[8]);\r
+    \r
+       printf("Using port %d %s %s %s NumberOfBuffers = %d , BufferSize = %d \n", \r
+        port, \r
+        Send == 1 ? "sending" : "recieving", \r
+        SendPacketNumber ? "Packets Are filed with data" : "Packets are not filled with data",\r
+        UseOverlappedSend ? "Using overlapped send" : "not using overlapped send", \r
+        NumberOfBuffers,\r
+        BufferSize\r
+        );\r
+\r
+    server_addr = addressFromString(argv[2], port);\r
+\r
+    s = socket (AF_INET_FAMILY, SOCK_STREAM, 0);\r
+    if (s == -1) {\r
+            printf("Error creating socket\n");\r
+    }\r
+\r
+    ret = connect(s, &server_addr, sizeof(server_addr));\r
+    if (ret ==0) {\r
+        printf("connect succeeded\n");\r
+    } else {\r
+        printf("connect failed\n");\r
+            result = 1;\r
+            goto cleanup;\r
+    }\r
+\r
+    // Send the other side the correct information: \r
+    // send(=1), UseOverlapped ,NumberOfBuffers, BufferSize,\r
+    // or recieve(=0),UseOverlapped ,NumberOfBuffers, BufferSize,\r
+    if (Send) {\r
+        char message[60];\r
+        sprintf(message, "1,%d,%d,%d,0,", UseOverlappedSend,NumberOfBuffers, BufferSize);\r
+        ret = send(s, message, strlen (message), 0);\r
+        if (ret != strlen (message)) {\r
+            printf("Error in send winsock error=%d\n", WSAGetLastError());\r
+            result = 1;\r
+            goto cleanup;\r
+        } \r
+    } else {\r
+        char message[60];\r
+        sprintf(message, "2,%d,%d,%d,%d,",UseOverlappedSend, NumberOfBuffers, BufferSize, SendPacketNumber);\r
+        ret = send(s, message, strlen (message), 0);\r
+        if (ret != strlen (message)) {\r
+            printf("Error in send winsock error=%d\n", WSAGetLastError());\r
+            result = 1;\r
+            goto cleanup;\r
+        }\r
+    }\r
+    if (Send) {\r
+        if (UseOverlappedSend) {\r
+            SendDataOverlapped(s, NumberOfBuffers, BufferSize, SendPacketNumber);\r
+        } else {\r
+            SendData(s, NumberOfBuffers, BufferSize, SendPacketNumber);\r
+        }\r
+    } else {\r
+        RecvData(s, NumberOfBuffers, BufferSize);\r
+    }\r
+cleanup:\r
+    ret = CloseSocket(s);\r
+    if (ret == 0) {\r
+        printf("CloseSocket succeeded\n");\r
+    } else {\r
+        printf("CloseSocket failed\n");\r
+    }\r
+    \r
+    \r
+\r
+    return result;\r
+\r
+\r
+\r
+}\r
+int ReadNumber(SOCKET fd)\r
+{\r
+    //???\r
+       char Data[20]; // Don't care about buffer overflows :-)\r
+       int i = 0, ret;\r
+//    printf("ReadNumber called\n");\r
+\r
+       while (1) {\r
+               ret = recv(fd,&Data[i],1,0);\r
+        if (ret != 1) {\r
+            printf("Error reading data 1\n");\r
+            exit(1);\r
+        }\r
+               if (Data[i] == ',') break;\r
+               i++;\r
+               if (i == 20) {\r
+                       printf("Error reading data 2\n");\r
+                       exit(1);\r
+               }       \r
+       }\r
+       return atoi(Data);\r
+}\r
+\r
+int Server(int argc, char *argv[])\r
+{\r
+\r
+    u_short port = (u_short)atoi(argv[2]);\r
+    int option;\r
+    SOCKET s, newfd;\r
+    int ret;\r
+    struct sockaddr_in my_addr, client_addr;\r
+    int addr_len = sizeof(client_addr);\r
+    int BufferSize, NumberOfBuffers;\r
+    BOOL SendPacketNumber;\r
+    int UseOverlappedSend;\r
+\r
+    printf("Using port %d\n", port);\r
+    \r
+    memset(&my_addr, 0, sizeof(my_addr));\r
+    my_addr.sin_family = AF_INET;\r
+    my_addr.sin_port = htons(port);\r
+    s = socket (AF_INET_FAMILY , SOCK_STREAM, IPPROTO_TCP);\r
+        if (s == -1) {\r
+            printf("Error creating socket\n");\r
+            return 1;\r
+        }\r
+\r
+    ret =      bind(s, (struct sockaddr*)&my_addr, sizeof(my_addr));\r
+    if (ret < 0) {\r
+        printf("Error in bind\n" );\r
+        return 2;\r
+       }\r
+\r
+       ret = listen(s, 5);\r
+       for(;;) {\r
+               struct sockaddr name;\r
+               int namelen;\r
+               unsigned char *p;\r
+               \r
+               newfd  = accept(s, (struct sockaddr *)&client_addr, &addr_len);\r
+               namelen = sizeof(name);\r
+               getpeername(newfd, &name, &namelen);\r
+               p = (unsigned char *) &client_addr;\r
+        printf("recieved a client %d from %d.%d.%d.%d:%d\n", newfd, \r
+            p[4], p[5],p[6], p[7], p[2] *256 +  p[3] ) ;\r
+\r
+\r
+        option = ReadNumber(newfd);\r
+        UseOverlappedSend = ReadNumber(newfd);\r
+        NumberOfBuffers = ReadNumber(newfd);\r
+        BufferSize = ReadNumber(newfd);\r
+        SendPacketNumber = ReadNumber(newfd);\r
+\r
+        printf("%s ,%s Using overlapped Send,NumberOfBuffers = %d , BufferSize =%d PacketsFilled=%s \n", \r
+            option == 2 ? "sending" : "recieving",\r
+            UseOverlappedSend ? "" : "not",\r
+            NumberOfBuffers,\r
+            BufferSize,\r
+            SendPacketNumber ? "true" : "false"\r
+            );\r
+        \r
+        if (option == 1) RecvData(newfd, NumberOfBuffers, BufferSize);\r
+        if (option == 2) SendData(newfd, NumberOfBuffers, BufferSize, SendPacketNumber);\r
+\r
+        ret = CloseSocket(newfd);\r
+        if (ret == 0) {\r
+            printf("CloseSocket succeeded\n");\r
+        } else {\r
+            printf("CloseSocket failed\n");\r
+        }\r
+        //break;    \r
+    }\r
+    return 0;\r
+}\r
+\r
+int __cdecl main(int argc, char *argv[])\r
+{\r
+    int error; \r
+\r
+       if (argc < 3) {\r
+               PrintUsage(argv[0]);\r
+               return 1;\r
+       }\r
+\r
+#ifdef _WIN32\r
+    {\r
+        WSADATA wsaData;\r
+\r
+       error = WSAStartup( MAKEWORD( 2, 2 ), &wsaData);\r
+       if (error) {\r
+               printf("Error in WSAStartup: winsock error=%d\n", WSAGetLastError());\r
+                       return 1;\r
+       }\r
+    }\r
+#endif    \r
+    if (!_stricmp(argv[1],"client")) {\r
+        Connect(argc, argv);\r
+        //sleep(2);\r
+    } else if (!_stricmp(argv[1],"server")) {\r
+        Server(argc, argv);\r
+    } else {\r
+        printf("Unknowion option %s", argv[1]);\r
+               PrintUsage(argv[0]);\r
+        \r
+    }\r
+       SOCKET s;\r
+    s = socket (AF_INET_FAMILY, SOCK_STREAM, 0);\r
+CloseSocket(s);        \r
+    //printf("calling WSACleanup\n ");\r
+    int ret = WSACleanup ();\r
+    //printf("ret = %d", ret);\r
+    //Sleep(10000);\r
+    return 0;\r
+}\r
+\r