[WSD] Fix collision between event handles used in connect/disconnect and
authorftillier <ftillier@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Tue, 27 Sep 2005 00:13:42 +0000 (00:13 +0000)
committerftillier <ftillier@ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86>
Tue, 27 Sep 2005 00:13:42 +0000 (00:13 +0000)
listen handle.  Eliminated cleanup thread, simplified and improvided socket
cleanup.

git-svn-id: svn://openib.tc.cornell.edu/gen1/trunk@85 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

ulp/wsd/user/ib_cm.c
ulp/wsd/user/ibsp_duplicate.c
ulp/wsd/user/ibsp_iblow.c
ulp/wsd/user/ibspdll.c
ulp/wsd/user/ibspproto.h
ulp/wsd/user/ibspstruct.h
ulp/wsd/user/misc.c
ulp/wsd/user/sockinfo.c

index c53b710..d3f6435 100644 (file)
@@ -137,7 +137,8 @@ cm_req_callback(
        switch( socket_info->socket_state )\r
        {\r
        case IBSP_LISTEN:\r
-               if( cl_qlist_count( &socket_info->info.listen.list ) >= socket_info->info.listen.backlog )\r
+               if( cl_qlist_count( &socket_info->listen.list ) >=\r
+                       socket_info->listen.backlog )\r
                {\r
                        /* Already too many connection requests are queued */\r
                        IBSP_TRACE1( IBSP_DBG_CM,\r
@@ -148,7 +149,7 @@ cm_req_callback(
                }\r
 \r
                /* Add to the waiting list */\r
-               cl_qlist_insert_head( &socket_info->info.listen.list, &incoming->item );\r
+               cl_qlist_insert_tail( &socket_info->listen.list, &incoming->item );\r
 \r
                /* Send MRA */\r
                mra.mra_length = 0;\r
@@ -168,8 +169,8 @@ cm_req_callback(
                        wait_cq_drain( socket_info );\r
 \r
                        /* Non-blocking cancel since we're in CM callback context */\r
-                       ib_cm_cancel( socket_info->info.listen.handle, NULL );\r
-                       socket_info->info.listen.handle = NULL;\r
+                       ib_cm_cancel( socket_info->listen.handle, NULL );\r
+                       socket_info->listen.handle = NULL;\r
 \r
 #if 0\r
                        cl_spinlock_release( &socket_info->mutex );\r
@@ -239,24 +240,25 @@ cm_rep_callback(
 \r
        cl_spinlock_acquire( &socket_info->mutex );\r
 \r
-       if( socket_info->socket_state == IBSP_CONNECT )\r
+       switch( socket_info->socket_state )\r
        {\r
+       case IBSP_CONNECT:\r
                status = ib_cm_rtu( p_cm_rep_rec->h_cm_rep, &cm_rtu );\r
                if( status != IB_SUCCESS )\r
                {\r
                        /* Note: a REJ has been automatically sent. */\r
                        CL_ERROR( IBSP_DBG_CM, gdbg_lvl, ("ib_cm_rtu failed (0x%d)\n", status) );\r
                        IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_BIND );\r
-                       ibsp_post_select_event( socket_info, FD_CONNECT, WSAENOBUFS );\r
+                       ibsp_post_select_event( socket_info, FD_CONNECT, WSAETIMEDOUT );\r
                }\r
                else\r
                {\r
                        IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_CONNECTED );\r
                        ibsp_post_select_event( socket_info, FD_CONNECT, 0 );\r
                }\r
-       }\r
-       else if( socket_info->socket_state == IBSP_DUPLICATING_NEW )\r
-       {\r
+               break;\r
+\r
+       case IBSP_DUPLICATING_NEW:\r
                status = ib_cm_rtu( p_cm_rep_rec->h_cm_rep, &cm_rtu );\r
                if( status != IB_SUCCESS )\r
                {\r
@@ -267,10 +269,9 @@ cm_rep_callback(
                {\r
                        IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_CONNECTED );\r
                }\r
-               SetEvent( socket_info->info.connect.event );\r
-       }\r
-       else\r
-       {\r
+               SetEvent( socket_info->h_event );\r
+\r
+       default:\r
                /* The socket might be closing */\r
                CL_ERROR( IBSP_DBG_CM, gdbg_lvl, ("socket %x not in connecting state (%s)\n",\r
                        socket_info, IBSP_SOCKET_STATE_STR( socket_info->socket_state )) );\r
@@ -302,7 +303,7 @@ cm_rtu_callback(
        if( socket_info->socket_state == IBSP_ACCEPT )\r
        {\r
                IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_CONNECTED );\r
-               SetEvent( socket_info->info.accept.event );\r
+               SetEvent( socket_info->h_event );\r
        }\r
        else if( socket_info->socket_state == IBSP_DUPLICATING_REMOTE )\r
        {\r
@@ -353,7 +354,6 @@ cm_rtu_callback(
 \r
                        status = ib_post_recv( socket_info->qp, &wr->recv, NULL );\r
 \r
-\r
                        if( status == IB_SUCCESS )\r
                        {\r
                                /* Update the index and wrap as needed */\r
@@ -442,7 +442,7 @@ cm_rej_callback(
        case IBSP_DUPLICATING_NEW:\r
                /* Leave in that state. IBSPSocket will eventually return \r
                 * an error becaus the socket is not connected. */\r
-               SetEvent( socket_info->info.connect.event );\r
+               SetEvent( socket_info->h_event );\r
                break;\r
 \r
        default:\r
@@ -513,9 +513,9 @@ cm_dreq_callback(
                                socket_info->duplicate.dwProcessId = reason->duplicating.dwProcessId;\r
 \r
                                /* Now, setup our listening callback. */\r
-                               socket_info->info.listen.listen_req_param.dwProcessId =\r
+                               socket_info->listen.listen_req_param.dwProcessId =\r
                                        reason->duplicating.dwProcessId;\r
-                               socket_info->info.listen.listen_req_param.identifier =\r
+                               socket_info->listen.listen_req_param.identifier =\r
                                        reason->duplicating.identifier;\r
 \r
                                ret = ib_listen( socket_info, &error );\r
@@ -533,9 +533,6 @@ cm_dreq_callback(
                        socket_info->qp_error = WSAECONNRESET;\r
                }\r
        }\r
-       else\r
-       {\r
-       }\r
        cl_spinlock_release( &socket_info->mutex );\r
 \r
        memset( &cm_drep, 0, sizeof(cm_drep) );\r
@@ -655,22 +652,22 @@ ib_listen(
        }\r
        param.lid = IB_ALL_LIDS;\r
 \r
-       param.p_compare_buffer = (uint8_t *) & socket_info->info.listen.listen_req_param;\r
+       param.p_compare_buffer = (uint8_t *) & socket_info->listen.listen_req_param;\r
        param.compare_length = sizeof(struct listen_req_param);\r
        param.compare_offset = offsetof(struct cm_req_params, listen_req_param);\r
 \r
        fzprint(("%s():%d:0x%x:0x%x: socket=0x%p params: %x %x\n", __FUNCTION__,\r
                         __LINE__, GetCurrentProcessId(),\r
                         GetCurrentThreadId(), socket_info,\r
-                        socket_info->info.listen.listen_req_param.dwProcessId,\r
-                        socket_info->info.listen.listen_req_param.identifier));\r
+                        socket_info->listen.listen_req_param.dwProcessId,\r
+                        socket_info->listen.listen_req_param.identifier));\r
 \r
        param.pfn_cm_req_cb = cm_req_callback;\r
 \r
        param.qp_type = IB_QPT_RELIABLE_CONN;\r
 \r
        status = ib_cm_listen( g_ibsp.al_handle, &param, listen_err_callback, socket_info,      /* context */\r
-               &socket_info->info.listen.handle );\r
+               &socket_info->listen.handle );\r
 \r
        if( status != IB_SUCCESS )\r
        {\r
@@ -699,12 +696,12 @@ ib_listen_backlog(
        cl_list_item_t *item;\r
        struct listen_incoming *incoming;\r
 \r
-       socket_info->info.listen.backlog = backlog;\r
+       socket_info->listen.backlog = backlog;\r
 \r
        while(\r
-               cl_qlist_count( &socket_info->info.listen.list ) > (uint32_t)backlog )\r
+               cl_qlist_count( &socket_info->listen.list ) > (uint32_t)backlog )\r
        {\r
-               item = cl_qlist_remove_tail( &socket_info->info.listen.list );\r
+               item = cl_qlist_remove_tail( &socket_info->listen.list );\r
 \r
                incoming = PARENT_STRUCT(item, struct listen_incoming, item);\r
 \r
@@ -724,10 +721,7 @@ ib_listen_cancel(
 \r
        IBSP_ENTER( IBSP_DBG_CM );\r
 \r
-       /* We should be in the CLOSING state */\r
-       CL_ASSERT( socket_info->socket_state == IBSP_CLOSING );\r
-\r
-       status = ib_cm_cancel( socket_info->info.listen.handle, ib_sync_destroy );\r
+       status = ib_cm_cancel( socket_info->listen.handle, ib_sync_destroy );\r
        if( status )\r
        {\r
                CL_ERROR( IBSP_DBG_CM, gdbg_lvl,\r
@@ -744,7 +738,7 @@ ib_listen_cancel(
        ib_listen_backlog( socket_info, 0 );\r
        cl_spinlock_release( &socket_info->mutex );\r
 \r
-       socket_info->info.listen.handle = NULL;\r
+       socket_info->listen.handle = NULL;\r
 \r
        IBSP_EXIT( IBSP_DBG_CM );\r
 }\r
index b5e3dbd..9497ae7 100644 (file)
@@ -182,7 +182,7 @@ setup_duplicate_socket(
        }\r
 \r
        IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_DUPLICATING_NEW );\r
-       socket_info->info.connect.event = CreateEvent( NULL, FALSE, FALSE, NULL );\r
+       socket_info->h_event = CreateEvent( NULL, FALSE, FALSE, NULL );\r
 \r
        /* Connects the QP. */\r
        ret = ib_connect( socket_info, &path_rec );\r
@@ -192,7 +192,7 @@ setup_duplicate_socket(
                goto done;\r
        }\r
 \r
-       if( WaitForSingleObject( socket_info->info.connect.event, INFINITE ) != WAIT_OBJECT_0 )\r
+       if( WaitForSingleObject( socket_info->h_event, INFINITE ) != WAIT_OBJECT_0 )\r
        {\r
                CL_ERROR( IBSP_DBG_CM, gdbg_lvl, ("WaitForSingleObject failed\n") );\r
        }\r
@@ -213,10 +213,10 @@ done:
        if( h )\r
                CloseHandle( h );\r
 \r
-       if( socket_info->info.connect.event )\r
+       if( socket_info->h_event )\r
        {\r
-               CloseHandle( socket_info->info.connect.event );\r
-               socket_info->info.connect.event = NULL;\r
+               CloseHandle( socket_info->h_event );\r
+               socket_info->h_event = NULL;\r
        }\r
 \r
        CL_EXIT( IBSP_DBG_CONN, gdbg_lvl );\r
index 8872081..38103a3 100644 (file)
@@ -37,9 +37,8 @@ static void ib_destroy_cq_tinfo( struct cq_thread_info *cq_tinfo );
 \r
 typedef struct _io_comp_info\r
 {\r
-       SOCKET                                  socket;\r
+       struct ibsp_socket_info *p_socket;\r
        LPWSAOVERLAPPED                 p_ov;\r
-       atomic32_t                              *p_io_cnt;\r
 \r
 } io_comp_info_t;\r
 \r
@@ -63,7 +62,7 @@ complete_wq(
        CL_ASSERT( wr );\r
 \r
        socket_info = wr->socket_info;\r
-       p_io_info->socket = socket_info->switch_socket;\r
+       p_io_info->p_socket = socket_info;\r
 \r
        lpOverlapped = wr->lpOverlapped;\r
 \r
@@ -126,7 +125,7 @@ complete_wq(
 \r
                        cl_atomic_inc( &socket_info->dup_cnt );\r
                        /* ib_cq_comp will decrement the receive count. */\r
-                       p_io_info->p_io_cnt = &socket_info->recv_cnt;\r
+                       cl_atomic_dec( &socket_info->recv_cnt );\r
 \r
                        cl_spinlock_release( &socket_info->recv_lock );\r
 \r
@@ -280,12 +279,13 @@ complete_wq(
                        lpOverlapped->InternalHigh, lpOverlapped->hEvent) );\r
 \r
                p_io_info->p_ov = lpOverlapped;\r
+               cl_atomic_inc( &socket_info->ref_cnt );\r
        }\r
 \r
        if( wc->wc_type == IB_WC_RECV )\r
-               p_io_info->p_io_cnt = &socket_info->recv_cnt;\r
+               cl_atomic_dec( &socket_info->recv_cnt );\r
        else\r
-               p_io_info->p_io_cnt = &socket_info->send_cnt;\r
+               cl_atomic_dec( &socket_info->send_cnt );\r
 \r
        IBSP_EXIT( IBSP_DBG_IO );\r
 }\r
@@ -366,25 +366,28 @@ ib_cq_comp(
                        done_wclist = done_wclist->p_next;\r
                }\r
 \r
-               while( cb_idx-- )\r
+               for( i = 0; i < cb_idx; i++ )\r
                {\r
                        int error;\r
                        int ret;\r
 \r
-                       if( info[cb_idx].p_ov )\r
+                       if( info[i].p_ov )\r
                        {\r
+                               IBSP_TRACE1( IBSP_DBG_IO,\r
+                                       ("Calling WPUCompleteOverlappedRequest for ov=%p\n",\r
+                                       info[i].p_ov) );\r
+\r
                                ret = g_ibsp.up_call_table.lpWPUCompleteOverlappedRequest(\r
-                                       info[cb_idx].socket, info[cb_idx].p_ov,\r
-                                       info[cb_idx].p_ov->OffsetHigh,\r
-                                       (DWORD)info[cb_idx].p_ov->InternalHigh, &error );\r
+                                       info[i].p_socket->switch_socket, info[i].p_ov,\r
+                                       info[i].p_ov->OffsetHigh,\r
+                                       (DWORD)info[i].p_ov->InternalHigh, &error );\r
                                if( ret != 0 )\r
                                {\r
                                        IBSP_ERROR( ("WPUCompleteOverlappedRequest for ov=%p "\r
-                                               "returned %d err %d\n", info[cb_idx].p_ov, ret, error) );\r
+                                               "returned %d err %d\n", info[i].p_ov, ret, error) );\r
                                }\r
+                               deref_socket_info( info[i].p_socket );\r
                        }\r
-\r
-                       cl_atomic_dec( info[cb_idx].p_io_cnt );\r
                }\r
 \r
 #ifdef _DEBUG_\r
@@ -762,38 +765,6 @@ ib_release(void)
 \r
                unregister_pnp();\r
 \r
-               if( g_ibsp.ib_cleanup_thread )\r
-               {\r
-                       /* Let thread know it's okay to exit after resources are freed */\r
-                       g_ibsp.ib_cleanup_thread_exit_wanted = TRUE;\r
-                       SetEvent( g_ibsp.ib_cleanup_event );\r
-\r
-                       fzprint(("%s():%d:0x%x:0x%x: Waiting for ib_cleanup_thread to die.\n",\r
-                                        __FUNCTION__, __LINE__, GetCurrentProcessId(),\r
-                                        GetCurrentThreadId()));\r
-\r
-                       /* Wait for ib_cleanup_thread to die */\r
-                       if( WaitForSingleObject( g_ibsp.ib_cleanup_thread, INFINITE ) != WAIT_OBJECT_0 )\r
-                       {\r
-                               CL_ERROR( IBSP_DBG_CM, gdbg_lvl, ("WaitForSingleObject failed\n") );\r
-                       }\r
-                       else\r
-                       {\r
-                               STAT_DEC( thread_num );\r
-                       }\r
-\r
-                       fzprint(("%s():%d:0x%x:0x%x: ib_cleanup_thread exited.\n", __FUNCTION__,\r
-                                        __LINE__, GetCurrentProcessId(), GetCurrentThreadId()));\r
-                       CloseHandle( g_ibsp.ib_cleanup_thread );\r
-                       g_ibsp.ib_cleanup_thread = NULL;\r
-               }\r
-\r
-               if( g_ibsp.ib_cleanup_event )\r
-               {\r
-                       CloseHandle( g_ibsp.ib_cleanup_event );\r
-                       g_ibsp.ib_cleanup_event = NULL;\r
-               }\r
-\r
                while( (item = cl_qlist_head( &g_ibsp.hca_list )) != cl_qlist_end( &g_ibsp.hca_list ) )\r
                {\r
                        struct ibsp_hca *hca = PARENT_STRUCT(item, struct ibsp_hca, item);\r
@@ -841,134 +812,6 @@ ib_release(void)
 }\r
 \r
 \r
-/* IP notify thread */\r
-static DWORD WINAPI\r
-ib_cleanup_thread(\r
-                                       LPVOID                                          lpParameter )\r
-{\r
-       cl_list_item_t *socket_item = NULL;\r
-\r
-       CL_ENTER( IBSP_DBG_HW, gdbg_lvl );\r
-\r
-       UNUSED_PARAM( lpParameter );\r
-\r
-       while( !g_ibsp.ib_cleanup_thread_exit_wanted ||\r
-               cl_qlist_count( &g_ibsp.socket_info_list ) )\r
-       {\r
-               if( g_ibsp.ib_cleanup_thread_exit_wanted == FALSE )\r
-               {\r
-                       if( WaitForSingleObject( g_ibsp.ib_cleanup_event, INFINITE ) != WAIT_OBJECT_0 )\r
-                       {\r
-                               CL_ERROR( IBSP_DBG_CM, gdbg_lvl, ("WaitForSingleObject failed\n") );\r
-                       }\r
-                       ResetEvent( g_ibsp.ib_cleanup_event );\r
-               }\r
-               else\r
-               {\r
-                       fzprint(("%s():%d:0x%x:0x%x: socket_info_list cnt=%d\n", __FUNCTION__,\r
-                                               __LINE__, GetCurrentProcessId(),\r
-                                               GetCurrentThreadId(),\r
-                                               cl_qlist_count( &g_ibsp.socket_info_list) == 0));\r
-                       Sleep( 1000 );\r
-               }\r
-\r
-               CL_TRACE( IBSP_DBG_WQ, gdbg_lvl, ("%s():%d:0x%x:0x%x: Wakeup\n",\r
-                                                                                __FUNCTION__,\r
-                                                                                __LINE__, GetCurrentProcessId(),\r
-                                                                                GetCurrentThreadId()));\r
-\r
-               cl_spinlock_acquire( &g_ibsp.closed_socket_info_mutex );\r
-               while( (socket_item = cl_qlist_remove_head( &g_ibsp.closed_socket_info_list )) !=\r
-                       cl_qlist_end( &g_ibsp.closed_socket_info_list ) )\r
-               {\r
-                       struct ibsp_socket_info *socket_info = NULL;\r
-\r
-                       cl_spinlock_release( &g_ibsp.closed_socket_info_mutex );\r
-\r
-                       socket_info = PARENT_STRUCT(socket_item, struct ibsp_socket_info, item);\r
-\r
-#ifdef _DEBUG_\r
-                       {\r
-                               uint8_t                 idx, i;\r
-                               LPOVERLAPPED    lpOverlapped;\r
-\r
-                               idx = socket_info->send_idx - (uint8_t)socket_info->send_cnt;\r
-                               if( idx >= QP_ATTRIB_SQ_DEPTH )\r
-                                       idx += QP_ATTRIB_SQ_DEPTH;\r
-\r
-                               for( i = 0; i < socket_info->send_cnt; i++ )\r
-                               {\r
-                                       lpOverlapped = socket_info->send_wr[idx].lpOverlapped;\r
-                                       fzprint(("%s():%d:0x%x:0x%x: socket=0x%p wr=0x%p overlapped=0x%p Internal=%d InternalHigh=%d hEvent=%d\n",\r
-                                               __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), socket_info, &socket_info->send_wr[idx], lpOverlapped, lpOverlapped->Internal, lpOverlapped->InternalHigh, lpOverlapped->hEvent));\r
-\r
-                                       if( ++idx == QP_ATTRIB_SQ_DEPTH )\r
-                                               idx = 0;\r
-                               }\r
-\r
-                               idx = socket_info->recv_idx - (uint8_t)socket_info->recv_cnt;\r
-                               if( idx >= QP_ATTRIB_RQ_DEPTH )\r
-                                       idx += QP_ATTRIB_RQ_DEPTH;\r
-\r
-                               for( i = 0; i < socket_info->recv_cnt; i++ )\r
-                               {\r
-                                       lpOverlapped = socket_info->recv_wr[idx].wr.lpOverlapped;\r
-                                       fzprint(("%s():%d:0x%x:0x%x: socket=0x%p wr=0x%p overlapped=0x%p Internal=%d InternalHigh=%d hEvent=%d\n",\r
-                                               __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), socket_info, &socket_info->recv_wr[idx], lpOverlapped, lpOverlapped->Internal, lpOverlapped->InternalHigh, lpOverlapped->hEvent));\r
-\r
-                                       if( ++idx == QP_ATTRIB_RQ_DEPTH )\r
-                                               idx = 0;\r
-                               }\r
-                       }\r
-#endif\r
-                       fzprint(("%s():%d:0x%x:0x%x: socket=0x%p\n",\r
-                                        __FUNCTION__,\r
-                                        __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), socket_info));\r
-\r
-                       wait_cq_drain( socket_info );\r
-\r
-                       if( socket_info->dup_cnt )\r
-                               ibsp_dup_overlap_abort( socket_info );\r
-\r
-                       /* Destroy the switch socket. */\r
-                       if( socket_info->switch_socket != INVALID_SOCKET )\r
-                       {\r
-                               int ret;\r
-                               int error;\r
-\r
-                               fzprint(("%s():%d:0x%x:0x%x: socket=0x%p calling lpWPUCloseSocketHandle=0x%p\n", __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), socket_info, socket_info->switch_socket));\r
-\r
-                               ret = g_ibsp.up_call_table.lpWPUCloseSocketHandle(\r
-                                       socket_info->switch_socket, &error );\r
-                               if( ret == SOCKET_ERROR )\r
-                               {\r
-                                       CL_ERROR( IBSP_DBG_EP, gdbg_lvl,\r
-                                               ("WPUCloseSocketHandle failed: %d\n", error) );\r
-                               }\r
-                               else\r
-                               {\r
-                                       STAT_DEC( wpusocket_num );\r
-                               }\r
-\r
-                               socket_info->switch_socket = INVALID_SOCKET;\r
-                       }\r
-\r
-                       ib_destroy_socket( socket_info );\r
-\r
-                       ib_deregister_all_mr( &socket_info->buf_mem_list );\r
-                       free_socket_info( socket_info );\r
-                       cl_spinlock_acquire( &g_ibsp.closed_socket_info_mutex );\r
-               }\r
-               cl_spinlock_release( &g_ibsp.closed_socket_info_mutex );\r
-       }\r
-\r
-       /* No special exit code, even on errors. */\r
-       CL_EXIT( IBSP_DBG_HW, gdbg_lvl );\r
-       ExitThread( 0 );\r
-}\r
-\r
-\r
-\r
 /* Initialize IB ressources. */\r
 int\r
 ibsp_initialize(void)\r
@@ -1007,25 +850,6 @@ ibsp_initialize(void)
        /* Populate IP list. */\r
        update_all_ip_addrs();\r
 \r
-       /* Create a cleanup event */\r
-       g_ibsp.ib_cleanup_event = CreateEvent( NULL, TRUE, FALSE, NULL );\r
-       if( g_ibsp.ib_cleanup_event == NULL )\r
-       {\r
-               CL_ERROR( IBSP_DBG_HW, gdbg_lvl, ("CreateEvent failed."));\r
-               ret = WSAEPROVIDERFAILEDINIT;\r
-               goto done;\r
-       }\r
-\r
-       /* Create a cleanup thread */\r
-       g_ibsp.ib_cleanup_thread = CreateThread( NULL, 0, ib_cleanup_thread, NULL, 0, NULL );\r
-\r
-       if( g_ibsp.ib_cleanup_thread == NULL )\r
-       {\r
-               CL_ERROR( IBSP_DBG_HW, gdbg_lvl, ("CreateThread failed.") );\r
-               ret = WSAEPROVIDERFAILEDINIT;\r
-               goto done;\r
-       }\r
-\r
        STAT_INC( thread_num );\r
 \r
        ret = 0;\r
@@ -1271,18 +1095,24 @@ ibsp_dup_overlap_abort(
 /* Closes a connection and release its ressources. */\r
 void\r
 shutdown_and_destroy_socket_info(\r
-       IN      OUT                     struct ibsp_socket_info         *socket_info,\r
-       IN                              int                                                     old_state )\r
+       IN      OUT                     struct ibsp_socket_info         *socket_info )\r
 {\r
+       enum ibsp_socket_state  old_state;\r
+\r
        CL_ENTER( IBSP_DBG_EP, gdbg_lvl );\r
 \r
+       cl_spinlock_acquire( &socket_info->mutex );\r
+       old_state = socket_info->socket_state;\r
+       IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_CLOSED );\r
+       cl_spinlock_release( &socket_info->mutex );\r
+\r
        if( socket_info->duplicate.mmap_handle )\r
        {\r
                CloseHandle( socket_info->duplicate.mmap_handle );\r
                socket_info->duplicate.mmap_handle = NULL;\r
        }\r
 \r
-       if( socket_info->info.listen.handle )\r
+       if( socket_info->listen.handle )\r
        {\r
                /* Stop listening and reject queued connections. */\r
                ib_listen_cancel( socket_info );\r
@@ -1290,13 +1120,6 @@ shutdown_and_destroy_socket_info(
 \r
        switch( old_state )\r
        {\r
-       case IBSP_CLOSING:\r
-               /* This function has already been called. Should not happen. */\r
-               CL_ERROR( IBSP_DBG_EP, gdbg_lvl,\r
-                                ("shutdown_and_destroy_socket_info already in closing socket_state\n") );\r
-               return;\r
-               break;\r
-\r
        case IBSP_CREATE:\r
                /* Nothing to do. */\r
                break;\r
@@ -1330,6 +1153,20 @@ shutdown_and_destroy_socket_info(
                break;\r
        }\r
 \r
+       /* Flush all completions. */\r
+       if( socket_info->dup_cnt )\r
+               ibsp_dup_overlap_abort( socket_info );\r
+\r
+       while( socket_info->send_cnt || socket_info->recv_cnt )\r
+               ib_cq_comp( socket_info->cq_tinfo );\r
+\r
+       ib_deregister_all_mr( &socket_info->buf_mem_list );\r
+\r
+       ib_destroy_socket( socket_info );\r
+\r
+       /* Release the initial reference and clean up. */\r
+       deref_socket_info( socket_info );\r
+\r
        CL_EXIT( IBSP_DBG_EP, gdbg_lvl );\r
 }\r
 \r
index 1d6c9bd..8d65528 100644 (file)
@@ -277,7 +277,7 @@ IBSPAccept(
                return INVALID_SOCKET;\r
        }\r
 \r
-       if( cl_qlist_count( &socket_info->info.listen.list ) == 0 )\r
+       if( cl_qlist_count( &socket_info->listen.list ) == 0 )\r
        {\r
                cl_spinlock_release( &socket_info->mutex );\r
 \r
@@ -289,7 +289,7 @@ IBSPAccept(
        CL_TRACE( IBSP_DBG_CONN, gdbg_lvl,\r
                ("IBSPAccept: Found pending connection on this socket\n") );\r
 \r
-       incoming = PARENT_STRUCT(cl_qlist_head( &socket_info->info.listen.list ),\r
+       incoming = PARENT_STRUCT(cl_qlist_head( &socket_info->listen.list ),\r
                                                         struct listen_incoming, item);\r
        port = socket_info->port;\r
 \r
@@ -318,7 +318,7 @@ IBSPAccept(
 \r
 reject:\r
                /* The request is invalid. Remove it from the list and reject it. */\r
-               cl_qlist_remove_item( &socket_info->info.listen.list, &incoming->item );\r
+               cl_qlist_remove_item( &socket_info->listen.list, &incoming->item );\r
                cl_spinlock_release( &socket_info->mutex );\r
 \r
                ib_reject( incoming->cm_req_received.h_cm_req, IB_REJ_INSUF_QP );\r
@@ -357,7 +357,7 @@ reject:
        switch( ret )\r
        {\r
        case CF_REJECT:\r
-               cl_qlist_remove_item( &socket_info->info.listen.list, &incoming->item );\r
+               cl_qlist_remove_item( &socket_info->listen.list, &incoming->item );\r
                cl_spinlock_release( &socket_info->mutex );\r
 \r
                IBSP_TRACE1( IBSP_DBG_CONN,\r
@@ -440,9 +440,9 @@ reject:
        /* Store the IP address and port number in the socket context */\r
        new_socket_info->local_addr = incoming->params.dest;\r
 \r
-       cl_qlist_remove_item( &socket_info->info.listen.list, &incoming->item );\r
+       cl_qlist_remove_item( &socket_info->listen.list, &incoming->item );\r
        /* Signal the event again if there are more connection requests. */\r
-       if( cl_qlist_count( &socket_info->info.listen.list ) )\r
+       if( cl_qlist_count( &socket_info->listen.list ) )\r
                ibsp_post_select_event( socket_info, FD_ACCEPT, 0 );\r
 \r
        cl_spinlock_release( &socket_info->mutex );\r
@@ -460,7 +460,7 @@ reject:
        new_socket_info->peer_addr = incoming->params.source;\r
        new_socket_info->switch_socket = new_socket;\r
 \r
-       new_socket_info->info.accept.event = CreateEvent( NULL, FALSE, FALSE, NULL );\r
+       new_socket_info->h_event = CreateEvent( NULL, FALSE, FALSE, NULL );\r
 \r
        cl_spinlock_acquire( &new_socket_info->mutex );\r
        /* Update the state of the socket context */\r
@@ -486,7 +486,7 @@ reject:
                        STAT_DEC( wpusocket_num );\r
                }\r
 \r
-               CloseHandle( new_socket_info->info.accept.event );\r
+               CloseHandle( new_socket_info->h_event );\r
 \r
                ib_destroy_socket( new_socket_info );\r
 \r
@@ -507,9 +507,10 @@ reject:
                cl_spinlock_release( &new_socket_info->mutex );\r
                HeapFree( g_ibsp.heap, 0, incoming );\r
 \r
-               if( WaitForSingleObject( new_socket_info->info.accept.event, INFINITE ) == WAIT_OBJECT_0 )\r
+               if( WaitForSingleObject( new_socket_info->h_event, INFINITE ) == WAIT_OBJECT_0 )\r
                {\r
-                       CloseHandle( new_socket_info->info.accept.event );\r
+                       CloseHandle( new_socket_info->h_event );\r
+                       new_socket_info->h_event = NULL;\r
 \r
                        cl_spinlock_acquire( &new_socket_info->mutex );\r
 \r
@@ -560,7 +561,8 @@ reject:
                }\r
                else\r
                {\r
-                       CloseHandle( new_socket_info->info.accept.event );\r
+                       CloseHandle( new_socket_info->h_event );\r
+                       new_socket_info->h_event = NULL;\r
 \r
                        CL_EXIT_ERROR( IBSP_DBG_CONN, gdbg_lvl, ("wait for ib_accept failed\n") );\r
 \r
@@ -697,7 +699,6 @@ IBSPCloseSocket(
                                        LPINT                                           lpErrno )\r
 {\r
        struct ibsp_socket_info *socket_info = (struct ibsp_socket_info *)s;\r
-       int old_state;\r
 \r
        CL_ENTER( IBSP_DBG_CONN, gdbg_lvl );\r
 \r
@@ -714,30 +715,12 @@ IBSPCloseSocket(
        cl_atomic_inc( &g_ibsp.CloseSocket_count );\r
 #endif\r
 \r
-       cl_spinlock_acquire( &socket_info->mutex );\r
-\r
-       old_state = socket_info->socket_state;\r
-       IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_CLOSING );\r
-\r
-       cl_spinlock_release( &socket_info->mutex );\r
-\r
-       shutdown_and_destroy_socket_info( socket_info, old_state );\r
-\r
-       cl_spinlock_acquire( &socket_info->mutex );\r
-       IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_CLOSED );\r
-       cl_spinlock_release( &socket_info->mutex );\r
-\r
-       /* Take off socket_info_list and put on closed_socket_info_list */\r
        cl_spinlock_acquire( &g_ibsp.socket_info_mutex );\r
        cl_qlist_remove_item( &g_ibsp.socket_info_list, &socket_info->item );\r
        cl_spinlock_release( &g_ibsp.socket_info_mutex );\r
 \r
-       cl_spinlock_acquire( &g_ibsp.closed_socket_info_mutex );\r
-       cl_qlist_insert_tail( &g_ibsp.closed_socket_info_list, &socket_info->item );\r
-       cl_spinlock_release( &g_ibsp.closed_socket_info_mutex );\r
+       shutdown_and_destroy_socket_info( socket_info );\r
 \r
-       /* Notify ib_cleanup_thread() to free this */\r
-       SetEvent( g_ibsp.ib_cleanup_event );\r
 \r
        CL_EXIT( IBSP_DBG_CONN, gdbg_lvl );\r
 \r
@@ -1500,11 +1483,11 @@ IBSPListen(
        case IBSP_BIND:\r
 \r
                /* Store the backlog value in the context */\r
-               socket_info->info.listen.backlog = backlog;\r
+               socket_info->listen.backlog = backlog;\r
                IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_LISTEN );\r
 \r
-               socket_info->info.listen.listen_req_param.dwProcessId = 0;\r
-               socket_info->info.listen.listen_req_param.identifier = 0;\r
+               socket_info->listen.listen_req_param.dwProcessId = 0;\r
+               socket_info->listen.listen_req_param.identifier = 0;\r
 \r
                ret = ib_listen( socket_info, lpErrno );\r
                if( ret )\r
index 67b7999..9a5f867 100644 (file)
@@ -37,6 +37,10 @@ void
 free_socket_info(\r
        struct ibsp_socket_info *socket_info );\r
 \r
+void\r
+deref_socket_info(\r
+       IN                              struct ibsp_socket_info         *p_socket );\r
+\r
 /* protos from extension.c */\r
 HANDLE WSPAPI\r
 IBSPRegisterMemory(\r
@@ -122,8 +126,7 @@ ib_destroy_socket(
 \r
 void\r
 shutdown_and_destroy_socket_info(\r
-       IN      OUT                     struct ibsp_socket_info         *socket_info,\r
-       IN                              int                                                     old_state );\r
+       IN      OUT                     struct ibsp_socket_info         *socket_info );\r
 \r
 void\r
 destroy_qp_and_release_ressources(\r
index 5cde26f..4588174 100644 (file)
@@ -48,7 +48,6 @@ enum ibsp_socket_state
        IBSP_DUPLICATING_NEW,           /* duplicating socket on the new controlling process */\r
        IBSP_DUPLICATING_REMOTE,        /* duplicating socket on the remote side */\r
        IBSP_DISCONNECTED,\r
-       IBSP_CLOSING,\r
        IBSP_CLOSED,\r
        IBSP_NUM_STATES\r
 };\r
@@ -230,36 +229,17 @@ struct ibsp_socket_info
 \r
        enum ibsp_socket_state socket_state;    /* represents current socket state */\r
 \r
-       union\r
+       struct\r
        {\r
-               struct\r
-               {\r
-                       /* Listening socket */\r
-                       unsigned int backlog;   /* Maximum number of pending connections */\r
-                       cl_qlist_t list;        /* list of pending connections */\r
-                       ib_listen_handle_t handle;\r
-                       struct listen_req_param listen_req_param;\r
-               } listen;\r
-\r
-               struct\r
-               {\r
-                       /* Accepting socket */\r
-                       /* TODO: ib_cm_rep will be blocking with some flags. \r
-                        * Use that instead of this event. */\r
-                       HANDLE event;\r
-               } accept;\r
-\r
-               struct\r
-               {\r
-                       HANDLE event;\r
-               } connect;\r
-\r
-               struct\r
-               {\r
-                       HANDLE event;\r
-               } disconnect;\r
-\r
-       }       info;\r
+               /* Listening socket */\r
+               unsigned int backlog;   /* Maximum number of pending connections */\r
+               cl_qlist_t list;        /* list of pending connections */\r
+               ib_listen_handle_t handle;\r
+               struct listen_req_param listen_req_param;\r
+       } listen;\r
+\r
+       /* Event for blocking accept, connect, and disconnect */\r
+       HANDLE h_event;\r
 \r
        /* Variables associated with IBSPSelectEvent */\r
        WSAEVENT event_select;          /* Handle to Event Object */\r
@@ -290,13 +270,19 @@ struct ibsp_socket_info
        ib_send_opt_t   send_opt;\r
        struct _wr              send_wr[QP_ATTRIB_SQ_DEPTH];\r
        uint8_t                 send_idx;\r
-       atomic32_t              send_cnt;\r
+       atomic32_t              send_cnt; /* Used to limit access to send_wr array. */\r
 \r
        /* Receive request processing. */\r
        cl_spinlock_t   recv_lock;\r
        struct _recv_wr recv_wr[QP_ATTRIB_RQ_DEPTH];\r
        uint8_t                 recv_idx;\r
-       atomic32_t              recv_cnt;\r
+       atomic32_t              recv_cnt; /* Used to limit access to recv_wr array. */\r
+\r
+       /*\r
+        * Used to stall destruction of switch socket until all completion\r
+        * upcalls have unwound.\r
+        */\r
+       atomic32_t              ref_cnt;\r
 \r
 #ifdef _DEBUG_\r
        atomic32_t              send_comp;\r
@@ -353,16 +339,6 @@ ibsp_css(
                return;\r
        }\r
 \r
-       /* \r
-        * Once in IBSP_CLOSING state, the only allowed new state\r
-        * is IBSP_CLOSING.\r
-        */\r
-       if( (old_state == IBSP_CLOSING) && (new_state != IBSP_CLOSED) )\r
-       {\r
-               fzprint(("%s():0x%x:0x%x: socket=0x%p cannot change from %s to %s, called by %s():%d\n", __FUNCTION__, GetCurrentProcessId(), GetCurrentThreadId(), s, IBSP_SOCKET_STATE_STR(old_state), IBSP_SOCKET_STATE_STR(new_state), calling_func, line));\r
-               return;\r
-       }\r
-\r
        fzprint(("%s():0x%x:0x%x: socket=0x%p %s to %s, called by %s():%d\n", __FUNCTION__,\r
                         GetCurrentProcessId(),\r
                         GetCurrentThreadId(), s,\r
@@ -434,9 +410,6 @@ struct ibspdll_globals
        cl_rbmap_t conn_map;    /* rb tree of all connections to ensure unique 4-tuple */\r
        cl_spinlock_t socket_info_mutex;\r
 \r
-       cl_qlist_t closed_socket_info_list;     /* List of sockets in IBSP_CLOSED state */\r
-       cl_spinlock_t closed_socket_info_mutex;\r
-\r
        WSAPROTOCOL_INFOW protocol_info;\r
 \r
        /* Infiniband */\r
@@ -450,11 +423,6 @@ struct ibspdll_globals
        cl_fmap_t               ip_map;                 /* list of all IP addresses supported by all the ports. */\r
        cl_spinlock_t   ip_mutex;\r
 \r
-       /* IB resource cleanup thread */\r
-       HANDLE ib_cleanup_thread;\r
-       HANDLE ib_cleanup_event;\r
-       BOOL ib_cleanup_thread_exit_wanted;\r
-\r
 #ifdef _DEBUG_\r
        /* Statistics */\r
        atomic32_t qp_num;\r
index 2ea7ac1..0430a9f 100644 (file)
@@ -42,7 +42,6 @@ char *ibsp_socket_state_str[IBSP_NUM_STATES] = {
        "IBSP_DUPLICATING_NEW",\r
        "IBSP_DUPLICATING_REMOTE",\r
        "IBSP_DISCONNECTED",\r
-       "IBSP_CLOSING",\r
        "IBSP_CLOSED"\r
 };\r
 \r
@@ -107,9 +106,6 @@ init_globals( void )
        cl_rbmap_init( &g_ibsp.conn_map );\r
        cl_spinlock_init( &g_ibsp.socket_info_mutex );\r
 \r
-       cl_qlist_init( &g_ibsp.closed_socket_info_list );\r
-       cl_spinlock_init( &g_ibsp.closed_socket_info_mutex );\r
-\r
        return 0;\r
 }\r
 \r
@@ -122,7 +118,6 @@ release_globals( void )
        g_ibsp.heap = NULL;\r
 \r
        cl_spinlock_destroy( &g_ibsp.socket_info_mutex );\r
-       cl_spinlock_destroy( &g_ibsp.closed_socket_info_mutex );\r
        cl_spinlock_destroy( &g_ibsp.hca_mutex );\r
        cl_spinlock_destroy( &g_ibsp.ip_mutex );\r
        cl_spinlock_destroy( &g_ibsp.mutex );\r
index 2926235..bdad2f7 100644 (file)
@@ -59,7 +59,7 @@ create_socket_info(void)
        cl_qlist_init( &socket_info->buf_mem_list.list );\r
        cl_spinlock_init( &socket_info->buf_mem_list.mutex );\r
 \r
-       cl_qlist_init( &socket_info->info.listen.list );\r
+       cl_qlist_init( &socket_info->listen.list );\r
 \r
        cl_spinlock_init( &socket_info->send_lock );\r
        cl_spinlock_init( &socket_info->recv_lock );\r
@@ -71,6 +71,11 @@ create_socket_info(void)
 #endif\r
 \r
        socket_info->switch_socket = INVALID_SOCKET;\r
+       /*\r
+        * Preset to 1, IBSPCloseSocket will decrement it, and switch socket\r
+        * will be freed once it goes to zero after an upcall.\r
+        */\r
+       socket_info->ref_cnt = 1;\r
 \r
        IBSP_TRACE( IBSP_DBG_SI, ("socket_info (%p)\n", socket_info) );\r
 \r
@@ -105,3 +110,44 @@ free_socket_info(
 \r
        IBSP_EXIT( IBSP_DBG_SI );\r
 }\r
+\r
+\r
+void\r
+deref_socket_info(\r
+       IN                              struct ibsp_socket_info         *p_socket )\r
+{\r
+       int             ret, error;\r
+\r
+       if( !cl_atomic_dec( &p_socket->ref_cnt ) )\r
+       {\r
+               if( p_socket->switch_socket != INVALID_SOCKET )\r
+               {\r
+                       /* ref_cnt hit zero - destroy the switch socket. */\r
+                       IBSP_TRACE1( IBSP_DBG_SI,\r
+                               ("socket=0x%p calling lpWPUCloseSocketHandle=0x%p\n",\r
+                               p_socket, p_socket->switch_socket) );\r
+\r
+                       ret = g_ibsp.up_call_table.lpWPUCloseSocketHandle(\r
+                               p_socket->switch_socket, &error );\r
+                       if( ret == SOCKET_ERROR )\r
+                       {\r
+                               IBSP_ERROR( ("WPUCloseSocketHandle failed: %d\n", error) );\r
+                       }\r
+                       else\r
+                       {\r
+                               STAT_DEC( wpusocket_num );\r
+                       }\r
+\r
+                       p_socket->switch_socket = INVALID_SOCKET;\r
+               }\r
+\r
+               /*\r
+                * This only works because the socket doesn't have its own CQ.  If it\r
+                * did CQ destruction would have to be async since IBSPCloseSocket can\r
+                * be invoked from the CQ callback thread context, and freeing the\r
+                * socket structure would need to be delayed until the CQ's destroy\r
+                * callback.\r
+                */\r
+               free_socket_info( p_socket );\r
+       }\r
+}\r