[WSD] Fix race between CQ thread info cleanup and thread exit.
[mirror/winof/.git] / ulp / wsd / user / ibsp_iblow.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  *\r
4  * This software is available to you under the OpenIB.org BSD license\r
5  * below:\r
6  *\r
7  *     Redistribution and use in source and binary forms, with or\r
8  *     without modification, are permitted provided that the following\r
9  *     conditions are met:\r
10  *\r
11  *      - Redistributions of source code must retain the above\r
12  *        copyright notice, this list of conditions and the following\r
13  *        disclaimer.\r
14  *\r
15  *      - Redistributions in binary form must reproduce the above\r
16  *        copyright notice, this list of conditions and the following\r
17  *        disclaimer in the documentation and/or other materials\r
18  *        provided with the distribution.\r
19  *\r
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
27  * SOFTWARE.\r
28  *\r
29  * $Id$\r
30  */\r
31 \r
32 #include "ibspdll.h"\r
33 \r
34 #ifdef PERFMON_ENABLED\r
35 #include "ibsp_perfmon.h"\r
36 #endif\r
37 \r
38 \r
39 typedef struct _io_comp_info\r
40 {\r
41         struct ibsp_socket_info *p_socket;\r
42         LPWSAOVERLAPPED                 p_ov;\r
43 \r
44 } io_comp_info_t;\r
45 \r
46 \r
47 /* Work queue entry completion routine. */\r
48 static void\r
49 complete_wq(\r
50         IN              const   ib_wc_t                                         *wc,\r
51                 OUT                     io_comp_info_t                          *p_io_info )\r
52 {\r
53         struct _wr                              *wr = NULL;\r
54         struct _recv_wr                 *p_recv_wr = NULL;\r
55         LPWSAOVERLAPPED                 lpOverlapped = NULL;\r
56         struct ibsp_socket_info *socket_info = NULL;\r
57 \r
58         IBSP_ENTER( IBSP_DBG_IO );\r
59 \r
60         wr = (struct _wr * __ptr64)wc->wr_id;\r
61         p_recv_wr = (struct _recv_wr * __ptr64)wc->wr_id;\r
62 \r
63         CL_ASSERT( wr );\r
64 \r
65         socket_info = wr->socket_info;\r
66         p_io_info->p_socket = socket_info;\r
67 \r
68         lpOverlapped = wr->lpOverlapped;\r
69 \r
70         IBSP_TRACE4( IBSP_DBG_IO,\r
71                 ("socket %p, ov %p, work completion status=%s, wc_type=%s\n",\r
72                 socket_info, lpOverlapped, ib_get_wc_status_str( wc->status ),\r
73                 ib_get_wc_type_str( wc->wc_type )) );\r
74 \r
75         /* Set the windows error code. It's not easy to find an easy\r
76          * correspondence between the IBAL error codes and windows error\r
77          * codes; but it probably does not matter, as long as it returns an\r
78          * error. */\r
79         switch( wc->status )\r
80         {\r
81         case IB_WCS_SUCCESS:\r
82                 /*\r
83                  * Set the length of the operation. Under Infiniband, the work\r
84                  * completion length is only valid for a receive\r
85                  * operation. Fortunately we had already set the length during the\r
86                  * send operation. \r
87                  *\r
88                  * lpWPUCompleteOverlappedRequest is supposed to store the length\r
89                  * into InternalHigh, however it will not be called if the low\r
90                  * order bit of lpOverlapped->hEvent is set. So we do it and hope\r
91                  * for the best. \r
92                  *\r
93                  * NOTE: Without a valid length, the switch doesn't seem to call \r
94                  * GetOverlappedResult() even if we call lpWPUCompleteOverlappedRequest()\r
95                  */\r
96                 switch ( wc->wc_type ) \r
97                 {\r
98                 case IB_WC_RECV:\r
99                         lpOverlapped->InternalHigh = wc->length;\r
100 #ifdef IBSP_LOGGING\r
101                         cl_spinlock_acquire( &socket_info->recv_lock );\r
102                         DataLogger_WriteData(&socket_info->RecvDataLogger,\r
103                                 p_recv_wr->idx, (void * __ptr64)p_recv_wr->ds_array[0].vaddr,\r
104                                 wc->length);\r
105                         cl_spinlock_release( &socket_info->recv_lock );\r
106 #endif\r
107 #ifdef PERFMON_ENABLED\r
108                         InterlockedIncrement64( &g_pm_stat.pdata[COMP_RECV] );\r
109                         InterlockedExchangeAdd64( &g_pm_stat.pdata[BYTES_RECV],\r
110                                 lpOverlapped->InternalHigh );\r
111 #endif\r
112 #ifdef _DEBUG_\r
113                         cl_atomic_inc(&g_ibsp.total_recv_compleated);\r
114 #endif\r
115                         break;\r
116 #ifdef PERFMON_ENABLED\r
117 \r
118                 case IB_WC_RDMA_READ:\r
119                         lpOverlapped->InternalHigh = wc->length;\r
120                         InterlockedIncrement64( &g_pm_stat.pdata[COMP_RECV] );\r
121                         InterlockedExchangeAdd64( &g_pm_stat.pdata[BYTES_READ],\r
122                                 lpOverlapped->InternalHigh );\r
123                         break;\r
124 \r
125                 case IB_WC_SEND:\r
126                         InterlockedIncrement64( &g_pm_stat.pdata[COMP_SEND] );\r
127                         InterlockedExchangeAdd64( &g_pm_stat.pdata[BYTES_SEND],\r
128                                 lpOverlapped->InternalHigh );\r
129                         break;\r
130 \r
131                 case IB_WC_RDMA_WRITE:\r
132                         InterlockedIncrement64( &g_pm_stat.pdata[COMP_SEND] );\r
133                         InterlockedExchangeAdd64( &g_pm_stat.pdata[BYTES_WRITE],\r
134                                 lpOverlapped->InternalHigh );\r
135                 default:\r
136                         break;\r
137                 }\r
138 \r
139 #endif /* PERFMON_ENABLED */\r
140 \r
141                 lpOverlapped->OffsetHigh = 0;\r
142                 break;\r
143 \r
144         case IB_WCS_WR_FLUSHED_ERR:\r
145                 cl_spinlock_acquire( &socket_info->mutex );\r
146 \r
147                 if( socket_info->socket_state == IBSP_DUPLICATING_REMOTE &&\r
148                         wc->wc_type == IB_WC_RECV )\r
149                 {\r
150                         /*\r
151                          * Take the wr off the wr_list, and place onto the\r
152                          * dup_wr_list.  We will post them later on the new QP. \r
153                          */\r
154                         cl_spinlock_acquire( &socket_info->recv_lock );\r
155 \r
156                         /* Copy to the duplicate WR array. */\r
157                         socket_info->dup_wr[socket_info->dup_idx] = *p_recv_wr;\r
158 \r
159 #if QP_ATTRIB_RQ_DEPTH == 256 || QP_ATTRIB_RQ_DEPTH == 128 || \\r
160         QP_ATTRIB_RQ_DEPTH == 64 || QP_ATTRIB_RQ_DEPTH == 32 || \\r
161         QP_ATTRIB_RQ_DEPTH == 16 || QP_ATTRIB_RQ_DEPTH == 8\r
162                         socket_info->dup_idx++;\r
163                         socket_info->dup_idx &= (QP_ATTRIB_RQ_DEPTH - 1);\r
164 #else\r
165                         if( ++socket_info->dup_idx == QP_ATTRIB_RQ_DEPTH )\r
166                                 socket_info->dup_idx = 0;\r
167 #endif\r
168 \r
169                         cl_atomic_inc( &socket_info->dup_cnt );\r
170                         /* ib_cq_comp will decrement the receive count. */\r
171                         cl_atomic_dec( &socket_info->recv_cnt );\r
172 \r
173                         cl_spinlock_release( &socket_info->recv_lock );\r
174 \r
175                         cl_spinlock_release( &socket_info->mutex );\r
176                         p_io_info->p_ov = NULL;\r
177                         IBSP_EXIT( IBSP_DBG_IO );\r
178                         return;\r
179                 }\r
180                 \r
181                 /* Check for flushing the receive buffers on purpose. */\r
182                 if( socket_info->socket_state == IBSP_DUPLICATING_OLD )\r
183                         wr->lpOverlapped->OffsetHigh = 0;\r
184                 else\r
185                         wr->lpOverlapped->OffsetHigh = WSA_OPERATION_ABORTED;\r
186 \r
187                 cl_spinlock_release( &socket_info->mutex );\r
188 \r
189                 /* Override the length, as per the WSD specs. */\r
190                 wr->lpOverlapped->InternalHigh = 0;\r
191                 break;\r
192 \r
193         case IB_WCS_LOCAL_LEN_ERR:\r
194         case IB_WCS_LOCAL_OP_ERR:\r
195         case IB_WCS_LOCAL_PROTECTION_ERR:\r
196         case IB_WCS_MEM_WINDOW_BIND_ERR:\r
197         case IB_WCS_REM_ACCESS_ERR:\r
198         case IB_WCS_REM_OP_ERR:\r
199         case IB_WCS_RNR_RETRY_ERR:\r
200         case IB_WCS_TIMEOUT_RETRY_ERR:\r
201         case IB_WCS_REM_INVALID_REQ_ERR:\r
202         default:\r
203                 IBSP_ERROR( ("%s error: %s\n",\r
204                         ib_get_wc_type_str( wc->wc_type ),\r
205                         ib_get_wc_status_str( wc->status )) );\r
206                 lpOverlapped->OffsetHigh = WSAECONNABORTED;\r
207                 wr->lpOverlapped->InternalHigh = 0;\r
208                 socket_info->qp_error = WSAECONNABORTED;\r
209                 break;\r
210         }\r
211 \r
212 #ifdef PERFMON_ENABLED\r
213         InterlockedIncrement64( &g_pm_stat.pdata[COMP_TOTAL] );\r
214 #endif\r
215 \r
216 #ifdef _DEBUG_\r
217         if( wc->wc_type == IB_WC_RECV )\r
218         {\r
219                 // This code requires the recv count to be decremented here, but it needs\r
220                 // to be decremented after any callbacks are invoked so socket destruction\r
221                 // gets delayed until all callbacks have been invoked.\r
222                 //{\r
223                 //      uint8_t idx;\r
224 \r
225                 //      cl_spinlock_acquire( &socket_info->recv_lock );\r
226                 //      idx = socket_info->recv_idx - (uint8_t)socket_info->recv_cnt;\r
227                 //      if( idx >= QP_ATTRIB_RQ_DEPTH )\r
228                 //              idx += QP_ATTRIB_RQ_DEPTH;\r
229 \r
230                 //      CL_ASSERT( wc->wr_id == (uint64_t)(void* __ptr64)&socket_info->recv_wr[idx] );\r
231                 //      cl_atomic_dec( &socket_info->recv_cnt );\r
232                 //      cl_spinlock_release( &socket_info->recv_lock );\r
233                 //}\r
234 \r
235                 if( wc->status == IB_SUCCESS && p_recv_wr->ds_array[0].length >= 40 )\r
236                 {\r
237                         debug_dump_buffer( IBSP_DBG_WQ | IBSP_DBG_LEVEL4, "RECV",\r
238                                 (void * __ptr64)p_recv_wr->ds_array[0].vaddr, 40 );\r
239                 }\r
240 \r
241                 cl_atomic_dec( &g_ibsp.recv_count );\r
242                 cl_atomic_inc( &socket_info->recv_comp );\r
243 \r
244                 memset( p_recv_wr, 0x33, sizeof(struct _recv_wr) );\r
245         }\r
246         else\r
247         {\r
248                 // This code requires the send count to be decremented here, but it needs\r
249                 // to be decremented after any callbacks are invoked so socket destruction\r
250                 // gets delayed until all callbacks have been invoked.\r
251                 //{\r
252                 //      uint8_t idx;\r
253 \r
254                 //      cl_spinlock_acquire( &socket_info->send_lock );\r
255                 //      idx = socket_info->send_idx - (uint8_t)socket_info->send_cnt;\r
256                 //      if( idx >= QP_ATTRIB_SQ_DEPTH )\r
257                 //              idx += QP_ATTRIB_SQ_DEPTH;\r
258                 //      CL_ASSERT( wc->wr_id == (uint64_t)(void* __ptr64)&socket_info->send_wr[idx] );\r
259                 //      cl_atomic_dec( &socket_info->send_cnt );\r
260                 //      cl_spinlock_release( &socket_info->send_lock );\r
261                 //}\r
262 \r
263                 if( wc->wc_type == IB_WC_SEND )\r
264                 {\r
265                         cl_atomic_dec( &g_ibsp.send_count );\r
266                         cl_atomic_inc( &socket_info->send_comp );\r
267 \r
268                         fzprint(("%s():%d:0x%x:0x%x: send_count=%d\n",\r
269                                 __FUNCTION__,\r
270                                 __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), g_ibsp.send_count));\r
271                 }\r
272 \r
273                 memset( wr, 0x33, sizeof(struct _wr) );\r
274         }\r
275 #endif\r
276 \r
277         IBSP_TRACE4( IBSP_DBG_IO,\r
278                 ("overlapped=%p, InternalHigh=%d, hEvent=%x\n",\r
279                 lpOverlapped, lpOverlapped->InternalHigh,\r
280                 (uintptr_t) lpOverlapped->hEvent) );\r
281 \r
282         /* Don't notify the switch for that completion only if:\r
283          *   - the switch don't want a notification\r
284          *   - the wq completed with success\r
285          *   - the socket is still connected\r
286          */\r
287         if( ((uintptr_t) lpOverlapped->hEvent) & 0x00000001 )\r
288         {\r
289                 /* Indicate this operation is complete. The switch will poll\r
290                  * with calls to WSPGetOverlappedResult(). */\r
291 \r
292 #ifdef _DEBUG_\r
293                 cl_atomic_dec( &g_ibsp.overlap_h1_comp_count );\r
294 \r
295                 fzprint(("%s():%d:0x%x:0x%x: ov=0x%p h0=%d h1=%d h1_c=%d send=%d recv=%d\n",\r
296                                  __FUNCTION__, __LINE__, GetCurrentProcessId(),\r
297                                  GetCurrentThreadId(), lpOverlapped,\r
298                                  g_ibsp.overlap_h0_count, g_ibsp.overlap_h1_count,\r
299                                  g_ibsp.overlap_h1_comp_count, g_ibsp.send_count, g_ibsp.recv_count));\r
300 #endif\r
301 \r
302                 IBSP_TRACE1( IBSP_DBG_IO,\r
303                         ("Not calling lpWPUCompleteOverlappedRequest: "\r
304                         "socket=%p, ov=%p OffsetHigh=%d, InternalHigh=%d hEvent=%p\n",\r
305                         socket_info, lpOverlapped, lpOverlapped->OffsetHigh,\r
306                         lpOverlapped->InternalHigh, lpOverlapped->hEvent) );\r
307 \r
308                 lpOverlapped->Internal = 0;\r
309                 p_io_info->p_ov = NULL;\r
310         }\r
311         else\r
312         {\r
313 #ifdef _DEBUG_\r
314                 cl_atomic_dec( &g_ibsp.overlap_h0_count );\r
315 \r
316                 fzprint(("%s():%d:0x%x:0x%x: ov=0x%p h0=%d h1=%d h1_c=%d send=%d recv=%d\n",\r
317                                  __FUNCTION__, __LINE__, GetCurrentProcessId(),\r
318                                  GetCurrentThreadId(), lpOverlapped,\r
319                                  g_ibsp.overlap_h0_count, g_ibsp.overlap_h1_count,\r
320                                  g_ibsp.overlap_h1_comp_count, g_ibsp.send_count, g_ibsp.recv_count));\r
321 #endif\r
322 \r
323                 p_io_info->p_ov = lpOverlapped;\r
324                 cl_atomic_inc( &socket_info->ref_cnt );\r
325         }\r
326 \r
327         if( wc->wc_type == IB_WC_RECV )\r
328         {\r
329                 cl_atomic_dec( &socket_info->recv_cnt );\r
330         }\r
331         else\r
332         {\r
333                 cl_atomic_dec( &socket_info->send_cnt );\r
334         }\r
335 \r
336         IBSP_EXIT( IBSP_DBG_IO );\r
337 }\r
338 \r
339 \r
340 /* CQ completion handler. */\r
341 int\r
342 ib_cq_comp(\r
343                                         void                                            *cq_context )\r
344 {\r
345         struct cq_thread_info   *cq_tinfo = cq_context;\r
346         ib_api_status_t                 status;\r
347         ib_wc_t                                 wclist[WC_LIST_SIZE];\r
348         ib_wc_t                                 *free_wclist;\r
349         ib_wc_t                                 *done_wclist;\r
350         io_comp_info_t                  info[WC_LIST_SIZE];\r
351         int                                             cb_idx;\r
352         int                                             i;\r
353         int                                             n_comp = 0;\r
354 #ifdef _DEBUG_\r
355         int                                             comp_count;\r
356 #endif\r
357 \r
358         IBSP_ENTER( IBSP_DBG_WQ );\r
359 \r
360         CL_ASSERT( WC_LIST_SIZE >= 1 );\r
361 \r
362         do\r
363         {\r
364                 /* Try to retrieve up to WC_LIST_SIZE completions at a time. */\r
365                 for( i = 0; i < (WC_LIST_SIZE - 1); i++ )\r
366                 {\r
367                         wclist[i].p_next = &wclist[i + 1];\r
368                 }\r
369                 wclist[(WC_LIST_SIZE - 1)].p_next = NULL;\r
370 \r
371                 free_wclist = &wclist[0];\r
372                 done_wclist = NULL;\r
373 \r
374                 status = ib_poll_cq( cq_tinfo->cq, &free_wclist, &done_wclist );\r
375 \r
376                 IBSP_TRACE( IBSP_DBG_WQ,\r
377                         ("%s():%d:0x%x:0x%x: poll CQ got status %d, free=%p, done=%p\n",\r
378                         __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(),\r
379                         status, free_wclist, done_wclist) );\r
380 \r
381                 switch( status )\r
382                 {\r
383                 case IB_NOT_FOUND:\r
384                 case IB_SUCCESS:\r
385                         break;\r
386 \r
387                 case IB_INVALID_CQ_HANDLE:\r
388                         /* This happens when the switch closes the socket while the \r
389                          * execution thread was calling lpWPUCompleteOverlappedRequest. */\r
390                         IBSP_ERROR(\r
391                                 ("ib_poll_cq returned IB_INVLALID_CQ_HANDLE\n") );\r
392                         goto done;\r
393 \r
394                 default:\r
395                         IBSP_ERROR(\r
396                                 ("ib_poll_cq failed returned %s\n", ib_get_err_str( status )) );\r
397                         break;\r
398                 }\r
399 \r
400 #ifdef _DEBUG_\r
401                 comp_count = 0;\r
402 #endif\r
403 \r
404                 /* We have some completions. */\r
405                 cb_idx = 0;\r
406                 while( done_wclist )\r
407                 {\r
408 #ifdef _DEBUG_\r
409                         comp_count++;\r
410 #endif\r
411                         complete_wq( done_wclist, &info[cb_idx++] );\r
412 \r
413                         done_wclist = done_wclist->p_next;\r
414                 }\r
415 \r
416                 for( i = 0; i < cb_idx; i++ )\r
417                 {\r
418                         int error;\r
419                         int ret;\r
420 \r
421                         if( info[i].p_ov )\r
422                         {\r
423                                 IBSP_TRACE1( IBSP_DBG_IO,\r
424                                         ("Calling lpWPUCompleteOverlappedRequest: "\r
425                                         "socket=%p, ov=%p OffsetHigh=%d "\r
426                                         "InternalHigh=%d hEvent=%p\n",\r
427                                         info[i].p_socket, info[i].p_ov, info[i].p_ov->OffsetHigh,\r
428                                         info[i].p_ov->InternalHigh, info[i].p_ov->hEvent) );\r
429 \r
430                                 ret = g_ibsp.up_call_table.lpWPUCompleteOverlappedRequest(\r
431                                         info[i].p_socket->switch_socket, info[i].p_ov,\r
432                                         info[i].p_ov->OffsetHigh,\r
433                                         (DWORD)info[i].p_ov->InternalHigh, &error );\r
434                                 if( ret != 0 )\r
435                                 {\r
436                                         IBSP_ERROR( ("WPUCompleteOverlappedRequest for ov=%p "\r
437                                                 "returned %d err %d\n", info[i].p_ov, ret, error) );\r
438                                 }\r
439                                 deref_socket_info( info[i].p_socket );\r
440                         }\r
441                 }\r
442 \r
443                 n_comp += i;\r
444 \r
445 #ifdef _DEBUG_\r
446                 if( comp_count > g_ibsp.max_comp_count )\r
447                 {\r
448                         g_ibsp.max_comp_count = comp_count;\r
449                 }\r
450 #endif\r
451         } while( !free_wclist );\r
452 \r
453 done:\r
454 \r
455 #ifdef _DEBUG_\r
456         fzprint(("%s():%d:0x%x:0x%x: overlap_h0_count=%d overlap_h1_count=%d\n",\r
457                          __FUNCTION__,\r
458                          __LINE__, GetCurrentProcessId(),\r
459                          GetCurrentThreadId(), g_ibsp.overlap_h0_count, g_ibsp.overlap_h1_count));\r
460 #endif\r
461 \r
462         IBSP_EXIT( IBSP_DBG_WQ );\r
463         return n_comp;\r
464 }\r
465 \r
466 \r
467 /* IB completion thread */\r
468 static DWORD WINAPI\r
469 ib_cq_thread(\r
470                                         LPVOID                                          lpParameter )\r
471 {\r
472         struct cq_thread_info   *cq_tinfo = (struct cq_thread_info *)lpParameter;\r
473         cl_status_t                             cl_status;\r
474         ib_api_status_t                 status;\r
475         int                                             i;\r
476 \r
477         IBSP_ENTER( IBSP_DBG_HW );\r
478 \r
479         fzprint(("%s():%d:0x%x:0x%x: cq_tinfo=0x%p\n", __FUNCTION__,\r
480                          __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), cq_tinfo));\r
481 \r
482         do\r
483         {\r
484                 cl_status = cl_waitobj_wait_on( cq_tinfo->cq_waitobj, EVENT_NO_TIMEOUT, TRUE );\r
485                 if( cl_status != CL_SUCCESS )\r
486                 {\r
487                         IBSP_ERROR(\r
488                                 ("cl_waitobj_wait_on() (%d)\n", cl_status) );\r
489                 }\r
490 \r
491                 /* \r
492                  * TODO: By rearranging thread creation and cq creation, this check\r
493                  * may be eliminated.\r
494                  */\r
495                 if( cq_tinfo->cq != NULL )\r
496                 {\r
497                         fzprint(("%s():%d:0x%x:0x%x: Calling ib_cq_comp().\n", __FUNCTION__,\r
498                                          __LINE__, GetCurrentProcessId(), GetCurrentThreadId()));\r
499 \r
500 #ifdef PERFMON_ENABLED\r
501                         InterlockedIncrement64( &g_pm_stat.pdata[INTR_TOTAL] );\r
502 #endif\r
503                         i = g_max_poll;\r
504                         do\r
505                         {\r
506                                 if( ib_cq_comp( cq_tinfo ) )\r
507                                         i = g_max_poll;\r
508 \r
509                         } while( i-- );\r
510 \r
511                         fzprint(("%s():%d:0x%x:0x%x: Done calling ib_cq_comp().\n", __FUNCTION__,\r
512                                          __LINE__, GetCurrentProcessId(), GetCurrentThreadId()));\r
513 \r
514                         status = ib_rearm_cq( cq_tinfo->cq, FALSE );\r
515                         if( status != IB_SUCCESS )\r
516                         {\r
517                                 IBSP_ERROR(\r
518                                         ("ib_rearm_cq returned %s)\n", ib_get_err_str( status )) );\r
519                         }\r
520                 }\r
521 \r
522         } while( !cq_tinfo->ib_cq_thread_exit_wanted );\r
523 \r
524         cl_status = cl_waitobj_destroy( cq_tinfo->cq_waitobj );\r
525         if( cl_status != CL_SUCCESS )\r
526         {\r
527                 IBSP_ERROR(\r
528                         ("cl_waitobj_destroy() returned %s\n", CL_STATUS_MSG(cl_status)) );\r
529         }\r
530         HeapFree( g_ibsp.heap, 0, cq_tinfo );\r
531 \r
532         /* No special exit code, even on errors. */\r
533         IBSP_EXIT( IBSP_DBG_HW );\r
534         ExitThread( 0 );\r
535 }\r
536 \r
537 \r
538 /* Called with the HCA's CQ lock held. */\r
539 static struct cq_thread_info *\r
540 ib_alloc_cq_tinfo(\r
541                                         struct ibsp_hca                         *hca )\r
542 {\r
543         struct cq_thread_info *cq_tinfo = NULL;\r
544         ib_cq_create_t cq_create;\r
545         ib_api_status_t status;\r
546         cl_status_t cl_status;\r
547 \r
548         IBSP_ENTER( IBSP_DBG_HW );\r
549 \r
550         cq_tinfo = HeapAlloc(\r
551                 g_ibsp.heap, HEAP_ZERO_MEMORY, sizeof(struct cq_thread_info) );\r
552 \r
553         if( !cq_tinfo )\r
554         {\r
555                 IBSP_ERROR_EXIT( ("HeapAlloc() Failed.\n") );\r
556                 return NULL;\r
557         }\r
558 \r
559         cl_status = cl_waitobj_create( FALSE, &cq_tinfo->cq_waitobj );\r
560         if( cl_status != CL_SUCCESS )\r
561         {\r
562                 cq_tinfo->cq_waitobj = NULL;\r
563                 ib_destroy_cq_tinfo( cq_tinfo );\r
564                 IBSP_ERROR_EXIT(\r
565                         ("cl_waitobj_create() returned %s\n", CL_STATUS_MSG(cl_status)) );\r
566                 return NULL;\r
567         }\r
568 \r
569         cq_tinfo->hca = hca;\r
570         cq_tinfo->ib_cq_thread_exit_wanted = FALSE;\r
571 \r
572         cq_tinfo->ib_cq_thread = CreateThread( NULL, 0, ib_cq_thread, cq_tinfo, 0,\r
573                 (LPDWORD)&cq_tinfo->ib_cq_thread_id );\r
574 \r
575         if( cq_tinfo->ib_cq_thread == NULL )\r
576         {\r
577                 ib_destroy_cq_tinfo( cq_tinfo );\r
578                 IBSP_ERROR_EXIT( ("CreateThread failed (%d)", GetLastError()) );\r
579                 return NULL;\r
580         }\r
581 \r
582         STAT_INC( thread_num );\r
583 \r
584         /* Completion queue */\r
585         cq_create.size = IB_CQ_SIZE;\r
586 \r
587         cq_create.pfn_comp_cb = NULL;\r
588         cq_create.h_wait_obj = cq_tinfo->cq_waitobj;\r
589 \r
590         status = ib_create_cq( hca->hca_handle, &cq_create, cq_tinfo,\r
591                 NULL, &cq_tinfo->cq );\r
592         if( status )\r
593         {\r
594                 ib_destroy_cq_tinfo( cq_tinfo );\r
595                 IBSP_ERROR_EXIT(\r
596                         ("ib_create_cq returned %s\n", ib_get_err_str( status )) );\r
597                 return NULL;\r
598         }\r
599 \r
600         STAT_INC( cq_num );\r
601 \r
602         status = ib_rearm_cq( cq_tinfo->cq, FALSE );\r
603         if( status )\r
604         {\r
605                 ib_destroy_cq_tinfo( cq_tinfo );\r
606                 IBSP_ERROR_EXIT(\r
607                         ("ib_rearm_cq returned %s\n", ib_get_err_str( status )) );\r
608                 return NULL;\r
609         }\r
610 \r
611         cq_tinfo->cqe_size = IB_CQ_SIZE;\r
612 \r
613         if( hca->cq_tinfo )\r
614         {\r
615                 __cl_primitive_insert(\r
616                         &hca->cq_tinfo->list_item, &cq_tinfo->list_item );\r
617         }\r
618         else\r
619         {\r
620                 /* Setup the list entry to point to itself. */\r
621                 cq_tinfo->list_item.p_next = &cq_tinfo->list_item;\r
622                 cq_tinfo->list_item.p_prev = &cq_tinfo->list_item;\r
623         }\r
624 \r
625         /* Upon allocation, the new CQ becomes the primary. */\r
626         hca->cq_tinfo = cq_tinfo;\r
627 \r
628         IBSP_EXIT( IBSP_DBG_HW );\r
629         return (cq_tinfo);\r
630 }\r
631 \r
632 \r
633 void\r
634 ib_destroy_cq_tinfo(\r
635                                         struct cq_thread_info           *cq_tinfo )\r
636 {\r
637         ib_wc_t wclist;\r
638         ib_wc_t *free_wclist;\r
639         ib_wc_t *done_wclist;\r
640         ib_api_status_t status;\r
641         HANDLE h_cq_thread;\r
642         DWORD cq_thread_id;\r
643 \r
644         IBSP_ENTER( IBSP_DBG_HW );\r
645 \r
646         CL_ASSERT( cq_tinfo );\r
647         CL_ASSERT( cq_tinfo->qp_count == 0 );\r
648 \r
649         if( cq_tinfo->cq )\r
650         {\r
651                 wclist.p_next = NULL;\r
652                 free_wclist = &wclist;\r
653 \r
654                 while( ib_poll_cq(\r
655                         cq_tinfo->cq, &free_wclist, &done_wclist ) == IB_SUCCESS )\r
656                 {\r
657                         IBSP_TRACE1( IBSP_DBG_WQ,\r
658                                 ("free=%p, done=%p\n", free_wclist, done_wclist) );\r
659                 }\r
660 \r
661                 IBSP_TRACE4( IBSP_DBG_WQ, ("ib_destroy_cq() start..\n") );\r
662 \r
663                 /*\r
664                  * Called from cleanup thread, okay to block.\r
665                  */\r
666                 status = ib_destroy_cq( cq_tinfo->cq, ib_sync_destroy );\r
667                 if( status )\r
668                 {\r
669                         IBSP_ERROR(\r
670                                 ("ib_destroy_cq returned %s\n", ib_get_err_str( status )) );\r
671                 }\r
672                 else\r
673                 {\r
674                         IBSP_TRACE4( IBSP_DBG_WQ, ("ib_destroy_cq() finished.\n") );\r
675 \r
676                         cq_tinfo->cq = NULL;\r
677 \r
678                         STAT_DEC( cq_num );\r
679                 }\r
680         }\r
681 \r
682         if( cq_tinfo->ib_cq_thread )\r
683         {\r
684                 /* ib_cq_thread() will release the cq_tinfo before exit. Don't\r
685                    reference cq_tinfo after signaling  */\r
686                 h_cq_thread = cq_tinfo->ib_cq_thread;\r
687                 cq_tinfo->ib_cq_thread = NULL;\r
688                 cq_thread_id = cq_tinfo->ib_cq_thread_id;\r
689 \r
690                 cq_tinfo->ib_cq_thread_exit_wanted = TRUE;\r
691                 cl_waitobj_signal( cq_tinfo->cq_waitobj );\r
692 \r
693                 /* Wait for ib_cq_thread to die, if we are not running on it */\r
694                 if( GetCurrentThreadId() != cq_thread_id )\r
695                 {\r
696                         fzprint(("%s():%d:0x%x:0x%x: Waiting for ib_cq_thread=0x%x to die\n",\r
697                                          __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(),\r
698                                          cq_thread_id ));\r
699                         if( WaitForSingleObject( h_cq_thread, INFINITE ) != WAIT_OBJECT_0 )\r
700                         {\r
701                                 IBSP_ERROR( ("WaitForSingleObject failed\n") );\r
702                         }\r
703                         else\r
704                         {\r
705                                 STAT_DEC( thread_num );\r
706                         }\r
707                 }\r
708                 else\r
709                 {\r
710                         fzprint(("%s():%d:0x%x:0x%x: Currently on ib_cq_thread.\n", __FUNCTION__,\r
711                                          __LINE__, GetCurrentProcessId(), GetCurrentThreadId()));\r
712                         STAT_DEC( thread_num );\r
713                 }\r
714                 CloseHandle( h_cq_thread );\r
715         }\r
716         else\r
717         {\r
718                 /* There was no thread created, destroy cq_waitobj and\r
719                    free memory */\r
720                 if( cq_tinfo->cq_waitobj )\r
721                 {\r
722                         cl_waitobj_destroy( cq_tinfo->cq_waitobj );\r
723                         cq_tinfo->cq_waitobj = NULL;\r
724                 }\r
725                 HeapFree( g_ibsp.heap, 0, cq_tinfo );\r
726         }\r
727 \r
728         IBSP_EXIT( IBSP_DBG_HW );\r
729 }\r
730 \r
731 \r
732 static struct cq_thread_info *\r
733 ib_acquire_cq_tinfo(\r
734                                         struct ibsp_hca                         *hca )\r
735 {\r
736         struct cq_thread_info   *cq_tinfo = NULL;\r
737         uint32_t                                cqe_size;\r
738         ib_api_status_t                 status;\r
739 \r
740         IBSP_ENTER( IBSP_DBG_HW );\r
741 \r
742         cl_spinlock_acquire( &hca->cq_lock );\r
743 \r
744         if( !hca->cq_tinfo )\r
745         {\r
746                 cq_tinfo = ib_alloc_cq_tinfo( hca );\r
747                 if( !cq_tinfo )\r
748                 {\r
749                         IBSP_ERROR_EXIT( ("ib_alloc_cq_tinfo() failed\n") );\r
750                         cl_spinlock_release( &hca->cq_lock );\r
751                         return (NULL);\r
752                 }\r
753         }\r
754         else\r
755         {\r
756                 cq_tinfo = hca->cq_tinfo;\r
757         }\r
758 \r
759         CL_ASSERT( cq_tinfo != NULL );\r
760 \r
761         cqe_size = (cq_tinfo->qp_count + 1) * IB_CQ_SIZE;\r
762 \r
763         if( cq_tinfo->cqe_size < cqe_size )\r
764         {\r
765                 status = ib_modify_cq( cq_tinfo->cq, &cqe_size );\r
766                 switch( status )\r
767                 {\r
768                 case IB_INVALID_CQ_SIZE:\r
769                 case IB_UNSUPPORTED:\r
770                         cq_tinfo = ib_alloc_cq_tinfo( hca );\r
771                         if( !cq_tinfo )\r
772                                 break;\r
773 \r
774                         cq_tinfo->qp_count++;\r
775                         break;\r
776 \r
777                 case IB_SUCCESS:\r
778                         cq_tinfo->cqe_size = cqe_size;\r
779 \r
780                         cq_tinfo->qp_count++;\r
781 \r
782                         fzprint(("%s():%d:0x%x:0x%x: New cq size=%d.\n",\r
783                                          __FUNCTION__,\r
784                                          __LINE__, GetCurrentProcessId(),\r
785                                          GetCurrentThreadId(), cq_tinfo->cqe_size));\r
786                         break;\r
787 \r
788                 default:\r
789                         IBSP_ERROR_EXIT(\r
790                                 ("ib_modify_cq() returned %s\n", ib_get_err_str(status)) );\r
791                         cq_tinfo = NULL;\r
792                 }\r
793         }\r
794         else\r
795         {\r
796                 cq_tinfo->qp_count++;\r
797         }\r
798 \r
799         cl_spinlock_release( &hca->cq_lock );\r
800         IBSP_EXIT( IBSP_DBG_HW );\r
801         return (cq_tinfo);\r
802 }\r
803 \r
804 void\r
805 ib_release_cq_tinfo(\r
806                                         struct cq_thread_info           *cq_tinfo )\r
807 {\r
808         IBSP_ENTER( IBSP_DBG_HW );\r
809 \r
810         CL_ASSERT( cq_tinfo );\r
811         CL_ASSERT( cq_tinfo->hca );\r
812 \r
813         cl_spinlock_acquire( &cq_tinfo->hca->cq_lock );\r
814         /* If this CQ now has fewer QPs than the primary, make it the primary. */\r
815         if( --cq_tinfo->qp_count < cq_tinfo->hca->cq_tinfo->qp_count )\r
816                 cq_tinfo->hca->cq_tinfo = cq_tinfo;\r
817         cl_spinlock_release( &cq_tinfo->hca->cq_lock );\r
818 \r
819         IBSP_EXIT( IBSP_DBG_HW );\r
820 }\r
821 \r
822 \r
823 /* Release IB ressources. */\r
824 void\r
825 ib_release(void)\r
826 {\r
827         cl_fmap_item_t                  *p_item;\r
828 \r
829         IBSP_ENTER( IBSP_DBG_HW );\r
830 \r
831         if( g_ibsp.al_handle )\r
832         {\r
833                 cl_list_item_t *item;\r
834                 ib_api_status_t status;\r
835 \r
836                 unregister_pnp();\r
837 \r
838                 while( (item = cl_qlist_head( &g_ibsp.hca_list )) != cl_qlist_end( &g_ibsp.hca_list ) )\r
839                 {\r
840                         struct ibsp_hca *hca = PARENT_STRUCT(item, struct ibsp_hca, item);\r
841 \r
842                         pnp_ca_remove( hca );\r
843                 }\r
844 \r
845                 fzprint(("%s():%d:0x%x:0x%x: Calling ib_close_al...\n", __FUNCTION__,\r
846                                  __LINE__, GetCurrentProcessId(), GetCurrentThreadId()));\r
847 \r
848                 status = ib_close_al( g_ibsp.al_handle );\r
849 \r
850                 fzprint(("%s():%d:0x%x:0x%x: Done calling ib_close_al, status=%d.\n",\r
851                                  __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(),\r
852                                  status));\r
853                 if( status != IB_SUCCESS )\r
854                 {\r
855                         IBSP_ERROR(\r
856                                 ("ib_close_al returned %s\n", ib_get_err_str( status )) );\r
857                 }\r
858                 else\r
859                 {\r
860                         IBSP_TRACE( IBSP_DBG_HW, ("ib_close_al success\n") );\r
861                         STAT_DEC( al_num );\r
862                 }\r
863                 g_ibsp.al_handle = NULL;\r
864         }\r
865 \r
866         for( p_item = cl_fmap_head( &g_ibsp.ip_map );\r
867                 p_item != cl_fmap_end( &g_ibsp.ip_map );\r
868                 p_item = cl_fmap_head( &g_ibsp.ip_map ) )\r
869         {\r
870                 cl_fmap_remove_item( &g_ibsp.ip_map, p_item );\r
871 \r
872                 HeapFree( g_ibsp.heap, 0,\r
873                         PARENT_STRUCT(p_item, struct ibsp_ip_addr, item) );\r
874         }\r
875 \r
876         IBSP_EXIT( IBSP_DBG_HW );\r
877 }\r
878 \r
879 \r
880 /* Initialize IB ressources. */\r
881 int\r
882 ibsp_initialize(void)\r
883 {\r
884         ib_api_status_t status;\r
885         int ret;\r
886 \r
887         IBSP_ENTER( IBSP_DBG_HW );\r
888 \r
889         CL_ASSERT( g_ibsp.al_handle == NULL );\r
890         CL_ASSERT( cl_qlist_count( &g_ibsp.hca_list ) == 0 );\r
891 \r
892         /* Open the IB library */\r
893         status = ib_open_al( &g_ibsp.al_handle );\r
894 \r
895         IBSP_TRACE( IBSP_DBG_HW, ("open is %d %p\n", status, g_ibsp.al_handle) );\r
896 \r
897         if( status != IB_SUCCESS )\r
898         {\r
899                 IBSP_ERROR( ("ib_open_al failed (%d)\n", status) );\r
900                 ret = WSAEPROVIDERFAILEDINIT;\r
901                 goto done;\r
902         }\r
903 \r
904         STAT_INC( al_num );\r
905 \r
906         /* Register for PNP events */\r
907         status = register_pnp();\r
908         if( status )\r
909         {\r
910                 IBSP_ERROR( ("register_pnp failed (%d)\n", status) );\r
911                 ret = WSAEPROVIDERFAILEDINIT;\r
912                 goto done;\r
913         }\r
914 \r
915         STAT_INC( thread_num );\r
916 \r
917         ret = 0;\r
918 done:\r
919         if( ret )\r
920         {\r
921                 /* Free up resources. */\r
922                 ib_release();\r
923         }\r
924 \r
925         IBSP_EXIT( IBSP_DBG_HW );\r
926 \r
927         return ret;\r
928 }\r
929 \r
930 \r
931 /* Destroys the infiniband ressources of a socket. */\r
932 void\r
933 ib_destroy_socket(\r
934         IN      OUT                     struct ibsp_socket_info         *socket_info )\r
935 {\r
936         ib_api_status_t status;\r
937 \r
938         IBSP_ENTER( IBSP_DBG_EP );\r
939 \r
940         if( socket_info->qp )\r
941         {\r
942                 cl_atomic_inc( &socket_info->ref_cnt );\r
943                 status = ib_destroy_qp( socket_info->qp, deref_socket_info );\r
944                 if( status != IB_SUCCESS )\r
945                 {\r
946                         IBSP_ERROR( ("ib_destroy_qp returned %s\n",\r
947                                 ib_get_err_str( status )) );\r
948                         deref_socket_info( socket_info );\r
949                 }\r
950 \r
951                 ib_release_cq_tinfo( socket_info->cq_tinfo );\r
952 \r
953                 socket_info->qp = NULL;\r
954         }\r
955 \r
956         IBSP_EXIT( IBSP_DBG_EP );\r
957 }\r
958 \r
959 \r
960 /*\r
961  * Creates the necessary IB ressources for a socket\r
962  */\r
963 int\r
964 ib_create_socket(\r
965         IN      OUT                     struct ibsp_socket_info         *socket_info)\r
966 {\r
967         ib_qp_create_t                  qp_create;\r
968         ib_api_status_t                 status;\r
969         ib_qp_attr_t                    qp_attr;\r
970 \r
971         IBSP_ENTER( IBSP_DBG_EP );\r
972 \r
973         CL_ASSERT( socket_info != NULL );\r
974         CL_ASSERT( socket_info->port != NULL );\r
975         CL_ASSERT( socket_info->qp == NULL );\r
976 \r
977         socket_info->hca_pd = socket_info->port->hca->pd;\r
978 \r
979         /* Get the completion queue and thread info for this socket */\r
980         socket_info->cq_tinfo = ib_acquire_cq_tinfo( socket_info->port->hca );\r
981         if( !socket_info->cq_tinfo )\r
982         {\r
983                 IBSP_ERROR_EXIT( ("ib_acquire_cq_tinfo failed\n") );\r
984                 return WSAENOBUFS;\r
985         }\r
986 \r
987         /* Queue pair */\r
988         qp_create.qp_type = IB_QPT_RELIABLE_CONN;\r
989         qp_create.sq_depth = QP_ATTRIB_SQ_DEPTH;\r
990         qp_create.rq_depth = QP_ATTRIB_RQ_DEPTH;\r
991         qp_create.sq_sge = QP_ATTRIB_SQ_SGE;\r
992         qp_create.rq_sge = 1;\r
993         qp_create.h_rq_cq = socket_info->cq_tinfo->cq;\r
994         qp_create.h_sq_cq = socket_info->cq_tinfo->cq;\r
995         qp_create.sq_signaled = TRUE;\r
996 \r
997         status = ib_create_qp( socket_info->hca_pd, &qp_create, socket_info,    /* context */\r
998                 NULL,   /* async handler */\r
999                 &socket_info->qp );\r
1000         if( status )\r
1001         {\r
1002                 ib_release_cq_tinfo( socket_info->cq_tinfo );\r
1003                 IBSP_ERROR_EXIT(\r
1004                         ("ib_create_qp returned %s\n", ib_get_err_str( status )) );\r
1005                 return WSAENOBUFS;\r
1006         }\r
1007 \r
1008         status = ib_query_qp( socket_info->qp, &qp_attr );\r
1009         if( status == IB_SUCCESS )\r
1010         {\r
1011                 socket_info->max_inline = min( g_max_inline, qp_attr.sq_max_inline );\r
1012         }\r
1013         else\r
1014         {\r
1015                 IBSP_ERROR( ("ib_query_qp returned %s\n", ib_get_err_str( status )) );\r
1016                 socket_info->max_inline = 0;\r
1017         }\r
1018 \r
1019         STAT_INC( qp_num );\r
1020 \r
1021         IBSP_EXIT( IBSP_DBG_EP );\r
1022         return 0;\r
1023 }\r
1024 \r
1025 \r
1026 void\r
1027 wait_cq_drain(\r
1028         IN      OUT                     struct ibsp_socket_info         *socket_info )\r
1029 {\r
1030         IBSP_ENTER( IBSP_DBG_EP );\r
1031 \r
1032         if( socket_info->cq_tinfo == NULL )\r
1033         {\r
1034                 IBSP_EXIT( IBSP_DBG_EP );\r
1035                 return;\r
1036         }\r
1037 \r
1038         /* Wait for the QP to be drained. */\r
1039         while( socket_info->send_cnt || socket_info->recv_cnt )\r
1040         {\r
1041                 fzprint(("%s():%d:0x%x:0x%x: socket=0x%p wr_list_count=%d qp state=%d\n",\r
1042                                  __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(),\r
1043                                  socket_info, cl_qlist_count(&socket_info->wr_list)));\r
1044 \r
1045                 Sleep(100);\r
1046         }\r
1047 \r
1048         IBSP_EXIT( IBSP_DBG_EP );\r
1049 }\r
1050 \r
1051 \r
1052 void\r
1053 ibsp_dup_overlap_abort(\r
1054         IN      OUT                     struct ibsp_socket_info         *socket_info )\r
1055 {\r
1056         LPWSAOVERLAPPED lpOverlapped = NULL;\r
1057         int error;\r
1058         int ret;\r
1059         uint8_t                         idx;\r
1060 \r
1061         IBSP_ENTER( IBSP_DBG_EP );\r
1062         CL_ASSERT( !socket_info->send_cnt && !socket_info->recv_cnt );\r
1063 \r
1064         /* Browse the list of all posted overlapped structures\r
1065          * to mark them as aborted. */\r
1066         idx = socket_info->dup_idx - (uint8_t)socket_info->dup_cnt;\r
1067         if( idx >= QP_ATTRIB_RQ_DEPTH )\r
1068                 idx += QP_ATTRIB_RQ_DEPTH;\r
1069 \r
1070         while( socket_info->dup_cnt )\r
1071         {\r
1072                 lpOverlapped = socket_info->dup_wr[idx].wr.lpOverlapped;\r
1073 \r
1074                 fzprint(("%s():%d:0x%x:0x%x: socket=0x%p wr=0x%p overlapped=0x%p Internal=%d InternalHigh=%d hEvent=%d\n",\r
1075                         __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), socket_info, &socket_info->dup_wr[idx], lpOverlapped, lpOverlapped->Internal, lpOverlapped->InternalHigh, lpOverlapped->hEvent));\r
1076 \r
1077                 lpOverlapped->OffsetHigh = WSAECONNABORTED;\r
1078                 lpOverlapped->InternalHigh = 0;\r
1079 \r
1080                 if( ((uintptr_t) lpOverlapped->hEvent) & 0x00000001 )\r
1081                 {\r
1082                         /* Indicate this operation is complete. The switch will poll\r
1083                          * with calls to WSPGetOverlappedResult(). */\r
1084 #ifdef _DEBUG_\r
1085                         cl_atomic_dec(&g_ibsp.overlap_h1_comp_count);\r
1086 \r
1087                         fzprint(("%s():%d:0x%x:0x%x: ov=0x%p h0=%d h1=%d h1_c=%d send=%d recv=%d\n",\r
1088                                          __FUNCTION__, __LINE__, GetCurrentProcessId(),\r
1089                                          GetCurrentThreadId(), lpOverlapped,\r
1090                                          g_ibsp.overlap_h0_count, g_ibsp.overlap_h1_count,\r
1091                                          g_ibsp.overlap_h1_comp_count, g_ibsp.send_count, g_ibsp.recv_count));\r
1092 #endif\r
1093 \r
1094                         CL_TRACE(IBSP_DBG_WQ, gdbg_lvl,\r
1095                                          ("%s: set internal overlapped=0x%p Internal=%d OffsetHigh=%d\n",\r
1096                                           __FUNCTION__, lpOverlapped, lpOverlapped->Internal,\r
1097                                           lpOverlapped->OffsetHigh));\r
1098 \r
1099                         lpOverlapped->Internal = 0;\r
1100                 }\r
1101                 else\r
1102                 {\r
1103 #ifdef _DEBUG_\r
1104                         cl_atomic_dec(&g_ibsp.overlap_h0_count);\r
1105 \r
1106 \r
1107                         fzprint(("%s():%d:0x%x:0x%x: ov=0x%p h0=%d h1=%d h1_c=%d send=%d recv=%d\n",\r
1108                                          __FUNCTION__, __LINE__, GetCurrentProcessId(),\r
1109                                          GetCurrentThreadId(), lpOverlapped,\r
1110                                          g_ibsp.overlap_h0_count, g_ibsp.overlap_h1_count,\r
1111                                          g_ibsp.overlap_h1_comp_count, g_ibsp.send_count, g_ibsp.recv_count));\r
1112 #endif\r
1113                         CL_TRACE(IBSP_DBG_WQ, gdbg_lvl,\r
1114                                          ("%s: calls lpWPUCompleteOverlappedRequest, overlapped=0x%p OffsetHigh=%d InternalHigh=%d hEvent=%d\n",\r
1115                                           __FUNCTION__, lpOverlapped, lpOverlapped->OffsetHigh,\r
1116                                           lpOverlapped->InternalHigh, lpOverlapped->hEvent));\r
1117 \r
1118                         ret = g_ibsp.up_call_table.lpWPUCompleteOverlappedRequest\r
1119                                 (socket_info->switch_socket,\r
1120                                  lpOverlapped,\r
1121                                  lpOverlapped->OffsetHigh, (DWORD) lpOverlapped->InternalHigh, &error);\r
1122 \r
1123                         if( ret != 0 )\r
1124                         {\r
1125                                 CL_ERROR(IBSP_DBG_EP, gdbg_lvl,\r
1126                                                  ("lpWPUCompleteOverlappedRequest failed with %d/%d\n", ret,\r
1127                                                   error));\r
1128                         }\r
1129                 }\r
1130                 cl_atomic_dec( &socket_info->dup_cnt );\r
1131         }\r
1132 \r
1133         IBSP_EXIT( IBSP_DBG_EP );\r
1134 }\r
1135 \r
1136 \r
1137 /* Closes a connection and release its ressources. */\r
1138 void\r
1139 shutdown_and_destroy_socket_info(\r
1140         IN      OUT                     struct ibsp_socket_info         *socket_info )\r
1141 {\r
1142         enum ibsp_socket_state  old_state;\r
1143 \r
1144         IBSP_ENTER( IBSP_DBG_EP );\r
1145 \r
1146         cl_spinlock_acquire( &socket_info->mutex );\r
1147         old_state = socket_info->socket_state;\r
1148         IBSP_CHANGE_SOCKET_STATE( socket_info, IBSP_CLOSED );\r
1149         cl_spinlock_release( &socket_info->mutex );\r
1150 \r
1151         if( socket_info->listen.handle )\r
1152         {\r
1153                 /* Stop listening and reject queued connections. */\r
1154                 ib_listen_cancel( socket_info );\r
1155         }\r
1156 \r
1157         cl_spinlock_acquire( &g_ibsp.socket_info_mutex );\r
1158         cl_qlist_remove_item( &g_ibsp.socket_info_list, &socket_info->item );\r
1159 \r
1160         switch( old_state )\r
1161         {\r
1162         case IBSP_CREATE:\r
1163         case IBSP_LISTEN:\r
1164                 /* Nothing to do. */\r
1165                 break;\r
1166 \r
1167         case IBSP_CONNECTED:\r
1168                 {\r
1169                         struct disconnect_reason reason;\r
1170 \r
1171                         memset( &reason, 0, sizeof(reason) );\r
1172                         reason.type = DISC_SHUTDOWN;\r
1173                         ib_disconnect( socket_info, &reason );\r
1174                 }\r
1175                 /* Fall through. */\r
1176 \r
1177         case IBSP_CONNECT:\r
1178         case IBSP_DISCONNECTED:\r
1179                 /* We changed the state - remove from connection map. */\r
1180                 CL_ASSERT( socket_info->conn_item.p_map );\r
1181                 cl_rbmap_remove_item( &g_ibsp.conn_map, &socket_info->conn_item );\r
1182                 break;\r
1183         }\r
1184         cl_spinlock_release( &g_ibsp.socket_info_mutex );\r
1185 \r
1186         /* Flush all completions. */\r
1187         if( socket_info->dup_cnt )\r
1188                 ibsp_dup_overlap_abort( socket_info );\r
1189 \r
1190         while( socket_info->send_cnt || socket_info->recv_cnt )\r
1191                 ib_cq_comp( socket_info->cq_tinfo );\r
1192 \r
1193         ibsp_dereg_socket( socket_info );\r
1194 \r
1195         ib_destroy_socket( socket_info );\r
1196 \r
1197 #ifdef IBSP_LOGGING\r
1198         DataLogger_Shutdown(&socket_info->SendDataLogger);\r
1199         DataLogger_Shutdown(&socket_info->RecvDataLogger);\r
1200 #endif\r
1201 \r
1202         /* Release the initial reference and clean up. */\r
1203         deref_socket_info( socket_info );\r
1204 \r
1205         IBSP_EXIT( IBSP_DBG_EP );\r
1206 }\r
1207 \r
1208 \r
1209 boolean_t\r
1210 ibsp_conn_insert(\r
1211         IN                              struct ibsp_socket_info         *s )\r
1212 {\r
1213         struct ibsp_socket_info         *p_sock;\r
1214         cl_rbmap_item_t                         *p_item, *p_insert_at;\r
1215         boolean_t                                       left = TRUE;\r
1216 \r
1217         cl_spinlock_acquire( &g_ibsp.socket_info_mutex );\r
1218         p_item = cl_rbmap_root( &g_ibsp.conn_map );\r
1219         p_insert_at = p_item;\r
1220 \r
1221         CL_ASSERT( !s->conn_item.p_map );\r
1222         while( p_item != cl_rbmap_end( &g_ibsp.conn_map ) )\r
1223         {\r
1224                 p_insert_at = p_item;\r
1225                 p_sock = PARENT_STRUCT( p_item, struct ibsp_socket_info, conn_item );\r
1226                 if( p_sock->local_addr.sin_family < s->local_addr.sin_family )\r
1227                         p_item = cl_rbmap_left( p_item ), left = TRUE;\r
1228                 else if( p_sock->local_addr.sin_family > s->local_addr.sin_family )\r
1229                         p_item = cl_rbmap_right( p_item ), left = FALSE;\r
1230                 else if( p_sock->local_addr.sin_addr.S_un.S_addr < s->local_addr.sin_addr.S_un.S_addr )\r
1231                         p_item = cl_rbmap_left( p_item ), left = TRUE;\r
1232                 else if( p_sock->local_addr.sin_addr.S_un.S_addr > s->local_addr.sin_addr.S_un.S_addr )\r
1233                         p_item = cl_rbmap_right( p_item ), left = FALSE;\r
1234                 else if( p_sock->local_addr.sin_port < s->local_addr.sin_port )\r
1235                         p_item = cl_rbmap_left( p_item ), left = TRUE;\r
1236                 else if( p_sock->local_addr.sin_port > s->local_addr.sin_port )\r
1237                         p_item = cl_rbmap_right( p_item ), left = FALSE;\r
1238                 else if( p_sock->peer_addr.sin_family < s->peer_addr.sin_family )\r
1239                         p_item = cl_rbmap_left( p_item ), left = TRUE;\r
1240                 else if( p_sock->peer_addr.sin_family > s->peer_addr.sin_family )\r
1241                         p_item = cl_rbmap_right( p_item ), left = FALSE;\r
1242                 else if( p_sock->peer_addr.sin_addr.S_un.S_addr < s->peer_addr.sin_addr.S_un.S_addr )\r
1243                         p_item = cl_rbmap_left( p_item ), left = TRUE;\r
1244                 else if( p_sock->peer_addr.sin_addr.S_un.S_addr > s->peer_addr.sin_addr.S_un.S_addr )\r
1245                         p_item = cl_rbmap_right( p_item ), left = FALSE;\r
1246                 else if( p_sock->peer_addr.sin_port < s->peer_addr.sin_port )\r
1247                         p_item = cl_rbmap_left( p_item ), left = TRUE;\r
1248                 else if( p_sock->peer_addr.sin_port > s->peer_addr.sin_port )\r
1249                         p_item = cl_rbmap_right( p_item ), left = FALSE;\r
1250                 else\r
1251                         goto done;\r
1252         }\r
1253 \r
1254         cl_rbmap_insert( &g_ibsp.conn_map, p_insert_at, &s->conn_item, left );\r
1255 \r
1256 done:\r
1257         cl_spinlock_release( &g_ibsp.socket_info_mutex );\r
1258         return p_item == cl_rbmap_end( &g_ibsp.conn_map );\r
1259 }\r
1260 \r
1261 \r
1262 void\r
1263 ibsp_conn_remove(\r
1264         IN                              struct ibsp_socket_info         *s )\r
1265 {\r
1266         cl_spinlock_acquire( &g_ibsp.socket_info_mutex );\r
1267         CL_ASSERT( s->conn_item.p_map );\r
1268         cl_rbmap_remove_item( &g_ibsp.conn_map, &s->conn_item );\r
1269         cl_spinlock_release( &g_ibsp.socket_info_mutex );\r
1270 }\r