initial implementation
[mirror/winof/.git] / ulp / wsd / user / ibsp_iblow.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  *\r
4  * This software is available to you under the OpenIB.org BSD license\r
5  * below:\r
6  *\r
7  *     Redistribution and use in source and binary forms, with or\r
8  *     without modification, are permitted provided that the following\r
9  *     conditions are met:\r
10  *\r
11  *      - Redistributions of source code must retain the above\r
12  *        copyright notice, this list of conditions and the following\r
13  *        disclaimer.\r
14  *\r
15  *      - Redistributions in binary form must reproduce the above\r
16  *        copyright notice, this list of conditions and the following\r
17  *        disclaimer in the documentation and/or other materials\r
18  *        provided with the distribution.\r
19  *\r
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
27  * SOFTWARE.\r
28  *\r
29  * $Id:$\r
30  */\r
31 \r
32 #include "ibspdll.h"\r
33 \r
34 \r
35 static void ib_destroy_cq_tinfo( struct cq_thread_info *cq_tinfo );\r
36 \r
37 \r
38 typedef struct _io_comp_info\r
39 {\r
40         SOCKET                                  socket;\r
41         LPWSAOVERLAPPED                 p_ov;\r
42         atomic32_t                              *p_io_cnt;\r
43 \r
44 } io_comp_info_t;\r
45 \r
46 \r
47 /* Work queue entry completion routine. */\r
48 static void\r
49 complete_wq(\r
50         IN              const   ib_wc_t                                         *wc,\r
51                 OUT                     io_comp_info_t                          *p_io_info )\r
52 {\r
53         struct _wr                              *wr = NULL;\r
54         struct _recv_wr                 *p_recv_wr = NULL;\r
55         LPWSAOVERLAPPED                 lpOverlapped = NULL;\r
56         struct ibsp_socket_info *socket_info = NULL;\r
57 \r
58         IBSP_ENTER( IBSP_DBG_IO );\r
59 \r
60         wr = (struct _wr * __ptr64)wc->wr_id;\r
61         p_recv_wr = (struct _recv_wr * __ptr64)wc->wr_id;\r
62 \r
63         CL_ASSERT( wr );\r
64 \r
65         socket_info = wr->socket_info;\r
66         p_io_info->socket = socket_info->switch_socket;\r
67 \r
68         lpOverlapped = wr->lpOverlapped;\r
69 \r
70         IBSP_TRACE4( IBSP_DBG_IO,\r
71                 ("socket %p, ov %p, work completion status=%s, wc_type=%s\n",\r
72                 socket_info, lpOverlapped, ib_get_wc_status_str( wc->status ),\r
73                 ib_get_wc_type_str( wc->wc_type )) );\r
74 \r
75         /* Set the windows error code. It's not easy to find an easy\r
76          * correspondence between the IBAL error codes and windows error\r
77          * codes; but it probably does not matter, as long as it returns an\r
78          * error. */\r
79         switch( wc->status )\r
80         {\r
81         case IB_WCS_SUCCESS:\r
82                 /*\r
83                  * Set the length of the operation. Under Infiniband, the work\r
84                  * completion length is only valid for a receive\r
85                  * operation. Fortunately we had already set the length during the\r
86                  * send operation. \r
87                  *\r
88                  * lpWPUCompleteOverlappedRequest is supposed to store the length\r
89                  * into InternalHigh, however it will not be called if the low\r
90                  * order bit of lpOverlapped->hEvent is set. So we do it and hope\r
91                  * for the best. \r
92                  *\r
93                  * NOTE: Without a valid length, the switch doesn't seem to call \r
94                  * GetOverlappedResult() even if we call lpWPUCompleteOverlappedRequest()\r
95                  */\r
96                 if( wc->wc_type == IB_WC_RECV )\r
97                         lpOverlapped->InternalHigh = wc->length;\r
98 \r
99                 lpOverlapped->OffsetHigh = 0;\r
100                 break;\r
101 \r
102         case IB_WCS_WR_FLUSHED_ERR:\r
103                 cl_spinlock_acquire( &socket_info->mutex );\r
104 \r
105                 if( socket_info->socket_state == IBSP_DUPLICATING_REMOTE &&\r
106                         wc->wc_type == IB_WC_RECV )\r
107                 {\r
108                         /*\r
109                          * Take the wr off the wr_list, and place onto the\r
110                          * dup_wr_list.  We will post them later on the new QP. \r
111                          */\r
112                         cl_spinlock_acquire( &socket_info->recv_lock );\r
113 \r
114                         /* Copy to the duplicate WR array. */\r
115                         socket_info->dup_wr[socket_info->dup_idx] = *p_recv_wr;\r
116 \r
117 #if QP_ATTRIB_RQ_DEPTH == 256 || QP_ATTRIB_RQ_DEPTH == 128 || \\r
118         QP_ATTRIB_RQ_DEPTH == 64 || QP_ATTRIB_RQ_DEPTH == 32 || \\r
119         QP_ATTRIB_RQ_DEPTH == 16 || QP_ATTRIB_RQ_DEPTH == 8\r
120                         socket_info->dup_idx++;\r
121                         socket_info->dup_idx &= (QP_ATTRIB_RQ_DEPTH - 1);\r
122 #else\r
123                         if( ++socket_info->dup_idx == QP_ATTRIB_RQ_DEPTH )\r
124                                 socket_info->dup_idx = 0;\r
125 #endif\r
126 \r
127                         cl_atomic_inc( &socket_info->dup_cnt );\r
128                         /* ib_cq_comp will decrement the receive count. */\r
129                         p_io_info->p_io_cnt = &socket_info->recv_cnt;\r
130 \r
131                         cl_spinlock_release( &socket_info->recv_lock );\r
132 \r
133                         cl_spinlock_release( &socket_info->mutex );\r
134                         IBSP_EXIT( IBSP_DBG_IO );\r
135                         return;\r
136                 }\r
137                 \r
138                 /* Check for flushing the receive buffers on purpose. */\r
139                 if( socket_info->socket_state == IBSP_DUPLICATING_OLD )\r
140                         wr->lpOverlapped->OffsetHigh = 0;\r
141                 else\r
142                         wr->lpOverlapped->OffsetHigh = WSA_OPERATION_ABORTED;\r
143 \r
144                 cl_spinlock_release( &socket_info->mutex );\r
145 \r
146                 /* Override the length, as per the WSD specs. */\r
147                 wr->lpOverlapped->InternalHigh = 0;\r
148                 socket_info->qp_error = WSAECONNABORTED;\r
149                 break;\r
150 \r
151         case IB_WCS_LOCAL_LEN_ERR:\r
152         case IB_WCS_LOCAL_OP_ERR:\r
153         case IB_WCS_LOCAL_PROTECTION_ERR:\r
154         case IB_WCS_MEM_WINDOW_BIND_ERR:\r
155         case IB_WCS_REM_ACCESS_ERR:\r
156         case IB_WCS_REM_OP_ERR:\r
157         case IB_WCS_RNR_RETRY_ERR:\r
158         case IB_WCS_TIMEOUT_RETRY_ERR:\r
159         case IB_WCS_REM_INVALID_REQ_ERR:\r
160         default:\r
161                 IBSP_ERROR( ("%s error: %s\n",\r
162                         ib_get_wc_type_str( wc->wc_type ),\r
163                         ib_get_wc_status_str( wc->status )) );\r
164                 lpOverlapped->OffsetHigh = WSAECONNABORTED;\r
165                 wr->lpOverlapped->InternalHigh = 0;\r
166                 socket_info->qp_error = WSAECONNABORTED;\r
167                 break;\r
168         }\r
169 \r
170 #ifdef _DEBUG_\r
171         if( wc->wc_type == IB_WC_RECV )\r
172         {\r
173                 // This code requires the recv count to be decremented here, but it needs\r
174                 // to be decremented after any callbacks are invoked so socket destruction\r
175                 // gets delayed until all callbacks have been invoked.\r
176                 //{\r
177                 //      uint8_t idx;\r
178 \r
179                 //      cl_spinlock_acquire( &socket_info->recv_lock );\r
180                 //      idx = socket_info->recv_idx - (uint8_t)socket_info->recv_cnt;\r
181                 //      if( idx >= QP_ATTRIB_RQ_DEPTH )\r
182                 //              idx += QP_ATTRIB_RQ_DEPTH;\r
183 \r
184                 //      CL_ASSERT( wc->wr_id == (uint64_t)(void* __ptr64)&socket_info->recv_wr[idx] );\r
185                 //      cl_atomic_dec( &socket_info->recv_cnt );\r
186                 //      cl_spinlock_release( &socket_info->recv_lock );\r
187                 //}\r
188 \r
189                 if( p_recv_wr->ds_array[0].length >= 40 )\r
190                 {\r
191                         debug_dump_buffer( IBSP_DBG_WQ, gdbg_lvl, "RECV",\r
192                                 (void * __ptr64)p_recv_wr->ds_array[0].vaddr, 40 );\r
193                 }\r
194 \r
195                 cl_atomic_dec( &g_ibsp.recv_count );\r
196                 cl_atomic_inc( &socket_info->recv_comp );\r
197 \r
198                 memset( p_recv_wr, 0x33, sizeof(struct _recv_wr) );\r
199         }\r
200         else\r
201         {\r
202                 // This code requires the send count to be decremented here, but it needs\r
203                 // to be decremented after any callbacks are invoked so socket destruction\r
204                 // gets delayed until all callbacks have been invoked.\r
205                 //{\r
206                 //      uint8_t idx;\r
207 \r
208                 //      cl_spinlock_acquire( &socket_info->send_lock );\r
209                 //      idx = socket_info->send_idx - (uint8_t)socket_info->send_cnt;\r
210                 //      if( idx >= QP_ATTRIB_SQ_DEPTH )\r
211                 //              idx += QP_ATTRIB_SQ_DEPTH;\r
212                 //      CL_ASSERT( wc->wr_id == (uint64_t)(void* __ptr64)&socket_info->send_wr[idx] );\r
213                 //      cl_atomic_dec( &socket_info->send_cnt );\r
214                 //      cl_spinlock_release( &socket_info->send_lock );\r
215                 //}\r
216 \r
217                 if( wc->wc_type == IB_WC_SEND )\r
218                 {\r
219                         cl_atomic_dec( &g_ibsp.send_count );\r
220                         cl_atomic_inc( &socket_info->send_comp );\r
221 \r
222                         fzprint(("%s():%d:0x%x:0x%x: send_count=%d\n",\r
223                                 __FUNCTION__,\r
224                                 __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), g_ibsp.send_count));\r
225                 }\r
226 \r
227                 memset( wr, 0x33, sizeof(struct _wr) );\r
228         }\r
229 #endif\r
230 \r
231         IBSP_TRACE4( IBSP_DBG_IO,\r
232                 ("overlapped=%p, InternalHigh=%d, hEvent=%x\n",\r
233                 lpOverlapped, lpOverlapped->InternalHigh,\r
234                 (uintptr_t) lpOverlapped->hEvent) );\r
235 \r
236         /* Don't notify the switch for that completion only if:\r
237          *   - the switch don't want a notification\r
238          *   - the wq completed with success\r
239          *   - the socket is still connected\r
240          */\r
241         if( ((uintptr_t) lpOverlapped->hEvent) & 0x00000001 )\r
242         {\r
243                 /* Indicate this operation is complete. The switch will poll\r
244                  * with calls to WSPGetOverlappedResult(). */\r
245 \r
246 #ifdef _DEBUG_\r
247                 cl_atomic_dec( &g_ibsp.overlap_h1_comp_count );\r
248 \r
249                 fzprint(("%s():%d:0x%x:0x%x: ov=0x%p h0=%d h1=%d h1_c=%d send=%d recv=%d\n",\r
250                                  __FUNCTION__, __LINE__, GetCurrentProcessId(),\r
251                                  GetCurrentThreadId(), lpOverlapped,\r
252                                  g_ibsp.overlap_h0_count, g_ibsp.overlap_h1_count,\r
253                                  g_ibsp.overlap_h1_comp_count, g_ibsp.send_count, g_ibsp.recv_count));\r
254 #endif\r
255 \r
256                 IBSP_TRACE1( IBSP_DBG_IO,\r
257                         ("Not calling lpWPUCompleteOverlappedRequest: "\r
258                         "socket=%p, ov=%p OffsetHigh=%d, InternalHigh=%d hEvent=%p\n",\r
259                         socket_info, lpOverlapped, lpOverlapped->OffsetHigh,\r
260                         lpOverlapped->InternalHigh, lpOverlapped->hEvent) );\r
261 \r
262                 lpOverlapped->Internal = 0;\r
263                 p_io_info->p_ov = NULL;\r
264         }\r
265         else\r
266         {\r
267 #ifdef _DEBUG_\r
268                 cl_atomic_dec( &g_ibsp.overlap_h0_count );\r
269 \r
270                 fzprint(("%s():%d:0x%x:0x%x: ov=0x%p h0=%d h1=%d h1_c=%d send=%d recv=%d\n",\r
271                                  __FUNCTION__, __LINE__, GetCurrentProcessId(),\r
272                                  GetCurrentThreadId(), lpOverlapped,\r
273                                  g_ibsp.overlap_h0_count, g_ibsp.overlap_h1_count,\r
274                                  g_ibsp.overlap_h1_comp_count, g_ibsp.send_count, g_ibsp.recv_count));\r
275 #endif\r
276 \r
277                 IBSP_TRACE1( IBSP_DBG_IO,\r
278                         ("Calling lpWPUCompleteOverlappedRequest: "\r
279                         "socket=%p, ov=%p OffsetHigh=%d InternalHigh=%d hEvent=%p\n",\r
280                         socket_info, lpOverlapped, lpOverlapped->OffsetHigh,\r
281                         lpOverlapped->InternalHigh, lpOverlapped->hEvent) );\r
282 \r
283                 p_io_info->p_ov = lpOverlapped;\r
284         }\r
285 \r
286         if( wc->wc_type == IB_WC_RECV )\r
287                 p_io_info->p_io_cnt = &socket_info->recv_cnt;\r
288         else\r
289                 p_io_info->p_io_cnt = &socket_info->send_cnt;\r
290 \r
291         IBSP_EXIT( IBSP_DBG_IO );\r
292 }\r
293 \r
294 \r
295 /* CQ completion handler. */\r
296 void\r
297 ib_cq_comp(\r
298                                         void                                            *cq_context )\r
299 {\r
300         struct cq_thread_info   *cq_tinfo = cq_context;\r
301         ib_api_status_t                 status;\r
302         ib_wc_t                                 wclist[WC_LIST_SIZE];\r
303         ib_wc_t                                 *free_wclist;\r
304         ib_wc_t                                 *done_wclist;\r
305         io_comp_info_t                  info[WC_LIST_SIZE];\r
306         int                                             cb_idx;\r
307         int                                             i;\r
308 #ifdef _DEBUG_\r
309         int                                             comp_count;\r
310 #endif\r
311 \r
312         CL_ENTER( IBSP_DBG_WQ, gdbg_lvl );\r
313 \r
314         CL_ASSERT( WC_LIST_SIZE >= 1 );\r
315 \r
316         do\r
317         {\r
318                 /* Try to retrieve up to WC_LIST_SIZE completions at a time. */\r
319                 for( i = 0; i < (WC_LIST_SIZE - 1); i++ )\r
320                 {\r
321                         wclist[i].p_next = &wclist[i + 1];\r
322                 }\r
323                 wclist[(WC_LIST_SIZE - 1)].p_next = NULL;\r
324 \r
325                 free_wclist = &wclist[0];\r
326                 done_wclist = NULL;\r
327 \r
328                 status = ib_poll_cq( cq_tinfo->cq, &free_wclist, &done_wclist );\r
329 \r
330                 CL_TRACE( IBSP_DBG_WQ, gdbg_lvl,\r
331                         ("%s():%d:0x%x:0x%x: poll CQ got status %d, free=%p, done=%p\n",\r
332                         __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(),\r
333                         status, free_wclist, done_wclist) );\r
334 \r
335                 switch( status )\r
336                 {\r
337                 case IB_NOT_FOUND:\r
338                 case IB_SUCCESS:\r
339                         break;\r
340 \r
341                 case IB_INVALID_CQ_HANDLE:\r
342                         /* This happens when the switch closes the socket while the \r
343                          * execution thread was calling lpWPUCompleteOverlappedRequest. */\r
344                         CL_ERROR( IBSP_DBG_WQ, gdbg_lvl,\r
345                                 ("ib_poll_cq returned IB_INVLALID_CQ_HANDLE\n") );\r
346                         goto done;\r
347 \r
348                 default:\r
349                         CL_ERROR( IBSP_DBG_WQ, gdbg_lvl,\r
350                                 ("ib_poll_cq failed returned %s\n", ib_get_err_str( status )) );\r
351                         break;\r
352                 }\r
353 \r
354 #ifdef _DEBUG_\r
355                 comp_count = 0;\r
356 #endif\r
357 \r
358                 /* We have some completions. */\r
359                 cb_idx = 0;\r
360                 while( done_wclist )\r
361                 {\r
362 #ifdef _DEBUG_\r
363                         comp_count++;\r
364 #endif\r
365                         complete_wq( done_wclist, &info[cb_idx++] );\r
366 \r
367                         done_wclist = done_wclist->p_next;\r
368                 }\r
369 \r
370                 while( cb_idx-- )\r
371                 {\r
372                         int error;\r
373                         int ret;\r
374 \r
375                         if( info[cb_idx].p_ov )\r
376                         {\r
377                                 ret = g_ibsp.up_call_table.lpWPUCompleteOverlappedRequest(\r
378                                         info[cb_idx].socket, info[cb_idx].p_ov,\r
379                                         info[cb_idx].p_ov->OffsetHigh,\r
380                                         (DWORD)info[cb_idx].p_ov->InternalHigh, &error );\r
381                                 if( ret != 0 )\r
382                                 {\r
383                                         IBSP_ERROR( ("WPUCompleteOverlappedRequest for ov=%p "\r
384                                                 "returned %d err %d\n", info[cb_idx].p_ov, ret, error) );\r
385                                 }\r
386                         }\r
387 \r
388                         cl_atomic_dec( info[cb_idx].p_io_cnt );\r
389                 }\r
390 \r
391 #ifdef _DEBUG_\r
392                 if( comp_count > g_ibsp.max_comp_count )\r
393                 {\r
394                         g_ibsp.max_comp_count = comp_count;\r
395                 }\r
396 #endif\r
397         } while( !free_wclist );\r
398 \r
399         status = ib_rearm_cq( cq_tinfo->cq, FALSE );\r
400         if( status != IB_SUCCESS )\r
401         {\r
402                 CL_ERROR( IBSP_DBG_WQ, gdbg_lvl,\r
403                         ("ib_rearm_cq returned %s)\n", ib_get_err_str( status )) );\r
404         }\r
405 \r
406 done:\r
407 \r
408 #ifdef _DEBUG_\r
409         fzprint(("%s():%d:0x%x:0x%x: overlap_h0_count=%d overlap_h1_count=%d\n",\r
410                          __FUNCTION__,\r
411                          __LINE__, GetCurrentProcessId(),\r
412                          GetCurrentThreadId(), g_ibsp.overlap_h0_count, g_ibsp.overlap_h1_count));\r
413 #endif\r
414 \r
415         CL_EXIT( IBSP_DBG_WQ, gdbg_lvl );\r
416 }\r
417 \r
418 \r
419 /* IB completion thread */\r
420 static DWORD WINAPI\r
421 ib_cq_thread(\r
422                                         LPVOID                                          lpParameter )\r
423 {\r
424         struct cq_thread_info *cq_tinfo = (struct cq_thread_info *)lpParameter;\r
425         cl_status_t cl_status;\r
426 \r
427         CL_ENTER( IBSP_DBG_HW, gdbg_lvl );\r
428 \r
429 \r
430         fzprint(("%s():%d:0x%x:0x%x: cq_tinfo=0x%p\n", __FUNCTION__,\r
431                          __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), cq_tinfo));\r
432 \r
433         do\r
434         {\r
435                 cl_status = cl_waitobj_wait_on( cq_tinfo->cq_waitobj, EVENT_NO_TIMEOUT, TRUE );\r
436                 if( cl_status != CL_SUCCESS )\r
437                 {\r
438                         CL_ERROR( IBSP_DBG_EP, gdbg_lvl,\r
439                                 ("cl_waitobj_wait_on() (%d)\n", cl_status) );\r
440                 }\r
441 \r
442                 /* \r
443                  * TODO: By rearanging thread creation and cq creation, this check\r
444                  * may be eliminated.\r
445                  */\r
446                 if( cq_tinfo->cq != NULL )\r
447                 {\r
448                         fzprint(("%s():%d:0x%x:0x%x: Calling ib_cq_comp().\n", __FUNCTION__,\r
449                                          __LINE__, GetCurrentProcessId(), GetCurrentThreadId()));\r
450 \r
451                         ib_cq_comp( cq_tinfo );\r
452                         fzprint(("%s():%d:0x%x:0x%x: Done calling ib_cq_comp().\n", __FUNCTION__,\r
453                                          __LINE__, GetCurrentProcessId(), GetCurrentThreadId()));\r
454                 }\r
455 \r
456         } while( (cq_tinfo->ib_cq_thread_exit_wanted != TRUE) ||\r
457                         cl_qlist_count( &cq_tinfo->done_wr_list ) );\r
458 \r
459         cl_status = cl_waitobj_destroy( cq_tinfo->cq_waitobj );\r
460         if( cl_status != CL_SUCCESS )\r
461         {\r
462                 CL_ERROR( IBSP_DBG_EP, gdbg_lvl, ("cl_waitobj_destroy() (%d)\n", cl_status) );\r
463         }\r
464         HeapFree( g_ibsp.heap, 0, cq_tinfo );\r
465 \r
466         /* No special exit code, even on errors. */\r
467         CL_EXIT( IBSP_DBG_HW, gdbg_lvl );\r
468         ExitThread( 0 );\r
469 }\r
470 \r
471 \r
472 static struct cq_thread_info *\r
473 ib_alloc_cq_tinfo(\r
474                                         struct ibsp_hca                         *hca )\r
475 {\r
476         struct cq_thread_info *cq_tinfo = NULL;\r
477         ib_cq_create_t cq_create;\r
478         ib_api_status_t status;\r
479         cl_status_t cl_status;\r
480         int error;\r
481 \r
482         CL_ENTER( IBSP_DBG_HW, gdbg_lvl );\r
483 \r
484         cq_tinfo = HeapAlloc( g_ibsp.heap, HEAP_ZERO_MEMORY, sizeof(struct cq_thread_info) );\r
485 \r
486         if( cq_tinfo == NULL )\r
487         {\r
488                 CL_ERROR( IBSP_DBG_EP, gdbg_lvl, ("HeapAlloc() Failed.\n") );\r
489                 error = TRUE;\r
490                 goto done;\r
491         }\r
492 \r
493         cl_status = cl_waitobj_create( FALSE, &cq_tinfo->cq_waitobj );\r
494         if( cl_status != CL_SUCCESS )\r
495         {\r
496                 cq_tinfo->cq_waitobj = NULL;\r
497                 CL_ERROR( IBSP_DBG_EP, gdbg_lvl, ("cl_waitobj_create() (%d)\n", cl_status) );\r
498                 error = TRUE;\r
499                 goto done;\r
500         }\r
501 \r
502         cq_tinfo->hca = hca;\r
503         cq_tinfo->ib_cq_thread_exit_wanted = FALSE;\r
504 \r
505         /* Create a cleanup thread */\r
506         cq_tinfo->ib_cq_thread = CreateThread( NULL, 0, ib_cq_thread, cq_tinfo, 0, (LPDWORD)&cq_tinfo->ib_cq_thread_id );\r
507 \r
508         if( cq_tinfo->ib_cq_thread == NULL )\r
509         {\r
510                 CL_ERROR( IBSP_DBG_HW, gdbg_lvl, ("CreateThread failed.") );\r
511                 error = TRUE;\r
512                 goto done;\r
513         }\r
514 \r
515         STAT_INC( thread_num );\r
516 \r
517         /* Completion queue */\r
518         cq_create.size = IB_CQ_SIZE;\r
519 \r
520         cq_create.pfn_comp_cb = NULL;\r
521         cq_create.h_wait_obj = cq_tinfo->cq_waitobj;\r
522 \r
523         status = ib_create_cq( hca->hca_handle, &cq_create, cq_tinfo,   /* context */\r
524                 NULL,   /* async handler */\r
525                 &cq_tinfo->cq );\r
526         if( status )\r
527         {\r
528                 CL_ERROR( IBSP_DBG_EP, gdbg_lvl, ("ib_create_cq failed (%d)\n", status) );\r
529                 error = TRUE;\r
530                 goto done;\r
531         }\r
532 \r
533         STAT_INC( cq_num );\r
534 \r
535         status = ib_rearm_cq( cq_tinfo->cq, FALSE );\r
536         if( status )\r
537         {\r
538                 CL_ERROR( IBSP_DBG_EP, gdbg_lvl, ("ib_rearm_cq failed (%d)\n", status) );\r
539                 error = TRUE;\r
540                 goto done;\r
541         }\r
542 \r
543         cl_spinlock_init( &cq_tinfo->wr_mutex );\r
544         cl_qlist_init( &cq_tinfo->done_wr_list );\r
545         cq_tinfo->cqe_size = IB_CQ_SIZE;\r
546 \r
547         /* Only one CQ per HCA now */\r
548         hca->cq_tinfo = cq_tinfo;\r
549 \r
550         error = FALSE;\r
551 \r
552 done:\r
553         if( error == TRUE )\r
554         {\r
555                 ib_destroy_cq_tinfo( cq_tinfo );\r
556                 cq_tinfo = NULL;\r
557         }\r
558 \r
559         CL_EXIT( IBSP_DBG_HW, gdbg_lvl );\r
560 \r
561         return (cq_tinfo);\r
562 }\r
563 \r
564 \r
565 static void\r
566 ib_destroy_cq_tinfo(\r
567                                         struct cq_thread_info           *cq_tinfo )\r
568 {\r
569         ib_wc_t wclist;\r
570         ib_wc_t *free_wclist;\r
571         ib_wc_t *done_wclist;\r
572         ib_api_status_t status;\r
573         HANDLE h_cq_thread;\r
574 \r
575         CL_ENTER( IBSP_DBG_HW, gdbg_lvl );\r
576 \r
577         if( cq_tinfo == NULL )\r
578         {\r
579                 return;\r
580         }\r
581 \r
582         if( cq_tinfo->cq )\r
583         {\r
584                 wclist.p_next = NULL;\r
585                 free_wclist = &wclist;\r
586 \r
587                 while( ib_poll_cq( cq_tinfo->cq, &free_wclist, &done_wclist ) == IB_SUCCESS )\r
588                 {\r
589                         CL_TRACE( IBSP_DBG_WQ, gdbg_lvl, ("%s():%d:0x%x:0x%x: free=%p, done=%p\n",\r
590                                 __FUNCTION__,\r
591                                 __LINE__, GetCurrentProcessId(),\r
592                                 GetCurrentThreadId(),\r
593                                 free_wclist, done_wclist) );\r
594                 }\r
595 \r
596 \r
597                 CL_TRACE( IBSP_DBG_WQ, gdbg_lvl, ("%s():%d:0x%x:0x%x: ib_destroy_cq() start..\n",\r
598                         __FUNCTION__,\r
599                         __LINE__, GetCurrentProcessId(),\r
600                         GetCurrentThreadId()) );\r
601 \r
602                 /*\r
603                  * Called from cleanup thread, okay to block.\r
604                  */\r
605                 status = ib_destroy_cq( cq_tinfo->cq, ib_sync_destroy );\r
606                 if( status )\r
607                 {\r
608                         CL_ERROR( IBSP_DBG_EP, gdbg_lvl, ("ib_destroy_cq failed (%d)\n", status) );\r
609                 }\r
610                 else\r
611                 {\r
612                         CL_TRACE( IBSP_DBG_WQ, gdbg_lvl,\r
613                                 ("%s():%d:0x%x:0x%x: ib_destroy_cq() finished.\n", __FUNCTION__,\r
614                                 __LINE__, GetCurrentProcessId(), GetCurrentThreadId()) );\r
615 \r
616                         cq_tinfo->cq = NULL;\r
617 \r
618                         STAT_DEC( cq_num );\r
619                 }\r
620         }\r
621 \r
622         /* Currently only 1 CQ per HCA */\r
623         cq_tinfo->hca = NULL;\r
624 \r
625         if( cq_tinfo->ib_cq_thread )\r
626         {\r
627                 /* ib_cq_thread() will release the cq_tinfo before exit. Don't\r
628                    reference cq_tinfo after signaling  */\r
629                 h_cq_thread = cq_tinfo->ib_cq_thread;\r
630                 cq_tinfo->ib_cq_thread = NULL;\r
631 \r
632                 cq_tinfo->ib_cq_thread_exit_wanted = TRUE;\r
633                 cl_waitobj_signal( cq_tinfo->cq_waitobj );\r
634 \r
635                 /* Wait for ib_cq_thread to die, if we are not running on it */\r
636                 if( GetCurrentThreadId() != cq_tinfo->ib_cq_thread_id )\r
637                 {\r
638                         fzprint(("%s():%d:0x%x:0x%x: Waiting for ib_cq_thread=0x%x to die\n",\r
639                                          __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(),\r
640                                          cq_tinfo->ib_cq_thread_id ));\r
641                         if( WaitForSingleObject( h_cq_thread, INFINITE ) != WAIT_OBJECT_0 )\r
642                         {\r
643                                 CL_ERROR( IBSP_DBG_CM, gdbg_lvl, ("WaitForSingleObject failed\n") );\r
644                         }\r
645                         else\r
646                         {\r
647                                 STAT_DEC( thread_num );\r
648                         }\r
649                 }\r
650                 else\r
651                 {\r
652                         fzprint(("%s():%d:0x%x:0x%x: Currently on ib_cq_thread.\n", __FUNCTION__,\r
653                                          __LINE__, GetCurrentProcessId(), GetCurrentThreadId()));\r
654                         STAT_DEC( thread_num );\r
655                 }\r
656                 CloseHandle( h_cq_thread );\r
657         }\r
658         else\r
659         {\r
660                 /* There was no thread created, destroy cq_waitobj and\r
661                    free memory */\r
662                 if( cq_tinfo->cq_waitobj )\r
663                 {\r
664                         cl_waitobj_destroy( cq_tinfo->cq_waitobj );\r
665                         cq_tinfo->cq_waitobj = NULL;\r
666                 }\r
667                 HeapFree( g_ibsp.heap, 0, cq_tinfo );\r
668         }\r
669 \r
670         CL_EXIT( IBSP_DBG_HW, gdbg_lvl );\r
671 }\r
672 \r
673 \r
674 static struct cq_thread_info *\r
675 ib_acquire_cq_tinfo(\r
676                                         struct ibsp_hca                         *hca )\r
677 {\r
678         struct cq_thread_info *cq_tinfo = NULL;\r
679         uint32_t current_cqe_size;\r
680 \r
681         CL_ENTER( IBSP_DBG_HW, gdbg_lvl );\r
682 \r
683         /* \r
684          * TODO: If future implementations require more than 1 cq_tinfo per HCA, then\r
685          *   search HCA cq_tinfo list for optimal cq_tinfo \r
686          */\r
687         if( hca->cq_tinfo == NULL )\r
688         {\r
689                 cq_tinfo = ib_alloc_cq_tinfo( hca );\r
690                 if( cq_tinfo == NULL )\r
691                 {\r
692                         CL_ERROR( IBSP_DBG_CM, gdbg_lvl, ("ib_alloc_cq_tinfo() failed\n") );\r
693                         return (NULL);\r
694                 }\r
695         }\r
696         else\r
697         {\r
698                 cq_tinfo = hca->cq_tinfo;\r
699         }\r
700 \r
701         CL_ASSERT( cq_tinfo != NULL );\r
702 \r
703         current_cqe_size = cq_tinfo->qp_count * IB_CQ_SIZE;\r
704 \r
705         cl_atomic_inc( &cq_tinfo->qp_count );\r
706 \r
707         if( cq_tinfo->cqe_size < current_cqe_size )\r
708         {\r
709                 ib_api_status_t status;\r
710                 status = ib_modify_cq( cq_tinfo->cq, &current_cqe_size );\r
711                 if( status )\r
712                 {\r
713                         /* \r
714                          * TODO: This could mean we are out of cqe and need to have\r
715                          * more than one cq per HCA in the future.\r
716                          */\r
717                         cl_atomic_dec( &cq_tinfo->qp_count );\r
718                         CL_EXIT_ERROR( IBSP_DBG_EP, gdbg_lvl,\r
719                                 ("ib_modify_cq() failed. (%d)\n", status) );\r
720                         return (NULL);\r
721                 }\r
722                 else\r
723                 {\r
724                         cq_tinfo->cqe_size = current_cqe_size;\r
725                         fzprint(("%s():%d:0x%x:0x%x: New cq size=%d.\n",\r
726                                          __FUNCTION__,\r
727                                          __LINE__, GetCurrentProcessId(),\r
728                                          GetCurrentThreadId(), cq_tinfo->cqe_size));\r
729 \r
730                 }\r
731         }\r
732 \r
733         CL_EXIT( IBSP_DBG_HW, gdbg_lvl );\r
734 \r
735         return (cq_tinfo);\r
736 }\r
737 \r
738 void\r
739 ib_release_cq_tinfo(\r
740                                         struct cq_thread_info           *cq_tinfo )\r
741 {\r
742         CL_ENTER( IBSP_DBG_HW, gdbg_lvl );\r
743 \r
744         cl_atomic_dec( &cq_tinfo->qp_count );\r
745 \r
746         /* TODO: downsize the cq  */\r
747 \r
748         CL_EXIT( IBSP_DBG_HW, gdbg_lvl );\r
749 }\r
750 \r
751 \r
752 /* Release IB ressources. */\r
753 void\r
754 ib_release(void)\r
755 {\r
756         cl_fmap_item_t                  *p_item;\r
757 \r
758         CL_ENTER( IBSP_DBG_HW, gdbg_lvl );\r
759 \r
760         if( g_ibsp.al_handle )\r
761         {\r
762                 cl_list_item_t *item;\r
763                 ib_api_status_t status;\r
764 \r
765                 unregister_pnp();\r
766 \r
767                 if( g_ibsp.ib_cleanup_thread )\r
768                 {\r
769                         /* Let thread know it's okay to exit after resources are freed */\r
770                         g_ibsp.ib_cleanup_thread_exit_wanted = TRUE;\r
771                         SetEvent( g_ibsp.ib_cleanup_event );\r
772 \r
773                         fzprint(("%s():%d:0x%x:0x%x: Waiting for ib_cleanup_thread to die.\n",\r
774                                          __FUNCTION__, __LINE__, GetCurrentProcessId(),\r
775                                          GetCurrentThreadId()));\r
776 \r
777                         /* Wait for ib_cleanup_thread to die */\r
778                         if( WaitForSingleObject( g_ibsp.ib_cleanup_thread, INFINITE ) != WAIT_OBJECT_0 )\r
779                         {\r
780                                 CL_ERROR( IBSP_DBG_CM, gdbg_lvl, ("WaitForSingleObject failed\n") );\r
781                         }\r
782                         else\r
783                         {\r
784                                 STAT_DEC( thread_num );\r
785                         }\r
786 \r
787                         fzprint(("%s():%d:0x%x:0x%x: ib_cleanup_thread exited.\n", __FUNCTION__,\r
788                                          __LINE__, GetCurrentProcessId(), GetCurrentThreadId()));\r
789                         CloseHandle( g_ibsp.ib_cleanup_thread );\r
790                         g_ibsp.ib_cleanup_thread = NULL;\r
791                 }\r
792 \r
793                 if( g_ibsp.ib_cleanup_event )\r
794                 {\r
795                         CloseHandle( g_ibsp.ib_cleanup_event );\r
796                         g_ibsp.ib_cleanup_event = NULL;\r
797                 }\r
798 \r
799                 while( (item = cl_qlist_head( &g_ibsp.hca_list )) != cl_qlist_end( &g_ibsp.hca_list ) )\r
800                 {\r
801                         struct ibsp_hca *hca = PARENT_STRUCT(item, struct ibsp_hca, item);\r
802 \r
803                         if( hca->cq_tinfo )\r
804                         {\r
805                                 CL_ASSERT( hca->cq_tinfo->qp_count == 0 );\r
806                                 ib_destroy_cq_tinfo( hca->cq_tinfo );\r
807                         }\r
808 \r
809                         pnp_ca_remove( hca );\r
810                 }\r
811 \r
812                 fzprint(("%s():%d:0x%x:0x%x: Calling ib_close_al...\n", __FUNCTION__,\r
813                                  __LINE__, GetCurrentProcessId(), GetCurrentThreadId()));\r
814 \r
815                 status = ib_close_al( g_ibsp.al_handle );\r
816 \r
817                 fzprint(("%s():%d:0x%x:0x%x: Done calling ib_close_al, status=%d.\n",\r
818                                  __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(),\r
819                                  status));\r
820                 if( status != IB_SUCCESS )\r
821                 {\r
822                         CL_ERROR( IBSP_DBG_HW, gdbg_lvl, ("ib_close_al failed (%d)\n", status) );\r
823                 }\r
824                 else\r
825                 {\r
826                         CL_TRACE( IBSP_DBG_HW, gdbg_lvl, ("ib_close_al success\n") );\r
827                         STAT_DEC( al_num );\r
828                 }\r
829                 g_ibsp.al_handle = NULL;\r
830         }\r
831 \r
832         for( p_item = cl_fmap_head( &g_ibsp.ip_map );\r
833                 p_item != cl_fmap_end( &g_ibsp.ip_map );\r
834                 p_item = cl_fmap_head( &g_ibsp.ip_map ) )\r
835         {\r
836                 cl_fmap_remove_item( &g_ibsp.ip_map, p_item );\r
837 \r
838                 HeapFree( g_ibsp.heap, 0,\r
839                         PARENT_STRUCT(p_item, struct ibsp_ip_addr, item) );\r
840         }\r
841 \r
842         CL_EXIT( IBSP_DBG_HW, gdbg_lvl );\r
843 }\r
844 \r
845 \r
846 /* IP notify thread */\r
847 static DWORD WINAPI\r
848 ib_cleanup_thread(\r
849                                         LPVOID                                          lpParameter )\r
850 {\r
851         cl_list_item_t *socket_item = NULL;\r
852 \r
853         CL_ENTER( IBSP_DBG_HW, gdbg_lvl );\r
854 \r
855         UNUSED_PARAM( lpParameter );\r
856 \r
857         while( !g_ibsp.ib_cleanup_thread_exit_wanted ||\r
858                 cl_qlist_count( &g_ibsp.socket_info_list ) )\r
859         {\r
860                 if( g_ibsp.ib_cleanup_thread_exit_wanted == FALSE )\r
861                 {\r
862                         if( WaitForSingleObject( g_ibsp.ib_cleanup_event, INFINITE ) != WAIT_OBJECT_0 )\r
863                         {\r
864                                 CL_ERROR( IBSP_DBG_CM, gdbg_lvl, ("WaitForSingleObject failed\n") );\r
865                         }\r
866                         ResetEvent( g_ibsp.ib_cleanup_event );\r
867                 }\r
868                 else\r
869                 {\r
870                         fzprint(("%s():%d:0x%x:0x%x: socket_info_list cnt=%d\n", __FUNCTION__,\r
871                                                 __LINE__, GetCurrentProcessId(),\r
872                                                 GetCurrentThreadId(),\r
873                                                 cl_qlist_count( &g_ibsp.socket_info_list) == 0));\r
874                         Sleep( 1000 );\r
875                 }\r
876 \r
877                 CL_TRACE( IBSP_DBG_WQ, gdbg_lvl, ("%s():%d:0x%x:0x%x: Wakeup\n",\r
878                                                                                  __FUNCTION__,\r
879                                                                                  __LINE__, GetCurrentProcessId(),\r
880                                                                                  GetCurrentThreadId()));\r
881 \r
882                 cl_spinlock_acquire( &g_ibsp.closed_socket_info_mutex );\r
883                 while( (socket_item = cl_qlist_remove_head( &g_ibsp.closed_socket_info_list )) !=\r
884                         cl_qlist_end( &g_ibsp.closed_socket_info_list ) )\r
885                 {\r
886                         struct ibsp_socket_info *socket_info = NULL;\r
887 \r
888                         cl_spinlock_release( &g_ibsp.closed_socket_info_mutex );\r
889 \r
890                         socket_info = PARENT_STRUCT(socket_item, struct ibsp_socket_info, item);\r
891 \r
892 #ifdef _DEBUG_\r
893                         {\r
894                                 uint8_t                 idx, i;\r
895                                 LPOVERLAPPED    lpOverlapped;\r
896 \r
897                                 idx = socket_info->send_idx - (uint8_t)socket_info->send_cnt;\r
898                                 if( idx >= QP_ATTRIB_SQ_DEPTH )\r
899                                         idx += QP_ATTRIB_SQ_DEPTH;\r
900 \r
901                                 for( i = 0; i < socket_info->send_cnt; i++ )\r
902                                 {\r
903                                         lpOverlapped = socket_info->send_wr[idx].lpOverlapped;\r
904                                         fzprint(("%s():%d:0x%x:0x%x: socket=0x%p wr=0x%p overlapped=0x%p Internal=%d InternalHigh=%d hEvent=%d\n",\r
905                                                 __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), socket_info, &socket_info->send_wr[idx], lpOverlapped, lpOverlapped->Internal, lpOverlapped->InternalHigh, lpOverlapped->hEvent));\r
906 \r
907                                         if( ++idx == QP_ATTRIB_SQ_DEPTH )\r
908                                                 idx = 0;\r
909                                 }\r
910 \r
911                                 idx = socket_info->recv_idx - (uint8_t)socket_info->recv_cnt;\r
912                                 if( idx >= QP_ATTRIB_RQ_DEPTH )\r
913                                         idx += QP_ATTRIB_RQ_DEPTH;\r
914 \r
915                                 for( i = 0; i < socket_info->recv_cnt; i++ )\r
916                                 {\r
917                                         lpOverlapped = socket_info->recv_wr[idx].wr.lpOverlapped;\r
918                                         fzprint(("%s():%d:0x%x:0x%x: socket=0x%p wr=0x%p overlapped=0x%p Internal=%d InternalHigh=%d hEvent=%d\n",\r
919                                                 __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), socket_info, &socket_info->recv_wr[idx], lpOverlapped, lpOverlapped->Internal, lpOverlapped->InternalHigh, lpOverlapped->hEvent));\r
920 \r
921                                         if( ++idx == QP_ATTRIB_RQ_DEPTH )\r
922                                                 idx = 0;\r
923                                 }\r
924                         }\r
925 #endif\r
926                         fzprint(("%s():%d:0x%x:0x%x: socket=0x%p\n",\r
927                                          __FUNCTION__,\r
928                                          __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), socket_info));\r
929 \r
930                         wait_cq_drain( socket_info );\r
931 \r
932                         if( socket_info->dup_cnt )\r
933                                 ibsp_dup_overlap_abort( socket_info );\r
934 \r
935                         /* Destroy the switch socket. */\r
936                         if( socket_info->switch_socket != INVALID_SOCKET )\r
937                         {\r
938                                 int ret;\r
939                                 int error;\r
940 \r
941                                 fzprint(("%s():%d:0x%x:0x%x: socket=0x%p calling lpWPUCloseSocketHandle=0x%p\n", __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), socket_info, socket_info->switch_socket));\r
942 \r
943                                 ret = g_ibsp.up_call_table.lpWPUCloseSocketHandle(\r
944                                         socket_info->switch_socket, &error );\r
945                                 if( ret == SOCKET_ERROR )\r
946                                 {\r
947                                         CL_ERROR( IBSP_DBG_EP, gdbg_lvl,\r
948                                                 ("WPUCloseSocketHandle failed: %d\n", error) );\r
949                                 }\r
950                                 else\r
951                                 {\r
952                                         STAT_DEC( wpusocket_num );\r
953                                 }\r
954 \r
955                                 socket_info->switch_socket = INVALID_SOCKET;\r
956                         }\r
957 \r
958                         ib_destroy_socket( socket_info );\r
959 \r
960                         ib_deregister_all_mr( &socket_info->buf_mem_list );\r
961                         free_socket_info( socket_info );\r
962                         cl_spinlock_acquire( &g_ibsp.closed_socket_info_mutex );\r
963                 }\r
964                 cl_spinlock_release( &g_ibsp.closed_socket_info_mutex );\r
965         }\r
966 \r
967         /* No special exit code, even on errors. */\r
968         CL_EXIT( IBSP_DBG_HW, gdbg_lvl );\r
969         ExitThread( 0 );\r
970 }\r
971 \r
972 \r
973 \r
974 /* Initialize IB ressources. */\r
975 int\r
976 ibsp_initialize(void)\r
977 {\r
978         ib_api_status_t status;\r
979         int ret;\r
980 \r
981         CL_ENTER( IBSP_DBG_HW, gdbg_lvl );\r
982 \r
983         CL_ASSERT( g_ibsp.al_handle == NULL );\r
984         CL_ASSERT( cl_qlist_count( &g_ibsp.hca_list ) == 0 );\r
985 \r
986         /* Open the IB library */\r
987         status = ib_open_al( &g_ibsp.al_handle );\r
988 \r
989         CL_TRACE( IBSP_DBG_HW, gdbg_lvl, ("open is %d %p\n", status, g_ibsp.al_handle) );\r
990 \r
991         if( status != IB_SUCCESS )\r
992         {\r
993                 CL_ERROR( IBSP_DBG_HW, gdbg_lvl, ("ib_open_al failed (%d)\n", status) );\r
994                 ret = WSAEPROVIDERFAILEDINIT;\r
995                 goto done;\r
996         }\r
997 \r
998         STAT_INC( al_num );\r
999 \r
1000         /* Register for PNP events */\r
1001         status = register_pnp();\r
1002         if( status )\r
1003         {\r
1004                 CL_ERROR( IBSP_DBG_HW, gdbg_lvl, ("register_pnp failed (%d)\n", status) );\r
1005                 ret = WSAEPROVIDERFAILEDINIT;\r
1006                 goto done;\r
1007         }\r
1008 \r
1009         /* Populate IP list. */\r
1010         update_all_ip_addrs();\r
1011 \r
1012         /* Create a cleanup event */\r
1013         g_ibsp.ib_cleanup_event = CreateEvent( NULL, TRUE, FALSE, NULL );\r
1014         if( g_ibsp.ib_cleanup_event == NULL )\r
1015         {\r
1016                 CL_ERROR( IBSP_DBG_HW, gdbg_lvl, ("CreateEvent failed."));\r
1017                 ret = WSAEPROVIDERFAILEDINIT;\r
1018                 goto done;\r
1019         }\r
1020 \r
1021         /* Create a cleanup thread */\r
1022         g_ibsp.ib_cleanup_thread = CreateThread( NULL, 0, ib_cleanup_thread, NULL, 0, NULL );\r
1023 \r
1024         if( g_ibsp.ib_cleanup_thread == NULL )\r
1025         {\r
1026                 CL_ERROR( IBSP_DBG_HW, gdbg_lvl, ("CreateThread failed.") );\r
1027                 ret = WSAEPROVIDERFAILEDINIT;\r
1028                 goto done;\r
1029         }\r
1030 \r
1031         STAT_INC( thread_num );\r
1032 \r
1033         ret = 0;\r
1034 done:\r
1035         if( ret )\r
1036         {\r
1037                 /* Free up resources. */\r
1038                 ib_release();\r
1039         }\r
1040 \r
1041         CL_EXIT( IBSP_DBG_HW, gdbg_lvl );\r
1042 \r
1043         return ret;\r
1044 }\r
1045 \r
1046 \r
1047 /* Destroys the infiniband ressources of a socket. */\r
1048 void\r
1049 ib_destroy_socket(\r
1050         IN      OUT                     struct ibsp_socket_info         *socket_info )\r
1051 {\r
1052         ib_api_status_t status;\r
1053 \r
1054         CL_ENTER( IBSP_DBG_EP, gdbg_lvl );\r
1055 \r
1056         if( socket_info->qp )\r
1057         {\r
1058                 status = ib_destroy_qp( socket_info->qp, ib_sync_destroy );\r
1059                 if( status )\r
1060                 {\r
1061                         CL_ERROR( IBSP_DBG_EP, gdbg_lvl, ("ib_destroy_qp failed (%d)\n", status) );\r
1062                 }\r
1063                 else\r
1064                 {\r
1065                         CL_TRACE( IBSP_DBG_WQ, gdbg_lvl,\r
1066                                 ("%s():%d:0x%x:0x%x: ib_destroy_qp() finished\n", __FUNCTION__,\r
1067                                 __LINE__, GetCurrentProcessId(), GetCurrentThreadId()) );\r
1068 \r
1069                         socket_info->qp = NULL;\r
1070 \r
1071                         STAT_DEC( qp_num );\r
1072 \r
1073                         ib_release_cq_tinfo( socket_info->cq_tinfo );\r
1074                 }\r
1075         }\r
1076 \r
1077         CL_EXIT( IBSP_DBG_EP, gdbg_lvl );\r
1078 }\r
1079 \r
1080 \r
1081 /*\r
1082  * Creates the necessary IB ressources for a socket\r
1083  */\r
1084 int\r
1085 ib_create_socket(\r
1086         IN      OUT                     struct ibsp_socket_info         *socket_info)\r
1087 {\r
1088         struct cq_thread_info   *cq_tinfo;\r
1089         ib_qp_create_t                  qp_create;\r
1090         ib_api_status_t                 status;\r
1091         int                                             ret;\r
1092         struct ibsp_hca                 *hca;\r
1093         ib_qp_attr_t                    qp_attr;\r
1094 \r
1095         CL_ENTER( IBSP_DBG_EP, gdbg_lvl );\r
1096 \r
1097         CL_ASSERT( socket_info != NULL );\r
1098         CL_ASSERT( socket_info->port != NULL );\r
1099         CL_ASSERT( socket_info->qp == NULL );\r
1100 \r
1101         hca = socket_info->port->hca;\r
1102         socket_info->hca_pd = hca->pd;\r
1103 \r
1104         /* Get the completion queue and thread info for this socket */\r
1105         cq_tinfo = ib_acquire_cq_tinfo( hca );\r
1106         if( cq_tinfo == NULL )\r
1107         {\r
1108                 CL_ERROR( IBSP_DBG_EP, gdbg_lvl, ("ib_acquire_cq_tinfo failed\n") );\r
1109                 ret = WSAEPROVIDERFAILEDINIT;\r
1110                 goto done;\r
1111         }\r
1112         socket_info->cq_tinfo = cq_tinfo;\r
1113 \r
1114         /* Queue pair */\r
1115         qp_create.qp_type = IB_QPT_RELIABLE_CONN;\r
1116         qp_create.sq_depth = QP_ATTRIB_SQ_DEPTH;\r
1117         qp_create.rq_depth = QP_ATTRIB_RQ_DEPTH;\r
1118         qp_create.sq_sge = QP_ATTRIB_SQ_SGE;\r
1119         qp_create.rq_sge = 1;\r
1120         qp_create.h_rq_cq = cq_tinfo->cq;\r
1121         qp_create.h_sq_cq = cq_tinfo->cq;\r
1122         qp_create.sq_signaled = TRUE;\r
1123 \r
1124         status = ib_create_qp( socket_info->hca_pd, &qp_create, socket_info,    /* context */\r
1125                 NULL,   /* async handler */\r
1126                 &socket_info->qp );\r
1127         if( status )\r
1128         {\r
1129                 CL_ERROR( IBSP_DBG_EP, gdbg_lvl, ("ib_create_qp failed (%d)\n", status));\r
1130                 ret = WSAEPROVIDERFAILEDINIT;\r
1131                 goto done;\r
1132         }\r
1133 \r
1134         status = ib_query_qp( socket_info->qp, &qp_attr );\r
1135         if( status == IB_SUCCESS )\r
1136         {\r
1137                 socket_info->max_inline = qp_attr.sq_max_inline;\r
1138         }\r
1139         else\r
1140         {\r
1141                 CL_ERROR( IBSP_DBG_EP, gdbg_lvl,\r
1142                         ("ib_query_qp returned %s\n", ib_get_err_str( status )) );\r
1143                 socket_info->max_inline = 0;\r
1144         }\r
1145 \r
1146         STAT_INC( qp_num );\r
1147 \r
1148         ret = 0;\r
1149 \r
1150   done:\r
1151         if( ret )\r
1152         {\r
1153                 ib_destroy_socket( socket_info );\r
1154         }\r
1155 \r
1156         CL_EXIT( IBSP_DBG_EP, gdbg_lvl );\r
1157 \r
1158         return ret;\r
1159 }\r
1160 \r
1161 \r
1162 void\r
1163 wait_cq_drain(\r
1164         IN      OUT                     struct ibsp_socket_info         *socket_info )\r
1165 {\r
1166         CL_ENTER( IBSP_DBG_EP, gdbg_lvl );\r
1167 \r
1168         if( socket_info->cq_tinfo == NULL )\r
1169         {\r
1170                 CL_EXIT( IBSP_DBG_EP, gdbg_lvl );\r
1171                 return;\r
1172         }\r
1173 \r
1174         /* Wait for the QP to be drained. */\r
1175         while( socket_info->send_cnt || socket_info->recv_cnt )\r
1176         {\r
1177                 fzprint(("%s():%d:0x%x:0x%x: socket=0x%p wr_list_count=%d qp state=%d\n",\r
1178                                  __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(),\r
1179                                  socket_info, cl_qlist_count(&socket_info->wr_list)));\r
1180 \r
1181                 Sleep(100);\r
1182         }\r
1183 \r
1184         CL_EXIT( IBSP_DBG_EP, gdbg_lvl );\r
1185 }\r
1186 \r
1187 \r
1188 void\r
1189 ibsp_dup_overlap_abort(\r
1190         IN      OUT                     struct ibsp_socket_info         *socket_info )\r
1191 {\r
1192         LPWSAOVERLAPPED lpOverlapped = NULL;\r
1193         int error;\r
1194         int ret;\r
1195         uint8_t                         idx;\r
1196 \r
1197         CL_ENTER( IBSP_DBG_EP, gdbg_lvl );\r
1198         CL_ASSERT( !socket_info->send_cnt && !socket_info->recv_cnt );\r
1199 \r
1200         /* Browse the list of all posted overlapped structures\r
1201          * to mark them as aborted. */\r
1202         idx = socket_info->dup_idx - (uint8_t)socket_info->dup_cnt;\r
1203         if( idx >= QP_ATTRIB_RQ_DEPTH )\r
1204                 idx += QP_ATTRIB_RQ_DEPTH;\r
1205 \r
1206         while( socket_info->dup_cnt )\r
1207         {\r
1208                 lpOverlapped = socket_info->dup_wr[idx].wr.lpOverlapped;\r
1209 \r
1210                 fzprint(("%s():%d:0x%x:0x%x: socket=0x%p wr=0x%p overlapped=0x%p Internal=%d InternalHigh=%d hEvent=%d\n",\r
1211                         __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), socket_info, &socket_info->dup_wr[idx], lpOverlapped, lpOverlapped->Internal, lpOverlapped->InternalHigh, lpOverlapped->hEvent));\r
1212 \r
1213                 lpOverlapped->OffsetHigh = WSAECONNABORTED;\r
1214                 lpOverlapped->InternalHigh = 0;\r
1215 \r
1216                 if( ((uintptr_t) lpOverlapped->hEvent) & 0x00000001 )\r
1217                 {\r
1218                         /* Indicate this operation is complete. The switch will poll\r
1219                          * with calls to WSPGetOverlappedResult(). */\r
1220 #ifdef _DEBUG_\r
1221                         cl_atomic_dec(&g_ibsp.overlap_h1_comp_count);\r
1222 \r
1223                         fzprint(("%s():%d:0x%x:0x%x: ov=0x%p h0=%d h1=%d h1_c=%d send=%d recv=%d\n",\r
1224                                          __FUNCTION__, __LINE__, GetCurrentProcessId(),\r
1225                                          GetCurrentThreadId(), lpOverlapped,\r
1226                                          g_ibsp.overlap_h0_count, g_ibsp.overlap_h1_count,\r
1227                                          g_ibsp.overlap_h1_comp_count, g_ibsp.send_count, g_ibsp.recv_count));\r
1228 #endif\r
1229 \r
1230                         CL_TRACE(IBSP_DBG_WQ, gdbg_lvl,\r
1231                                          ("%s: set internal overlapped=0x%p Internal=%d OffsetHigh=%d\n",\r
1232                                           __FUNCTION__, lpOverlapped, lpOverlapped->Internal,\r
1233                                           lpOverlapped->OffsetHigh));\r
1234 \r
1235                         lpOverlapped->Internal = 0;\r
1236                 }\r
1237                 else\r
1238                 {\r
1239 #ifdef _DEBUG_\r
1240                         cl_atomic_dec(&g_ibsp.overlap_h0_count);\r
1241 \r
1242 \r
1243                         fzprint(("%s():%d:0x%x:0x%x: ov=0x%p h0=%d h1=%d h1_c=%d send=%d recv=%d\n",\r
1244                                          __FUNCTION__, __LINE__, GetCurrentProcessId(),\r
1245                                          GetCurrentThreadId(), lpOverlapped,\r
1246                                          g_ibsp.overlap_h0_count, g_ibsp.overlap_h1_count,\r
1247                                          g_ibsp.overlap_h1_comp_count, g_ibsp.send_count, g_ibsp.recv_count));\r
1248 #endif\r
1249                         CL_TRACE(IBSP_DBG_WQ, gdbg_lvl,\r
1250                                          ("%s: calls lpWPUCompleteOverlappedRequest, overlapped=0x%p OffsetHigh=%d InternalHigh=%d hEvent=%d\n",\r
1251                                           __FUNCTION__, lpOverlapped, lpOverlapped->OffsetHigh,\r
1252                                           lpOverlapped->InternalHigh, lpOverlapped->hEvent));\r
1253 \r
1254                         ret = g_ibsp.up_call_table.lpWPUCompleteOverlappedRequest\r
1255                                 (socket_info->switch_socket,\r
1256                                  lpOverlapped,\r
1257                                  lpOverlapped->OffsetHigh, (DWORD) lpOverlapped->InternalHigh, &error);\r
1258 \r
1259                         if( ret != 0 )\r
1260                         {\r
1261                                 CL_ERROR(IBSP_DBG_EP, gdbg_lvl,\r
1262                                                  ("lpWPUCompleteOverlappedRequest failed with %d/%d\n", ret,\r
1263                                                   error));\r
1264                         }\r
1265                 }\r
1266                 cl_atomic_dec( &socket_info->dup_cnt );\r
1267         }\r
1268 \r
1269         CL_EXIT( IBSP_DBG_EP, gdbg_lvl );\r
1270 }\r
1271 \r
1272 \r
1273 /* Closes a connection and release its ressources. */\r
1274 void\r
1275 shutdown_and_destroy_socket_info(\r
1276         IN      OUT                     struct ibsp_socket_info         *socket_info,\r
1277         IN                              int                                                     old_state )\r
1278 {\r
1279         CL_ENTER( IBSP_DBG_EP, gdbg_lvl );\r
1280 \r
1281         if( socket_info->duplicate.mmap_handle )\r
1282         {\r
1283                 CloseHandle( socket_info->duplicate.mmap_handle );\r
1284                 socket_info->duplicate.mmap_handle = NULL;\r
1285         }\r
1286 \r
1287         if( socket_info->info.listen.handle )\r
1288         {\r
1289                 /* Stop listening and reject queued connections. */\r
1290                 ib_listen_cancel( socket_info );\r
1291         }\r
1292 \r
1293         switch( old_state )\r
1294         {\r
1295         case IBSP_CLOSING:\r
1296                 /* This function has already been called. Should not happen. */\r
1297                 CL_ERROR( IBSP_DBG_EP, gdbg_lvl,\r
1298                                  ("shutdown_and_destroy_socket_info already in closing socket_state\n") );\r
1299                 return;\r
1300                 break;\r
1301 \r
1302         case IBSP_CREATE:\r
1303                 /* Nothing to do. */\r
1304                 break;\r
1305 \r
1306         case IBSP_CONNECT:\r
1307                 // TODO\r
1308                 break;\r
1309 \r
1310         case IBSP_ACCEPT:\r
1311                 // TODO\r
1312                 break;\r
1313 \r
1314         case IBSP_LISTEN:\r
1315                 break;\r
1316 \r
1317         case IBSP_CONNECTED:\r
1318                 {\r
1319                         struct disconnect_reason reason;\r
1320                         memset( &reason, 0, sizeof(reason) );\r
1321                         reason.type = DISC_SHUTDOWN;\r
1322                         ib_disconnect( socket_info, &reason );\r
1323                 }\r
1324                 break;\r
1325 \r
1326         case IBSP_DISCONNECTED:\r
1327                 /* Nothing to do. */\r
1328                 break;\r
1329         }\r
1330 \r
1331         CL_EXIT( IBSP_DBG_EP, gdbg_lvl );\r
1332 }\r