[WSD] Fix MR caching to properly flush both local and RDMA registrations.
[mirror/winof/.git] / ulp / wsd / user / extensions.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  *\r
4  * This software is available to you under the OpenIB.org BSD license\r
5  * below:\r
6  *\r
7  *     Redistribution and use in source and binary forms, with or\r
8  *     without modification, are permitted provided that the following\r
9  *     conditions are met:\r
10  *\r
11  *      - Redistributions of source code must retain the above\r
12  *        copyright notice, this list of conditions and the following\r
13  *        disclaimer.\r
14  *\r
15  *      - Redistributions in binary form must reproduce the above\r
16  *        copyright notice, this list of conditions and the following\r
17  *        disclaimer in the documentation and/or other materials\r
18  *        provided with the distribution.\r
19  *\r
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
27  * SOFTWARE.\r
28  *\r
29  * $Id$\r
30  */\r
31 \r
32 #include "ibspdll.h"\r
33 \r
34 \r
35 /* Function: IBSPRegisterMemory\r
36  *  Description:\r
37  *    Registers buffer memory\r
38  */\r
39 HANDLE WSPAPI\r
40 IBSPRegisterMemory(\r
41         IN                              SOCKET                                          s,\r
42         IN                              PVOID                                           lpBuffer,\r
43         IN                              DWORD                                           dwBufferLength,\r
44         IN                              DWORD                                           dwFlags,\r
45                 OUT                     LPINT                                           lpErrno )\r
46 {\r
47         struct ibsp_socket_info *socket_info = (struct ibsp_socket_info *)s;\r
48         ib_access_t access_ctrl;\r
49         struct memory_node *node;\r
50 \r
51         IBSP_ENTER( IBSP_DBG_MEM );\r
52 \r
53         fzprint(("%s():%d:0x%x:0x%x: socket=0x%p \n", __FUNCTION__,\r
54                          __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), s));\r
55 \r
56         if( lpBuffer == NULL )\r
57         {\r
58                 IBSP_ERROR_EXIT( ("invalid buffer %p\n", lpBuffer) );\r
59                 *lpErrno = WSAEFAULT;\r
60                 return NULL;\r
61         }\r
62 \r
63         if( dwBufferLength > socket_info->socket_options.max_msg_size )\r
64         {\r
65                 IBSP_ERROR_EXIT( ("invalid buffer length %d\n", dwBufferLength) );\r
66                 *lpErrno = WSAEFAULT;\r
67                 return NULL;\r
68         }\r
69 \r
70         switch( dwFlags )\r
71         {\r
72         case MEM_READ:\r
73                 access_ctrl = 0;\r
74                 break;\r
75 \r
76         case MEM_WRITE:\r
77                 access_ctrl = IB_AC_LOCAL_WRITE;\r
78                 break;\r
79 \r
80         case MEM_READWRITE:\r
81                 access_ctrl = IB_AC_LOCAL_WRITE;\r
82                 break;\r
83 \r
84         default:\r
85                 IBSP_ERROR_EXIT( ("invalid flags %x\n", dwFlags) );\r
86                 *lpErrno = WSAEINVAL;\r
87                 return NULL;\r
88         }\r
89 \r
90         node = ibsp_reg_mem( socket_info, socket_info->hca_pd,\r
91                 lpBuffer, dwBufferLength, access_ctrl, lpErrno );\r
92 \r
93         fzprint(("%s():%d:0x%x:0x%x: registering MEM from %p to %p, len %d, handle %p\n",\r
94                          __FUNCTION__, __LINE__, GetCurrentProcessId(), GetCurrentThreadId(),\r
95                          lpBuffer, (unsigned char *)lpBuffer + dwBufferLength, dwBufferLength, node));\r
96 \r
97 \r
98         if( node == NULL )\r
99         {\r
100                 IBSP_ERROR_EXIT(\r
101                         ("ibsp_reg_mem failed (pd=%p)\n", socket_info->hca_pd) );\r
102                 *lpErrno = WSAENOBUFS;\r
103         }\r
104         else\r
105         {\r
106                 IBSP_TRACE_EXIT( IBSP_DBG_MEM, ("returning node %p\n", node) );\r
107                 *lpErrno = 0;\r
108         }\r
109 \r
110         IBSP_EXIT( IBSP_DBG_MEM );\r
111 \r
112         return (HANDLE) node;\r
113 }\r
114 \r
115 /* Function: IBSPDeregisterMemory\r
116  *  Description:\r
117  *    This is our provider's DeregisterMemory function.\r
118  */\r
119 int WSPAPI\r
120 IBSPDeregisterMemory(\r
121         IN                              SOCKET                                          s,\r
122         IN                              HANDLE                                          handle,\r
123                 OUT                     LPINT                                           lpErrno )\r
124 {\r
125         struct memory_node *node = handle;\r
126         struct ibsp_socket_info *socket_info = (struct ibsp_socket_info *)s;\r
127         int ret;\r
128 \r
129         IBSP_ENTER( IBSP_DBG_MEM );\r
130 \r
131         fzprint(("%s():%d:0x%x:0x%x: handle=0x%p socket=0x%p \n", __FUNCTION__,\r
132                          __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), handle, s));\r
133 \r
134         if( s == INVALID_SOCKET )\r
135         {\r
136                 IBSP_ERROR( ("invalid socket handle %x\n", s) );\r
137                 *lpErrno = WSAENOTSOCK;\r
138                 return SOCKET_ERROR;\r
139         }\r
140 \r
141         ret = ibsp_dereg_mem( socket_info, node, lpErrno );\r
142 \r
143         fzprint(("%s():%d:0x%x:0x%x: unregistering MEM %p, mr_num=%d, ret=%d\n",\r
144                          __FUNCTION__,\r
145                          __LINE__, GetCurrentProcessId(),\r
146                          GetCurrentThreadId(), node, g_ibsp.mr_num, ret));\r
147 \r
148         IBSP_EXIT( IBSP_DBG_MEM );\r
149         return ret;\r
150 }\r
151 \r
152 /* Function: IBSPRegisterRdmaMemory\r
153  *  Description:\r
154  *    This is our provider's RegisterRdmaMemory function.\r
155 */\r
156 int WSPAPI\r
157 IBSPRegisterRdmaMemory(\r
158         IN                              SOCKET                                          s,\r
159         IN                              PVOID                                           lpBuffer,\r
160         IN                              DWORD                                           dwBufferLength,\r
161         IN                              DWORD                                           dwFlags,\r
162                 OUT                     LPVOID                                          lpRdmaBufferDescriptor,\r
163         IN      OUT                     LPDWORD                                         lpdwDescriptorLength,\r
164                 OUT                     LPINT                                           lpErrno )\r
165 {\r
166         struct memory_node *node;\r
167         struct rdma_memory_desc *desc;\r
168         struct ibsp_socket_info *socket_info = (struct ibsp_socket_info *)s;\r
169         ib_access_t access_ctrl;\r
170         struct ibsp_hca *hca;\r
171 \r
172         IBSP_ENTER( IBSP_DBG_MEM );\r
173 \r
174         fzprint(("%s():%d:0x%x:0x%x: socket=0x%p \n", __FUNCTION__,\r
175                          __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), s));\r
176 \r
177         if( *lpdwDescriptorLength < sizeof(struct rdma_memory_desc) )\r
178         {\r
179                 /* This is the probe from the switch to learn the length of the descriptor. */\r
180                 IBSP_ERROR_EXIT(\r
181                         ("invalid descriptor length %d (usually not an error)\n",\r
182                         *lpdwDescriptorLength) );\r
183                 *lpdwDescriptorLength = sizeof(struct rdma_memory_desc);\r
184                 *lpErrno = WSAEFAULT;\r
185                 return SOCKET_ERROR;\r
186         }\r
187 \r
188         if( lpBuffer == NULL )\r
189         {\r
190                 IBSP_ERROR_EXIT( ("invalid buffer %p\n", lpBuffer) );\r
191                 *lpErrno = WSAEFAULT;\r
192                 return SOCKET_ERROR;\r
193         }\r
194 \r
195         if( dwBufferLength > socket_info->socket_options.max_msg_size )\r
196         {\r
197                 IBSP_ERROR_EXIT( ("invalid buffer length %d\n", dwBufferLength) );\r
198                 *lpErrno = WSAEFAULT;\r
199                 return SOCKET_ERROR;\r
200         }\r
201 \r
202         access_ctrl = IB_AC_LOCAL_WRITE;\r
203 \r
204         switch( dwFlags )\r
205         {\r
206         case MEM_READ:\r
207                 access_ctrl |= IB_AC_RDMA_READ;\r
208                 break;\r
209 \r
210         case MEM_WRITE:\r
211                 access_ctrl |= IB_AC_RDMA_WRITE;\r
212                 break;\r
213 \r
214         case MEM_READWRITE:\r
215                 access_ctrl |= (IB_AC_RDMA_READ | IB_AC_RDMA_WRITE);\r
216                 break;\r
217 \r
218         default:\r
219                 IBSP_ERROR_EXIT( ("invalid flags %x\n", dwFlags) );\r
220                 *lpErrno = WSAEINVAL;\r
221                 return SOCKET_ERROR;\r
222         }\r
223 \r
224         hca = socket_info->port->hca;\r
225 \r
226         /** TODO: Fix locking so we dont' dereference node outside of mutex. */\r
227         node = ibsp_reg_mem( socket_info, hca->pd,\r
228                 lpBuffer, dwBufferLength, access_ctrl, lpErrno );\r
229 \r
230         if( !node )\r
231         {\r
232                 IBSP_ERROR_EXIT( ("ibsp_reg_mem failed %d\n", *lpErrno) );\r
233                 *lpErrno = WSAENOBUFS;\r
234                 return SOCKET_ERROR;\r
235         }\r
236 \r
237         desc = lpRdmaBufferDescriptor;\r
238 \r
239         desc->iova = (uint64_t) (uintptr_t) lpBuffer;\r
240         desc->lkey = node->p_reg->lkey;\r
241         desc->rkey = node->p_reg->rkey;\r
242         desc->node = node;\r
243 \r
244         *lpErrno = 0;\r
245 \r
246         IBSP_TRACE1( IBSP_DBG_MEM,\r
247                 ("Socket %p registered RDMA MEM at %p, len %d, for access %d, "\r
248                 "returning handle %p, rkey %08x\n",\r
249                 s, lpBuffer, dwBufferLength, dwFlags, node, desc->rkey));\r
250 \r
251         IBSP_EXIT( IBSP_DBG_MEM );\r
252 \r
253         return 0;\r
254 }\r
255 \r
256 /* Function: IBSPDeregisterRdmaMemory\r
257  *  Description:\r
258  *    This is our provider's DeregisterRdmaMemory function.\r
259  */\r
260 int WSPAPI\r
261 IBSPDeregisterRdmaMemory(\r
262         IN                              SOCKET                                          s,\r
263         IN                              LPVOID                                          lpRdmaBufferDescriptor,\r
264         IN                              DWORD                                           dwDescriptorLength,\r
265                 OUT                     LPINT                                           lpErrno )\r
266 {\r
267         struct rdma_memory_desc *desc;\r
268         struct ibsp_socket_info *socket_info = (struct ibsp_socket_info *)s;\r
269         int ret;\r
270 \r
271         IBSP_ENTER( IBSP_DBG_MEM );\r
272 \r
273         fzprint(("%s():%d:0x%x:0x%x: socket=0x%p \n", __FUNCTION__,\r
274                          __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), s));\r
275 \r
276         if( s == INVALID_SOCKET )\r
277         {\r
278                 /* Seen in real life with overlap/client test.\r
279                  * The switch closes a socket then calls this. Why? */\r
280                 IBSP_ERROR_EXIT( ("invalid socket handle %x\n", s) );\r
281                 *lpErrno = WSAENOTSOCK;\r
282                 return SOCKET_ERROR;\r
283         }\r
284 \r
285         CL_ASSERT( lpRdmaBufferDescriptor );\r
286 \r
287         if( dwDescriptorLength < sizeof(struct rdma_memory_desc) )\r
288         {\r
289                 IBSP_ERROR_EXIT(\r
290                         ("invalid descriptor length %d)\n", dwDescriptorLength) );\r
291                 *lpErrno = WSAEINVAL;\r
292                 return SOCKET_ERROR;\r
293         }\r
294 \r
295         desc = lpRdmaBufferDescriptor;\r
296 \r
297         ret = ibsp_dereg_mem( socket_info, desc->node, lpErrno );\r
298 \r
299         fzprint(("%s():%d:0x%x:0x%x: Unregistering RDMA MEM %p\n",\r
300                          __FUNCTION__, __LINE__, GetCurrentProcessId(),\r
301                          GetCurrentThreadId(), desc->node));\r
302 \r
303         IBSP_EXIT( IBSP_DBG_MEM );\r
304         return ret;\r
305 }\r
306 \r
307 \r
308 /*\r
309  * Do a RDMA read or write operation since the code for both is very close. \r
310  */\r
311 static int\r
312 do_rdma_op(\r
313         IN                              SOCKET                                          s,\r
314         IN                              LPWSABUFEX                                      lpBuffers,\r
315         IN                              DWORD                                           dwBufferCount,\r
316         IN                              LPVOID                                          lpTargetBufferDescriptor,\r
317         IN                              DWORD                                           dwTargetDescriptorLength,\r
318         IN                              DWORD                                           dwTargetBufferOffset,\r
319         IN                              LPWSAOVERLAPPED                         lpOverlapped,\r
320         IN                              ib_wr_type_t                            wr_type,\r
321                 OUT                     LPINT                                           lpErrno )\r
322 {\r
323         struct ibsp_socket_info *socket_info = (struct ibsp_socket_info *)s;\r
324         ib_api_status_t                 status;\r
325         struct rdma_memory_desc *desc;  /* remote descriptor */\r
326         struct _wr                              *wr;\r
327         ib_send_wr_t                    send_wr;\r
328         ib_local_ds_t                   local_ds[QP_ATTRIB_SQ_SGE];\r
329         DWORD                                   ds_idx;\r
330 \r
331         IBSP_ENTER( IBSP_DBG_IO );\r
332 \r
333         CL_ASSERT( wr_type == WR_RDMA_WRITE || wr_type == WR_RDMA_READ );\r
334 \r
335         cl_spinlock_acquire( &socket_info->mutex );\r
336         switch( socket_info->socket_state )\r
337         {\r
338         case IBSP_CONNECTED:\r
339         case IBSP_DISCONNECTED:\r
340                 break;\r
341 \r
342         default:\r
343                 cl_spinlock_release( &socket_info->mutex );\r
344                 IBSP_ERROR_EXIT(\r
345                         ("Socket is not in connected socket_state state=%s\n",\r
346                         IBSP_SOCKET_STATE_STR( socket_info->socket_state )) );\r
347                 *lpErrno = WSAENOTCONN;\r
348                 return SOCKET_ERROR;\r
349         }\r
350         cl_spinlock_release( &socket_info->mutex );\r
351 \r
352         if( socket_info->qp_error )\r
353         {\r
354                 IBSP_ERROR_EXIT(\r
355                         ("QP is in error state %d\n", socket_info->qp_error) );\r
356                 *lpErrno = socket_info->qp_error;\r
357                 return SOCKET_ERROR;\r
358         }\r
359 \r
360         /* This function only works for that case. */\r
361         if( dwBufferCount > QP_ATTRIB_SQ_SGE )\r
362         {\r
363                 CL_ASSERT( dwBufferCount <= QP_ATTRIB_SQ_SGE );\r
364                 /* TODO - support splitting large requests into multiple RDMA operations. */\r
365                 IBSP_ERROR_EXIT(\r
366                         ("dwBufferCount is greater than %d\n", QP_ATTRIB_SQ_SGE) );\r
367                 *lpErrno = WSAEINVAL;\r
368                 return SOCKET_ERROR;\r
369         }\r
370 \r
371         if( dwTargetDescriptorLength != sizeof(struct rdma_memory_desc) )\r
372         {\r
373                 IBSP_ERROR_EXIT(\r
374                         ("invalid descriptor length %d)\n", dwTargetDescriptorLength) );\r
375                 *lpErrno = WSAEINVAL;\r
376                 return SOCKET_ERROR;\r
377         }\r
378 \r
379         desc = lpTargetBufferDescriptor;\r
380 \r
381         /* The send lock is only used to serialize posting. */\r
382         cl_spinlock_acquire( &socket_info->send_lock );\r
383         if( socket_info->send_cnt == QP_ATTRIB_SQ_DEPTH )\r
384         {\r
385                 /* TODO: queue requests. */\r
386                 cl_spinlock_release( &socket_info->send_lock );\r
387                 IBSP_ERROR_EXIT( ("not enough wr on the free list\n") );\r
388                 *lpErrno = WSAENETDOWN;\r
389                 return SOCKET_ERROR;\r
390         }\r
391 \r
392         wr = &socket_info->send_wr[socket_info->send_idx];\r
393 \r
394         wr->lpOverlapped = lpOverlapped;\r
395         wr->socket_info = socket_info;\r
396 \r
397         /* Format the send work request and post. */\r
398         send_wr.p_next = NULL;\r
399         send_wr.wr_id = (uint64_t)(void* __ptr64)wr;\r
400         send_wr.wr_type = wr_type;\r
401         send_wr.send_opt = 0;\r
402         send_wr.num_ds = dwBufferCount;\r
403         send_wr.ds_array = local_ds;\r
404 \r
405         send_wr.remote_ops.vaddr = desc->iova + dwTargetBufferOffset;\r
406         send_wr.remote_ops.rkey = desc->rkey;\r
407 \r
408         lpOverlapped->InternalHigh = 0;\r
409         for( ds_idx = 0; ds_idx < dwBufferCount; ds_idx++ )\r
410         {\r
411                 local_ds[ds_idx].vaddr = (uint64_t)(void* __ptr64)lpBuffers[ds_idx].buf;\r
412                 local_ds[ds_idx].length = lpBuffers[ds_idx].len;\r
413                 local_ds[ds_idx].lkey =\r
414                         ((struct memory_node*)lpBuffers[ds_idx].handle)->p_reg->lkey;\r
415 \r
416                 lpOverlapped->InternalHigh += lpBuffers[ds_idx].len;\r
417         }\r
418 \r
419         if( wr_type == WR_RDMA_READ )\r
420         {\r
421                 /*\r
422                  * Next send must be fenced since it could indicate that this\r
423                  * RDMA READ is complete.\r
424                  */\r
425                 socket_info->send_opt = IB_SEND_OPT_FENCE;\r
426         }\r
427         else if( lpOverlapped->InternalHigh <= socket_info->max_inline )\r
428         {\r
429                 send_wr.send_opt |= IB_SEND_OPT_INLINE;\r
430         }\r
431 \r
432         /*\r
433          * We must set this now, because the operation could complete\r
434          * before ib_post_send returns.\r
435          */\r
436         lpOverlapped->Internal = WSS_OPERATION_IN_PROGRESS;\r
437 \r
438         cl_atomic_inc( &socket_info->send_cnt );\r
439 \r
440 #ifdef _DEBUG_\r
441         if( lpOverlapped->hEvent == 0 )\r
442         {\r
443                 cl_atomic_inc( &g_ibsp.overlap_h0_count );\r
444         }\r
445         else\r
446         {\r
447                 cl_atomic_inc( &g_ibsp.overlap_h1_count );\r
448                 cl_atomic_inc( &g_ibsp.overlap_h1_comp_count );\r
449         }\r
450 \r
451         fzprint(("%s():%d:0x%x:0x%x: ov=0x%p h0_cnt=%d h1_cnt=%d\n",\r
452                          __FUNCTION__, __LINE__, GetCurrentProcessId(),\r
453                          GetCurrentThreadId(), lpOverlapped,\r
454                          g_ibsp.overlap_h0_count, g_ibsp.overlap_h1_count));\r
455 \r
456 #endif\r
457 \r
458         status = ib_post_send( socket_info->qp, &send_wr, NULL );\r
459 \r
460         if( status == IB_SUCCESS )\r
461         {\r
462                 /* Update the index and wrap as needed */\r
463 #if QP_ATTRIB_SQ_DEPTH == 256 || QP_ATTRIB_SQ_DEPTH == 128 || \\r
464         QP_ATTRIB_SQ_DEPTH == 64 || QP_ATTRIB_SQ_DEPTH == 32 || \\r
465         QP_ATTRIB_SQ_DEPTH == 16 || QP_ATTRIB_SQ_DEPTH == 8\r
466                 socket_info->send_idx++;\r
467                 socket_info->send_idx &= (QP_ATTRIB_SQ_DEPTH - 1);\r
468 #else\r
469                 if( ++socket_info->send_idx == QP_ATTRIB_SQ_DEPTH )\r
470                         socket_info->send_idx = 0;\r
471 #endif\r
472 \r
473                 *lpErrno = WSA_IO_PENDING;\r
474 \r
475                 IBSP_TRACE1( IBSP_DBG_IO,\r
476                         ("Posted RDMA: socket=%p, ov=%p, type=%d, local=%p, len=%d, "\r
477                         "dest=%016I64x, rkey=%08x\n",\r
478                         s, lpOverlapped, wr_type, lpBuffers[0].buf, lpBuffers[0].len,\r
479                         send_wr.remote_ops.vaddr, send_wr.remote_ops.rkey) );\r
480 \r
481                 fzprint(("posted RDMA %p, len=%d, op=%d, mr handle=%p\n",\r
482                                 lpOverlapped, lpBuffers[0].len, wr_type, node));\r
483         }\r
484         else\r
485         {\r
486                 IBSP_ERROR(\r
487                         ("ib_post_send returned %s\n", ib_get_err_str( status )) );\r
488 \r
489 #ifdef _DEBUG_\r
490 \r
491                 if( lpOverlapped->hEvent == 0 )\r
492                 {\r
493                         cl_atomic_dec( &g_ibsp.overlap_h0_count );\r
494                 }\r
495                 else\r
496                 {\r
497                         cl_atomic_dec( &g_ibsp.overlap_h1_count );\r
498                         cl_atomic_dec( &g_ibsp.overlap_h1_comp_count );\r
499                 }\r
500 \r
501                 memset( wr, 0x44, sizeof(struct _wr) );\r
502 #endif\r
503                 cl_atomic_dec( &socket_info->send_cnt );\r
504 \r
505                 *lpErrno = ibal_to_wsa_error( status );\r
506         }\r
507 \r
508         cl_spinlock_release( &socket_info->send_lock );\r
509 \r
510         /* We never complete the operation here. */\r
511         IBSP_EXIT( IBSP_DBG_IO );\r
512         return SOCKET_ERROR;\r
513 }\r
514 \r
515 \r
516 /* Function: IBSPRdmaWrite\r
517  Description:\r
518     This is our provider's RdmaWrite function. When an app calls WSAIoctl\r
519     to request the function pointer to RdmaWrite, we return pointer to this \r
520         function and this function is called by application directly using the function pointer.\r
521 */\r
522 int WSPAPI\r
523 IBSPRdmaWrite(\r
524         IN                              SOCKET                                          s,\r
525         IN                              LPWSABUFEX                                      lpBuffers,\r
526         IN                              DWORD                                           dwBufferCount,\r
527         IN                              LPVOID                                          lpTargetBufferDescriptor,\r
528         IN                              DWORD                                           dwTargetDescriptorLength,\r
529         IN                              DWORD                                           dwTargetBufferOffset,\r
530                 OUT                     LPDWORD                                         lpdwNumberOfBytesWritten,\r
531         IN                              DWORD                                           dwFlags,\r
532         IN                              LPWSAOVERLAPPED                         lpOverlapped,\r
533         IN                              LPWSAOVERLAPPED_COMPLETION_ROUTINE      lpCompletionRoutine,\r
534         IN                              LPWSATHREADID                           lpThreadId,\r
535                 OUT                     LPINT                                           lpErrno )\r
536 {\r
537         int ret;\r
538 \r
539         IBSP_ENTER( IBSP_DBG_IO );\r
540 \r
541         UNUSED_PARAM( lpThreadId );\r
542         UNUSED_PARAM( lpCompletionRoutine );\r
543         UNUSED_PARAM( lpdwNumberOfBytesWritten );\r
544 \r
545         if( s == INVALID_SOCKET )\r
546         {\r
547                 IBSP_ERROR_EXIT( ("invalid socket handle %x\n", s) );\r
548                 *lpErrno = WSAENOTSOCK;\r
549                 return SOCKET_ERROR;\r
550         }\r
551 \r
552         fzprint(("%s():%d:0x%x:0x%x: socket=0x%p overlapped=0x%p\n", __FUNCTION__,\r
553                          __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), s, lpOverlapped));\r
554 \r
555         /* Store the flags for reporting back in IBSPGetOverlappedResult */\r
556         lpOverlapped->Offset = dwFlags;\r
557 \r
558         ret = do_rdma_op( s, lpBuffers, dwBufferCount, lpTargetBufferDescriptor,\r
559                 dwTargetDescriptorLength, dwTargetBufferOffset,\r
560                 lpOverlapped, WR_RDMA_WRITE, lpErrno );\r
561 \r
562         IBSP_EXIT( IBSP_DBG_IO );\r
563 \r
564         return ret;\r
565 }\r
566 \r
567 \r
568 /* Function: IBSPRdmaRead\r
569  Description:\r
570     This is our provider's RdmaRead function. When an app calls WSAIoctl\r
571     to request the function pointer to RdmaRead, we return pointer to this \r
572         function and this function is called by application directly using the function pointer.\r
573 */\r
574 int WSPAPI\r
575 IBSPRdmaRead(\r
576         IN                              SOCKET                                          s,\r
577         IN                              LPWSABUFEX                                      lpBuffers,\r
578         IN                              DWORD                                           dwBufferCount,\r
579         IN                              LPVOID                                          lpTargetBufferDescriptor,\r
580         IN                              DWORD                                           dwTargetDescriptorLength,\r
581         IN                              DWORD                                           dwTargetBufferOffset,\r
582                 OUT                     LPDWORD                                         lpdwNumberOfBytesRead,\r
583         IN                              DWORD                                           dwFlags,\r
584         IN                              LPWSAOVERLAPPED                         lpOverlapped,\r
585         IN                              LPWSAOVERLAPPED_COMPLETION_ROUTINE      lpCompletionRoutine,\r
586         IN                              LPWSATHREADID                           lpThreadId,\r
587                 OUT                     LPINT                                           lpErrno )\r
588 {\r
589         int ret;\r
590 \r
591         IBSP_ENTER( IBSP_DBG_IO );\r
592 \r
593         UNUSED_PARAM( lpThreadId );\r
594         UNUSED_PARAM( lpCompletionRoutine );\r
595         UNUSED_PARAM( lpdwNumberOfBytesRead );\r
596 \r
597         fzprint(("%s():%d:0x%x:0x%x: socket=0x%p overlapped=0x%p \n", __FUNCTION__,\r
598                          __LINE__, GetCurrentProcessId(), GetCurrentThreadId(), s, lpOverlapped));\r
599 \r
600         /* Store the flags for reporting back in IBSPGetOverlappedResult */\r
601         lpOverlapped->Offset = dwFlags;\r
602 \r
603         ret = do_rdma_op( s, lpBuffers, dwBufferCount, lpTargetBufferDescriptor,\r
604                 dwTargetDescriptorLength, dwTargetBufferOffset,\r
605                 lpOverlapped, WR_RDMA_READ, lpErrno );\r
606 \r
607         IBSP_EXIT( IBSP_DBG_IO );\r
608 \r
609         return ret;\r
610 }\r
611 \r
612 \r
613 /* Function: IBSPMemoryRegistrationCacheCallback\r
614  *  Description:\r
615  *   This is our provider's MemoryRegistrationCacheCallback\r
616  *   function. When an app calls WSAIoctl to request the function\r
617  *   pointer to MemoryRegistrationCacheCallback, we return pointer to\r
618  *   this function and this function is called by application directly\r
619  *   using the function pointer.\r
620  */\r
621 int WSPAPI\r
622 IBSPMemoryRegistrationCacheCallback(\r
623         IN                              LPVOID                                          lpvAddress,\r
624         IN                              SIZE_T                                          Size,\r
625                 OUT                     LPINT                                           lpErrno )\r
626 {\r
627         cl_list_item_t          *p_item;\r
628 \r
629         IBSP_ENTER( IBSP_DBG_MEM );\r
630 \r
631         UNUSED_PARAM( lpErrno );\r
632 \r
633         cl_spinlock_acquire( &g_ibsp.hca_mutex );\r
634         for( p_item = cl_qlist_head( &g_ibsp.hca_list );\r
635                 p_item != cl_qlist_end( &g_ibsp.hca_list );\r
636                 p_item = cl_qlist_next( p_item ) )\r
637         {\r
638                 ibsp_hca_flush_mr_cache(\r
639                         PARENT_STRUCT( p_item, struct ibsp_hca, item ), lpvAddress, Size );\r
640         }\r
641         cl_spinlock_release( &g_ibsp.hca_mutex );\r
642 \r
643         IBSP_EXIT( IBSP_DBG_MEM );\r
644         return 0;\r
645 }\r