[WSD] Rename some structures to make the code more readable.
[mirror/winof/.git] / ulp / wsd / user / ibsp_mem.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  * Copyright (c) 2006 Mellanox Technologies.  All rights reserved.\r
4  *\r
5  * This software is available to you under the OpenIB.org BSD license\r
6  * below:\r
7  *\r
8  *     Redistribution and use in source and binary forms, with or\r
9  *     without modification, are permitted provided that the following\r
10  *     conditions are met:\r
11  *\r
12  *      - Redistributions of source code must retain the above\r
13  *        copyright notice, this list of conditions and the following\r
14  *        disclaimer.\r
15  *\r
16  *      - Redistributions in binary form must reproduce the above\r
17  *        copyright notice, this list of conditions and the following\r
18  *        disclaimer in the documentation and/or other materials\r
19  *        provided with the distribution.\r
20  *\r
21  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
22  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
23  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
24  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
25  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
26  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
27  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
28  * SOFTWARE.\r
29  *\r
30  * $Id$\r
31  */\r
32 \r
33 /* Registers a memory region */\r
34 #include "ibspdebug.h"\r
35 #if defined(EVENT_TRACING)\r
36 #ifdef offsetof\r
37 #undef offsetof\r
38 #endif\r
39 #include "ibsp_mem.tmh"\r
40 #endif\r
41 \r
42 #include "ibspdll.h"\r
43 \r
44 \r
45 __forceinline boolean_t\r
46 __check_mr(\r
47         IN                              struct memory_reg                       *p_reg,\r
48         IN                              ib_access_t                                     acl_mask,\r
49         IN                              void                                            *start,\r
50         IN                              size_t                                          len )\r
51 {\r
52         return( (p_reg->type.access_ctrl & acl_mask) == acl_mask &&\r
53                 start >= p_reg->type.vaddr &&\r
54                 ((uintn_t)start) + len <=\r
55                 ((uintn_t)(uint64_t)p_reg->type.vaddr) + p_reg->type.length );\r
56 }\r
57 \r
58 \r
59 /* Find the first registered mr that matches the given region. \r
60  * mem_list is either socket_info->buf_mem_list or socket_info->rdma_mem_list.\r
61  */\r
62 struct memory_node *\r
63 lookup_partial_mr(\r
64         IN                              struct ibsp_socket_info         *s,\r
65         IN                              ib_access_t                                     acl_mask,\r
66         IN                              void                                            *start,\r
67         IN                              size_t                                          len )\r
68 {\r
69         struct memory_node      *p_node;\r
70         cl_list_item_t          *p_item;\r
71 \r
72         IBSP_ENTER( IBSP_DBG_MEM );\r
73 \r
74         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
75 \r
76         for( p_item = cl_qlist_head( &s->mr_list );\r
77                 p_item != cl_qlist_end( &s->mr_list );\r
78                 p_item = cl_qlist_next( p_item ) )\r
79         {\r
80                 p_node = PARENT_STRUCT( p_item, struct memory_node, socket_item );\r
81                 if(p_node->p_reg1 &&\r
82                         __check_mr( p_node->p_reg1, acl_mask, start, len ) )\r
83                 {\r
84                         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
85                         IBSP_EXIT( IBSP_DBG_MEM );\r
86                         return p_node;\r
87                 }\r
88         }\r
89 \r
90         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
91 \r
92         IBSP_PRINT_EXIT(TRACE_LEVEL_INFORMATION, IBSP_DBG_MEM, ("mr not found\n") );\r
93         return NULL;\r
94 }\r
95 \r
96 \r
97 /* Registers a memory region. The memory region might be cached.\r
98  * mem_list is either socket_info->buf_mem_list or hca->rdma_mem_list.\r
99  */\r
100 struct memory_node *\r
101 ibsp_reg_mem(\r
102         IN                              struct ibsp_socket_info         *s,\r
103         IN                              ib_pd_handle_t                          pd,\r
104         IN                              void                                            *start,\r
105         IN                              size_t                                          len,\r
106         IN                              ib_access_t                                     access_ctrl,\r
107                 OUT                     LPINT                                           lpErrno )\r
108 {\r
109         struct memory_node      *p_node;\r
110         struct memory_reg       *p_reg;\r
111         cl_list_item_t          *p_item;\r
112         ib_api_status_t         status;\r
113 \r
114         IBSP_ENTER( IBSP_DBG_MEM );\r
115 \r
116         CL_ASSERT( start != NULL );\r
117         CL_ASSERT( len != 0 );\r
118         CL_ASSERT( (access_ctrl & ~(IB_AC_RDMA_READ | IB_AC_RDMA_WRITE | IB_AC_LOCAL_WRITE)) ==\r
119                           0 );\r
120 \r
121         /* Optimistically allocate a tracking structure. */\r
122         p_node = HeapAlloc( g_ibsp.heap, 0, sizeof(struct memory_node) );\r
123         if( !p_node )\r
124         {\r
125                 IBSP_ERROR_EXIT( ("AllocateOverlappedBuf:HeapAlloc() failed: %d\n",\r
126                         GetLastError()) );\r
127                 *lpErrno = WSAENOBUFS;\r
128                 return NULL;\r
129         }\r
130 \r
131         /* First, try to find a suitable MR */\r
132         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
133 \r
134         /* Find the first registered mr that matches the given region. */\r
135         for( p_item = cl_qlist_head( &s->port->hca->rdma_mem_list.list );\r
136                 p_item != cl_qlist_end( &s->port->hca->rdma_mem_list.list );\r
137                 p_item = cl_qlist_next( p_item ) )\r
138         {\r
139                 p_reg = PARENT_STRUCT(p_item, struct memory_reg, item);\r
140 \r
141                 if( __check_mr( p_reg, access_ctrl, start, len ) )\r
142                 {\r
143                         p_node->p_reg1 = p_reg;\r
144                         p_node->s = s;\r
145                         cl_qlist_insert_tail( &p_reg->node_list, &p_node->mr_item );\r
146                         cl_qlist_insert_head(\r
147                                 &s->mr_list, &p_node->socket_item );\r
148                         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
149                         IBSP_EXIT( IBSP_DBG_MEM );\r
150                         return p_node;\r
151                 }\r
152         }\r
153 \r
154         /* No corresponding MR has been found. Create a new one. */\r
155         p_reg = HeapAlloc( g_ibsp.heap, 0, sizeof(struct memory_reg) );\r
156 \r
157         if( !p_reg )\r
158         {\r
159                 IBSP_ERROR_EXIT( ("AllocateOverlappedBuf:HeapAlloc() failed: %d\n",\r
160                         GetLastError()) );\r
161                 cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
162                 HeapFree( g_ibsp.heap, 0, p_node );\r
163                 *lpErrno = WSAENOBUFS;\r
164                 return NULL;\r
165         }\r
166 \r
167         /* The node is not initialized yet. All the parameters given are\r
168          * supposed to be valid so we don't check them. */\r
169         cl_qlist_init( &p_reg->node_list );\r
170         p_reg->type.vaddr = start;\r
171         p_reg->type.length = len;\r
172         p_reg->type.access_ctrl = access_ctrl;\r
173 \r
174         IBSP_PRINT(TRACE_LEVEL_INFORMATION, IBSP_DBG_MEM, ("pinning memory node %p\n", p_node) );\r
175         status = ib_reg_mem(\r
176                 pd, &p_reg->type, &p_reg->lkey, &p_reg->rkey, &p_reg->mr_handle );\r
177 \r
178         if( status )\r
179         {\r
180                 cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
181                 HeapFree( g_ibsp.heap, 0, p_reg );\r
182                 HeapFree( g_ibsp.heap, 0, p_node );\r
183 \r
184                 IBSP_ERROR_EXIT( ("ib_reg_mem returned %s\n", ib_get_err_str(status)) );\r
185 \r
186                 *lpErrno = WSAEFAULT;\r
187                 return NULL;\r
188         }\r
189 \r
190         STAT_INC( mr_num );\r
191 \r
192         p_node->p_reg1 = p_reg;\r
193         p_node->s = s;\r
194 \r
195         /* Link to the list of nodes. */\r
196         cl_qlist_insert_head( &s->port->hca->rdma_mem_list.list, &p_reg->item );\r
197         cl_qlist_insert_head( &s->mr_list, &p_node->socket_item );\r
198         cl_qlist_insert_tail( &p_reg->node_list, &p_node->mr_item );\r
199         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
200 \r
201         IBSP_EXIT( IBSP_DBG_MEM );\r
202 \r
203         *lpErrno = 0;\r
204         return p_node;\r
205 }\r
206 \r
207 \r
208 static inline int __ibsp_dereg_mem_mr(\r
209         IN                              struct memory_node                      *node )\r
210 {\r
211         IBSP_ENTER( IBSP_DBG_MEM );\r
212 \r
213         // Underlying registration could be freed before the node.\r
214         if( node->p_reg1 )\r
215                 cl_qlist_remove_item( &node->p_reg1->node_list, &node->mr_item );\r
216 \r
217         cl_qlist_remove_item( &node->s->mr_list, &node->socket_item );\r
218 \r
219 \r
220         memset(node,0x45,sizeof node);\r
221         HeapFree( g_ibsp.heap, 0, node );\r
222 \r
223         IBSP_EXIT( IBSP_DBG_MEM );\r
224         return 0;\r
225 }\r
226 \r
227 \r
228 /* Deregisters a memory region */\r
229 int\r
230 ibsp_dereg_mem(\r
231         IN                              struct ibsp_socket_info         *s,\r
232         IN                              struct memory_node                      *node,\r
233                 OUT                     LPINT                                           lpErrno )\r
234 {\r
235         IBSP_ENTER( IBSP_DBG_MEM );\r
236 \r
237         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
238         *lpErrno = __ibsp_dereg_mem_mr( node );\r
239         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
240 \r
241         IBSP_EXIT( IBSP_DBG_MEM );\r
242         return (*lpErrno? SOCKET_ERROR : 0);\r
243 }\r
244 \r
245 \r
246 /*\r
247  * Deregister the remaining memory regions on an HCA. This function should\r
248  * only be called before destroying the PD. In normal case, the list should\r
249  * be empty because the switch should have done it.\r
250  */\r
251 void\r
252 ibsp_dereg_hca(\r
253         IN                              struct mr_list                          *mem_list )\r
254 {\r
255         cl_list_item_t *item;\r
256         cl_list_item_t *item1;\r
257 \r
258         IBSP_ENTER( IBSP_DBG_MEM );\r
259 \r
260         cl_spinlock_acquire( &mem_list->mutex );\r
261         IBSP_PRINT(TRACE_LEVEL_INFORMATION, IBSP_DBG_MEM,\r
262                 ("%Id registrations.\n", cl_qlist_count( &mem_list->list )) );\r
263 \r
264         for( item = cl_qlist_remove_head( &mem_list->list );\r
265                 item != cl_qlist_end( &mem_list->list );\r
266                 item = cl_qlist_remove_head( &mem_list->list ) )\r
267         {\r
268                 struct memory_reg *p_reg = PARENT_STRUCT(item, struct memory_reg, item);\r
269                 ib_api_status_t status;\r
270 \r
271                 /*\r
272                  * Clear the pointer from the node to this registration.  No need\r
273                  * to remove from the list as we're about to free the registration.\r
274                  */\r
275                 for( item1 = cl_qlist_head( &p_reg->node_list );\r
276                         item1 != cl_qlist_end( &p_reg->node_list );\r
277                         item1 = cl_qlist_next( item1 ) )\r
278                 {\r
279                         struct memory_node *p_node =\r
280                                 PARENT_STRUCT( item1, struct memory_node, mr_item );\r
281                         p_node->p_reg1 = NULL;\r
282                 }\r
283 \r
284                 IBSP_PRINT(TRACE_LEVEL_INFORMATION, IBSP_DBG_MEM, ("unpinning ,memory reg %p\n", p_reg) );\r
285                 status = ib_dereg_mr( p_reg->mr_handle );\r
286                 if( status )\r
287                 {\r
288                         IBSP_ERROR( (\r
289                                 "ib_dereg_mem returned %s\n", ib_get_err_str( status )) );\r
290                 }\r
291                 else\r
292                 {\r
293                         STAT_DEC( mr_num );\r
294                 }\r
295 \r
296                 HeapFree( g_ibsp.heap, 0, p_reg );\r
297         }\r
298 \r
299         cl_spinlock_release( &mem_list->mutex );\r
300 \r
301         IBSP_EXIT( IBSP_DBG_MEM );\r
302 }\r
303 \r
304 \r
305 /* Deregister the remaining memory regions. This function should only \r
306  * be called when destroying the socket. In normal case, the list should \r
307  * be empty because the switch should have done it. */\r
308 void\r
309 ibsp_dereg_socket(\r
310         IN                              struct ibsp_socket_info         *s )\r
311 {\r
312         IBSP_ENTER( IBSP_DBG_MEM );\r
313 \r
314         if( !s->port )\r
315         {\r
316                 CL_ASSERT( !cl_qlist_count( &s->mr_list ) );\r
317                 IBSP_EXIT( IBSP_DBG_MEM );\r
318                 return;\r
319         }\r
320 \r
321         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
322         IBSP_PRINT(TRACE_LEVEL_INFORMATION, IBSP_DBG_MEM,\r
323                 ("%Id registrations.\n", cl_qlist_count( &s->mr_list )) );\r
324 \r
325         while( cl_qlist_count( &s->mr_list ) )\r
326         {\r
327                 __ibsp_dereg_mem_mr( PARENT_STRUCT( cl_qlist_head( &s->mr_list ),\r
328                         struct memory_node, socket_item) );\r
329         }\r
330 \r
331         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
332 \r
333         IBSP_EXIT( IBSP_DBG_MEM );\r
334 }\r
335 \r
336 \r
337 /*\r
338  * Loop through all the memory registrations on an HCA and release\r
339  * all that fall within the specified range.\r
340  */\r
341 void\r
342 ibsp_hca_flush_mr_cache(\r
343         IN                              struct ibsp_hca                         *p_hca,\r
344         IN                              LPVOID                                          lpvAddress,\r
345         IN                              SIZE_T                                          Size )\r
346 {\r
347         struct memory_reg       *p_reg;\r
348         cl_list_item_t          *p_item;\r
349         cl_list_item_t          *p_item1;\r
350         ib_api_status_t         status;\r
351 \r
352         IBSP_ENTER( IBSP_DBG_MEM );\r
353 \r
354         cl_spinlock_acquire( &p_hca->rdma_mem_list.mutex );\r
355         p_item = cl_qlist_head( &p_hca->rdma_mem_list.list );\r
356         while( p_item != cl_qlist_end( &p_hca->rdma_mem_list.list ) )\r
357         {\r
358                 p_reg = PARENT_STRUCT( p_item, struct memory_reg, item );\r
359 \r
360                 /* Move to the next item now so we can remove the current. */\r
361                 p_item = cl_qlist_next( p_item );\r
362 \r
363                 if( lpvAddress > p_reg->type.vaddr ||\r
364                         ((uintn_t)lpvAddress) + Size <\r
365                         ((uintn_t)(uint64_t)p_reg->type.vaddr) + p_reg->type.length )\r
366                 {\r
367                         continue;\r
368                 }\r
369 \r
370                 /*\r
371                  * Clear the pointer from all sockets' nodes to this registration.\r
372                  * No need to remove from the list as we're about to free the\r
373                  * registration.\r
374                  */\r
375                 for( p_item1 = cl_qlist_head( &p_reg->node_list );\r
376                         p_item1 != cl_qlist_end( &p_reg->node_list );\r
377                         p_item1 = cl_qlist_next( p_item1 ) )\r
378                 {\r
379                         struct memory_node *p_node =\r
380                                 PARENT_STRUCT( p_item1, struct memory_node, mr_item );\r
381 \r
382                         p_node->p_reg1 = NULL;\r
383                 }\r
384 \r
385                 cl_qlist_remove_item( &p_hca->rdma_mem_list.list, &p_reg->item );\r
386 \r
387                 status = ib_dereg_mr( p_reg->mr_handle );\r
388                 if( status != IB_SUCCESS )\r
389                 {\r
390                         IBSP_ERROR( (\r
391                                 "ib_dereg_mr returned %s\n", ib_get_err_str(status)) );\r
392                 }\r
393 \r
394                 HeapFree( g_ibsp.heap, 0, p_reg );\r
395         }\r
396         cl_spinlock_release( &p_hca->rdma_mem_list.mutex );\r
397 \r
398         IBSP_EXIT( IBSP_DBG_MEM );\r
399 }\r