[WSD] Fix memory registration code:
[mirror/winof/.git] / ulp / wsd / user / ibsp_mem.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  *\r
4  * This software is available to you under the OpenIB.org BSD license\r
5  * below:\r
6  *\r
7  *     Redistribution and use in source and binary forms, with or\r
8  *     without modification, are permitted provided that the following\r
9  *     conditions are met:\r
10  *\r
11  *      - Redistributions of source code must retain the above\r
12  *        copyright notice, this list of conditions and the following\r
13  *        disclaimer.\r
14  *\r
15  *      - Redistributions in binary form must reproduce the above\r
16  *        copyright notice, this list of conditions and the following\r
17  *        disclaimer in the documentation and/or other materials\r
18  *        provided with the distribution.\r
19  *\r
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
27  * SOFTWARE.\r
28  *\r
29  * $Id$\r
30  */\r
31 \r
32 /* Registers a memory region */\r
33 \r
34 #include "ibspdll.h"\r
35 \r
36 \r
37 __forceinline boolean_t\r
38 __check_mr(\r
39         IN                              struct memory_reg                       *p_reg,\r
40         IN                              ib_access_t                                     acl_mask,\r
41         IN                              void                                            *start,\r
42         IN                              size_t                                          len )\r
43 {\r
44         return( (p_reg->type.access_ctrl & acl_mask) == acl_mask &&\r
45                 start >= p_reg->type.vaddr &&\r
46                 ((uintn_t)start) + len <=\r
47                 ((uintn_t)(uint64_t)p_reg->type.vaddr) + p_reg->type.length );\r
48 }\r
49 \r
50 \r
51 /* Find the first registered mr that matches the given region. \r
52  * mem_list is either socket_info->buf_mem_list or socket_info->rdma_mem_list.\r
53  */\r
54 struct memory_node *\r
55 lookup_partial_mr(\r
56         IN                              struct ibsp_socket_info         *s,\r
57         IN                              ib_access_t                                     acl_mask,\r
58         IN                              void                                            *start,\r
59         IN                              size_t                                          len )\r
60 {\r
61         struct memory_node      *p_node;\r
62         cl_list_item_t          *p_item;\r
63 \r
64         IBSP_ENTER( IBSP_DBG_MEM );\r
65 \r
66         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
67 \r
68         for( p_item = cl_qlist_head( &s->mr_list );\r
69                 p_item != cl_qlist_end( &s->mr_list );\r
70                 p_item = cl_qlist_next( p_item ) )\r
71         {\r
72                 p_node = PARENT_STRUCT( p_item, struct memory_node, socket_item );\r
73                 \r
74                 if( __check_mr( p_node->p_reg, acl_mask, start, len ) )\r
75                 {\r
76                         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
77                         IBSP_EXIT( IBSP_DBG_MEM );\r
78                         return p_node;\r
79                 }\r
80         }\r
81 \r
82         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
83 \r
84         IBSP_TRACE_EXIT( IBSP_DBG_MEM, ("mr not found\n") );\r
85         return NULL;\r
86 }\r
87 \r
88 \r
89 /* Registers a memory region. The memory region might be cached.\r
90  * mem_list is either socket_info->buf_mem_list or hca->rdma_mem_list.\r
91  */\r
92 struct memory_node *\r
93 ibsp_reg_mem(\r
94         IN                              struct ibsp_socket_info         *s,\r
95         IN                              ib_pd_handle_t                          pd,\r
96         IN                              void                                            *start,\r
97         IN                              size_t                                          len,\r
98         IN                              ib_access_t                                     access_ctrl,\r
99                 OUT                     LPINT                                           lpErrno )\r
100 {\r
101         struct memory_node      *p_node;\r
102         struct memory_reg       *p_reg;\r
103         cl_list_item_t          *p_item;\r
104         ib_api_status_t         status;\r
105 \r
106         IBSP_ENTER( IBSP_DBG_MEM );\r
107 \r
108         CL_ASSERT( start != NULL );\r
109         CL_ASSERT( len != 0 );\r
110         CL_ASSERT( (access_ctrl & ~(IB_AC_RDMA_READ | IB_AC_RDMA_WRITE | IB_AC_LOCAL_WRITE)) ==\r
111                           0 );\r
112 \r
113         /* Optimistically allocate a tracking structure. */\r
114         p_node = HeapAlloc( g_ibsp.heap, 0, sizeof(struct memory_node) );\r
115         if( !p_node )\r
116         {\r
117                 IBSP_ERROR_EXIT(\r
118                         ("AllocateOverlappedBuf:HeapAlloc() failed: %d\n",\r
119                         GetLastError()) );\r
120                 *lpErrno = WSAENOBUFS;\r
121                 return NULL;\r
122         }\r
123 \r
124         /* First, try to find a suitable MR */\r
125         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
126 \r
127         /* Find the first registered mr that matches the given region. */\r
128         for( p_item = cl_qlist_head( &s->port->hca->rdma_mem_list.list );\r
129                 p_item != cl_qlist_end( &s->port->hca->rdma_mem_list.list );\r
130                 p_item = cl_qlist_next( p_item ) )\r
131         {\r
132                 p_reg = PARENT_STRUCT(p_item, struct memory_reg, item);\r
133 \r
134                 if( __check_mr( p_reg, access_ctrl, start, len ) )\r
135                 {\r
136                         p_node->p_reg = p_reg;\r
137                         p_node->s = s;\r
138                         cl_qlist_insert_tail( &p_reg->node_list, &p_node->mr_item );\r
139                         cl_qlist_insert_head(\r
140                                 &s->mr_list, &p_node->socket_item );\r
141                         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
142                         IBSP_EXIT( IBSP_DBG_MEM );\r
143                         return p_node;\r
144                 }\r
145         }\r
146 \r
147         /* No corresponding MR has been found. Create a new one. */\r
148         p_reg = HeapAlloc( g_ibsp.heap, 0, sizeof(struct memory_reg) );\r
149 \r
150         if( !p_reg )\r
151         {\r
152                 IBSP_ERROR_EXIT(\r
153                         ("AllocateOverlappedBuf:HeapAlloc() failed: %d\n",\r
154                         GetLastError()) );\r
155                 cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
156                 HeapFree( g_ibsp.heap, 0, p_node );\r
157                 *lpErrno = WSAENOBUFS;\r
158                 return NULL;\r
159         }\r
160 \r
161         /* The node is not initialized yet. All the parameters given are\r
162          * supposed to be valid so we don't check them. */\r
163         cl_qlist_init( &p_reg->node_list );\r
164         p_reg->type.vaddr = start;\r
165         p_reg->type.length = len;\r
166         p_reg->type.access_ctrl = access_ctrl;\r
167 \r
168         IBSP_TRACE2( IBSP_DBG_MEM, ("pinning memory node %p\n", p_node) );\r
169         status = ib_reg_mem(\r
170                 pd, &p_reg->type, &p_reg->lkey, &p_reg->rkey, &p_reg->mr_handle );\r
171 \r
172         if( status )\r
173         {\r
174                 cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
175                 HeapFree( g_ibsp.heap, 0, p_reg );\r
176                 HeapFree( g_ibsp.heap, 0, p_node );\r
177 \r
178                 IBSP_ERROR_EXIT(\r
179                         ("ib_reg_mem returned %s\n", ib_get_err_str(status)) );\r
180 \r
181                 *lpErrno = WSAEFAULT;\r
182                 return NULL;\r
183         }\r
184 \r
185         STAT_INC( mr_num );\r
186 \r
187         p_node->p_reg = p_reg;\r
188         p_node->s = s;\r
189 \r
190         /* Link to the list of nodes. */\r
191         cl_qlist_insert_head( &s->port->hca->rdma_mem_list.list, &p_reg->item );\r
192         cl_qlist_insert_head( &s->mr_list, &p_node->socket_item );\r
193         cl_qlist_insert_tail( &p_reg->node_list, &p_node->mr_item );\r
194         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
195 \r
196         IBSP_EXIT( IBSP_DBG_MEM );\r
197 \r
198         *lpErrno = 0;\r
199         return p_node;\r
200 }\r
201 \r
202 \r
203 static inline int __ibsp_dereg_mem_mr(\r
204         IN                              struct memory_node                      *node )\r
205 {\r
206         IBSP_ENTER( IBSP_DBG_MEM );\r
207 \r
208         // Underlying registration could be freed before the node.\r
209         if( node->p_reg )\r
210                 cl_qlist_remove_item( &node->p_reg->node_list, &node->mr_item );\r
211 \r
212         cl_qlist_remove_item( &node->s->mr_list, &node->socket_item );\r
213 \r
214         HeapFree( g_ibsp.heap, 0, node );\r
215 \r
216         IBSP_EXIT( IBSP_DBG_MEM );\r
217         return 0;\r
218 }\r
219 \r
220 \r
221 /* Deregisters a memory region */\r
222 int\r
223 ibsp_dereg_mem(\r
224         IN                              struct ibsp_socket_info         *s,\r
225         IN                              struct memory_node                      *node,\r
226                 OUT                     LPINT                                           lpErrno )\r
227 {\r
228         IBSP_ENTER( IBSP_DBG_MEM );\r
229 \r
230         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
231         *lpErrno = __ibsp_dereg_mem_mr( node );\r
232         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
233 \r
234         IBSP_EXIT( IBSP_DBG_MEM );\r
235         return (*lpErrno? SOCKET_ERROR : 0);\r
236 }\r
237 \r
238 \r
239 /*\r
240  * Deregister the remaining memory regions on an HCA. This function should\r
241  * only be called before destroying the PD. In normal case, the list should\r
242  * be empty because the switch should have done it.\r
243  */\r
244 void\r
245 ibsp_dereg_hca(\r
246         IN                              struct mr_list                          *mem_list )\r
247 {\r
248         cl_list_item_t *item;\r
249 \r
250         IBSP_ENTER( IBSP_DBG_MEM );\r
251 \r
252         cl_spinlock_acquire( &mem_list->mutex );\r
253         IBSP_TRACE1( IBSP_DBG_MEM,\r
254                 ("%d registrations.\n", cl_qlist_count( &mem_list->list )) );\r
255 \r
256         for( item = cl_qlist_remove_head( &mem_list->list );\r
257                 item != cl_qlist_end( &mem_list->list );\r
258                 item = cl_qlist_remove_head( &mem_list->list ) )\r
259         {\r
260                 struct memory_reg *p_reg = PARENT_STRUCT(item, struct memory_reg, item);\r
261                 ib_api_status_t status;\r
262 \r
263                 /*\r
264                  * Clear the pointer from the node to this registration.  No need\r
265                  * to remove from the list as we're about to free the registration.\r
266                  */\r
267                 for( item = cl_qlist_head( &p_reg->node_list );\r
268                         item != cl_qlist_end( &p_reg->node_list );\r
269                         item = cl_qlist_next( item ) )\r
270                 {\r
271                         struct memory_node *p_node =\r
272                                 PARENT_STRUCT( item, struct memory_node, mr_item );\r
273 \r
274                         p_node->p_reg = NULL;\r
275                 }\r
276 \r
277                 IBSP_TRACE2( IBSP_DBG_MEM, ("unpinning ,memory reg %p\n", p_reg) );\r
278                 status = ib_dereg_mr( p_reg->mr_handle );\r
279                 if( status )\r
280                 {\r
281                         IBSP_ERROR(\r
282                                 ("ib_dereg_mem returned %s\n", ib_get_err_str( status )) );\r
283                 }\r
284                 else\r
285                 {\r
286                         STAT_DEC( mr_num );\r
287                 }\r
288 \r
289                 HeapFree( g_ibsp.heap, 0, p_reg );\r
290         }\r
291 \r
292         cl_spinlock_release( &mem_list->mutex );\r
293 \r
294         IBSP_EXIT( IBSP_DBG_MEM );\r
295 }\r
296 \r
297 \r
298 /* Deregister the remaining memory regions. This function should only \r
299  * be called when destroying the socket. In normal case, the list should \r
300  * be empty because the switch should have done it. */\r
301 void\r
302 ibsp_dereg_socket(\r
303         IN                              struct ibsp_socket_info         *s )\r
304 {\r
305         IBSP_ENTER( IBSP_DBG_MEM );\r
306 \r
307         if( !s->port )\r
308         {\r
309                 CL_ASSERT( !cl_qlist_count( &s->mr_list ) );\r
310                 IBSP_EXIT( IBSP_DBG_MEM );\r
311                 return;\r
312         }\r
313 \r
314         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
315         IBSP_TRACE1( IBSP_DBG_MEM,\r
316                 ("%d registrations.\n", cl_qlist_count( &s->mr_list )) );\r
317 \r
318         while( cl_qlist_count( &s->mr_list ) )\r
319         {\r
320                 __ibsp_dereg_mem_mr( PARENT_STRUCT( cl_qlist_head( &s->mr_list ),\r
321                         struct memory_node, socket_item) );\r
322         }\r
323 \r
324         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
325 \r
326         IBSP_EXIT( IBSP_DBG_MEM );\r
327 }\r
328 \r
329 \r
330 /*\r
331  * Loop through all the memory registrations on an HCA and release\r
332  * all that fall within the specified range.\r
333  */\r
334 void\r
335 ibsp_hca_flush_mr_cache(\r
336         IN                              struct ibsp_hca                         *p_hca,\r
337         IN                              LPVOID                                          lpvAddress,\r
338         IN                              SIZE_T                                          Size )\r
339 {\r
340         struct memory_reg       *p_reg;\r
341         cl_list_item_t          *p_item;\r
342         ib_api_status_t         status;\r
343 \r
344         IBSP_ENTER( IBSP_DBG_MEM );\r
345 \r
346         cl_spinlock_acquire( &p_hca->rdma_mem_list.mutex );\r
347         p_item = cl_qlist_head( &p_hca->rdma_mem_list.list );\r
348         while( p_item != cl_qlist_end( &p_hca->rdma_mem_list.list ) )\r
349         {\r
350                 p_reg = PARENT_STRUCT( p_item, struct memory_reg, item );\r
351 \r
352                 /* Move to the next item now so we can remove the current. */\r
353                 p_item = cl_qlist_next( p_item );\r
354 \r
355                 if( lpvAddress > p_reg->type.vaddr ||\r
356                         ((uintn_t)lpvAddress) + Size <\r
357                         ((uintn_t)(uint64_t)p_reg->type.vaddr) + p_reg->type.length )\r
358                 {\r
359                         continue;\r
360                 }\r
361 \r
362                 /*\r
363                  * Clear the pointer from all sockets' nodes to this registration.\r
364                  * No need to remove from the list as we're about to free the\r
365                  * registration.\r
366                  */\r
367                 for( p_item = cl_qlist_head( &p_reg->node_list );\r
368                         p_item != cl_qlist_end( &p_reg->node_list );\r
369                         p_item = cl_qlist_next( p_item ) )\r
370                 {\r
371                         struct memory_node *p_node =\r
372                                 PARENT_STRUCT( p_item, struct memory_node, mr_item );\r
373 \r
374                         p_node->p_reg = NULL;\r
375                 }\r
376 \r
377                 cl_qlist_remove_item( &p_hca->rdma_mem_list.list, &p_reg->item );\r
378 \r
379                 status = ib_dereg_mr( p_reg->mr_handle );\r
380                 if( status != IB_SUCCESS )\r
381                 {\r
382                         IBSP_ERROR(\r
383                                 ("ib_dereg_mr returned %s\n", ib_get_err_str(status)) );\r
384                 }\r
385 \r
386                 HeapFree( g_ibsp.heap, 0, p_reg );\r
387         }\r
388         cl_spinlock_release( &p_hca->rdma_mem_list.mutex );\r
389 \r
390         IBSP_EXIT( IBSP_DBG_MEM );\r
391 }\r