3e13c7f2d9341a9e276654a6f22e400fd1c37217
[mirror/winof/.git] / ulp / wsd / user / ibsp_mem.c
1 /*\r
2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
3  *\r
4  * This software is available to you under the OpenIB.org BSD license\r
5  * below:\r
6  *\r
7  *     Redistribution and use in source and binary forms, with or\r
8  *     without modification, are permitted provided that the following\r
9  *     conditions are met:\r
10  *\r
11  *      - Redistributions of source code must retain the above\r
12  *        copyright notice, this list of conditions and the following\r
13  *        disclaimer.\r
14  *\r
15  *      - Redistributions in binary form must reproduce the above\r
16  *        copyright notice, this list of conditions and the following\r
17  *        disclaimer in the documentation and/or other materials\r
18  *        provided with the distribution.\r
19  *\r
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
27  * SOFTWARE.\r
28  *\r
29  * $Id$\r
30  */\r
31 \r
32 /* Registers a memory region */\r
33 \r
34 #include "ibspdll.h"\r
35 \r
36 \r
37 __forceinline boolean_t\r
38 __check_mr(\r
39         IN                              struct memory_reg                       *p_reg,\r
40         IN                              ib_access_t                                     acl_mask,\r
41         IN                              void                                            *start,\r
42         IN                              size_t                                          len )\r
43 {\r
44         return( (p_reg->type.access_ctrl & acl_mask) == acl_mask &&\r
45                 start >= p_reg->type.vaddr &&\r
46                 ((uintn_t)start) + len <=\r
47                 ((uintn_t)(uint64_t)p_reg->type.vaddr) + p_reg->type.length );\r
48 }\r
49 \r
50 \r
51 /* Find the first registered mr that matches the given region. \r
52  * mem_list is either socket_info->buf_mem_list or socket_info->rdma_mem_list.\r
53  */\r
54 struct memory_node *\r
55 lookup_partial_mr(\r
56         IN                              struct ibsp_socket_info         *s,\r
57         IN                              ib_access_t                                     acl_mask,\r
58         IN                              void                                            *start,\r
59         IN                              size_t                                          len )\r
60 {\r
61         struct memory_node      *p_node;\r
62         cl_list_item_t          *p_item;\r
63 \r
64         IBSP_ENTER( IBSP_DBG_MEM );\r
65 \r
66         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
67 \r
68         for( p_item = cl_qlist_head( &s->mr_list );\r
69                 p_item != cl_qlist_end( &s->mr_list );\r
70                 p_item = cl_qlist_next( p_item ) )\r
71         {\r
72                 p_node = PARENT_STRUCT( p_item, struct memory_node, socket_item );\r
73                 \r
74                 if( __check_mr( p_node->p_reg, acl_mask, start, len ) )\r
75                 {\r
76                         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
77                         IBSP_EXIT( IBSP_DBG_MEM );\r
78                         return p_node;\r
79                 }\r
80         }\r
81 \r
82         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
83 \r
84         IBSP_TRACE_EXIT( IBSP_DBG_MEM, ("mr not found\n") );\r
85         return NULL;\r
86 }\r
87 \r
88 \r
89 /* Registers a memory region. The memory region might be cached.\r
90  * mem_list is either socket_info->buf_mem_list or hca->rdma_mem_list.\r
91  */\r
92 struct memory_node *\r
93 ibsp_reg_mem(\r
94         IN                              struct ibsp_socket_info         *s,\r
95         IN                              ib_pd_handle_t                          pd,\r
96         IN                              void                                            *start,\r
97         IN                              size_t                                          len,\r
98         IN                              ib_access_t                                     access_ctrl,\r
99                 OUT                     LPINT                                           lpErrno )\r
100 {\r
101         struct memory_node      *p_node;\r
102         struct memory_reg       *p_reg;\r
103         cl_list_item_t          *p_item;\r
104         ib_api_status_t         status;\r
105 \r
106         IBSP_ENTER( IBSP_DBG_MEM );\r
107 \r
108         CL_ASSERT( start != NULL );\r
109         CL_ASSERT( len != 0 );\r
110         CL_ASSERT( (access_ctrl & ~(IB_AC_RDMA_READ | IB_AC_RDMA_WRITE | IB_AC_LOCAL_WRITE)) ==\r
111                           0 );\r
112 \r
113         /* Optimistically allocate a tracking structure. */\r
114         p_node = HeapAlloc( g_ibsp.heap, 0, sizeof(struct memory_node) );\r
115         if( !p_node )\r
116         {\r
117                 IBSP_ERROR_EXIT(\r
118                         ("AllocateOverlappedBuf:HeapAlloc() failed: %d\n",\r
119                         GetLastError()) );\r
120                 *lpErrno = WSAENOBUFS;\r
121                 return NULL;\r
122         }\r
123 \r
124         /* First, try to find a suitable MR */\r
125         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
126 \r
127         /* Find the first registered mr that matches the given region. */\r
128         for( p_item = cl_qlist_head( &s->port->hca->rdma_mem_list.list );\r
129                 p_item != cl_qlist_end( &s->port->hca->rdma_mem_list.list );\r
130                 p_item = cl_qlist_next( p_item ) )\r
131         {\r
132                 p_reg = PARENT_STRUCT(p_item, struct memory_reg, item);\r
133 \r
134                 if( __check_mr( p_reg, access_ctrl, start, len ) )\r
135                 {\r
136                         p_node->p_reg = p_reg;\r
137                         p_node->s = s;\r
138                         cl_qlist_insert_tail( &p_reg->node_list, &p_node->mr_item );\r
139                         cl_qlist_insert_head(\r
140                                 &s->mr_list, &p_node->socket_item );\r
141                         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
142                         IBSP_EXIT( IBSP_DBG_MEM );\r
143                         return p_node;\r
144                 }\r
145         }\r
146 \r
147         /* No corresponding MR has been found. Create a new one. */\r
148         p_reg = HeapAlloc( g_ibsp.heap, 0, sizeof(struct memory_reg) );\r
149 \r
150         if( !p_reg )\r
151         {\r
152                 IBSP_ERROR_EXIT(\r
153                         ("AllocateOverlappedBuf:HeapAlloc() failed: %d\n",\r
154                         GetLastError()) );\r
155                 cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
156                 HeapFree( g_ibsp.heap, 0, p_node );\r
157                 *lpErrno = WSAENOBUFS;\r
158                 return NULL;\r
159         }\r
160 \r
161         /* The node is not initialized yet. All the parameters given are\r
162          * supposed to be valid so we don't check them. */\r
163         cl_qlist_init( &p_reg->node_list );\r
164         p_reg->type.vaddr = start;\r
165         p_reg->type.length = len;\r
166         p_reg->type.access_ctrl = access_ctrl;\r
167 \r
168         IBSP_TRACE2( IBSP_DBG_MEM, ("pinning memory node %p\n", p_node) );\r
169         status = ib_reg_mem(\r
170                 pd, &p_reg->type, &p_reg->lkey, &p_reg->rkey, &p_reg->mr_handle );\r
171 \r
172         if( status )\r
173         {\r
174                 cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
175                 HeapFree( g_ibsp.heap, 0, p_reg );\r
176                 HeapFree( g_ibsp.heap, 0, p_node );\r
177 \r
178                 IBSP_ERROR_EXIT(\r
179                         ("ib_reg_mem returned %s\n", ib_get_err_str(status)) );\r
180 \r
181                 *lpErrno = WSAEFAULT;\r
182                 return NULL;\r
183         }\r
184 \r
185         STAT_INC( mr_num );\r
186 \r
187         p_node->p_reg = p_reg;\r
188         p_node->s = s;\r
189 \r
190         /* Link to the list of nodes. */\r
191         cl_qlist_insert_head( &s->port->hca->rdma_mem_list.list, &p_reg->item );\r
192         cl_qlist_insert_head( &s->mr_list, &p_node->socket_item );\r
193         cl_qlist_insert_tail( &p_reg->node_list, &p_node->mr_item );\r
194         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
195 \r
196         IBSP_EXIT( IBSP_DBG_MEM );\r
197 \r
198         *lpErrno = 0;\r
199         return p_node;\r
200 }\r
201 \r
202 \r
203 static inline int __ibsp_dereg_mem_mr(\r
204         IN                              struct memory_node                      *node )\r
205 {\r
206         IBSP_ENTER( IBSP_DBG_MEM );\r
207 \r
208         cl_qlist_remove_item( &node->p_reg->node_list, &node->mr_item );\r
209         cl_qlist_remove_item( &node->s->mr_list, &node->socket_item );\r
210 \r
211         HeapFree( g_ibsp.heap, 0, node );\r
212 \r
213         IBSP_EXIT( IBSP_DBG_MEM );\r
214         return 0;\r
215 }\r
216 \r
217 \r
218 /* Deregisters a memory region */\r
219 int\r
220 ibsp_dereg_mem(\r
221         IN                              struct ibsp_socket_info         *s,\r
222         IN                              struct memory_node                      *node,\r
223                 OUT                     LPINT                                           lpErrno )\r
224 {\r
225         IBSP_ENTER( IBSP_DBG_MEM );\r
226 \r
227         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
228         *lpErrno = __ibsp_dereg_mem_mr( node );\r
229         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
230 \r
231         IBSP_EXIT( IBSP_DBG_MEM );\r
232         return (*lpErrno? SOCKET_ERROR : 0);\r
233 }\r
234 \r
235 \r
236 /*\r
237  * Deregister the remaining memory regions on an HCA. This function should\r
238  * only be called before destroying the PD. In normal case, the list should\r
239  * be empty because the switch should have done it.\r
240  */\r
241 void\r
242 ibsp_dereg_hca(\r
243         IN                              struct mr_list                          *mem_list )\r
244 {\r
245         cl_list_item_t *item;\r
246 \r
247         IBSP_ENTER( IBSP_DBG_MEM );\r
248 \r
249         cl_spinlock_acquire( &mem_list->mutex );\r
250         IBSP_TRACE1( IBSP_DBG_MEM,\r
251                 ("%d registrations.\n", cl_qlist_count( &mem_list->list )) );\r
252 \r
253         for( item = cl_qlist_remove_head( &mem_list->list );\r
254                 item != cl_qlist_end( &mem_list->list );\r
255                 item = cl_qlist_remove_head( &mem_list->list ) )\r
256         {\r
257                 struct memory_reg *p_reg = PARENT_STRUCT(item, struct memory_reg, item);\r
258                 ib_api_status_t status;\r
259 \r
260                 while( cl_qlist_count( &p_reg->node_list ) )\r
261                 {\r
262                         struct memory_node *p_node =\r
263                                 PARENT_STRUCT( cl_qlist_head( &p_reg->node_list ),\r
264                                 struct memory_node, mr_item );\r
265 \r
266                         __ibsp_dereg_mem_mr( p_node );\r
267                 }\r
268 \r
269                 IBSP_TRACE2( IBSP_DBG_MEM, ("unpinning ,memory reg %p\n", p_reg) );\r
270                 status = ib_dereg_mr( p_reg->mr_handle );\r
271                 if( status )\r
272                 {\r
273                         IBSP_ERROR(\r
274                                 ("ib_dereg_mem returned %s\n", ib_get_err_str( status )) );\r
275                 }\r
276                 else\r
277                 {\r
278                         STAT_DEC( mr_num );\r
279                 }\r
280 \r
281                 HeapFree( g_ibsp.heap, 0, p_reg );\r
282         }\r
283 \r
284         cl_spinlock_release( &mem_list->mutex );\r
285 \r
286         IBSP_EXIT( IBSP_DBG_MEM );\r
287 }\r
288 \r
289 \r
290 /* Deregister the remaining memory regions. This function should only \r
291  * be called when destroying the socket. In normal case, the list should \r
292  * be empty because the switch should have done it. */\r
293 void\r
294 ibsp_dereg_socket(\r
295         IN                              struct ibsp_socket_info         *s )\r
296 {\r
297         IBSP_ENTER( IBSP_DBG_MEM );\r
298 \r
299         if( !s->port )\r
300         {\r
301                 CL_ASSERT( !cl_qlist_count( &s->mr_list ) );\r
302                 IBSP_EXIT( IBSP_DBG_MEM );\r
303                 return;\r
304         }\r
305 \r
306         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
307         IBSP_TRACE1( IBSP_DBG_MEM,\r
308                 ("%d registrations.\n", cl_qlist_count( &s->mr_list )) );\r
309 \r
310         while( cl_qlist_count( &s->mr_list ) )\r
311         {\r
312                 __ibsp_dereg_mem_mr( PARENT_STRUCT( cl_qlist_head( &s->mr_list ),\r
313                         struct memory_node, socket_item) );\r
314         }\r
315 \r
316         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
317 \r
318         IBSP_EXIT( IBSP_DBG_MEM );\r
319 }\r
320 \r
321 \r
322 /*\r
323  * Loop through all the memory registrations on an HCA and release\r
324  * all that fall within the specified range.\r
325  */\r
326 void\r
327 ibsp_hca_flush_mr_cache(\r
328         IN                              struct ibsp_hca                         *p_hca,\r
329         IN                              LPVOID                                          lpvAddress,\r
330         IN                              SIZE_T                                          Size )\r
331 {\r
332         struct memory_reg       *p_reg;\r
333         cl_list_item_t          *p_item;\r
334         ib_api_status_t         status;\r
335 \r
336         IBSP_ENTER( IBSP_DBG_MEM );\r
337 \r
338         cl_spinlock_acquire( &p_hca->rdma_mem_list.mutex );\r
339         for( p_item = cl_qlist_head( &p_hca->rdma_mem_list.list );\r
340                 p_item != cl_qlist_end( &p_hca->rdma_mem_list.list );\r
341                 p_item = cl_qlist_next( p_item ) )\r
342         {\r
343                 p_reg = PARENT_STRUCT( p_item, struct memory_reg, item );\r
344 \r
345                 if( lpvAddress > p_reg->type.vaddr ||\r
346                         ((uintn_t)lpvAddress) + Size <\r
347                         ((uintn_t)(uint64_t)p_reg->type.vaddr) + p_reg->type.length )\r
348                 {\r
349                         continue;\r
350                 }\r
351 \r
352                 /* Release all socket's nodes that reference this registration. */\r
353                 while( cl_qlist_count( &p_reg->node_list ) )\r
354                 {\r
355                         struct memory_node      *p_node =\r
356                                 PARENT_STRUCT( cl_qlist_head( &p_reg->node_list ),\r
357                                 struct memory_node, mr_item );\r
358 \r
359                         __ibsp_dereg_mem_mr( p_node );\r
360                 }\r
361 \r
362                 /* Move to the previous item so the for loop properly moves forward. */\r
363                 p_item = cl_qlist_prev( p_item );\r
364 \r
365                 cl_qlist_remove_item( &p_hca->rdma_mem_list.list, &p_reg->item );\r
366 \r
367                 status = ib_dereg_mr( p_reg->mr_handle );\r
368                 if( status != IB_SUCCESS )\r
369                 {\r
370                         IBSP_ERROR(\r
371                                 ("ib_dereg_mr returned %s\n", ib_get_err_str(status)) );\r
372                 }\r
373 \r
374                 HeapFree( g_ibsp.heap, 0, p_reg );\r
375         }\r
376         cl_spinlock_release( &p_hca->rdma_mem_list.mutex );\r
377 \r
378         IBSP_EXIT( IBSP_DBG_MEM );\r
379 }\r