ulp/wsd/user/ibsp_mem.c

   1 /*\r
   2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
   3  *\r
   4  * This software is available to you under the OpenIB.org BSD license\r
   5  * below:\r
   6  *\r
   7  *     Redistribution and use in source and binary forms, with or\r
   8  *     without modification, are permitted provided that the following\r
   9  *     conditions are met:\r
  10  *\r
  11  *      - Redistributions of source code must retain the above\r
  12  *        copyright notice, this list of conditions and the following\r
  13  *        disclaimer.\r
  14  *\r
  15  *      - Redistributions in binary form must reproduce the above\r
  16  *        copyright notice, this list of conditions and the following\r
  17  *        disclaimer in the documentation and/or other materials\r
  18  *        provided with the distribution.\r
  19  *\r
  20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
  21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
  22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
  23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
  24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
  25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
  26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
  27  * SOFTWARE.\r
  28  *\r
  29  * $Id$\r
  30  */\r
  31 \r
  32 /* Registers a memory region */\r
  33 \r
  34 #include "ibspdll.h"\r
  35 \r
  36 \r
  37 __forceinline boolean_t\r
  38 __check_mr(\r
  39         IN                              struct memory_reg                       *p_reg,\r
  40         IN                              ib_access_t                                     acl_mask,\r
  41         IN                              void                                            *start,\r
  42         IN                              size_t                                          len )\r
  43 {\r
  44         return( (p_reg->type.access_ctrl & acl_mask) == acl_mask &&\r
  45                 start >= p_reg->type.vaddr &&\r
  46                 ((uintn_t)start) + len <=\r
  47                 ((uintn_t)(uint64_t)p_reg->type.vaddr) + p_reg->type.length );\r
  48 }\r
  49 \r
  50 \r
  51 /* Find the first registered mr that matches the given region. \r
  52  * mem_list is either socket_info->buf_mem_list or socket_info->rdma_mem_list.\r
  53  */\r
  54 struct memory_node *\r
  55 lookup_partial_mr(\r
  56         IN                              struct ibsp_socket_info         *s,\r
  57         IN                              ib_access_t                                     acl_mask,\r
  58         IN                              void                                            *start,\r
  59         IN                              size_t                                          len )\r
  60 {\r
  61         struct memory_node      *p_node;\r
  62         cl_list_item_t          *p_item;\r
  63 \r
  64         IBSP_ENTER( IBSP_DBG_MEM );\r
  65 \r
  66         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
  67 \r
  68         for( p_item = cl_qlist_head( &s->mr_list );\r
  69                 p_item != cl_qlist_end( &s->mr_list );\r
  70                 p_item = cl_qlist_next( p_item ) )\r
  71         {\r
  72                 p_node = PARENT_STRUCT( p_item, struct memory_node, socket_item );\r
  73                 \r
  74                 if( __check_mr( p_node->p_reg, acl_mask, start, len ) )\r
  75                 {\r
  76                         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
  77                         IBSP_EXIT( IBSP_DBG_MEM );\r
  78                         return p_node;\r
  79                 }\r
  80         }\r
  81 \r
  82         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
  83 \r
  84         IBSP_TRACE_EXIT( IBSP_DBG_MEM, ("mr not found\n") );\r
  85         return NULL;\r
  86 }\r
  87 \r
  88 \r
  89 /* Registers a memory region. The memory region might be cached.\r
  90  * mem_list is either socket_info->buf_mem_list or hca->rdma_mem_list.\r
  91  */\r
  92 struct memory_node *\r
  93 ibsp_reg_mem(\r
  94         IN                              struct ibsp_socket_info         *s,\r
  95         IN                              ib_pd_handle_t                          pd,\r
  96         IN                              void                                            *start,\r
  97         IN                              size_t                                          len,\r
  98         IN                              ib_access_t                                     access_ctrl,\r
  99                 OUT                     LPINT                                           lpErrno )\r
 100 {\r
 101         struct memory_node      *p_node;\r
 102         struct memory_reg       *p_reg;\r
 103         cl_list_item_t          *p_item;\r
 104         ib_api_status_t         status;\r
 105 \r
 106         IBSP_ENTER( IBSP_DBG_MEM );\r
 107 \r
 108         CL_ASSERT( start != NULL );\r
 109         CL_ASSERT( len != 0 );\r
 110         CL_ASSERT( (access_ctrl & ~(IB_AC_RDMA_READ | IB_AC_RDMA_WRITE | IB_AC_LOCAL_WRITE)) ==\r
 111                           0 );\r
 112 \r
 113         /* Optimistically allocate a tracking structure. */\r
 114         p_node = HeapAlloc( g_ibsp.heap, 0, sizeof(struct memory_node) );\r
 115         if( !p_node )\r
 116         {\r
 117                 IBSP_ERROR_EXIT(\r
 118                         ("AllocateOverlappedBuf:HeapAlloc() failed: %d\n",\r
 119                         GetLastError()) );\r
 120                 *lpErrno = WSAENOBUFS;\r
 121                 return NULL;\r
 122         }\r
 123 \r
 124         /* First, try to find a suitable MR */\r
 125         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
 126 \r
 127         /* Find the first registered mr that matches the given region. */\r
 128         for( p_item = cl_qlist_head( &s->port->hca->rdma_mem_list.list );\r
 129                 p_item != cl_qlist_end( &s->port->hca->rdma_mem_list.list );\r
 130                 p_item = cl_qlist_next( p_item ) )\r
 131         {\r
 132                 p_reg = PARENT_STRUCT(p_item, struct memory_reg, item);\r
 133 \r
 134                 if( __check_mr( p_reg, access_ctrl, start, len ) )\r
 135                 {\r
 136                         p_node->p_reg = p_reg;\r
 137                         p_node->s = s;\r
 138                         cl_qlist_insert_tail( &p_reg->node_list, &p_node->mr_item );\r
 139                         cl_qlist_insert_head(\r
 140                                 &s->mr_list, &p_node->socket_item );\r
 141                         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
 142                         IBSP_EXIT( IBSP_DBG_MEM );\r
 143                         return p_node;\r
 144                 }\r
 145         }\r
 146 \r
 147         /* No corresponding MR has been found. Create a new one. */\r
 148         p_reg = HeapAlloc( g_ibsp.heap, 0, sizeof(struct memory_reg) );\r
 149 \r
 150         if( !p_reg )\r
 151         {\r
 152                 IBSP_ERROR_EXIT(\r
 153                         ("AllocateOverlappedBuf:HeapAlloc() failed: %d\n",\r
 154                         GetLastError()) );\r
 155                 cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
 156                 HeapFree( g_ibsp.heap, 0, p_node );\r
 157                 *lpErrno = WSAENOBUFS;\r
 158                 return NULL;\r
 159         }\r
 160 \r
 161         /* The node is not initialized yet. All the parameters given are\r
 162          * supposed to be valid so we don't check them. */\r
 163         cl_qlist_init( &p_reg->node_list );\r
 164         p_reg->type.vaddr = start;\r
 165         p_reg->type.length = len;\r
 166         p_reg->type.access_ctrl = access_ctrl;\r
 167 \r
 168         IBSP_TRACE2( IBSP_DBG_MEM, ("pinning memory node %p\n", p_node) );\r
 169         status = ib_reg_mem(\r
 170                 pd, &p_reg->type, &p_reg->lkey, &p_reg->rkey, &p_reg->mr_handle );\r
 171 \r
 172         if( status )\r
 173         {\r
 174                 cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
 175                 HeapFree( g_ibsp.heap, 0, p_reg );\r
 176                 HeapFree( g_ibsp.heap, 0, p_node );\r
 177 \r
 178                 IBSP_ERROR_EXIT(\r
 179                         ("ib_reg_mem returned %s\n", ib_get_err_str(status)) );\r
 180 \r
 181                 *lpErrno = WSAEFAULT;\r
 182                 return NULL;\r
 183         }\r
 184 \r
 185         STAT_INC( mr_num );\r
 186 \r
 187         p_node->p_reg = p_reg;\r
 188         p_node->s = s;\r
 189 \r
 190         /* Link to the list of nodes. */\r
 191         cl_qlist_insert_head( &s->port->hca->rdma_mem_list.list, &p_reg->item );\r
 192         cl_qlist_insert_head( &s->mr_list, &p_node->socket_item );\r
 193         cl_qlist_insert_tail( &p_reg->node_list, &p_node->mr_item );\r
 194         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
 195 \r
 196         IBSP_EXIT( IBSP_DBG_MEM );\r
 197 \r
 198         *lpErrno = 0;\r
 199         return p_node;\r
 200 }\r
 201 \r
 202 \r
 203 static inline int __ibsp_dereg_mem_mr(\r
 204         IN                              struct memory_node                      *node )\r
 205 {\r
 206         IBSP_ENTER( IBSP_DBG_MEM );\r
 207 \r
 208         cl_qlist_remove_item( &node->p_reg->node_list, &node->mr_item );\r
 209         cl_qlist_remove_item( &node->s->mr_list, &node->socket_item );\r
 210 \r
 211         HeapFree( g_ibsp.heap, 0, node );\r
 212 \r
 213         IBSP_EXIT( IBSP_DBG_MEM );\r
 214         return 0;\r
 215 }\r
 216 \r
 217 \r
 218 /* Deregisters a memory region */\r
 219 int\r
 220 ibsp_dereg_mem(\r
 221         IN                              struct ibsp_socket_info         *s,\r
 222         IN                              struct memory_node                      *node,\r
 223                 OUT                     LPINT                                           lpErrno )\r
 224 {\r
 225         IBSP_ENTER( IBSP_DBG_MEM );\r
 226 \r
 227         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
 228         *lpErrno = __ibsp_dereg_mem_mr( node );\r
 229         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
 230 \r
 231         IBSP_EXIT( IBSP_DBG_MEM );\r
 232         return (*lpErrno? SOCKET_ERROR : 0);\r
 233 }\r
 234 \r
 235 \r
 236 /*\r
 237  * Deregister the remaining memory regions on an HCA. This function should\r
 238  * only be called before destroying the PD. In normal case, the list should\r
 239  * be empty because the switch should have done it.\r
 240  */\r
 241 void\r
 242 ibsp_dereg_hca(\r
 243         IN                              struct mr_list                          *mem_list )\r
 244 {\r
 245         cl_list_item_t *item;\r
 246 \r
 247         IBSP_ENTER( IBSP_DBG_MEM );\r
 248 \r
 249         cl_spinlock_acquire( &mem_list->mutex );\r
 250         IBSP_TRACE1( IBSP_DBG_MEM,\r
 251                 ("%d registrations.\n", cl_qlist_count( &mem_list->list )) );\r
 252 \r
 253         for( item = cl_qlist_remove_head( &mem_list->list );\r
 254                 item != cl_qlist_end( &mem_list->list );\r
 255                 item = cl_qlist_remove_head( &mem_list->list ) )\r
 256         {\r
 257                 struct memory_reg *p_reg = PARENT_STRUCT(item, struct memory_reg, item);\r
 258                 ib_api_status_t status;\r
 259 \r
 260                 while( cl_qlist_count( &p_reg->node_list ) )\r
 261                 {\r
 262                         struct memory_node *p_node =\r
 263                                 PARENT_STRUCT( cl_qlist_head( &p_reg->node_list ),\r
 264                                 struct memory_node, mr_item );\r
 265 \r
 266                         __ibsp_dereg_mem_mr( p_node );\r
 267                 }\r
 268 \r
 269                 IBSP_TRACE2( IBSP_DBG_MEM, ("unpinning ,memory reg %p\n", p_reg) );\r
 270                 status = ib_dereg_mr( p_reg->mr_handle );\r
 271                 if( status )\r
 272                 {\r
 273                         IBSP_ERROR(\r
 274                                 ("ib_dereg_mem returned %s\n", ib_get_err_str( status )) );\r
 275                 }\r
 276                 else\r
 277                 {\r
 278                         STAT_DEC( mr_num );\r
 279                 }\r
 280 \r
 281                 HeapFree( g_ibsp.heap, 0, p_reg );\r
 282         }\r
 283 \r
 284         cl_spinlock_release( &mem_list->mutex );\r
 285 \r
 286         IBSP_EXIT( IBSP_DBG_MEM );\r
 287 }\r
 288 \r
 289 \r
 290 /* Deregister the remaining memory regions. This function should only \r
 291  * be called when destroying the socket. In normal case, the list should \r
 292  * be empty because the switch should have done it. */\r
 293 void\r
 294 ibsp_dereg_socket(\r
 295         IN                              struct ibsp_socket_info         *s )\r
 296 {\r
 297         IBSP_ENTER( IBSP_DBG_MEM );\r
 298 \r
 299         if( !s->port )\r
 300         {\r
 301                 CL_ASSERT( !cl_qlist_count( &s->mr_list ) );\r
 302                 IBSP_EXIT( IBSP_DBG_MEM );\r
 303                 return;\r
 304         }\r
 305 \r
 306         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
 307         IBSP_TRACE1( IBSP_DBG_MEM,\r
 308                 ("%d registrations.\n", cl_qlist_count( &s->mr_list )) );\r
 309 \r
 310         while( cl_qlist_count( &s->mr_list ) )\r
 311         {\r
 312                 __ibsp_dereg_mem_mr( PARENT_STRUCT( cl_qlist_head( &s->mr_list ),\r
 313                         struct memory_node, socket_item) );\r
 314         }\r
 315 \r
 316         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
 317 \r
 318         IBSP_EXIT( IBSP_DBG_MEM );\r
 319 }\r
 320 \r
 321 \r
 322 /*\r
 323  * Loop through all the memory registrations on an HCA and release\r
 324  * all that fall within the specified range.\r
 325  */\r
 326 void\r
 327 ibsp_hca_flush_mr_cache(\r
 328         IN                              struct ibsp_hca                         *p_hca,\r
 329         IN                              LPVOID                                          lpvAddress,\r
 330         IN                              SIZE_T                                          Size )\r
 331 {\r
 332         struct memory_reg       *p_reg;\r
 333         cl_list_item_t          *p_item;\r
 334         ib_api_status_t         status;\r
 335 \r
 336         IBSP_ENTER( IBSP_DBG_MEM );\r
 337 \r
 338         cl_spinlock_acquire( &p_hca->rdma_mem_list.mutex );\r
 339         for( p_item = cl_qlist_head( &p_hca->rdma_mem_list.list );\r
 340                 p_item != cl_qlist_end( &p_hca->rdma_mem_list.list );\r
 341                 p_item = cl_qlist_next( p_item ) )\r
 342         {\r
 343                 p_reg = PARENT_STRUCT( p_item, struct memory_reg, item );\r
 344 \r
 345                 if( lpvAddress > p_reg->type.vaddr ||\r
 346                         ((uintn_t)lpvAddress) + Size <\r
 347                         ((uintn_t)(uint64_t)p_reg->type.vaddr) + p_reg->type.length )\r
 348                 {\r
 349                         continue;\r
 350                 }\r
 351 \r
 352                 /* Release all socket's nodes that reference this registration. */\r
 353                 while( cl_qlist_count( &p_reg->node_list ) )\r
 354                 {\r
 355                         struct memory_node      *p_node =\r
 356                                 PARENT_STRUCT( cl_qlist_head( &p_reg->node_list ),\r
 357                                 struct memory_node, mr_item );\r
 358 \r
 359                         __ibsp_dereg_mem_mr( p_node );\r
 360                 }\r
 361 \r
 362                 /* Move to the previous item so the for loop properly moves forward. */\r
 363                 p_item = cl_qlist_prev( p_item );\r
 364 \r
 365                 cl_qlist_remove_item( &p_hca->rdma_mem_list.list, &p_reg->item );\r
 366 \r
 367                 status = ib_dereg_mr( p_reg->mr_handle );\r
 368                 if( status != IB_SUCCESS )\r
 369                 {\r
 370                         IBSP_ERROR(\r
 371                                 ("ib_dereg_mr returned %s\n", ib_get_err_str(status)) );\r
 372                 }\r
 373 \r
 374                 HeapFree( g_ibsp.heap, 0, p_reg );\r
 375         }\r
 376         cl_spinlock_release( &p_hca->rdma_mem_list.mutex );\r
 377 \r
 378         IBSP_EXIT( IBSP_DBG_MEM );\r
 379 }\r