ulp/wsd/user/ibsp_mem.c

   1 /*\r
   2  * Copyright (c) 2005 SilverStorm Technologies.  All rights reserved.\r
   3  *\r
   4  * This software is available to you under the OpenIB.org BSD license\r
   5  * below:\r
   6  *\r
   7  *     Redistribution and use in source and binary forms, with or\r
   8  *     without modification, are permitted provided that the following\r
   9  *     conditions are met:\r
  10  *\r
  11  *      - Redistributions of source code must retain the above\r
  12  *        copyright notice, this list of conditions and the following\r
  13  *        disclaimer.\r
  14  *\r
  15  *      - Redistributions in binary form must reproduce the above\r
  16  *        copyright notice, this list of conditions and the following\r
  17  *        disclaimer in the documentation and/or other materials\r
  18  *        provided with the distribution.\r
  19  *\r
  20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
  21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
  22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
  23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
  24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
  25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
  26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
  27  * SOFTWARE.\r
  28  *\r
  29  * $Id$\r
  30  */\r
  31 \r
  32 /* Registers a memory region */\r
  33 \r
  34 #include "ibspdll.h"\r
  35 \r
  36 \r
  37 __forceinline boolean_t\r
  38 __check_mr(\r
  39         IN                              struct memory_reg                       *p_reg,\r
  40         IN                              ib_access_t                                     acl_mask,\r
  41         IN                              void                                            *start,\r
  42         IN                              size_t                                          len )\r
  43 {\r
  44         return( (p_reg->type.access_ctrl & acl_mask) == acl_mask &&\r
  45                 start >= p_reg->type.vaddr &&\r
  46                 ((uintn_t)start) + len <=\r
  47                 ((uintn_t)(uint64_t)p_reg->type.vaddr) + p_reg->type.length );\r
  48 }\r
  49 \r
  50 \r
  51 /* Find the first registered mr that matches the given region. \r
  52  * mem_list is either socket_info->buf_mem_list or socket_info->rdma_mem_list.\r
  53  */\r
  54 struct memory_node *\r
  55 lookup_partial_mr(\r
  56         IN                              struct ibsp_socket_info         *s,\r
  57         IN                              ib_access_t                                     acl_mask,\r
  58         IN                              void                                            *start,\r
  59         IN                              size_t                                          len )\r
  60 {\r
  61         struct memory_node      *p_node;\r
  62         cl_list_item_t          *p_item;\r
  63 \r
  64         IBSP_ENTER( IBSP_DBG_MEM );\r
  65 \r
  66         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
  67 \r
  68         for( p_item = cl_qlist_head( &s->mr_list );\r
  69                 p_item != cl_qlist_end( &s->mr_list );\r
  70                 p_item = cl_qlist_next( p_item ) )\r
  71         {\r
  72                 p_node = PARENT_STRUCT( p_item, struct memory_node, socket_item );\r
  73                 \r
  74                 if( __check_mr( p_node->p_reg, acl_mask, start, len ) )\r
  75                 {\r
  76                         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
  77                         IBSP_EXIT( IBSP_DBG_MEM );\r
  78                         return p_node;\r
  79                 }\r
  80         }\r
  81 \r
  82         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
  83 \r
  84         IBSP_TRACE_EXIT( IBSP_DBG_MEM, ("mr not found\n") );\r
  85         return NULL;\r
  86 }\r
  87 \r
  88 \r
  89 /* Registers a memory region. The memory region might be cached.\r
  90  * mem_list is either socket_info->buf_mem_list or hca->rdma_mem_list.\r
  91  */\r
  92 struct memory_node *\r
  93 ibsp_reg_mem(\r
  94         IN                              struct ibsp_socket_info         *s,\r
  95         IN                              ib_pd_handle_t                          pd,\r
  96         IN                              void                                            *start,\r
  97         IN                              size_t                                          len,\r
  98         IN                              ib_access_t                                     access_ctrl,\r
  99                 OUT                     LPINT                                           lpErrno )\r
 100 {\r
 101         struct memory_node      *p_node;\r
 102         struct memory_reg       *p_reg;\r
 103         cl_list_item_t          *p_item;\r
 104         ib_api_status_t         status;\r
 105 \r
 106         IBSP_ENTER( IBSP_DBG_MEM );\r
 107 \r
 108         CL_ASSERT( start != NULL );\r
 109         CL_ASSERT( len != 0 );\r
 110         CL_ASSERT( (access_ctrl & ~(IB_AC_RDMA_READ | IB_AC_RDMA_WRITE | IB_AC_LOCAL_WRITE)) ==\r
 111                           0 );\r
 112 \r
 113         /* Optimistically allocate a tracking structure. */\r
 114         p_node = HeapAlloc( g_ibsp.heap, 0, sizeof(struct memory_node) );\r
 115         if( !p_node )\r
 116         {\r
 117                 IBSP_ERROR_EXIT(\r
 118                         ("AllocateOverlappedBuf:HeapAlloc() failed: %d\n",\r
 119                         GetLastError()) );\r
 120                 *lpErrno = WSAENOBUFS;\r
 121                 return NULL;\r
 122         }\r
 123 \r
 124         /* First, try to find a suitable MR */\r
 125         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
 126 \r
 127         /* Find the first registered mr that matches the given region. */\r
 128         for( p_item = cl_qlist_head( &s->port->hca->rdma_mem_list.list );\r
 129                 p_item != cl_qlist_end( &s->port->hca->rdma_mem_list.list );\r
 130                 p_item = cl_qlist_next( p_item ) )\r
 131         {\r
 132                 p_reg = PARENT_STRUCT(p_item, struct memory_reg, item);\r
 133 \r
 134                 if( __check_mr( p_reg, access_ctrl, start, len ) )\r
 135                 {\r
 136                         p_node->p_reg = p_reg;\r
 137                         p_node->s = s;\r
 138                         cl_qlist_insert_tail( &p_reg->node_list, &p_node->mr_item );\r
 139                         cl_qlist_insert_head(\r
 140                                 &s->mr_list, &p_node->socket_item );\r
 141                         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
 142                         IBSP_EXIT( IBSP_DBG_MEM );\r
 143                         return p_node;\r
 144                 }\r
 145         }\r
 146 \r
 147         /* No corresponding MR has been found. Create a new one. */\r
 148         p_reg = HeapAlloc( g_ibsp.heap, 0, sizeof(struct memory_reg) );\r
 149 \r
 150         if( !p_reg )\r
 151         {\r
 152                 IBSP_ERROR_EXIT(\r
 153                         ("AllocateOverlappedBuf:HeapAlloc() failed: %d\n",\r
 154                         GetLastError()) );\r
 155                 cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
 156                 HeapFree( g_ibsp.heap, 0, p_node );\r
 157                 *lpErrno = WSAENOBUFS;\r
 158                 return NULL;\r
 159         }\r
 160 \r
 161         /* The node is not initialized yet. All the parameters given are\r
 162          * supposed to be valid so we don't check them. */\r
 163         cl_qlist_init( &p_reg->node_list );\r
 164         p_reg->type.vaddr = start;\r
 165         p_reg->type.length = len;\r
 166         p_reg->type.access_ctrl = access_ctrl;\r
 167 \r
 168         IBSP_TRACE2( IBSP_DBG_MEM, ("pinning memory node %p\n", p_node) );\r
 169         status = ib_reg_mem(\r
 170                 pd, &p_reg->type, &p_reg->lkey, &p_reg->rkey, &p_reg->mr_handle );\r
 171 \r
 172         if( status )\r
 173         {\r
 174                 cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
 175                 HeapFree( g_ibsp.heap, 0, p_reg );\r
 176                 HeapFree( g_ibsp.heap, 0, p_node );\r
 177 \r
 178                 IBSP_ERROR_EXIT(\r
 179                         ("ib_reg_mem returned %s\n", ib_get_err_str(status)) );\r
 180 \r
 181                 *lpErrno = WSAEFAULT;\r
 182                 return NULL;\r
 183         }\r
 184 \r
 185         STAT_INC( mr_num );\r
 186 \r
 187         p_node->p_reg = p_reg;\r
 188         p_node->s = s;\r
 189 \r
 190         /* Link to the list of nodes. */\r
 191         cl_qlist_insert_head( &s->port->hca->rdma_mem_list.list, &p_reg->item );\r
 192         cl_qlist_insert_head( &s->mr_list, &p_node->socket_item );\r
 193         cl_qlist_insert_tail( &p_reg->node_list, &p_node->mr_item );\r
 194         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
 195 \r
 196         IBSP_EXIT( IBSP_DBG_MEM );\r
 197 \r
 198         *lpErrno = 0;\r
 199         return p_node;\r
 200 }\r
 201 \r
 202 \r
 203 static inline int __ibsp_dereg_mem_mr(\r
 204         IN                              struct memory_node                      *node )\r
 205 {\r
 206         IBSP_ENTER( IBSP_DBG_MEM );\r
 207 \r
 208         // Underlying registration could be freed before the node.\r
 209         if( node->p_reg )\r
 210                 cl_qlist_remove_item( &node->p_reg->node_list, &node->mr_item );\r
 211 \r
 212         cl_qlist_remove_item( &node->s->mr_list, &node->socket_item );\r
 213 \r
 214         HeapFree( g_ibsp.heap, 0, node );\r
 215 \r
 216         IBSP_EXIT( IBSP_DBG_MEM );\r
 217         return 0;\r
 218 }\r
 219 \r
 220 \r
 221 /* Deregisters a memory region */\r
 222 int\r
 223 ibsp_dereg_mem(\r
 224         IN                              struct ibsp_socket_info         *s,\r
 225         IN                              struct memory_node                      *node,\r
 226                 OUT                     LPINT                                           lpErrno )\r
 227 {\r
 228         IBSP_ENTER( IBSP_DBG_MEM );\r
 229 \r
 230         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
 231         *lpErrno = __ibsp_dereg_mem_mr( node );\r
 232         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
 233 \r
 234         IBSP_EXIT( IBSP_DBG_MEM );\r
 235         return (*lpErrno? SOCKET_ERROR : 0);\r
 236 }\r
 237 \r
 238 \r
 239 /*\r
 240  * Deregister the remaining memory regions on an HCA. This function should\r
 241  * only be called before destroying the PD. In normal case, the list should\r
 242  * be empty because the switch should have done it.\r
 243  */\r
 244 void\r
 245 ibsp_dereg_hca(\r
 246         IN                              struct mr_list                          *mem_list )\r
 247 {\r
 248         cl_list_item_t *item;\r
 249 \r
 250         IBSP_ENTER( IBSP_DBG_MEM );\r
 251 \r
 252         cl_spinlock_acquire( &mem_list->mutex );\r
 253         IBSP_TRACE1( IBSP_DBG_MEM,\r
 254                 ("%d registrations.\n", cl_qlist_count( &mem_list->list )) );\r
 255 \r
 256         for( item = cl_qlist_remove_head( &mem_list->list );\r
 257                 item != cl_qlist_end( &mem_list->list );\r
 258                 item = cl_qlist_remove_head( &mem_list->list ) )\r
 259         {\r
 260                 struct memory_reg *p_reg = PARENT_STRUCT(item, struct memory_reg, item);\r
 261                 ib_api_status_t status;\r
 262 \r
 263                 /*\r
 264                  * Clear the pointer from the node to this registration.  No need\r
 265                  * to remove from the list as we're about to free the registration.\r
 266                  */\r
 267                 for( item = cl_qlist_head( &p_reg->node_list );\r
 268                         item != cl_qlist_end( &p_reg->node_list );\r
 269                         item = cl_qlist_next( item ) )\r
 270                 {\r
 271                         struct memory_node *p_node =\r
 272                                 PARENT_STRUCT( item, struct memory_node, mr_item );\r
 273 \r
 274                         p_node->p_reg = NULL;\r
 275                 }\r
 276 \r
 277                 IBSP_TRACE2( IBSP_DBG_MEM, ("unpinning ,memory reg %p\n", p_reg) );\r
 278                 status = ib_dereg_mr( p_reg->mr_handle );\r
 279                 if( status )\r
 280                 {\r
 281                         IBSP_ERROR(\r
 282                                 ("ib_dereg_mem returned %s\n", ib_get_err_str( status )) );\r
 283                 }\r
 284                 else\r
 285                 {\r
 286                         STAT_DEC( mr_num );\r
 287                 }\r
 288 \r
 289                 HeapFree( g_ibsp.heap, 0, p_reg );\r
 290         }\r
 291 \r
 292         cl_spinlock_release( &mem_list->mutex );\r
 293 \r
 294         IBSP_EXIT( IBSP_DBG_MEM );\r
 295 }\r
 296 \r
 297 \r
 298 /* Deregister the remaining memory regions. This function should only \r
 299  * be called when destroying the socket. In normal case, the list should \r
 300  * be empty because the switch should have done it. */\r
 301 void\r
 302 ibsp_dereg_socket(\r
 303         IN                              struct ibsp_socket_info         *s )\r
 304 {\r
 305         IBSP_ENTER( IBSP_DBG_MEM );\r
 306 \r
 307         if( !s->port )\r
 308         {\r
 309                 CL_ASSERT( !cl_qlist_count( &s->mr_list ) );\r
 310                 IBSP_EXIT( IBSP_DBG_MEM );\r
 311                 return;\r
 312         }\r
 313 \r
 314         cl_spinlock_acquire( &s->port->hca->rdma_mem_list.mutex );\r
 315         IBSP_TRACE1( IBSP_DBG_MEM,\r
 316                 ("%d registrations.\n", cl_qlist_count( &s->mr_list )) );\r
 317 \r
 318         while( cl_qlist_count( &s->mr_list ) )\r
 319         {\r
 320                 __ibsp_dereg_mem_mr( PARENT_STRUCT( cl_qlist_head( &s->mr_list ),\r
 321                         struct memory_node, socket_item) );\r
 322         }\r
 323 \r
 324         cl_spinlock_release( &s->port->hca->rdma_mem_list.mutex );\r
 325 \r
 326         IBSP_EXIT( IBSP_DBG_MEM );\r
 327 }\r
 328 \r
 329 \r
 330 /*\r
 331  * Loop through all the memory registrations on an HCA and release\r
 332  * all that fall within the specified range.\r
 333  */\r
 334 void\r
 335 ibsp_hca_flush_mr_cache(\r
 336         IN                              struct ibsp_hca                         *p_hca,\r
 337         IN                              LPVOID                                          lpvAddress,\r
 338         IN                              SIZE_T                                          Size )\r
 339 {\r
 340         struct memory_reg       *p_reg;\r
 341         cl_list_item_t          *p_item;\r
 342         ib_api_status_t         status;\r
 343 \r
 344         IBSP_ENTER( IBSP_DBG_MEM );\r
 345 \r
 346         cl_spinlock_acquire( &p_hca->rdma_mem_list.mutex );\r
 347         p_item = cl_qlist_head( &p_hca->rdma_mem_list.list );\r
 348         while( p_item != cl_qlist_end( &p_hca->rdma_mem_list.list ) )\r
 349         {\r
 350                 p_reg = PARENT_STRUCT( p_item, struct memory_reg, item );\r
 351 \r
 352                 /* Move to the next item now so we can remove the current. */\r
 353                 p_item = cl_qlist_next( p_item );\r
 354 \r
 355                 if( lpvAddress > p_reg->type.vaddr ||\r
 356                         ((uintn_t)lpvAddress) + Size <\r
 357                         ((uintn_t)(uint64_t)p_reg->type.vaddr) + p_reg->type.length )\r
 358                 {\r
 359                         continue;\r
 360                 }\r
 361 \r
 362                 /*\r
 363                  * Clear the pointer from all sockets' nodes to this registration.\r
 364                  * No need to remove from the list as we're about to free the\r
 365                  * registration.\r
 366                  */\r
 367                 for( p_item = cl_qlist_head( &p_reg->node_list );\r
 368                         p_item != cl_qlist_end( &p_reg->node_list );\r
 369                         p_item = cl_qlist_next( p_item ) )\r
 370                 {\r
 371                         struct memory_node *p_node =\r
 372                                 PARENT_STRUCT( p_item, struct memory_node, mr_item );\r
 373 \r
 374                         p_node->p_reg = NULL;\r
 375                 }\r
 376 \r
 377                 cl_qlist_remove_item( &p_hca->rdma_mem_list.list, &p_reg->item );\r
 378 \r
 379                 status = ib_dereg_mr( p_reg->mr_handle );\r
 380                 if( status != IB_SUCCESS )\r
 381                 {\r
 382                         IBSP_ERROR(\r
 383                                 ("ib_dereg_mr returned %s\n", ib_get_err_str(status)) );\r
 384                 }\r
 385 \r
 386                 HeapFree( g_ibsp.heap, 0, p_reg );\r
 387         }\r
 388         cl_spinlock_release( &p_hca->rdma_mem_list.mutex );\r
 389 \r
 390         IBSP_EXIT( IBSP_DBG_MEM );\r
 391 }\r