core/winverbs/kernel/wv_ep.c

   1 /*\r
   2  * Copyright (c) 2008 Intel Corporation. All rights reserved.\r
   3  *\r
   4  * This software is available to you under the OpenIB.org BSD license\r
   5  * below:\r
   6  *\r
   7  *     Redistribution and use in source and binary forms, with or\r
   8  *     without modification, are permitted provided that the following\r
   9  *     conditions are met:\r
  10  *\r
  11  *      - Redistributions of source code must retain the above\r
  12  *        copyright notice, this list of conditions and the following\r
  13  *        disclaimer.\r
  14  *\r
  15  *      - Redistributions in binary form must reproduce the above\r
  16  *        copyright notice, this list of conditions and the following\r
  17  *        disclaimer in the documentation and/or other materials\r
  18  *        provided with the distribution.\r
  19  *\r
  20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
  21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
  22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV\r
  23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
  24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
  25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
  26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
  27  * SOFTWARE.\r
  28  */\r
  29 \r
  30 #include <ntifs.h>\r
  31 #include <iba/ib_rdma_cm.h>\r
  32 \r
  33 #include "wv_ep.h"\r
  34 #include "wv_qp.h"\r
  35 #include "wv_ioctl.h"\r
  36 #include "wv_driver.h"\r
  37 \r
  38 #define WV_AF_INET      2\r
  39 #define WV_AF_INET6     23\r
  40 \r
  41 static void WvEpWorkHandler(WORK_ENTRY *pWork);\r
  42 \r
  43 static void WvEpGet(WV_ENDPOINT *pEndpoint)\r
  44 {\r
  45         InterlockedIncrement(&pEndpoint->Ref);\r
  46 }\r
  47 \r
  48 static void WvEpPut(WV_ENDPOINT *pEndpoint)\r
  49 {\r
  50         if (InterlockedDecrement(&pEndpoint->Ref) == 0) {\r
  51                 KeSetEvent(&pEndpoint->Event, 0, FALSE);\r
  52         }\r
  53 }\r
  54 \r
  55 WV_ENDPOINT *WvEpAcquire(WV_PROVIDER *pProvider, UINT64 Id)\r
  56 {\r
  57         WV_ENDPOINT *ep;\r
  58 \r
  59         KeAcquireGuardedMutex(&pProvider->Lock);\r
  60         WvProviderDisableRemove(pProvider);\r
  61         ep = IndexListAt(&pProvider->EpIndex, (SIZE_T) Id);\r
  62         if (ep != NULL && ep->State != WvEpDestroying) {\r
  63                 WvEpGet(ep);\r
  64         } else {\r
  65                 ep = NULL;\r
  66                 WvProviderEnableRemove(pProvider);\r
  67         }\r
  68         KeReleaseGuardedMutex(&pProvider->Lock);\r
  69 \r
  70         return ep;\r
  71 }\r
  72 \r
  73 void WvEpRelease(WV_ENDPOINT *pEndpoint)\r
  74 {\r
  75         WvProviderEnableRemove(pEndpoint->pProvider);\r
  76         WvEpPut(pEndpoint);\r
  77 }\r
  78 \r
  79 static NTSTATUS WvEpAllocate(WV_PROVIDER *pProvider, UINT16 EpType,\r
  80                                                          WV_ENDPOINT **ppEndpoint)\r
  81 {\r
  82         WV_ENDPOINT                     *ep;\r
  83         NTSTATUS                        status;\r
  84         WDF_IO_QUEUE_CONFIG     config;\r
  85 \r
  86         ep = ExAllocatePoolWithTag(NonPagedPool, sizeof(WV_ENDPOINT), 'pevw');\r
  87         if (ep == NULL) {\r
  88                 return STATUS_NO_MEMORY;\r
  89         }\r
  90 \r
  91         RtlZeroMemory(ep, sizeof(WV_ENDPOINT));\r
  92         ep->pWork = ExAllocatePoolWithTag(NonPagedPool, sizeof(WV_WORK_ENTRY), 'wevw');\r
  93         if (ep->pWork == NULL) {\r
  94                 status = STATUS_NO_MEMORY;\r
  95                 goto err1;\r
  96         }\r
  97 \r
  98         ep->Ref = 1;\r
  99         ep->pProvider = pProvider;\r
 100         ep->EpType = EpType;\r
 101         KeInitializeEvent(&ep->Event, NotificationEvent, FALSE);\r
 102         InitializeListHead(&ep->Entry);\r
 103 \r
 104         WDF_IO_QUEUE_CONFIG_INIT(&config, WdfIoQueueDispatchManual);\r
 105         status = WdfIoQueueCreate(ControlDevice, &config,\r
 106                                                           WDF_NO_OBJECT_ATTRIBUTES, &ep->Queue);\r
 107         if (!NT_SUCCESS(status)) {\r
 108                 goto err2;\r
 109         }\r
 110 \r
 111         *ppEndpoint = ep;\r
 112         return STATUS_SUCCESS;\r
 113 \r
 114 err2:\r
 115         ExFreePoolWithTag(ep->pWork, 'wevw');\r
 116 err1:\r
 117         ExFreePoolWithTag(ep, 'pevw');\r
 118         return status;\r
 119 }\r
 120 \r
 121 void WvEpCreate(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
 122 {\r
 123         UINT64                          *pId;\r
 124         UINT64                          *type;\r
 125         WV_ENDPOINT                     *ep;\r
 126         NTSTATUS                        status;\r
 127 \r
 128         status = WdfRequestRetrieveInputBuffer(Request, sizeof(UINT64), &type, NULL);\r
 129         if (!NT_SUCCESS(status)) {\r
 130                 goto err1;\r
 131         }\r
 132         status = WdfRequestRetrieveOutputBuffer(Request, sizeof(UINT64), &pId, NULL);\r
 133         if (!NT_SUCCESS(status)) {\r
 134                 goto err1;\r
 135         }\r
 136 \r
 137         status = WvEpAllocate(pProvider, (UINT16) *type, &ep);\r
 138         if (!NT_SUCCESS(status)) {\r
 139                 goto err1;\r
 140         }\r
 141 \r
 142         KeAcquireGuardedMutex(&pProvider->Lock);\r
 143         *pId = IndexListInsertHead(&pProvider->EpIndex, ep);\r
 144         if (*pId == 0) {\r
 145                 status = STATUS_NO_MEMORY;\r
 146                 goto err2;\r
 147         }\r
 148         KeReleaseGuardedMutex(&pProvider->Lock);\r
 149 \r
 150         WvWorkEntryInit(ep->pWork, *pId, WvEpWorkHandler, pProvider);\r
 151         WdfRequestCompleteWithInformation(Request, status, sizeof(UINT64));\r
 152         return;\r
 153 \r
 154 err2:\r
 155         KeReleaseGuardedMutex(&pProvider->Lock);\r
 156         WvEpFree(ep);\r
 157 err1:\r
 158         WdfRequestComplete(Request, status);\r
 159 }\r
 160 \r
 161 void WvEpDestroy(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
 162 {\r
 163         WV_ENDPOINT             *ep;\r
 164         UINT64                  *id;\r
 165         NTSTATUS                status;\r
 166 \r
 167         status = WdfRequestRetrieveInputBuffer(Request, sizeof(UINT64), &id, NULL);\r
 168         if (!NT_SUCCESS(status)) {\r
 169                 goto out;\r
 170         }\r
 171 \r
 172         KeAcquireGuardedMutex(&pProvider->Lock);\r
 173         WvProviderDisableRemove(pProvider);\r
 174         ep = IndexListAt(&pProvider->EpIndex, (SIZE_T) *id);\r
 175         if (ep == NULL) {\r
 176                 status = STATUS_NOT_FOUND;\r
 177         } else if (ep->Ref > 1) {\r
 178                 status = STATUS_ACCESS_DENIED;\r
 179         } else {\r
 180                 IndexListRemove(&pProvider->EpIndex, (SIZE_T) *id);\r
 181                 status = STATUS_SUCCESS;\r
 182         }\r
 183         KeReleaseGuardedMutex(&pProvider->Lock);\r
 184 \r
 185         if (NT_SUCCESS(status)) {\r
 186                 WvEpFree(ep);\r
 187         }\r
 188         WvProviderEnableRemove(pProvider);\r
 189 out:\r
 190         WdfRequestComplete(Request, status);\r
 191 }\r
 192 \r
 193 void WvEpFree(WV_ENDPOINT *pEndpoint)\r
 194 {\r
 195         WdfObjectAcquireLock(pEndpoint->Queue);\r
 196         pEndpoint->State = WvEpDestroying;\r
 197         WdfObjectReleaseLock(pEndpoint->Queue);\r
 198 \r
 199         if (InterlockedDecrement(&pEndpoint->Ref) > 0) {\r
 200                 KeWaitForSingleObject(&pEndpoint->Event, Executive, KernelMode, FALSE, NULL);\r
 201         }\r
 202 \r
 203         if (pEndpoint->pIbCmId != NULL) {\r
 204                 IbCmInterface.CM.destroy_id(pEndpoint->pIbCmId);\r
 205         }\r
 206 \r
 207         WdfIoQueuePurgeSynchronously(pEndpoint->Queue);\r
 208         WdfObjectDelete(pEndpoint->Queue);\r
 209         if (pEndpoint->pWork != NULL) {\r
 210                 ExFreePoolWithTag(pEndpoint->pWork, 'wevw');\r
 211         }\r
 212         ExFreePoolWithTag(pEndpoint, 'pevw');\r
 213 }\r
 214 \r
 215 void WvEpQuery(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
 216 {\r
 217         UINT64                          *id;\r
 218         WV_IO_EP_ATTRIBUTES     *pattr;\r
 219         WV_ENDPOINT                     *ep;\r
 220         size_t                          len = 0;\r
 221         NTSTATUS                        status;\r
 222 \r
 223         status = WdfRequestRetrieveInputBuffer(Request, sizeof(UINT64), &id, NULL);\r
 224         if (!NT_SUCCESS(status)) {\r
 225                 goto complete;\r
 226         }\r
 227         status = WdfRequestRetrieveOutputBuffer(Request, sizeof(WV_IO_EP_ATTRIBUTES),\r
 228                                                                                         &pattr, NULL);\r
 229         if (!NT_SUCCESS(status)) {\r
 230                 goto complete;\r
 231         }\r
 232 \r
 233         ep = WvEpAcquire(pProvider, *id);\r
 234         if (ep == NULL) {\r
 235                 status = STATUS_NOT_FOUND;\r
 236                 goto complete;\r
 237         }\r
 238 \r
 239         *pattr = ep->Attributes;\r
 240         WvEpRelease(ep);\r
 241         len = sizeof(WV_IO_EP_ATTRIBUTES);\r
 242 \r
 243 complete:\r
 244         WdfRequestCompleteWithInformation(Request, status, len);\r
 245 }\r
 246 \r
 247 void WvEpModify(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
 248 {\r
 249         WV_IO_ID                                *pId;\r
 250         size_t                                  inlen;\r
 251         WV_ENDPOINT                             *ep;\r
 252         NTSTATUS                                status;\r
 253 \r
 254         status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_ID), &pId, &inlen);\r
 255         if (!NT_SUCCESS(status)) {\r
 256                 goto complete;\r
 257         }\r
 258 \r
 259         if (pId->Data != WV_IO_EP_OPTION_ROUTE) {\r
 260                 status = STATUS_INVALID_PARAMETER;\r
 261                 goto complete;\r
 262         }\r
 263 \r
 264         if (inlen < sizeof(WV_IO_ID) + sizeof(ib_path_rec_t)) {\r
 265                 status = STATUS_BUFFER_TOO_SMALL;\r
 266                 goto complete;\r
 267         }\r
 268 \r
 269         ep = WvEpAcquire(pProvider, pId->Id);\r
 270         if (ep == NULL) {\r
 271                 status = STATUS_NOT_FOUND;\r
 272                 goto complete;\r
 273         }\r
 274 \r
 275         WdfObjectAcquireLock(ep->Queue);\r
 276         if (ep->State != WvEpAddressBound) {\r
 277                 status = STATUS_NOT_SUPPORTED;\r
 278                 goto release;\r
 279         }\r
 280 \r
 281         RtlCopyMemory(&ep->Route, pId + 1, sizeof ep->Route);\r
 282         ep->State = WvEpRouteResolved;\r
 283 \r
 284 release:\r
 285         WdfObjectReleaseLock(ep->Queue);\r
 286         WvEpRelease(ep);\r
 287 complete:\r
 288         WdfRequestComplete(Request, status);\r
 289 }\r
 290 \r
 291 void WvEpBind(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
 292 {\r
 293         WV_IO_EP_BIND           *pattr;\r
 294         WV_ENDPOINT                     *ep;\r
 295         size_t                          len = 0;\r
 296         NTSTATUS                        status;\r
 297 \r
 298         status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_EP_BIND),\r
 299                                                                                    &pattr, NULL);\r
 300         if (!NT_SUCCESS(status)) {\r
 301                 goto complete;\r
 302         }\r
 303         status = WdfRequestRetrieveOutputBuffer(Request, sizeof(WV_IO_EP_BIND),\r
 304                                                                                         &pattr, NULL);\r
 305         if (!NT_SUCCESS(status)) {\r
 306                 goto complete;\r
 307         }\r
 308 \r
 309         ep = WvEpAcquire(pProvider, pattr->Id);\r
 310         if (ep == NULL) {\r
 311                 status = STATUS_NOT_FOUND;\r
 312                 goto complete;\r
 313         }\r
 314 \r
 315         WdfObjectAcquireLock(ep->Queue);\r
 316         if (ep->State != WvEpIdle) {\r
 317                 status = STATUS_NOT_SUPPORTED;\r
 318                 goto release;\r
 319         }\r
 320 \r
 321         ep->Attributes.LocalAddress = pattr->Address;\r
 322         ep->Attributes.Device = pattr->Device;\r
 323         len = sizeof(WV_IO_EP_BIND);\r
 324         ep->State = WvEpAddressBound;\r
 325 \r
 326 release:\r
 327         WdfObjectReleaseLock(ep->Queue);\r
 328         WvEpRelease(ep);\r
 329 complete:\r
 330         WdfRequestCompleteWithInformation(Request, status, len);\r
 331 }\r
 332 \r
 333 static UINT64 WvGetServiceId(UINT16 EpType, WV_IO_SOCKADDR_DATA *pAddress)\r
 334 {\r
 335         return RtlUlonglongByteSwap(((UINT64)EpType << 16) +\r
 336                    RtlUshortByteSwap(pAddress->SockAddr.In.SinPort));\r
 337 }\r
 338 \r
 339 static int WvAnyAddress(WV_IO_SOCKADDR_DATA *pAddress)\r
 340 {\r
 341         if (pAddress->SockAddr.Sa.SaFamily == WV_AF_INET) {\r
 342                 return (pAddress->SockAddr.In.SinAddr == 0) ||\r
 343                            ((pAddress->SockAddr.In.SinAddr & 0xff) == 0x7f);\r
 344         } else {\r
 345                 return (RtlCompareMemoryUlong(pAddress->SockAddr.In6.Sin6Addr, 16, 0) == 16);\r
 346         }\r
 347 }\r
 348 \r
 349 static void WvFormatCmaHeader(IB_CMA_HEADER *pHeader,\r
 350                                                           WV_IO_SOCKADDR_DATA *pLocalAddress,\r
 351                                                           WV_IO_SOCKADDR_DATA *pPeerAddress)\r
 352 {\r
 353         pHeader->CmaVersion = IB_CMA_VERSION;\r
 354         if (pLocalAddress->SockAddr.Sa.SaFamily == WV_AF_INET) {\r
 355                 pHeader->IpVersion = 4 << 4;\r
 356                 pHeader->SrcAddress.Ip4.Address = pLocalAddress->SockAddr.In.SinAddr;\r
 357                 pHeader->DstAddress.Ip4.Address = pPeerAddress->SockAddr.In.SinAddr;\r
 358                 pHeader->Port = pLocalAddress->SockAddr.In.SinPort;\r
 359         } else {\r
 360                 pHeader->IpVersion = 6 << 4;\r
 361                 RtlCopyMemory(pHeader->SrcAddress.Ip6Address,\r
 362                                           pLocalAddress->SockAddr.In6.Sin6Addr, 16);\r
 363                 RtlCopyMemory(pHeader->DstAddress.Ip6Address,\r
 364                                           pPeerAddress->SockAddr.In6.Sin6Addr, 16);\r
 365                 pHeader->Port = pLocalAddress->SockAddr.In6.Sin6Port;\r
 366         }\r
 367 }\r
 368 \r
 369 static void WvEpSaveReply(WV_ENDPOINT *pEndpoint, iba_cm_rep_event *pReply)\r
 370 {\r
 371         UINT8   len;\r
 372 \r
 373         len = sizeof(pEndpoint->Attributes.Param.Connect.Data);\r
 374         RtlCopyMemory(pEndpoint->Attributes.Param.Connect.Data, pReply->rep.p_pdata, len);\r
 375         pEndpoint->Attributes.Param.Connect.DataLength = len;\r
 376         pEndpoint->Attributes.Param.Connect.InitiatorDepth = pReply->rep.resp_res;\r
 377         pEndpoint->Attributes.Param.Connect.ResponderResources = pReply->rep.init_depth;\r
 378         pEndpoint->Attributes.Param.Connect.RnrRetryCount = pReply->rep.rnr_retry_cnt;\r
 379 }\r
 380 \r
 381 static void WvEpSaveReject(WV_ENDPOINT *pEndpoint, iba_cm_rej_event *pReject)\r
 382 {\r
 383         UINT8   len;\r
 384 \r
 385         len = sizeof(pEndpoint->Attributes.Param.Connect.Data);\r
 386         RtlCopyMemory(pEndpoint->Attributes.Param.Connect.Data, pReject->p_pdata, len);\r
 387         pEndpoint->Attributes.Param.Connect.DataLength = len;\r
 388 }\r
 389 \r
 390 static NTSTATUS WvEpModifyQpErr(WV_QUEUE_PAIR *pQp,\r
 391                                                                 UINT8 *pVerbsData, UINT32 VerbsSize)\r
 392 {\r
 393         ib_qp_mod_t                     attr;\r
 394         ib_api_status_t         ib_status;\r
 395         NTSTATUS                        status;\r
 396 \r
 397         attr.req_state = IB_QPS_ERROR;\r
 398         ib_status = pQp->pVerbs->ndi_modify_qp(pQp->hVerbsQp, &attr, NULL,\r
 399                                                                                    VerbsSize, pVerbsData);\r
 400         if (ib_status == IB_SUCCESS) {\r
 401                 status = STATUS_SUCCESS;\r
 402         } else {        \r
 403                 status = STATUS_UNSUCCESSFUL;\r
 404         }\r
 405 \r
 406         return status;\r
 407 }\r
 408 \r
 409 static NTSTATUS WvEpDisconnectQp(WV_PROVIDER *pProvider, UINT64 QpId,\r
 410                                                                  UINT8 *pVerbsData, UINT32 VerbsSize)\r
 411 {\r
 412         WV_QUEUE_PAIR   *qp;\r
 413         NTSTATUS                status;\r
 414 \r
 415         if (QpId == 0) {\r
 416                 return STATUS_SUCCESS;\r
 417         }\r
 418 \r
 419         qp = WvQpAcquire(pProvider, QpId);\r
 420         if (qp == NULL) {\r
 421                 return STATUS_NOT_FOUND;\r
 422         }\r
 423 \r
 424         status = WvEpModifyQpErr(qp, pVerbsData, VerbsSize);\r
 425         WvQpRelease(qp);\r
 426 \r
 427         return status;\r
 428 }\r
 429 \r
 430 static NTSTATUS WvEpAsyncDisconnect(WV_ENDPOINT *pEndpoint, WDFREQUEST Request)\r
 431 {\r
 432         WV_IO_EP_DISCONNECT     *pattr;\r
 433         UINT8                           *out;\r
 434         size_t                          outlen = 0;\r
 435         NTSTATUS                        status;\r
 436 \r
 437         status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_EP_DISCONNECT),\r
 438                                                                                    &pattr, NULL);\r
 439         if (!NT_SUCCESS(status)) {\r
 440                 return status;\r
 441         }\r
 442 \r
 443         status = WdfRequestRetrieveOutputBuffer(Request, 0, &out, &outlen);\r
 444         if (!NT_SUCCESS(status) && status != STATUS_BUFFER_TOO_SMALL) {\r
 445                 return status;\r
 446         }\r
 447 \r
 448         status = (NTSTATUS) WdfRequestGetInformation(Request);\r
 449         if (NT_SUCCESS(status)) {\r
 450                 status = WvEpDisconnectQp(pEndpoint->pProvider, pattr->QpId, out, outlen);\r
 451         } else {\r
 452                 WvEpDisconnectQp(pEndpoint->pProvider, pattr->QpId, out, outlen);\r
 453         }\r
 454 \r
 455         WdfRequestCompleteWithInformation(Request, status, outlen);\r
 456         return STATUS_SUCCESS;\r
 457 }\r
 458 \r
 459 static void WvEpCompleteDisconnect(WV_ENDPOINT *pEndpoint, NTSTATUS DiscStatus)\r
 460 {\r
 461         WDFREQUEST                              request;\r
 462         WDFREQUEST                              disc_req = NULL;\r
 463         WDF_REQUEST_PARAMETERS  param;\r
 464         NTSTATUS                                status;\r
 465 \r
 466         WdfObjectAcquireLock(pEndpoint->Queue);\r
 467         if (pEndpoint->State == WvEpDestroying || !pEndpoint->pWork) {\r
 468                 goto release;\r
 469         }\r
 470         pEndpoint->State = WvEpDisconnected;\r
 471 \r
 472         status = WdfIoQueueRetrieveNextRequest(pEndpoint->Queue, &request);\r
 473         while (NT_SUCCESS(status)) {\r
 474 \r
 475                 WDF_REQUEST_PARAMETERS_INIT(&param);\r
 476                 WdfRequestGetParameters(request, &param);\r
 477                 if (param.Parameters.DeviceIoControl.IoControlCode == WV_IOCTL_EP_DISCONNECT) {\r
 478                         WdfRequestSetInformation(request, DiscStatus);\r
 479                         WvProviderGet(pEndpoint->pProvider);\r
 480                         WorkQueueInsert(&pEndpoint->pProvider->WorkQueue, &pEndpoint->pWork->Work);\r
 481                         pEndpoint->pWork = NULL;\r
 482                         disc_req = request;\r
 483                 } else {\r
 484                         WdfRequestComplete(request, DiscStatus);\r
 485                 }\r
 486 \r
 487                 status = WdfIoQueueRetrieveNextRequest(pEndpoint->Queue, &request);\r
 488         }\r
 489 \r
 490         if (disc_req != NULL) {\r
 491                 WdfRequestRequeue(disc_req);\r
 492         }\r
 493 release:\r
 494         WdfObjectReleaseLock(pEndpoint->Queue);\r
 495 \r
 496 }\r
 497 \r
 498 static NTSTATUS WvEpIbCmHandler(iba_cm_id *pId, iba_cm_event *pEvent)\r
 499 {\r
 500         WV_ENDPOINT     *ep;\r
 501 \r
 502         ep = pId->context;\r
 503         switch (pEvent->type) {\r
 504         case iba_cm_req_error:\r
 505                 WdfObjectAcquireLock(ep->Queue);\r
 506                 if (ep->State == WvEpActiveConnect) {\r
 507                         ep->State = WvEpDisconnected;\r
 508                         WvCompleteRequests(ep->Queue, STATUS_IO_TIMEOUT);\r
 509                 }\r
 510                 WdfObjectReleaseLock(ep->Queue);\r
 511                 break;\r
 512         case iba_cm_rep_error:\r
 513                 WdfObjectAcquireLock(ep->Queue);\r
 514                 if (ep->State == WvEpPassiveConnect) {\r
 515                         ep->State = WvEpDisconnected;\r
 516                         WvCompleteRequests(ep->Queue, STATUS_IO_TIMEOUT);\r
 517                 }\r
 518                 WdfObjectReleaseLock(ep->Queue);\r
 519                 break;\r
 520         case iba_cm_dreq_error:\r
 521                 WvEpCompleteDisconnect(ep, STATUS_IO_TIMEOUT);\r
 522                 break;\r
 523         case iba_cm_rep_received:\r
 524                 WdfObjectAcquireLock(ep->Queue);\r
 525                 if (ep->State == WvEpActiveConnect) {\r
 526                         WvEpSaveReply(ep, &pEvent->data.rep);\r
 527                         WvCompleteRequests(ep->Queue, STATUS_SUCCESS);\r
 528                 }\r
 529                 WdfObjectReleaseLock(ep->Queue);\r
 530                 break;\r
 531         case iba_cm_rtu_received:\r
 532                 WdfObjectAcquireLock(ep->Queue);\r
 533                 if (ep->State == WvEpPassiveConnect) {\r
 534                         ep->State = WvEpConnected;\r
 535                         WvCompleteRequestsWithInformation(ep->Queue, STATUS_SUCCESS);\r
 536                 }\r
 537                 WdfObjectReleaseLock(ep->Queue);\r
 538                 break;\r
 539         case iba_cm_dreq_received:\r
 540                 WdfObjectAcquireLock(ep->Queue);\r
 541                 if (ep->State == WvEpConnected) {\r
 542                         ep->State = WvEpPassiveDisconnect;\r
 543                         WvCompleteRequests(ep->Queue, STATUS_SUCCESS);\r
 544                         WdfObjectReleaseLock(ep->Queue);\r
 545                 } else if (ep->State == WvEpPassiveConnect) {\r
 546                         ep->State = WvEpPassiveDisconnect;\r
 547                         WvCompleteRequestsWithInformation(ep->Queue, STATUS_SUCCESS);\r
 548                         WdfObjectReleaseLock(ep->Queue);\r
 549                 } else {\r
 550                         WdfObjectReleaseLock(ep->Queue);\r
 551                         WvEpCompleteDisconnect(ep, STATUS_SUCCESS);\r
 552                 }\r
 553                 break;\r
 554         case iba_cm_drep_received:\r
 555                 WvEpCompleteDisconnect(ep, STATUS_SUCCESS);\r
 556                 break;\r
 557         case iba_cm_rej_received:\r
 558                 WdfObjectAcquireLock(ep->Queue);\r
 559                 if (ep->State == WvEpPassiveConnect || ep->State == WvEpActiveConnect) {\r
 560                         ep->State = WvEpDisconnected;\r
 561                         WvEpSaveReject(ep, &pEvent->data.rej);\r
 562                         WvCompleteRequests(ep->Queue, STATUS_CONNECTION_REFUSED);\r
 563                 }\r
 564                 WdfObjectReleaseLock(ep->Queue);\r
 565                 break;\r
 566         case iba_cm_mra_received:\r
 567                 break;\r
 568         default:\r
 569                 WdfObjectAcquireLock(ep->Queue);\r
 570                 if (ep->State != WvEpDestroying) {\r
 571                         ep->State = WvEpDisconnected;\r
 572                         WvCompleteRequests(ep->Queue, STATUS_NOT_IMPLEMENTED);\r
 573                 }\r
 574                 WdfObjectReleaseLock(ep->Queue);\r
 575                 break;\r
 576         }\r
 577 \r
 578         return STATUS_SUCCESS;\r
 579 }\r
 580 \r
 581 static NTSTATUS WvEpAsyncConnect(WV_ENDPOINT *pEndpoint, WDFREQUEST Request)\r
 582 {\r
 583         WV_IO_EP_CONNECT        *pattr;\r
 584         WV_QUEUE_PAIR           *qp;\r
 585         iba_cm_req                      req;\r
 586         NTSTATUS                        status;\r
 587         UINT8                           data[IB_REQ_PDATA_SIZE];\r
 588 \r
 589         status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_EP_CONNECT),\r
 590                                                                                    &pattr, NULL);\r
 591         if (!NT_SUCCESS(status)) {\r
 592                 return status;\r
 593         }\r
 594 \r
 595         qp = WvQpAcquire(pEndpoint->pProvider, pattr->QpId);\r
 596         if (qp == NULL) {\r
 597                 return STATUS_NOT_FOUND;\r
 598         }\r
 599 \r
 600         pEndpoint->Attributes.PeerAddress = pattr->PeerAddress;\r
 601         WvFormatCmaHeader((IB_CMA_HEADER *) data, &pEndpoint->Attributes.LocalAddress,\r
 602                                           &pEndpoint->Attributes.PeerAddress);\r
 603 \r
 604         req.service_id = WvGetServiceId(pEndpoint->EpType, &pEndpoint->Attributes.PeerAddress);\r
 605         req.p_primary_path = &pEndpoint->Route;\r
 606         req.p_alt_path = NULL;\r
 607         req.qpn = qp->Qpn;\r
 608         req.qp_type = IB_QPT_RELIABLE_CONN;\r
 609         req.starting_psn = (net32_t) RtlRandomEx(&RandomSeed);\r
 610         req.p_pdata = data;\r
 611         RtlCopyMemory(data + sizeof(IB_CMA_HEADER), pattr->Param.Data,\r
 612                                   pattr->Param.DataLength);\r
 613         req.pdata_len = sizeof(IB_CMA_HEADER) + pattr->Param.DataLength;\r
 614         req.max_cm_retries = IB_CMA_MAX_CM_RETRIES;\r
 615         req.resp_res = (UINT8) pattr->Param.ResponderResources;\r
 616         req.init_depth = (UINT8) pattr->Param.InitiatorDepth;\r
 617         req.remote_resp_timeout = IB_CMA_CM_RESPONSE_TIMEOUT;\r
 618         req.flow_ctrl = 1;\r
 619         req.local_resp_timeout = IB_CMA_CM_RESPONSE_TIMEOUT;\r
 620         req.rnr_retry_cnt = pattr->Param.RnrRetryCount;\r
 621         req.retry_cnt = pattr->Param.RetryCount;\r
 622         req.srq = (qp->pSrq != NULL);\r
 623 \r
 624         WvQpRelease(qp);\r
 625         RtlCopyMemory(&pEndpoint->Attributes.Param.Connect, &pattr->Param,\r
 626                                   sizeof(pattr->Param));\r
 627 \r
 628         WdfObjectAcquireLock(pEndpoint->Queue);\r
 629         if (pEndpoint->State != WvEpRouteResolved) {\r
 630                 status = STATUS_NOT_SUPPORTED;\r
 631                 goto out;\r
 632         }\r
 633 \r
 634         status = IbCmInterface.CM.create_id(WvEpIbCmHandler, pEndpoint, &pEndpoint->pIbCmId);\r
 635         if (!NT_SUCCESS(status)) {\r
 636                 goto out;\r
 637         }\r
 638 \r
 639         pEndpoint->State = WvEpActiveConnect;\r
 640         status = IbCmInterface.CM.send_req(pEndpoint->pIbCmId, &req);\r
 641         if (NT_SUCCESS(status)) {\r
 642                 status = WdfRequestRequeue(Request);\r
 643         }\r
 644 \r
 645         if (!NT_SUCCESS(status)) {\r
 646                 pEndpoint->State = WvEpDisconnected;\r
 647         }\r
 648 \r
 649 out:\r
 650         WdfObjectReleaseLock(pEndpoint->Queue);\r
 651         return status;\r
 652 }\r
 653 \r
 654 static NTSTATUS WvEpProcessAsync(WV_PROVIDER *pProvider, UINT64 Id, WDFREQUEST Request)\r
 655 {\r
 656         WV_ENDPOINT     *ep;\r
 657         NTSTATUS        status;\r
 658 \r
 659         ep = WvEpAcquire(pProvider, Id);\r
 660         if (ep == NULL) {\r
 661                 return STATUS_NOT_FOUND;\r
 662         }\r
 663 \r
 664         WdfObjectAcquireLock(ep->Queue);\r
 665         if (!ep->pWork) {\r
 666                 status = STATUS_TOO_MANY_COMMANDS;\r
 667                 goto out;\r
 668         }\r
 669 \r
 670         status = WdfRequestForwardToIoQueue(Request, ep->Queue);\r
 671         if (NT_SUCCESS(status)) {\r
 672                 WvProviderGet(pProvider);\r
 673                 WorkQueueInsert(&pProvider->WorkQueue, &ep->pWork->Work);\r
 674                 ep->pWork = NULL;\r
 675         }\r
 676 \r
 677 out:\r
 678         WdfObjectReleaseLock(ep->Queue);\r
 679         WvEpRelease(ep);\r
 680         return status;\r
 681 }\r
 682 \r
 683 void WvEpConnect(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
 684 {\r
 685         WV_IO_EP_CONNECT        *pattr;\r
 686         NTSTATUS                        status;\r
 687 \r
 688         status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_EP_CONNECT),\r
 689                                                                                    &pattr, NULL);\r
 690         if (!NT_SUCCESS(status)) {\r
 691                 goto out;\r
 692         }\r
 693 \r
 694         if (pattr->Param.DataLength > sizeof(pattr->Param.Data)) {\r
 695                 status = STATUS_INVALID_BUFFER_SIZE;\r
 696                 goto out;\r
 697         }\r
 698 \r
 699         status = WvEpProcessAsync(pProvider, pattr->Id, Request);\r
 700 \r
 701 out:\r
 702         if (!NT_SUCCESS(status)) {\r
 703                 WdfRequestComplete(Request, status);\r
 704         }\r
 705 }\r
 706 \r
 707 static NTSTATUS WvEpModifyQpRtr(WV_ENDPOINT *pEndpoint, WV_QUEUE_PAIR *pQp,\r
 708                                                                 UINT64 ResponderResources, UINT32 Psn,\r
 709                                                                 UINT8 *pVerbsData, UINT32 VerbsSize)\r
 710 {\r
 711         ib_qp_mod_t                     attr;\r
 712         ib_api_status_t         ib_status;\r
 713         NTSTATUS                        status;\r
 714 \r
 715         status =IbCmInterface.CM.get_qp_attr(pEndpoint->pIbCmId, IB_QPS_INIT, &attr);\r
 716         if (!NT_SUCCESS(status)) {\r
 717                 return status;\r
 718         }\r
 719         \r
 720         ib_status = pQp->pVerbs->ndi_modify_qp(pQp->hVerbsQp, &attr, NULL,\r
 721                                                                                    VerbsSize, pVerbsData);\r
 722         if (ib_status != IB_SUCCESS) {\r
 723                 return STATUS_UNSUCCESSFUL;\r
 724         }\r
 725 \r
 726         status = IbCmInterface.CM.get_qp_attr(pEndpoint->pIbCmId, IB_QPS_RTR, &attr);\r
 727         if (!NT_SUCCESS(status)) {\r
 728                 return status;\r
 729         }\r
 730         \r
 731         if (pEndpoint->State == WvEpPassiveConnect) {\r
 732                 attr.state.rtr.resp_res = (UINT8) ResponderResources;\r
 733                 attr.state.rtr.rq_psn = Psn;\r
 734         }\r
 735 \r
 736         ib_status = pQp->pVerbs->ndi_modify_qp(pQp->hVerbsQp, &attr, NULL,\r
 737                                                                                    VerbsSize, pVerbsData);\r
 738         if (ib_status != IB_SUCCESS) {\r
 739                 return STATUS_UNSUCCESSFUL;\r
 740         }\r
 741 \r
 742         return STATUS_SUCCESS;\r
 743 }\r
 744 \r
 745 static NTSTATUS WvEpModifyQpRts(WV_ENDPOINT *pEndpoint, WV_QUEUE_PAIR *pQp,\r
 746                                                                 UINT64 InitiatorDepth,\r
 747                                                                 UINT8 *pVerbsData, UINT32 VerbsSize)\r
 748 {\r
 749         ib_qp_mod_t                     attr;\r
 750         ib_api_status_t         ib_status;\r
 751         NTSTATUS                        status;\r
 752 \r
 753         status = IbCmInterface.CM.get_qp_attr(pEndpoint->pIbCmId, IB_QPS_RTS, &attr);\r
 754         if (!NT_SUCCESS(status)) {\r
 755                 return status;\r
 756         }\r
 757         \r
 758         if (pEndpoint->State == WvEpPassiveConnect) {\r
 759                 attr.state.rts.init_depth = (UINT8) InitiatorDepth;\r
 760         }\r
 761 \r
 762         ib_status = pQp->pVerbs->ndi_modify_qp(pQp->hVerbsQp, &attr, NULL,\r
 763                                                                                    VerbsSize, pVerbsData);\r
 764         if (ib_status != IB_SUCCESS) {\r
 765                 return STATUS_UNSUCCESSFUL;\r
 766         }\r
 767 \r
 768         return STATUS_SUCCESS;\r
 769 }\r
 770 \r
 771 static NTSTATUS WvEpAcceptActive(WDFREQUEST Request, UINT8 *pVerbsData, size_t VerbsSize,\r
 772                                                                  WV_ENDPOINT *pEndpoint, WV_IO_EP_ACCEPT *pAttr)\r
 773 {\r
 774         WV_QUEUE_PAIR           *qp;\r
 775         NTSTATUS                        status;\r
 776 \r
 777         qp = WvQpAcquire(pEndpoint->pProvider, pAttr->QpId);\r
 778         if (qp == NULL) {\r
 779                 return STATUS_NOT_FOUND;\r
 780         }\r
 781 \r
 782         status = WvEpModifyQpRtr(pEndpoint, qp, 0, 0, pVerbsData, VerbsSize);\r
 783         if (NT_SUCCESS(status)) {\r
 784                 status = WvEpModifyQpRts(pEndpoint, qp, 0, pVerbsData, VerbsSize);\r
 785         }\r
 786 \r
 787         WvQpRelease(qp);\r
 788 \r
 789         WdfObjectAcquireLock(pEndpoint->Queue);\r
 790         if (pEndpoint->State != WvEpActiveConnect) {\r
 791                 status = STATUS_NOT_SUPPORTED;\r
 792                 goto release;\r
 793         }\r
 794 \r
 795         pEndpoint->State = WvEpConnected;\r
 796         status = IbCmInterface.CM.send_rtu(pEndpoint->pIbCmId, pAttr->Param.Data,\r
 797                                                                            pAttr->Param.DataLength);\r
 798         if (NT_SUCCESS(status)) {\r
 799                 WdfRequestCompleteWithInformation(Request, status, VerbsSize);\r
 800         } else {\r
 801                 pEndpoint->State = WvEpDisconnected;\r
 802         }\r
 803 \r
 804 release:\r
 805         WdfObjectReleaseLock(pEndpoint->Queue);\r
 806         return status;\r
 807 }\r
 808 \r
 809 static NTSTATUS WvEpAcceptPassive(WDFREQUEST Request, UINT8 *pVerbsData, size_t VerbsSize,\r
 810                                                                   WV_ENDPOINT *pEndpoint, WV_IO_EP_ACCEPT *pAttr)\r
 811 {\r
 812         WV_QUEUE_PAIR           *qp;\r
 813         iba_cm_rep                      rep;\r
 814         NTSTATUS                        status;\r
 815 \r
 816         qp = WvQpAcquire(pEndpoint->pProvider, pAttr->QpId);\r
 817         if (qp == NULL) {\r
 818                 return STATUS_NOT_FOUND;\r
 819         }\r
 820 \r
 821         rep.qpn = qp->Qpn;\r
 822         rep.starting_psn = (net32_t) RtlRandomEx(&RandomSeed);\r
 823         rep.p_pdata = pAttr->Param.Data;\r
 824         rep.pdata_len = pAttr->Param.DataLength;\r
 825         rep.failover_accepted = IB_FAILOVER_ACCEPT_UNSUPPORTED;\r
 826         rep.resp_res = (UINT8) pAttr->Param.ResponderResources;\r
 827         rep.init_depth = (UINT8) pAttr->Param.InitiatorDepth;\r
 828         rep.flow_ctrl = 1;\r
 829         rep.rnr_retry_cnt = pAttr->Param.RnrRetryCount;\r
 830         rep.srq = (qp->pSrq != NULL);\r
 831 \r
 832         status = WvEpModifyQpRtr(pEndpoint, qp, pAttr->Param.ResponderResources,\r
 833                                                          rep.starting_psn, pVerbsData, VerbsSize);\r
 834         if (NT_SUCCESS(status)) {\r
 835                 status = WvEpModifyQpRts(pEndpoint, qp, pAttr->Param.InitiatorDepth,\r
 836                                                                  pVerbsData, VerbsSize);\r
 837         }\r
 838 \r
 839         WvQpRelease(qp);\r
 840 \r
 841         if (!NT_SUCCESS(status)) {\r
 842                 goto out;\r
 843         }\r
 844 \r
 845         WdfObjectAcquireLock(pEndpoint->Queue);\r
 846         if (pEndpoint->State != WvEpPassiveConnect) {\r
 847                 status = STATUS_NOT_SUPPORTED;\r
 848                 goto release;\r
 849         }\r
 850 \r
 851         status = IbCmInterface.CM.send_rep(pEndpoint->pIbCmId, &rep);\r
 852         if (NT_SUCCESS(status)) {\r
 853                 status = WdfRequestRequeue(Request);\r
 854         }\r
 855         \r
 856         if (!NT_SUCCESS(status)) {\r
 857                 pEndpoint->State = WvEpDisconnected;\r
 858         }\r
 859 \r
 860 release:\r
 861         WdfObjectReleaseLock(pEndpoint->Queue);\r
 862 out:\r
 863         return status;\r
 864 }\r
 865 \r
 866 static NTSTATUS WvEpAsyncAccept(WV_ENDPOINT *pEndpoint, WDFREQUEST Request)\r
 867 {\r
 868         WV_IO_EP_ACCEPT         *pattr;\r
 869         NTSTATUS                        status;\r
 870         UINT8                           *out;\r
 871         size_t                          outlen;\r
 872 \r
 873         status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_EP_ACCEPT),\r
 874                                                                                    &pattr, NULL);\r
 875         if (!NT_SUCCESS(status)) {\r
 876                 return status;\r
 877         }\r
 878 \r
 879         status = WdfRequestRetrieveOutputBuffer(Request, 0, &out, &outlen);\r
 880         if (!NT_SUCCESS(status) && status != STATUS_BUFFER_TOO_SMALL) {\r
 881                 return status;\r
 882         }\r
 883 \r
 884         /* EP state is re-checked under lock in WvEpAccept* calls */\r
 885         switch (pEndpoint->State) {\r
 886         case WvEpActiveConnect:\r
 887                 status = WvEpAcceptActive(Request, out, outlen, pEndpoint, pattr);\r
 888                 break;\r
 889         case WvEpPassiveConnect:\r
 890                 status = WvEpAcceptPassive(Request, out, outlen, pEndpoint, pattr);\r
 891                 break;\r
 892         default:\r
 893                 status = STATUS_NOT_SUPPORTED;\r
 894                 break;\r
 895         }\r
 896 \r
 897         return status;\r
 898 }\r
 899 \r
 900 static void WvEpWorkHandler(WORK_ENTRY *pWork)\r
 901 {\r
 902         WV_PROVIDER                             *prov;\r
 903         WV_ENDPOINT                             *ep;\r
 904         WV_WORK_ENTRY                   *work;\r
 905         WDFREQUEST                              request;\r
 906         WDF_REQUEST_PARAMETERS  param;\r
 907         NTSTATUS                                status;\r
 908 \r
 909         work = CONTAINING_RECORD(pWork, WV_WORK_ENTRY, Work);\r
 910         prov = (WV_PROVIDER *) pWork->Context;\r
 911 \r
 912         ep = WvEpAcquire(prov, work->Id);\r
 913         if (ep == NULL) {\r
 914                 ExFreePoolWithTag(work, 'wevw');\r
 915                 goto out;\r
 916         }\r
 917 \r
 918         WdfObjectAcquireLock(ep->Queue);\r
 919         ep->pWork = work;\r
 920         status = WdfIoQueueRetrieveNextRequest(ep->Queue, &request);\r
 921         WdfObjectReleaseLock(ep->Queue);\r
 922 \r
 923         if (!NT_SUCCESS(status)) {\r
 924                 goto put;\r
 925         }\r
 926 \r
 927         WDF_REQUEST_PARAMETERS_INIT(&param);\r
 928         WdfRequestGetParameters(request, &param);\r
 929         switch (param.Parameters.DeviceIoControl.IoControlCode) {\r
 930         case WV_IOCTL_EP_CONNECT:\r
 931                 status = WvEpAsyncConnect(ep, request);\r
 932                 break;\r
 933         case WV_IOCTL_EP_ACCEPT:\r
 934                 status = WvEpAsyncAccept(ep, request);\r
 935                 break;\r
 936         case WV_IOCTL_EP_DISCONNECT:\r
 937                 status = WvEpAsyncDisconnect(ep, request);\r
 938                 break;\r
 939         default:\r
 940                 status = STATUS_NOT_IMPLEMENTED;\r
 941         }\r
 942 \r
 943         if (!NT_SUCCESS(status)) {\r
 944                 WdfRequestComplete(request, status);\r
 945         }\r
 946 put:\r
 947         WvEpRelease(ep);\r
 948 out:\r
 949         WvProviderPut(prov);\r
 950 }\r
 951 \r
 952 void WvEpAccept(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
 953 {\r
 954         WV_IO_EP_ACCEPT         *pattr;\r
 955         NTSTATUS                        status;\r
 956 \r
 957         status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_EP_ACCEPT),\r
 958                                                                                    &pattr, NULL);\r
 959         if (!NT_SUCCESS(status)) {\r
 960                 goto out;\r
 961         }\r
 962 \r
 963         if (pattr->Param.DataLength > sizeof(pattr->Param.Data)) {\r
 964                 status = STATUS_INVALID_BUFFER_SIZE;\r
 965                 goto out;\r
 966         }\r
 967 \r
 968         status = WvEpProcessAsync(pProvider, pattr->Id, Request);\r
 969 \r
 970 out:\r
 971         if (!NT_SUCCESS(status)) {\r
 972                 WdfRequestComplete(Request, status);\r
 973         }\r
 974 }\r
 975 \r
 976 void WvEpReject(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
 977 {\r
 978         WV_IO_ID                        *id;\r
 979         WV_ENDPOINT                     *ep;\r
 980         NTSTATUS                        status;\r
 981         size_t                          len;\r
 982 \r
 983         status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_ID), &id, &len);\r
 984         if (!NT_SUCCESS(status)) {\r
 985                 goto complete;\r
 986         }\r
 987 \r
 988         ep = WvEpAcquire(pProvider, id->Id);\r
 989         if (ep == NULL) {\r
 990                 status = STATUS_NOT_FOUND;\r
 991                 goto complete;\r
 992         }\r
 993 \r
 994         WdfObjectAcquireLock(ep->Queue);\r
 995         if (ep->State != WvEpActiveConnect && ep->State != WvEpPassiveConnect) {\r
 996                 status = STATUS_NOT_SUPPORTED;\r
 997                 goto release;\r
 998         }\r
 999 \r
1000         ep->State = WvEpDisconnected;\r
1001         status = IbCmInterface.CM.send_rej(ep->pIbCmId, IB_REJ_USER_DEFINED,\r
1002                                                                            NULL, 0, id + 1, len - sizeof(WV_IO_ID));\r
1003 \r
1004 release:\r
1005         WdfObjectReleaseLock(ep->Queue);\r
1006         WvEpRelease(ep);\r
1007 complete:\r
1008         WdfRequestComplete(Request, status);\r
1009 }\r
1010 \r
1011 // The IB CM could have received and processed a DREQ that we haven't seen yet.\r
1012 void WvEpDisconnect(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
1013 {\r
1014         WV_IO_EP_DISCONNECT     *pattr;\r
1015         WV_ENDPOINT                     *ep;\r
1016         NTSTATUS                        status;\r
1017         UINT8                           *out;\r
1018         size_t                          outlen;\r
1019 \r
1020         status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_EP_DISCONNECT),\r
1021                                                                                    &pattr, NULL);\r
1022         if (!NT_SUCCESS(status)) {\r
1023                 goto complete;\r
1024         }\r
1025 \r
1026         status = WdfRequestRetrieveOutputBuffer(Request, 0, &out, &outlen);\r
1027         if (!NT_SUCCESS(status) && status != STATUS_BUFFER_TOO_SMALL) {\r
1028                 goto complete;\r
1029         }\r
1030 \r
1031         ep = WvEpAcquire(pProvider, pattr->Id);\r
1032         if (ep == NULL) {\r
1033                 status = STATUS_NOT_FOUND;\r
1034                 goto complete;\r
1035         }\r
1036 \r
1037         WdfObjectAcquireLock(ep->Queue);\r
1038         switch (ep->State) {\r
1039         case WvEpConnected:\r
1040                 status = IbCmInterface.CM.send_dreq(ep->pIbCmId, NULL, 0);\r
1041                 if (NT_SUCCESS(status)) {\r
1042                         status = WdfRequestForwardToIoQueue(Request, ep->Queue);\r
1043                         if (NT_SUCCESS(status)) {\r
1044                                 ep->State = WvEpActiveDisconnect;\r
1045                                 break;\r
1046                         }\r
1047                 }\r
1048                 /* Fall through to passive disconnect case on failure */\r
1049         case WvEpPassiveDisconnect:\r
1050                 ep->State = WvEpDisconnected;\r
1051                 WdfObjectReleaseLock(ep->Queue);\r
1052 \r
1053                 IbCmInterface.CM.send_drep(ep->pIbCmId, NULL, 0);\r
1054 \r
1055                 status = WvEpDisconnectQp(ep->pProvider, pattr->QpId, out, outlen);\r
1056                 if (NT_SUCCESS(status)) {\r
1057                         WdfRequestCompleteWithInformation(Request, status, outlen);\r
1058                 }\r
1059                 goto release;\r
1060         default:\r
1061                 status = STATUS_INVALID_DEVICE_STATE;\r
1062                 break;\r
1063         }\r
1064         WdfObjectReleaseLock(ep->Queue);\r
1065 \r
1066 release:\r
1067         WvEpRelease(ep);\r
1068 complete:\r
1069         if (!NT_SUCCESS(status)) {\r
1070                 WdfRequestComplete(Request, status);\r
1071         }\r
1072 }\r
1073 \r
1074 void WvEpDisconnectNotify(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
1075 {\r
1076         UINT64                          *id;\r
1077         WV_ENDPOINT                     *ep;\r
1078         NTSTATUS                        status;\r
1079 \r
1080         status = WdfRequestRetrieveInputBuffer(Request, sizeof(UINT64), &id, NULL);\r
1081         if (!NT_SUCCESS(status)) {\r
1082                 goto complete;\r
1083         }\r
1084 \r
1085         ep = WvEpAcquire(pProvider, *id);\r
1086         if (ep == NULL) {\r
1087                 status = STATUS_NOT_FOUND;\r
1088                 goto complete;\r
1089         }\r
1090 \r
1091         WdfObjectAcquireLock(ep->Queue);\r
1092         switch (ep->State) {\r
1093         case WvEpConnected:\r
1094         case WvEpActiveDisconnect:\r
1095                 status = WdfRequestForwardToIoQueue(Request, ep->Queue);\r
1096                 if (NT_SUCCESS(status)) {\r
1097                         WdfObjectReleaseLock(ep->Queue);\r
1098                         WvEpRelease(ep);\r
1099                         return;\r
1100                 }\r
1101                 break;\r
1102         case WvEpPassiveDisconnect:\r
1103         case WvEpDisconnected:\r
1104                 status = STATUS_SUCCESS;\r
1105                 break;\r
1106         default:\r
1107                 status = STATUS_NOT_SUPPORTED;\r
1108                 break;\r
1109         }\r
1110         WdfObjectReleaseLock(ep->Queue);\r
1111 \r
1112         WvEpRelease(ep);\r
1113 complete:\r
1114         WdfRequestComplete(Request, status);\r
1115 }\r
1116 \r
1117 static void WvEpGetIbRequest(WV_ENDPOINT *pListen)\r
1118 {\r
1119         WV_ENDPOINT             *ep;\r
1120         WDFREQUEST              request;\r
1121         NTSTATUS                status;\r
1122         IB_CMA_HEADER   *hdr;\r
1123         iba_cm_id               *id;\r
1124         iba_cm_event    event;\r
1125 \r
1126         WdfObjectAcquireLock(pListen->Queue);\r
1127         while (1) {\r
1128                 status = WdfIoQueueRetrieveNextRequest(pListen->Queue, &request);\r
1129                 if (!NT_SUCCESS(status)) {\r
1130                         break;\r
1131                 }\r
1132 \r
1133                 status = IbCmInterface.CM.get_request(pListen->pIbCmId, &id, &event);\r
1134                 if (!NT_SUCCESS(status)) {\r
1135                         WdfRequestRequeue(request);\r
1136                         break;\r
1137                 }\r
1138 \r
1139                 ASSERT(!IsListEmpty(&pListen->Entry));\r
1140                 ep = CONTAINING_RECORD(RemoveHeadList(&pListen->Entry), WV_ENDPOINT, Entry);\r
1141                 ep->pIbCmId = id;\r
1142                 id->callback = WvEpIbCmHandler;\r
1143                 id->context = ep;\r
1144 \r
1145                 hdr = (IB_CMA_HEADER *) event.data.req.pdata;\r
1146                 if ((hdr->IpVersion >> 4) == 4) {\r
1147                         ep->Attributes.LocalAddress.SockAddr.In.SinFamily = WV_AF_INET;\r
1148                         ep->Attributes.LocalAddress.SockAddr.In.SinAddr = hdr->DstAddress.Ip4.Address;\r
1149                         ep->Attributes.PeerAddress.SockAddr.In.SinFamily = WV_AF_INET;\r
1150                         ep->Attributes.PeerAddress.SockAddr.In.SinAddr = hdr->SrcAddress.Ip4.Address;\r
1151                 } else {\r
1152                         ep->Attributes.LocalAddress.SockAddr.In6.Sin6Family = WV_AF_INET6; \r
1153                         RtlCopyMemory(ep->Attributes.LocalAddress.SockAddr.In6.Sin6Addr,\r
1154                                                   hdr->DstAddress.Ip6Address, 16);\r
1155                         ep->Attributes.PeerAddress.SockAddr.In6.Sin6Family = WV_AF_INET6;\r
1156                         RtlCopyMemory(ep->Attributes.PeerAddress.SockAddr.In6.Sin6Addr,\r
1157                                                   hdr->SrcAddress.Ip6Address, 16);\r
1158                 }\r
1159                 ep->Attributes.Device.DeviceGuid = event.data.req.local_ca_guid;\r
1160                 ep->Attributes.Device.Pkey = event.data.req.primary_path.pkey;\r
1161                 ep->Attributes.Device.PortNumber = event.data.req.port_num;\r
1162                 ep->Attributes.Param.Connect.ResponderResources = event.data.req.resp_res;\r
1163                 ep->Attributes.Param.Connect.InitiatorDepth = event.data.req.init_depth;\r
1164                 ep->Attributes.Param.Connect.RetryCount = event.data.req.retry_cnt;\r
1165                 ep->Attributes.Param.Connect.RnrRetryCount = event.data.req.rnr_retry_cnt;\r
1166                 ep->Attributes.Param.Connect.DataLength = sizeof(ep->Attributes.Param.Connect.Data);\r
1167                 RtlCopyMemory(ep->Attributes.Param.Connect.Data, hdr + 1,\r
1168                                           sizeof(ep->Attributes.Param.Connect.Data));\r
1169                 ep->Route = event.data.req.primary_path;\r
1170 \r
1171                 ep->State = WvEpPassiveConnect;\r
1172                 WvEpPut(ep);\r
1173 \r
1174                 WdfRequestComplete(request, STATUS_SUCCESS);\r
1175         }\r
1176         WdfObjectReleaseLock(pListen->Queue);\r
1177 }\r
1178 \r
1179 static NTSTATUS WvEpIbListenHandler(iba_cm_id *pId, iba_cm_event *pEvent)\r
1180 {\r
1181         WV_ENDPOINT             *listen;\r
1182 \r
1183         listen = pId->context;\r
1184         WvEpGetIbRequest(listen);\r
1185         return STATUS_SUCCESS;\r
1186 }\r
1187 \r
1188 void WvEpListen(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
1189 {\r
1190         WV_ENDPOINT                     *ep;\r
1191         WV_IO_EP_LISTEN         *pattr;\r
1192         NTSTATUS                        status;\r
1193         void                            *buf;\r
1194         UINT8                           offset, len;\r
1195         UINT64                          sid;\r
1196 \r
1197         status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_EP_LISTEN),\r
1198                                                                                    &pattr, NULL);\r
1199         if (!NT_SUCCESS(status)) {\r
1200                 goto complete;\r
1201         }\r
1202 \r
1203         ep = WvEpAcquire(pProvider, pattr->Id);\r
1204         if (ep == NULL) {\r
1205                 status = STATUS_NOT_FOUND;\r
1206                 goto complete;\r
1207         }\r
1208 \r
1209         if (WvAnyAddress(&ep->Attributes.LocalAddress)) {\r
1210                 buf = NULL;\r
1211                 offset = 0;\r
1212                 len = 0;\r
1213         } else {\r
1214                 if (ep->Attributes.LocalAddress.SockAddr.Sa.SaFamily == WV_AF_INET) {\r
1215                         buf = &ep->Attributes.LocalAddress.SockAddr.In.SinAddr;\r
1216                         len = sizeof ep->Attributes.LocalAddress.SockAddr.In.SinAddr;\r
1217                         offset = FIELD_OFFSET(IB_CMA_HEADER, DstAddress.Ip4.Address);\r
1218                 } else {\r
1219                         buf = ep->Attributes.LocalAddress.SockAddr.In6.Sin6Addr;\r
1220                         len = sizeof ep->Attributes.LocalAddress.SockAddr.In6.Sin6Addr;\r
1221                         offset = FIELD_OFFSET(IB_CMA_HEADER, DstAddress.Ip6Address);\r
1222                 }\r
1223         }\r
1224 \r
1225         WdfObjectAcquireLock(ep->Queue);\r
1226         if (ep->State != WvEpAddressBound) {\r
1227                 status = STATUS_NOT_SUPPORTED;\r
1228                 goto release;\r
1229         }\r
1230 \r
1231         status = IbCmInterface.CM.create_id(WvEpIbListenHandler, ep, &ep->pIbCmId);\r
1232         if (!NT_SUCCESS(status)) {\r
1233                 goto release;\r
1234         }\r
1235 \r
1236         ep->Attributes.Param.Backlog = pattr->Backlog;\r
1237         ep->State = WvEpListening;\r
1238         sid = WvGetServiceId(ep->EpType, &ep->Attributes.LocalAddress);\r
1239         status = IbCmInterface.CM.listen(ep->pIbCmId, sid, buf, len, offset);\r
1240 \r
1241 release:\r
1242         WdfObjectReleaseLock(ep->Queue);\r
1243         WvEpRelease(ep);\r
1244 complete:\r
1245         WdfRequestComplete(Request, status);\r
1246 }\r
1247 \r
1248 void WvEpGetRequest(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
1249 {\r
1250         WV_ENDPOINT                             *listen, *ep;\r
1251         WV_IO_EP_GET_REQUEST    *req;\r
1252         NTSTATUS                                status;\r
1253 \r
1254         status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_EP_GET_REQUEST),\r
1255                                                                                    &req, NULL);\r
1256         if (!NT_SUCCESS(status)) {\r
1257                 goto err1;\r
1258         }\r
1259 \r
1260         listen = WvEpAcquire(pProvider, req->Id);\r
1261         if (listen == NULL) {\r
1262                 status = STATUS_NOT_FOUND;\r
1263                 goto err1;\r
1264         }\r
1265 \r
1266         if (listen->State != WvEpListening) {\r
1267                 status = STATUS_NOT_SUPPORTED;\r
1268                 goto err2;\r
1269         }\r
1270 \r
1271         ep = WvEpAcquire(pProvider, req->EpId);\r
1272         if (ep == NULL) {\r
1273                 status = STATUS_NOT_FOUND;\r
1274                 goto err2;\r
1275         }\r
1276 \r
1277         WdfObjectAcquireLock(ep->Queue);\r
1278         if (ep->State == WvEpIdle) {\r
1279                 ep->State = WvEpQueued;\r
1280         } else {\r
1281                 status = STATUS_CONNECTION_IN_USE;\r
1282         }\r
1283         WdfObjectReleaseLock(ep->Queue);\r
1284         if (!NT_SUCCESS(status)) {\r
1285                 goto err3;\r
1286         }\r
1287 \r
1288         WdfObjectAcquireLock(listen->Queue);\r
1289         status = WdfRequestForwardToIoQueue(Request, listen->Queue);\r
1290         if (NT_SUCCESS(status)) {\r
1291                 InsertTailList(&listen->Entry, &ep->Entry);\r
1292                 WvEpGet(ep);\r
1293         }\r
1294         WdfObjectReleaseLock(listen->Queue);\r
1295         if (!NT_SUCCESS(status)) {\r
1296                 goto err3;\r
1297         }\r
1298 \r
1299         WvEpRelease(ep);\r
1300         WvEpGetIbRequest(listen);\r
1301         WvEpRelease(listen);\r
1302         return;\r
1303 \r
1304 err3:\r
1305         WvEpRelease(ep);\r
1306 err2:\r
1307         WvEpRelease(listen);\r
1308 err1:\r
1309         WdfRequestComplete(Request, status);\r
1310 }\r
1311 \r
1312 void WvEpLookup(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
1313 {\r
1314         UNUSED_PARAM(pProvider);\r
1315         WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED);\r
1316 }\r
1317 \r
1318 void WvEpMulticastJoin(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
1319 {\r
1320         UNUSED_PARAM(pProvider);\r
1321         WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED);\r
1322 }\r
1323 \r
1324 void WvEpMulticastLeave(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
1325 {\r
1326         UNUSED_PARAM(pProvider);\r
1327         WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED);\r
1328 }\r
1329 \r
1330 //\r
1331 // Note that the framework may have already canceled outstanding requests.\r
1332 //\r
1333 void WvEpCancelListen(WV_ENDPOINT *pListen)\r
1334 {\r
1335         WV_ENDPOINT                     *ep;\r
1336 \r
1337         WdfObjectAcquireLock(pListen->Queue);\r
1338         WvCompleteRequests(pListen->Queue, STATUS_CANCELLED);\r
1339 \r
1340         while (!IsListEmpty(&pListen->Entry)) {\r
1341                 ep = CONTAINING_RECORD(RemoveHeadList(&pListen->Entry), WV_ENDPOINT, Entry);\r
1342                 ep->State = WvEpIdle;\r
1343                 WvEpPut(ep);\r
1344         }\r
1345         WdfObjectReleaseLock(pListen->Queue);\r
1346 }\r
1347 \r
1348 void WvEpCancel(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
1349 {\r
1350         UINT64                          *id;\r
1351         WV_ENDPOINT                     *ep;\r
1352         NTSTATUS                        status;\r
1353 \r
1354         status = WdfRequestRetrieveInputBuffer(Request, sizeof(UINT64), &id, NULL);\r
1355         if (!NT_SUCCESS(status)) {\r
1356                 goto out;\r
1357         }\r
1358 \r
1359         ep = WvEpAcquire(pProvider, *id);\r
1360         if (ep == NULL) {\r
1361                 status = STATUS_NOT_FOUND;\r
1362                 goto out;\r
1363         }\r
1364 \r
1365         if (ep->State == WvEpListening) {\r
1366                 WvEpCancelListen(ep);\r
1367         } else {\r
1368                 WvFlushQueue(ep->Queue, STATUS_CANCELLED);\r
1369         }\r
1370         WvEpRelease(ep);\r
1371 \r
1372 out:\r
1373         WdfRequestComplete(Request, status);\r
1374 }\r