winverbs: fix crash accessing freed memory from async thread
[mirror/winof/.git] / core / winverbs / kernel / wv_qp.c
1 /*\r
2  * Copyright (c) 2008 Intel Corporation. All rights reserved.\r
3  *\r
4  * This software is available to you under the OpenIB.org BSD license\r
5  * below:\r
6  *\r
7  *     Redistribution and use in source and binary forms, with or\r
8  *     without modification, are permitted provided that the following\r
9  *     conditions are met:\r
10  *\r
11  *      - Redistributions of source code must retain the above\r
12  *        copyright notice, this list of conditions and the following\r
13  *        disclaimer.\r
14  *\r
15  *      - Redistributions in binary form must reproduce the above\r
16  *        copyright notice, this list of conditions and the following\r
17  *        disclaimer in the documentation and/or other materials\r
18  *        provided with the distribution.\r
19  *\r
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV\r
23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
27  * SOFTWARE.\r
28  */\r
29 \r
30 #include "wv_qp.h"\r
31 #include "wv_ioctl.h"\r
32 \r
33 typedef struct _WV_MULTICAST\r
34 {\r
35         LIST_ENTRY                      Entry;\r
36         WV_IO_GID                       Gid;\r
37         NET16                           Lid;\r
38         ib_mcast_handle_t       hVerbsMc;\r
39 \r
40 }       WV_MULTICAST;\r
41 \r
42 static void WvVerbsConvertCreate(ib_qp_create_t *pVerbsAttr,\r
43                                                                  WV_IO_QP_CREATE *pAttr, WV_QUEUE_PAIR *pQp)\r
44 {\r
45         pVerbsAttr->h_sq_cq = pQp->pSendCq->hVerbsCq;\r
46         pVerbsAttr->h_rq_cq = pQp->pReceiveCq->hVerbsCq;\r
47         pVerbsAttr->h_srq = (pQp->pSrq != NULL) ? pQp->pSrq->hVerbsSrq : NULL;\r
48 \r
49         pVerbsAttr->qp_type = (ib_qp_type_t) pAttr->QpType;\r
50         pVerbsAttr->sq_max_inline = pAttr->MaxInlineSend;\r
51         pVerbsAttr->sq_depth = pAttr->SendDepth;\r
52         pVerbsAttr->rq_depth = pAttr->ReceiveDepth;\r
53         pVerbsAttr->sq_sge = pAttr->SendSge;\r
54         pVerbsAttr->rq_sge = pAttr->ReceiveSge;\r
55 \r
56         pVerbsAttr->sq_signaled = ((pAttr->QpFlags & WV_IO_QP_SIGNAL_SENDS) != 0);\r
57 }\r
58 \r
59 static UINT8 WvQpStateConvertVerbs(ib_qp_state_t State)\r
60 {\r
61         switch (State) {\r
62         case IB_QPS_RESET:      return WV_IO_QP_STATE_RESET;\r
63         case IB_QPS_INIT:       return WV_IO_QP_STATE_INIT;\r
64         case IB_QPS_RTR:        return WV_IO_QP_STATE_RTR;\r
65         case IB_QPS_RTS:        return WV_IO_QP_STATE_RTS;\r
66         case IB_QPS_SQD:        return WV_IO_QP_STATE_SQD;\r
67         case IB_QPS_SQERR:      return WV_IO_QP_STATE_SQERROR;\r
68         default:                        return WV_IO_QP_STATE_ERROR;\r
69         }\r
70 }\r
71 \r
72 static UINT8 WvVerbsConvertMtu(UINT32 Mtu)\r
73 {\r
74         switch (Mtu) {\r
75         case 256:       return IB_MTU_LEN_256;\r
76         case 512:       return IB_MTU_LEN_512;\r
77         case 1024:      return IB_MTU_LEN_1024;\r
78         case 2048:      return IB_MTU_LEN_2048;\r
79         case 4096:      return IB_MTU_LEN_4096;\r
80         default:        return 0;\r
81         }\r
82 }\r
83 static void WvAvConvertVerbs(WV_IO_AV *pAv, ib_av_attr_t *pVerbsAv)\r
84 {\r
85         pAv->NetworkRouteValid = (uint8_t) pVerbsAv->grh_valid;\r
86         if (pAv->NetworkRouteValid) {\r
87                 pAv->FlowLabel = pVerbsAv->grh.ver_class_flow & RtlUlongByteSwap(0x000FFFFF);\r
88                 pAv->TrafficClass = (UINT8) (RtlUlongByteSwap(pVerbsAv->grh.ver_class_flow) >> 20);\r
89                 pAv->HopLimit = pVerbsAv->grh.hop_limit;\r
90                 RtlCopyMemory(pAv->SGid, &pVerbsAv->grh.src_gid, sizeof(pAv->SGid));\r
91                 RtlCopyMemory(pAv->DGid, &pVerbsAv->grh.dest_gid, sizeof(pAv->DGid));\r
92         }\r
93 \r
94         pAv->PortNumber = pVerbsAv->port_num;\r
95         pAv->ServiceLevel = pVerbsAv->sl;\r
96         pAv->DLid = pVerbsAv->dlid;\r
97         pAv->StaticRate = pVerbsAv->static_rate;\r
98         pAv->SourcePathBits = pVerbsAv->path_bits;\r
99 }\r
100 \r
101 static void WvVerbsConvertAv(ib_av_attr_t *pVerbsAv, WV_IO_AV *pAv,\r
102                                                          WV_IO_QP_ATTRIBUTES *pAttr)\r
103 {\r
104         pVerbsAv->grh_valid = pAv->NetworkRouteValid;\r
105         if (pVerbsAv->grh_valid) {\r
106                 pVerbsAv->grh.ver_class_flow =\r
107                         RtlUlongByteSwap(RtlUlongByteSwap(pAv->FlowLabel) | (pAv->TrafficClass << 20));\r
108                 pVerbsAv->grh.hop_limit = pAv->HopLimit;\r
109                 RtlCopyMemory(&pVerbsAv->grh.src_gid, pAv->SGid, sizeof(pAv->SGid));\r
110                 RtlCopyMemory(&pVerbsAv->grh.dest_gid, pAv->DGid, sizeof(pAv->DGid));\r
111         }\r
112 \r
113         pVerbsAv->port_num = pAv->PortNumber;\r
114         pVerbsAv->sl = pAv->ServiceLevel;\r
115         pVerbsAv->dlid = pAv->DLid;\r
116         pVerbsAv->static_rate = pAv->StaticRate;\r
117         pVerbsAv->path_bits = pAv->SourcePathBits;\r
118 \r
119         pVerbsAv->conn.path_mtu = WvVerbsConvertMtu(pAttr->PathMtu);\r
120         pVerbsAv->conn.local_ack_timeout = pAttr->LocalAckTimeout;\r
121         pVerbsAv->conn.seq_err_retry_cnt = pAttr->SequenceErrorRetryCount;\r
122         pVerbsAv->conn.rnr_retry_cnt = pAttr->RnrRetryCount;\r
123 }\r
124 \r
125 static void WvQpAttrConvertVerbs(WV_IO_QP_ATTRIBUTES *pAttr, ib_qp_attr_t *pVerbsAttr)\r
126 {\r
127         pAttr->MaxInlineSend = pVerbsAttr->sq_max_inline;\r
128         pAttr->SendDepth = pVerbsAttr->sq_depth;\r
129         pAttr->ReceiveDepth = pVerbsAttr->rq_depth;\r
130         pAttr->SendSge = pVerbsAttr->sq_sge;\r
131         pAttr->ReceiveSge = pVerbsAttr->rq_sge;\r
132         pAttr->InitiatorDepth = pVerbsAttr->init_depth;\r
133         pAttr->ResponderResources = pVerbsAttr->resp_res;\r
134 \r
135         pAttr->Options = 0;\r
136         pAttr->QpType = pVerbsAttr->qp_type;\r
137         pAttr->CurrentQpState = WvQpStateConvertVerbs(pVerbsAttr->state);\r
138         pAttr->QpState = pAttr->CurrentQpState;\r
139         pAttr->ApmState = pVerbsAttr->apm_state;\r
140         pAttr->Qpn = pVerbsAttr->num;\r
141         pAttr->DestinationQpn = pVerbsAttr->dest_num;\r
142         pAttr->Qkey = pVerbsAttr->qkey;\r
143         pAttr->SendPsn = pVerbsAttr->sq_psn;\r
144         pAttr->ReceivePsn = pVerbsAttr->rq_psn;\r
145 \r
146         pAttr->QpFlags = pVerbsAttr->sq_signaled ? WV_IO_QP_SIGNAL_SENDS : 0;\r
147         pAttr->AccessFlags = (UINT8) pVerbsAttr->access_ctrl;\r
148 \r
149         pAttr->AddressVector.PortNumber = pVerbsAttr->primary_port;\r
150         pAttr->AlternateAddressVector.PortNumber = pVerbsAttr->alternate_port;\r
151         WvAvConvertVerbs(&pAttr->AddressVector, &pVerbsAttr->primary_av);\r
152         WvAvConvertVerbs(&pAttr->AlternateAddressVector, &pVerbsAttr->alternate_av);\r
153         pAttr->PathMtu = 0x80 << pVerbsAttr->primary_av.conn.path_mtu;\r
154         pAttr->AlternatePathMtu = 0x80 << pVerbsAttr->alternate_av.conn.path_mtu;\r
155         pAttr->PkeyIndex = pVerbsAttr->pkey_index;\r
156         pAttr->AlternatePkeyIndex = pAttr->PkeyIndex;   // TODO: missing in ib_qp_attr_t\r
157         pAttr->LocalAckTimeout = pVerbsAttr->primary_av.conn.local_ack_timeout;\r
158         pAttr->AlternateLocalAckTimeout = pVerbsAttr->alternate_av.conn.local_ack_timeout;\r
159 \r
160         pAttr->RnrNakTimeout = 0;                                               // TODO: missing in ib_ap_attr_t\r
161         pAttr->SequenceErrorRetryCount = pVerbsAttr->primary_av.conn.seq_err_retry_cnt;\r
162         pAttr->RnrRetryCount = pVerbsAttr->primary_av.conn.rnr_retry_cnt;\r
163 }\r
164 \r
165 static ib_qp_opts_t WvVerbsConvertOptions(UINT32 Options)\r
166 {\r
167         UINT32 opt = 0;\r
168 \r
169         if ((Options & WV_IO_QP_ATTR_CAPABILITIES) != 0) {\r
170                 opt |= IB_MOD_QP_SQ_DEPTH | IB_MOD_QP_RQ_DEPTH;\r
171         }\r
172         if ((Options & WV_IO_QP_ATTR_INITIATOR_DEPTH) != 0) {\r
173                 opt |= IB_MOD_QP_INIT_DEPTH;\r
174         }\r
175         if ((Options & WV_IO_QP_ATTR_RESPONDER_RESOURCES) != 0) {\r
176                 opt |= IB_MOD_QP_RESP_RES;\r
177         }\r
178         if ((Options & WV_IO_QP_ATTR_CURRENT_STATE) != 0) {\r
179                 opt |= IB_MOD_QP_CURRENT_STATE;\r
180         }\r
181         if ((Options & WV_IO_QP_ATTR_PATH_MIG_STATE) != 0) {\r
182                 opt |= IB_MOD_QP_APM_STATE;\r
183         }\r
184         if ((Options & WV_IO_QP_ATTR_QKEY) != 0) {\r
185                 opt |= IB_MOD_QP_QKEY;\r
186         }\r
187         if ((Options & WV_IO_QP_ATTR_ACCESS_FLAGS) != 0) {\r
188                 opt |= IB_MOD_QP_ACCESS_CTRL;\r
189         }\r
190         if ((Options & WV_IO_QP_ATTR_AV) != 0) {\r
191                 opt |= IB_MOD_QP_PRIMARY_AV;\r
192         }\r
193         if ((Options & WV_IO_QP_ATTR_ALTERNATE_AV) != 0) {\r
194                 opt |= IB_MOD_QP_ALTERNATE_AV;\r
195         }\r
196         if ((Options & WV_IO_QP_ATTR_PORT_NUMBER) != 0) {\r
197                 opt |= IB_MOD_QP_PRIMARY_PORT;\r
198         }\r
199         if ((Options & WV_IO_QP_ATTR_PKEY_INDEX) != 0) {\r
200                 opt |= IB_MOD_QP_PKEY;\r
201         }\r
202         if ((Options & WV_IO_QP_ATTR_ACK_TIMEOUT) != 0) {\r
203                 opt |= IB_MOD_QP_LOCAL_ACK_TIMEOUT;\r
204         }\r
205         if ((Options & WV_IO_QP_ATTR_RNR_NAK_TIMEOUT) != 0) {\r
206                 opt |= IB_MOD_QP_RNR_NAK_TIMEOUT;\r
207         }\r
208         if ((Options & WV_IO_QP_ATTR_ERROR_RETRY_COUNT) != 0) {\r
209                 opt |= IB_MOD_QP_RETRY_CNT;\r
210         }\r
211         if ((Options & WV_IO_QP_ATTR_RNR_RETRY_COUNT) != 0) {\r
212                 opt |= IB_MOD_QP_RNR_RETRY_CNT;\r
213         }\r
214 \r
215         return (ib_qp_opts_t) opt;\r
216 }\r
217 \r
218 static ib_qp_state_t WvVerbsConvertState(UINT8 State)\r
219 {\r
220         switch (State) {\r
221         case WV_IO_QP_STATE_RESET:              return IB_QPS_RESET;\r
222         case WV_IO_QP_STATE_INIT:               return IB_QPS_INIT;\r
223         case WV_IO_QP_STATE_RTR:                return IB_QPS_RTR;\r
224         case WV_IO_QP_STATE_RTS:                return IB_QPS_RTS;\r
225         case WV_IO_QP_STATE_SQD:                return IB_QPS_SQD;\r
226         case WV_IO_QP_STATE_SQERROR:    return IB_QPS_SQERR;\r
227         default:                                                return IB_QPS_ERROR;\r
228         }\r
229 }\r
230 \r
231 static void WvVerbsConvertAttr(ib_qp_mod_t *pVerbsAttr, WV_IO_QP_ATTRIBUTES *pAttr)\r
232 {\r
233         switch (pAttr->QpState) {\r
234         case WV_IO_QP_STATE_RESET:\r
235                 pVerbsAttr->req_state = IB_QPS_RESET;\r
236                 break;\r
237         case WV_IO_QP_STATE_INIT:\r
238                 pVerbsAttr->req_state = IB_QPS_INIT;\r
239 \r
240                 pVerbsAttr->state.init.primary_port = pAttr->AddressVector.PortNumber;\r
241                 pVerbsAttr->state.init.qkey = pAttr->Qkey;\r
242                 pVerbsAttr->state.init.pkey_index = pAttr->PkeyIndex;\r
243                 pVerbsAttr->state.init.access_ctrl = pAttr->AccessFlags;\r
244                 break;\r
245         case WV_IO_QP_STATE_RTR:\r
246                 pVerbsAttr->req_state = IB_QPS_RTR;\r
247 \r
248                 pVerbsAttr->state.rtr.rq_psn = pAttr->ReceivePsn;\r
249                 pVerbsAttr->state.rtr.dest_qp = pAttr->DestinationQpn;\r
250                 WvVerbsConvertAv(&pVerbsAttr->state.rtr.primary_av,\r
251                                                  &pAttr->AddressVector, pAttr);\r
252                 pVerbsAttr->state.rtr.resp_res = (UINT8) pAttr->ResponderResources;\r
253                 pVerbsAttr->state.rtr.rnr_nak_timeout = pAttr->RnrNakTimeout;\r
254 \r
255                 pVerbsAttr->state.rtr.opts = WvVerbsConvertOptions(pAttr->Options);\r
256                 WvVerbsConvertAv(&pVerbsAttr->state.rtr.alternate_av,\r
257                                                  &pAttr->AlternateAddressVector, pAttr);\r
258                 pVerbsAttr->state.rtr.qkey = pAttr->Qkey;\r
259                 pVerbsAttr->state.rtr.pkey_index = pAttr->PkeyIndex;\r
260                 pVerbsAttr->state.rtr.access_ctrl = pAttr->AccessFlags;\r
261                 pVerbsAttr->state.rtr.sq_depth = pAttr->SendDepth;\r
262                 pVerbsAttr->state.rtr.rq_depth = pAttr->ReceiveDepth;\r
263                 break;\r
264         case WV_IO_QP_STATE_RTS:\r
265                 pVerbsAttr->req_state = IB_QPS_RTS;\r
266 \r
267                 pVerbsAttr->state.rts.sq_psn = pAttr->SendPsn;\r
268                 pVerbsAttr->state.rts.retry_cnt = pAttr->SequenceErrorRetryCount;\r
269                 pVerbsAttr->state.rts.rnr_retry_cnt     = pAttr->RnrRetryCount;\r
270                 pVerbsAttr->state.rts.local_ack_timeout = pAttr->LocalAckTimeout;\r
271                 pVerbsAttr->state.rts.init_depth = (UINT8) pAttr->InitiatorDepth;\r
272 \r
273                 pVerbsAttr->state.rts.opts = WvVerbsConvertOptions(pAttr->Options);\r
274                 pVerbsAttr->state.rts.rnr_nak_timeout = pAttr->RnrNakTimeout;\r
275                 pVerbsAttr->state.rts.current_state = WvVerbsConvertState(pAttr->CurrentQpState);\r
276                 pVerbsAttr->state.rts.qkey = pAttr->Qkey;\r
277                 pVerbsAttr->state.rts.access_ctrl = pAttr->AccessFlags;\r
278                 pVerbsAttr->state.rts.resp_res = (UINT8) pAttr->ResponderResources;\r
279 \r
280                 WvVerbsConvertAv(&pVerbsAttr->state.rts.primary_av,\r
281                                                  &pAttr->AddressVector, pAttr);\r
282                 WvVerbsConvertAv(&pVerbsAttr->state.rts.alternate_av,\r
283                                                  &pAttr->AlternateAddressVector, pAttr);\r
284 \r
285                 pVerbsAttr->state.rts.sq_depth = pAttr->SendDepth;\r
286                 pVerbsAttr->state.rts.rq_depth = pAttr->ReceiveDepth;\r
287 \r
288                 pVerbsAttr->state.rts.apm_state = pAttr->ApmState;\r
289                 pVerbsAttr->state.rts.primary_port = pAttr->AddressVector.PortNumber;\r
290                 pVerbsAttr->state.rts.pkey_index = pAttr->PkeyIndex;\r
291                 break;\r
292         case WV_IO_QP_STATE_SQD:\r
293                 pVerbsAttr->req_state = IB_QPS_SQD;\r
294                 pVerbsAttr->state.sqd.sqd_event = 1;\r
295                 break;\r
296         case WV_IO_QP_STATE_SQERROR:\r
297                 pVerbsAttr->req_state = IB_QPS_SQERR;\r
298                 break;\r
299         default:\r
300                 pVerbsAttr->req_state = IB_QPS_ERROR;\r
301                 break;\r
302         }\r
303 }\r
304 \r
305 static void WvQpGet(WV_QUEUE_PAIR *pQp)\r
306 {\r
307         InterlockedIncrement(&pQp->Ref);\r
308 }\r
309 \r
310 static void WvQpPut(WV_QUEUE_PAIR *pQp)\r
311 {\r
312         if (InterlockedDecrement(&pQp->Ref) == 0) {\r
313                 KeSetEvent(&pQp->Event, 0, FALSE);\r
314         }\r
315 }\r
316 \r
317 WV_QUEUE_PAIR *WvQpAcquire(WV_PROVIDER *pProvider, UINT64 Id)\r
318 {\r
319         WV_QUEUE_PAIR *qp;\r
320 \r
321         KeAcquireGuardedMutex(&pProvider->Lock);\r
322         WvProviderDisableRemove(pProvider);\r
323         qp = IndexListAt(&pProvider->QpIndex, (SIZE_T) Id);\r
324         if (qp != NULL && qp->hVerbsQp != NULL) {\r
325                 WvQpGet(qp);\r
326         } else {\r
327                 qp = NULL;\r
328                 WvProviderEnableRemove(pProvider);\r
329         }\r
330         KeReleaseGuardedMutex(&pProvider->Lock);\r
331 \r
332         return qp;\r
333 }\r
334 \r
335 void WvQpRelease(WV_QUEUE_PAIR *pQp)\r
336 {\r
337         WvProviderEnableRemove(pQp->pProvider);\r
338         WvQpPut(pQp);\r
339 }\r
340 \r
341 static NTSTATUS WvQpCreateAcquire(WV_PROVIDER *pProvider, WV_QUEUE_PAIR *pQp,\r
342                                                                   WV_IO_QP_CREATE *pAttributes)\r
343 {\r
344         KeAcquireGuardedMutex(&pProvider->Lock);\r
345         WvProviderDisableRemove(pProvider);\r
346         pQp->pPd = IndexListAt(&pProvider->PdIndex, (SIZE_T) pAttributes->Id.Id);\r
347         if (pQp->pPd != NULL && pQp->pPd->hVerbsPd != NULL) {\r
348                 WvPdGet(pQp->pPd);\r
349                 pQp->pVerbs = pQp->pPd->pVerbs;\r
350         } else {\r
351                 goto err1;\r
352         }\r
353 \r
354         pQp->pSendCq = IndexListAt(&pProvider->CqIndex, (SIZE_T) pAttributes->SendCqId);\r
355         if (pQp->pSendCq != NULL && pQp->pSendCq->hVerbsCq != NULL) {\r
356                 WvCqGet(pQp->pSendCq);\r
357         } else {\r
358                 goto err2;\r
359         }\r
360 \r
361         pQp->pReceiveCq = IndexListAt(&pProvider->CqIndex,\r
362                                                                   (SIZE_T) pAttributes->ReceiveCqId);\r
363         if (pQp->pReceiveCq != NULL && pQp->pReceiveCq->hVerbsCq != NULL) {\r
364                 WvCqGet(pQp->pReceiveCq);\r
365         } else {\r
366                 goto err3;\r
367         }\r
368 \r
369         if (pAttributes->SrqId != 0) {\r
370                 pQp->pSrq = IndexListAt(&pProvider->SrqIndex, (SIZE_T) pAttributes->SrqId);\r
371                 if (pQp->pSrq != NULL && pQp->pSrq->hVerbsSrq != NULL) {\r
372                         WvSrqGet(pQp->pSrq);\r
373                 } else {\r
374                         goto err4;\r
375                 }\r
376         } else {\r
377                 pQp->pSrq = NULL;\r
378         }\r
379         KeReleaseGuardedMutex(&pProvider->Lock);\r
380 \r
381         return STATUS_SUCCESS;\r
382 \r
383 err4:\r
384         WvCqPut(pQp->pReceiveCq);\r
385 err3:\r
386         WvCqPut(pQp->pSendCq);\r
387 err2:\r
388         WvPdPut(pQp->pPd);\r
389 err1:\r
390         WvProviderEnableRemove(pProvider);\r
391         KeReleaseGuardedMutex(&pProvider->Lock);\r
392         return STATUS_NOT_FOUND;\r
393 }\r
394 \r
395 static void WvQpCreateRelease(WV_PROVIDER *pProvider, WV_QUEUE_PAIR *pQp)\r
396 {\r
397         WvProviderEnableRemove(pProvider);\r
398         if (pQp->pSrq != NULL) {\r
399                 WvSrqPut(pQp->pSrq);\r
400         }\r
401         WvCqPut(pQp->pReceiveCq);\r
402         WvCqPut(pQp->pSendCq);\r
403         WvPdPut(pQp->pPd);\r
404 }\r
405 \r
406 static void WvQpEventHandler(ib_event_rec_t *pEvent)\r
407 {\r
408         // TODO: Handle QP events when adding connection handling\r
409         // IB_AE_QP_FATAL\r
410         // IB_AE_QP_COMM\r
411         // IB_AE_QP_APM\r
412         UNREFERENCED_PARAMETER(pEvent);\r
413 }\r
414 \r
415 void WvQpCreate(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
416 {\r
417         WV_IO_QP_CREATE                 *inattr, *outattr;\r
418         size_t                                  inlen, outlen;\r
419         WV_QUEUE_PAIR                   *qp;\r
420         ib_qp_create_t                  create;\r
421         ib_qp_attr_t                    ib_attr;\r
422         NTSTATUS                                status;\r
423         ib_api_status_t                 ib_status;\r
424         ci_umv_buf_t                    verbsData;\r
425 \r
426         status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_QP_CREATE),\r
427                                                                                    &inattr, &inlen);\r
428         if (!NT_SUCCESS(status)) {\r
429                 goto err1;\r
430         }\r
431         status = WdfRequestRetrieveOutputBuffer(Request, sizeof(WV_IO_QP_CREATE),\r
432                                                                                         &outattr, &outlen);\r
433         if (!NT_SUCCESS(status)) {\r
434                 goto err1;\r
435         }\r
436 \r
437         qp = ExAllocatePoolWithTag(NonPagedPool, sizeof(WV_QUEUE_PAIR), 'pqvw');\r
438         if (qp == NULL) {\r
439                 status = STATUS_NO_MEMORY;\r
440                 goto err1;\r
441         }\r
442 \r
443         qp->Ref = 1;\r
444         qp->pProvider = pProvider;\r
445         KeInitializeEvent(&qp->Event, NotificationEvent, FALSE);\r
446         InitializeListHead(&qp->McList);\r
447         KeInitializeGuardedMutex(&qp->Lock);\r
448         status = WvQpCreateAcquire(pProvider, qp, inattr);\r
449         if (!NT_SUCCESS(status)) {\r
450                 goto err2;\r
451         }\r
452 \r
453         WvVerbsConvertCreate(&create, inattr, qp);\r
454         WvInitVerbsData(&verbsData, inattr->Id.VerbInfo, inlen - sizeof(WV_IO_QP_CREATE),\r
455                                         outlen - sizeof(WV_IO_QP_CREATE), inattr + 1);\r
456         ib_status = qp->pVerbs->create_qp(qp->pPd->hVerbsPd, qp, WvQpEventHandler,\r
457                                                                           &create, &ib_attr, &qp->hVerbsQp, &verbsData);\r
458         if (ib_status != IB_SUCCESS) {\r
459                 status = STATUS_UNSUCCESSFUL;\r
460                 goto err3;\r
461         }\r
462 \r
463         qp->Qpn = ib_attr.num;\r
464         KeAcquireGuardedMutex(&pProvider->Lock);\r
465         outattr->Id.Id = IndexListInsertHead(&pProvider->QpIndex, qp);\r
466         if (outattr->Id.Id == 0) {\r
467                 status = STATUS_NO_MEMORY;\r
468                 goto err4;\r
469         }\r
470         InsertHeadList(&qp->pPd->QpList, &qp->Entry);\r
471         KeReleaseGuardedMutex(&pProvider->Lock);\r
472 \r
473         WvProviderEnableRemove(pProvider);\r
474         outattr->Id.VerbInfo = verbsData.status;\r
475         WdfRequestCompleteWithInformation(Request, status, outlen);\r
476         return;\r
477 \r
478 err4:\r
479         KeReleaseGuardedMutex(&pProvider->Lock);\r
480         qp->pVerbs->destroy_qp(qp->hVerbsQp, 0);\r
481 err3:\r
482         WvQpCreateRelease(pProvider, qp);\r
483 err2:\r
484         ExFreePoolWithTag(qp, 'pqvw');\r
485 err1:\r
486         WdfRequestComplete(Request, status);\r
487 }\r
488 \r
489 void WvQpDestroy(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
490 {\r
491         WV_QUEUE_PAIR   *qp;\r
492         UINT64                  *id;\r
493         NTSTATUS                status;\r
494 \r
495         status = WdfRequestRetrieveInputBuffer(Request, sizeof(UINT64), &id, NULL);\r
496         if (!NT_SUCCESS(status)) {\r
497                 goto out;\r
498         }\r
499 \r
500         KeAcquireGuardedMutex(&pProvider->Lock);\r
501         WvProviderDisableRemove(pProvider);\r
502         qp = IndexListAt(&pProvider->QpIndex, (SIZE_T) *id);\r
503         if (qp == NULL) {\r
504                 status = STATUS_NOT_FOUND;\r
505         } else if (qp->Ref > 1) {\r
506                 status = STATUS_ACCESS_DENIED;\r
507         } else {\r
508                 IndexListRemove(&pProvider->QpIndex, (SIZE_T) *id);\r
509                 RemoveEntryList(&qp->Entry);\r
510                 status = STATUS_SUCCESS;\r
511         }\r
512         KeReleaseGuardedMutex(&pProvider->Lock);\r
513 \r
514         if (NT_SUCCESS(status)) {\r
515                 WvQpFree(qp);\r
516         }\r
517         WvProviderEnableRemove(pProvider);\r
518 out:\r
519         WdfRequestComplete(Request, status);\r
520 }\r
521 \r
522 void WvQpFree(WV_QUEUE_PAIR *pQp)\r
523 {\r
524         WV_MULTICAST            *mc;\r
525         LIST_ENTRY                      *entry;\r
526 \r
527         while ((entry = RemoveHeadList(&pQp->McList)) != &pQp->McList) {\r
528                 mc = CONTAINING_RECORD(entry, WV_MULTICAST, Entry);\r
529                 if (mc->hVerbsMc != NULL) {\r
530                         pQp->pVerbs->detach_mcast(mc->hVerbsMc);\r
531                 }\r
532                 ExFreePoolWithTag(mc, 'cmvw');\r
533         }\r
534 \r
535         if (InterlockedDecrement(&pQp->Ref) > 0) {\r
536                 KeWaitForSingleObject(&pQp->Event, Executive, KernelMode, FALSE, NULL);\r
537         }\r
538 \r
539         if (pQp->hVerbsQp != NULL) {\r
540                 pQp->pVerbs->destroy_qp(pQp->hVerbsQp, 0);\r
541         }\r
542 \r
543         if (pQp->pSrq != NULL) {\r
544                 WvSrqPut(pQp->pSrq);\r
545         }\r
546         WvCqPut(pQp->pReceiveCq);\r
547         WvCqPut(pQp->pSendCq);\r
548         WvPdPut(pQp->pPd);\r
549         ExFreePoolWithTag(pQp, 'pqvw');\r
550 }\r
551 \r
552 void WvQpQuery(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
553 {\r
554         UINT64                                  *id;\r
555         WV_IO_QP_ATTRIBUTES             *pattr;\r
556         size_t                                  inlen, outlen, len = 0;\r
557         WV_QUEUE_PAIR                   *qp;\r
558         ib_qp_attr_t                    attr;\r
559         NTSTATUS                                status;\r
560 \r
561         status = WdfRequestRetrieveInputBuffer(Request, sizeof(UINT64), &id, &inlen);\r
562         if (!NT_SUCCESS(status)) {\r
563                 goto complete;\r
564         }\r
565         status = WdfRequestRetrieveOutputBuffer(Request, sizeof(WV_IO_QP_ATTRIBUTES),\r
566                                                                                         &pattr, &outlen);\r
567         if (!NT_SUCCESS(status)) {\r
568                 goto complete;\r
569         }\r
570 \r
571         qp = WvQpAcquire(pProvider, *id);\r
572         if (qp == NULL) {\r
573                 status = STATUS_NOT_FOUND;\r
574                 goto complete;\r
575         }\r
576 \r
577         RtlZeroMemory(&attr, sizeof attr);\r
578         qp->pVerbs->query_qp(qp->hVerbsQp, &attr, NULL);\r
579         WvQpRelease(qp);\r
580 \r
581         WvQpAttrConvertVerbs(pattr, &attr);\r
582         len = outlen;\r
583 \r
584 complete:\r
585         WdfRequestCompleteWithInformation(Request, status, len);\r
586 }\r
587 \r
588 void WvQpModify(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
589 {\r
590         WV_IO_QP_ATTRIBUTES             *pattr;\r
591         UINT8                                   *out;\r
592         size_t                                  outlen, len = 0;\r
593         WV_QUEUE_PAIR                   *qp;\r
594         ib_qp_mod_t                             attr;\r
595         NTSTATUS                                status;\r
596         ib_api_status_t                 ib_status;\r
597 \r
598         status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_QP_ATTRIBUTES),\r
599                                                                                    &pattr, NULL);\r
600         if (!NT_SUCCESS(status)) {\r
601                 goto complete;\r
602         }\r
603         status = WdfRequestRetrieveOutputBuffer(Request, 0, &out, &outlen);\r
604         if (!NT_SUCCESS(status) && status != STATUS_BUFFER_TOO_SMALL) {\r
605                 goto complete;\r
606         }\r
607 \r
608         WvVerbsConvertAttr(&attr, pattr);\r
609         qp = WvQpAcquire(pProvider, pattr->Id.Id);\r
610         if (qp == NULL) {\r
611                 status = STATUS_NOT_FOUND;\r
612                 goto complete;\r
613         }\r
614 \r
615         ib_status = qp->pVerbs->ndi_modify_qp(qp->hVerbsQp, &attr, NULL, (UINT32) outlen, out);\r
616         if (ib_status != IB_SUCCESS) {\r
617                 status = STATUS_UNSUCCESSFUL;\r
618         }\r
619         WvQpRelease(qp);\r
620         len = outlen;\r
621 \r
622 complete:\r
623         WdfRequestCompleteWithInformation(Request, status, len);\r
624 }\r
625 \r
626 void WvQpAttach(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
627 {\r
628         WV_MULTICAST                    *pmc;\r
629         WV_IO_QP_MULTICAST              *mc;\r
630         WV_QUEUE_PAIR                   *qp;\r
631         NTSTATUS                                status;\r
632         ib_api_status_t                 ib_status;\r
633 \r
634         status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_QP_MULTICAST),\r
635                                                                                    &mc, NULL);\r
636         if (!NT_SUCCESS(status)) {\r
637                 goto err1;\r
638         }\r
639 \r
640         pmc = ExAllocatePoolWithTag(PagedPool, sizeof(WV_MULTICAST), 'cmvw');\r
641         if (pmc == NULL) {\r
642                 status = STATUS_NO_MEMORY;\r
643                 goto err1;\r
644         }\r
645 \r
646         qp = WvQpAcquire(pProvider, mc->Id.Id);\r
647         if (qp == NULL) {\r
648                 status = STATUS_NOT_FOUND;\r
649                 goto err2;\r
650         }\r
651 \r
652         pmc->Gid = mc->Gid;\r
653         pmc->Lid = (NET16) mc->Id.Data;\r
654         ib_status = qp->pVerbs->attach_mcast(qp->hVerbsQp, (ib_gid_t *) &mc->Gid,\r
655                                                                                  (NET16) mc->Id.Data, &pmc->hVerbsMc, NULL);\r
656 \r
657         if (ib_status != IB_SUCCESS) {\r
658                 status = STATUS_UNSUCCESSFUL;\r
659                 goto err3;\r
660         }\r
661 \r
662         KeAcquireGuardedMutex(&qp->Lock);\r
663         InsertHeadList(&qp->McList, &pmc->Entry);\r
664         KeReleaseGuardedMutex(&qp->Lock);\r
665 \r
666         WvQpRelease(qp);\r
667         WdfRequestComplete(Request, STATUS_SUCCESS);\r
668         return;\r
669 \r
670 err3:\r
671         WvQpRelease(qp);\r
672 err2:\r
673         ExFreePoolWithTag(pmc, 'cmvw');\r
674 err1:\r
675         WdfRequestComplete(Request, status);\r
676 }\r
677 \r
678 void WvQpDetach(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
679 {\r
680         WV_MULTICAST                    *pmc;\r
681         WV_IO_QP_MULTICAST              *mc;\r
682         WV_QUEUE_PAIR                   *qp;\r
683         LIST_ENTRY                              *entry;\r
684         NTSTATUS                                status;\r
685 \r
686         status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_QP_MULTICAST),\r
687                                                                                    &mc, NULL);\r
688         if (!NT_SUCCESS(status)) {\r
689                 goto complete;\r
690         }\r
691 \r
692         qp = WvQpAcquire(pProvider, mc->Id.Id);\r
693         if (qp == NULL) {\r
694                 status = STATUS_NOT_FOUND;\r
695                 goto complete;\r
696         }\r
697 \r
698         KeAcquireGuardedMutex(&qp->Lock);\r
699         for (entry = qp->McList.Flink; entry != &qp->McList; entry = entry->Flink) {\r
700                 pmc = CONTAINING_RECORD(entry, WV_MULTICAST, Entry);\r
701 \r
702                 if (RtlCompareMemory(&pmc->Gid, &mc->Gid, sizeof(pmc->Gid)) == sizeof(pmc->Gid) &&\r
703                         pmc->Lid == (NET16) mc->Id.Data) {\r
704                         RemoveEntryList(&pmc->Entry);\r
705                         KeReleaseGuardedMutex(&qp->Lock);\r
706 \r
707                         if (pmc->hVerbsMc != NULL) {\r
708                                 qp->pVerbs->detach_mcast(pmc->hVerbsMc);\r
709                         }\r
710 \r
711                         ExFreePoolWithTag(pmc, 'cmvw');\r
712                         status = STATUS_SUCCESS;\r
713                         goto release;\r
714                 }\r
715         }\r
716         KeReleaseGuardedMutex(&qp->Lock);\r
717         status = STATUS_NOT_FOUND;\r
718 \r
719 release:\r
720         WvQpRelease(qp);\r
721 complete:\r
722         WdfRequestComplete(Request, status);\r
723 }\r
724 \r
725 void WvQpRemoveHandler(WV_QUEUE_PAIR *pQp)\r
726 {\r
727         WV_MULTICAST            *mc;\r
728         LIST_ENTRY                      *entry;\r
729 \r
730         for (entry = pQp->McList.Flink; entry != &pQp->McList; entry = entry->Flink) {\r
731                 mc = CONTAINING_RECORD(entry, WV_MULTICAST, Entry);\r
732                 pQp->pVerbs->detach_mcast(mc->hVerbsMc);\r
733                 mc->hVerbsMc = NULL;\r
734         }\r
735 \r
736         pQp->pVerbs->destroy_qp(pQp->hVerbsQp, 0);\r
737         pQp->hVerbsQp = NULL;\r
738 }\r
739 \r
740 void WvQpCancel(WV_PROVIDER *pProvider, WDFREQUEST Request)\r
741 {\r
742         UINT64                          *id;\r
743         WV_QUEUE_PAIR           *qp;\r
744         NTSTATUS                        status;\r
745 \r
746         status = WdfRequestRetrieveInputBuffer(Request, sizeof(UINT64), &id, NULL);\r
747         if (!NT_SUCCESS(status)) {\r
748                 goto out;\r
749         }\r
750 \r
751         qp = WvQpAcquire(pProvider, *id);\r
752         if (qp == NULL) {\r
753                 status = STATUS_NOT_FOUND;\r
754                 goto out;\r
755         }\r
756 \r
757         // CI does not yet support async operations\r
758         WvQpRelease(qp);\r
759 \r
760 out:\r
761         WdfRequestComplete(Request, status);\r
762 }\r