[MTHCA] 1. A revised version of AV creation/destroy mechanism.
[mirror/winof/.git] / hw / mthca / kernel / mt_verbs.c
1 /*
2  * Copyright (c) 2004 Mellanox Technologies Ltd.  All rights reserved.
3  * Copyright (c) 2004 Infinicon Corporation.  All rights reserved.
4  * Copyright (c) 2004 Intel Corporation.  All rights reserved.
5  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
6  * Copyright (c) 2004 Voltaire Corporation.  All rights reserved.
7  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
8  * Copyright (c) 2005 Cisco Systems.  All rights reserved.
9  *
10  * This software is available to you under a choice of one of two
11  * licenses.  You may choose to be licensed under the terms of the GNU
12  * General Public License (GPL) Version 2, available from the file
13  * COPYING in the main directory of this source tree, or the
14  * OpenIB.org BSD license below:
15  *
16  *     Redistribution and use in source and binary forms, with or
17  *     without modification, are permitted provided that the following
18  *     conditions are met:
19  *
20  *      - Redistributions of source code must retain the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer.
23  *
24  *      - Redistributions in binary form must reproduce the above
25  *        copyright notice, this list of conditions and the following
26  *        disclaimer in the documentation and/or other materials
27  *        provided with the distribution.
28  *
29  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
33  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
34  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
35  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36  * SOFTWARE.
37  *
38  * $Id: verbs.c 2934 2005-07-29 17:31:49Z roland $
39  */
40
41 #include <ib_verbs.h>
42 #include <ib_cache.h>
43 #include "mthca_dev.h"
44 #include "mx_abi.h"
45
46 #if defined(EVENT_TRACING)
47 #ifdef offsetof
48 #undef offsetof
49 #endif
50 #include "mt_verbs.tmh"
51 #endif
52
53
54 void ibv_um_close(      struct ib_ucontext * h_um_ca )
55 {
56         int err;
57         ib_api_status_t         status;
58         struct ib_ucontext *context_p = (struct ib_ucontext *)h_um_ca;
59         PREP_IBDEV_FOR_PRINT(context_p->device)
60
61         HCA_ENTER(HCA_DBG_SHIM);
62
63         context_p->is_removing = TRUE;
64
65         if (atomic_read(&context_p->usecnt)) {
66                 HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_SHIM,
67                         ("resources are not released (cnt %d)\n", context_p->usecnt));
68                 status = IB_RESOURCE_BUSY;
69                 goto err_usage;
70         }
71         
72         err = ibv_dealloc_pd( context_p->pd );
73         if (err) {
74                 HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_SHIM,
75                         ("ibv_dealloc_pd failed (%d)\n", err));
76                 status = errno_to_iberr(err);
77         }
78
79         err = mthca_dealloc_ucontext(context_p);
80         if (err) {
81                 HCA_PRINT(TRACE_LEVEL_ERROR   ,HCA_DBG_SHIM,
82                         ("mthca_dealloc_ucontext failed (%d)\n", err));
83                 status = errno_to_iberr(err);
84                 goto err_dealloc_ucontext;
85         }
86
87         HCA_PRINT(TRACE_LEVEL_INFORMATION       ,HCA_DBG_SHIM,
88                 ("pcs %p\n", PsGetCurrentProcess()) );
89         status = IB_SUCCESS;
90         goto end;
91         
92 err_dealloc_ucontext: 
93 err_usage:
94 end:
95         HCA_PRINT_EXIT(TRACE_LEVEL_ERROR  , HCA_DBG_SHIM,
96                 ("completes with ERROR status %s\n", IB_GET_ERR_STR(status)));
97         return;
98 }
99
100 /* Protection domains */
101
102 struct ib_pd *ibv_alloc_pd(struct ib_device *device,
103         struct ib_ucontext *context, ci_umv_buf_t* const p_umv_buf)
104 {
105         struct ib_pd *pd;
106
107         pd = device->alloc_pd(device, context, p_umv_buf);
108
109         if (!IS_ERR(pd)) {
110                 pd->device  = device;
111                 pd->ucontext = context;
112                 atomic_set(&pd->usecnt, 0);
113                 KeInitializeMutex( &pd->mutex, 0 );
114                 INIT_LIST_HEAD( &pd->list );
115                 HCA_PRINT(TRACE_LEVEL_INFORMATION ,HCA_DBG_CQ ,("PD%d use cnt %d, pd_handle %p, ctx %p \n", 
116                         ((struct mthca_pd*)pd)->pd_num, pd->usecnt, pd, pd->ucontext));
117         }
118
119         return pd;
120 }
121
122 int ibv_dealloc_pd(struct ib_pd *pd)
123 {
124         // we need first to release list of AV MRs to decrease pd->usecnt
125         if (pd->ucontext) {
126                 struct ib_mr *ib_mr, *tmp;
127                 down(&pd->mutex );
128                 list_for_each_entry_safe(ib_mr, tmp, &pd->list, list,struct ib_mr,struct ib_mr) {
129                         ibv_dereg_mr( ib_mr );
130                 }
131                 up(&pd->mutex );
132         }
133
134         if (atomic_read(&pd->usecnt)) {
135                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_CQ,("resources are not released (cnt %d)\n", pd->usecnt));
136                 return -EBUSY;
137         }               
138
139         HCA_PRINT(TRACE_LEVEL_INFORMATION ,HCA_DBG_CQ ,("PD%d use cnt %d, pd_handle %p, ctx %p \n", 
140                 ((struct mthca_pd*)pd)->pd_num, pd->usecnt, pd, pd->ucontext));
141         return pd->device->dealloc_pd(pd);
142 }
143
144 /* Address handles */
145
146 struct ib_ah *ibv_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
147         struct ib_ucontext *context, ci_umv_buf_t* const p_umv_buf)
148 {
149         int err;
150         struct ib_ah *ah;
151         struct ib_mr *ib_mr = NULL;
152         u64 start = 0;
153         u64 user_handle = 0;
154
155         // for user call we need also allocate MR
156         if (context && p_umv_buf && p_umv_buf->p_inout_buf) {
157                 struct ibv_create_ah *create_ah = (struct ibv_create_ah *)(void*)p_umv_buf->p_inout_buf;
158                 
159                 // create region; destroy will be done on dealloc_pd
160                 ib_mr   = ibv_reg_mr( 
161                         pd, 
162                         create_ah->mr.access_flags, 
163                         (void*)(ULONG_PTR)create_ah->mr.start,
164                         create_ah->mr.length, create_ah->mr.hca_va, TRUE );
165                 if (IS_ERR(ib_mr)) {
166                         err = PTR_ERR(ib_mr);
167                         HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_AV  ,("ibv_reg_mr failed (%d)\n", err));
168                         goto err_alloc_mr;
169                 }
170
171                 start = create_ah->mr.start;
172                 user_handle = create_ah->user_handle;
173
174                 // chain this MR to PD list
175                 down(&pd->mutex );
176                 list_add_tail(&ib_mr->list, &pd->list);
177                 up(&pd->mutex );
178         }
179
180         ah = pd->device->create_ah(pd, ah_attr);
181
182         if (IS_ERR(ah)) {
183                 err = PTR_ERR(ah);
184                 HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_AV ,("create_ah failed (%d)\n", err));
185                 goto err_create_ah;
186         }
187
188         // fill results
189         ah->device  = pd->device;
190         ah->pd      = pd;
191         ah->ucontext = context;
192         atomic_inc(&pd->usecnt);
193         HCA_PRINT(TRACE_LEVEL_INFORMATION  ,HCA_DBG_AV  ,("PD%d use cnt %d, pd_handle %p, ctx %p \n", 
194                 ((struct mthca_pd*)pd)->pd_num, pd->usecnt, pd, pd->ucontext));
195
196         // fill results for user
197         if (context && p_umv_buf && p_umv_buf->p_inout_buf) {
198                 struct ibv_create_ah_resp *create_ah_resp = (struct ibv_create_ah_resp *)(void*)p_umv_buf->p_inout_buf;
199                 create_ah_resp->start = start;
200                 create_ah_resp->user_handle = user_handle;
201                 create_ah_resp->mr.lkey = ib_mr->lkey;
202                 create_ah_resp->mr.rkey = ib_mr->rkey;
203                 create_ah_resp->mr.mr_handle = (u64)(ULONG_PTR)ib_mr;
204                 p_umv_buf->output_size = sizeof(struct ibv_create_ah_resp);
205         }
206
207         return ah;
208         
209 err_create_ah:
210         if (ib_mr)
211                 ibv_dereg_mr(ib_mr);
212 err_alloc_mr:
213         if( p_umv_buf && p_umv_buf->command ) 
214                 p_umv_buf->status = IB_ERROR;
215         return ERR_PTR(ib_mr);
216 }
217
218 struct ib_ah *ibv_create_ah_from_wc(struct ib_pd *pd, struct _ib_wc *wc,
219                                    struct ib_grh *grh, u8 port_num)
220 {
221         struct ib_ah_attr ah_attr;
222         u32 flow_class;
223         u16 gid_index;
224         int ret;
225
226         memset(&ah_attr, 0, sizeof ah_attr);
227         ah_attr.dlid = wc->recv.ud.remote_lid;
228         ah_attr.sl = wc->recv.ud.remote_sl;
229         ah_attr.src_path_bits = wc->recv.ud.path_bits;
230         ah_attr.port_num = port_num;
231
232         if (wc->recv.ud.recv_opt & IB_RECV_OPT_GRH_VALID) {
233                 ah_attr.ah_flags = IB_AH_GRH;
234                 ah_attr.grh.dgid = grh->dgid;
235
236                 ret = ib_find_cached_gid(pd->device, &grh->sgid, &port_num,
237                                          &gid_index);
238                 if (ret)
239                         return ERR_PTR(ret);
240
241                 ah_attr.grh.sgid_index = (u8) gid_index;
242                 flow_class = cl_ntoh32(grh->version_tclass_flow);
243                 ah_attr.grh.flow_label = flow_class & 0xFFFFF;
244                 ah_attr.grh.traffic_class = (u8)((flow_class >> 20) & 0xFF);
245                 ah_attr.grh.hop_limit = grh->hop_limit;
246         }
247
248         return ibv_create_ah(pd, &ah_attr, NULL, NULL);
249 }
250
251 int ibv_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
252 {
253         return ah->device->modify_ah ?
254                 ah->device->modify_ah(ah, ah_attr) :
255                 -ENOSYS;
256 }
257
258 int ibv_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
259 {
260         return ah->device->query_ah ?
261                 ah->device->query_ah(ah, ah_attr) :
262                 -ENOSYS;
263 }
264
265
266 static void release_user_cq_qp_resources(
267         struct ib_ucontext      *ucontext,
268         struct ib_mr * ib_mr)
269 {
270         if (ucontext) {
271                 ibv_dereg_mr( ib_mr );
272                 atomic_dec(&ucontext->usecnt);
273                 if (!atomic_read(&ucontext->usecnt) && ucontext->is_removing) {
274                         HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_SHIM      ,("User resources are released. Removing context\n"));
275                         ibv_um_close(ucontext);
276                 }
277         }
278 }
279
280 int ibv_destroy_ah(struct ib_ah *ah)
281 {
282         struct ib_pd *pd;
283         int ret;
284
285         HCA_ENTER(HCA_DBG_AV);
286         pd = ah->pd;
287
288         ret = ah->device->destroy_ah(ah);
289         if (!ret) {
290                 atomic_dec(&pd->usecnt);
291                 HCA_PRINT(TRACE_LEVEL_INFORMATION  ,HCA_DBG_AV  ,("PD%d use cnt %d, pd_handle %p, ctx %p \n", 
292                         ((struct mthca_pd*)pd)->pd_num, pd->usecnt, pd, pd->ucontext));
293         }
294         HCA_EXIT(HCA_DBG_AV);
295         return ret;
296 }
297
298 /* Shared receive queues */
299
300 struct ib_srq *ibv_create_srq(struct ib_pd *pd,
301                              struct ib_srq_init_attr *srq_init_attr)
302 {
303         struct ib_srq *srq;
304
305         if (!pd->device->create_srq)
306                 return ERR_PTR(-ENOSYS);
307
308         srq = pd->device->create_srq(pd, srq_init_attr, NULL);
309
310         if (!IS_ERR(srq)) {
311                 srq->device        = pd->device;
312                 srq->pd            = pd;
313                 srq->uobject       = NULL;
314                 srq->event_handler = srq_init_attr->event_handler;
315                 srq->srq_context   = srq_init_attr->srq_context;
316                 atomic_inc(&pd->usecnt);
317                 atomic_set(&srq->usecnt, 0);
318                 HCA_PRINT(TRACE_LEVEL_INFORMATION ,HCA_DBG_QP ,("PD%d use cnt %d \n", 
319                         ((struct mthca_pd*)pd)->pd_num, pd->usecnt));
320         }
321
322         return srq;
323 }
324
325 int ibv_modify_srq(struct ib_srq *srq,
326                   struct ib_srq_attr *srq_attr,
327                   enum ib_srq_attr_mask srq_attr_mask)
328 {
329         return srq->device->modify_srq(srq, srq_attr, srq_attr_mask);
330 }
331
332 int ibv_query_srq(struct ib_srq *srq,
333                  struct ib_srq_attr *srq_attr)
334 {
335         return srq->device->query_srq ?
336                 srq->device->query_srq(srq, srq_attr) : -ENOSYS;
337 }
338
339 int ibv_destroy_srq(struct ib_srq *srq)
340 {
341         struct ib_pd *pd;
342         int ret;
343
344         if (atomic_read(&srq->usecnt))
345                 return -EBUSY;
346
347         pd = srq->pd;
348
349         ret = srq->device->destroy_srq(srq);
350         if (!ret) {
351                 atomic_dec(&pd->usecnt);
352                 HCA_PRINT(TRACE_LEVEL_INFORMATION  ,HCA_DBG_SHIM        ,("PD%d use cnt %d \n", 
353                         ((struct mthca_pd*)pd)->pd_num, pd->usecnt));
354         }
355
356         return ret;
357 }
358
359 /* Queue pairs */
360
361 struct ib_qp *ibv_create_qp(struct ib_pd *pd,
362         struct ib_qp_init_attr *qp_init_attr,
363         struct ib_ucontext *context, ci_umv_buf_t* const p_umv_buf)
364 {
365         int err;
366         struct ib_qp *ib_qp;
367         struct ib_mr *ib_mr = NULL;
368         u64 user_handle = 0;
369
370         HCA_ENTER(HCA_DBG_QP);
371
372         // for user call we need also allocate MR
373         if (context && p_umv_buf && p_umv_buf->p_inout_buf) {
374                 struct ibv_create_qp *create_qp = (struct ibv_create_qp *)(void*)p_umv_buf->p_inout_buf;
375                 
376                 // create region
377                 ib_mr   = ibv_reg_mr( 
378                         (struct ib_pd *)(ULONG_PTR)create_qp->mr.pd_handle, 
379                         create_qp->mr.access_flags, 
380                         (void*)(ULONG_PTR)create_qp->mr.start,
381                         create_qp->mr.length, create_qp->mr.hca_va, TRUE );
382                 if (IS_ERR(ib_mr)) {
383                         err = PTR_ERR(ib_mr);
384                         HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP ,("ibv_reg_mr failed (%d)\n", err));
385                         goto err_alloc_mr;
386                 }
387                 create_qp->lkey = ib_mr->lkey;
388                 user_handle = create_qp->user_handle;
389         }
390
391         ib_qp = pd->device->create_qp(pd, qp_init_attr, p_umv_buf);
392
393         if (IS_ERR(ib_qp)) {
394                 err = PTR_ERR(ib_qp);
395                 HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_QP ,("create_qp failed (%d)\n", err));
396                 goto err_create_qp;
397         }
398
399         // fill results
400         ib_qp->device                           = pd->device;
401         ib_qp->pd                                       = pd;
402         ib_qp->send_cq                          = qp_init_attr->send_cq;
403         ib_qp->recv_cq                          = qp_init_attr->recv_cq;
404         ib_qp->srq                              = qp_init_attr->srq;
405         ib_qp->ucontext                         = context;
406         ib_qp->event_handler    = qp_init_attr->event_handler;
407         ib_qp->qp_context       = qp_init_attr->qp_context;
408         ib_qp->qp_type                          = qp_init_attr->qp_type;
409         atomic_inc(&pd->usecnt);
410         atomic_inc(&qp_init_attr->send_cq->usecnt);
411         atomic_inc(&qp_init_attr->recv_cq->usecnt);
412         if (qp_init_attr->srq)
413                 atomic_inc(&qp_init_attr->srq->usecnt);
414         if (context)
415                 atomic_inc(&context->usecnt);
416                 HCA_PRINT(TRACE_LEVEL_INFORMATION ,HCA_DBG_QP ,("PD%d use cnt %d, pd_handle %p, ctx %p \n", 
417                         ((struct mthca_pd*)pd)->pd_num, pd->usecnt, pd, pd->ucontext));
418
419         HCA_PRINT(TRACE_LEVEL_INFORMATION, HCA_DBG_QP ,
420                 ("uctx %p, qhndl %p, qnum %#x, q_num  %#x, scq %#x:%#x, rcq %#x:%#x \n",
421                 pd->ucontext, ib_qp, ((struct mthca_qp*)ib_qp)->qpn, ib_qp->qp_num,
422                 ((struct mthca_cq*)ib_qp->send_cq)->cqn, ib_qp->send_cq->cqe,
423                 ((struct mthca_cq*)ib_qp->recv_cq)->cqn, ib_qp->recv_cq->cqe ) );
424
425         // fill results for user
426         if (context && p_umv_buf && p_umv_buf->p_inout_buf) {
427                 struct mthca_qp *qp = (struct mthca_qp *)ib_qp;
428                 struct ibv_create_qp_resp *create_qp_resp = (struct ibv_create_qp_resp *)(void*)p_umv_buf->p_inout_buf;
429                 ib_qp->ib_mr = ib_mr;
430                 create_qp_resp->qpn = ib_qp->qp_num;
431                 create_qp_resp->user_handle = user_handle;
432                 create_qp_resp->mr.lkey = ib_mr->lkey;
433                 create_qp_resp->mr.rkey = ib_mr->rkey;
434                 create_qp_resp->mr.mr_handle = (u64)(ULONG_PTR)ib_mr;
435                 create_qp_resp->qp_handle = (__u64)(ULONG_PTR)qp;
436                 create_qp_resp->max_send_wr = qp->sq.max;
437                 create_qp_resp->max_recv_wr = qp->rq.max;
438                 create_qp_resp->max_send_sge = qp->sq.max_gs;
439                 create_qp_resp->max_recv_sge = qp->rq.max_gs;
440                 create_qp_resp->max_inline_data = qp->max_inline_data;
441                 p_umv_buf->output_size = sizeof(struct ibv_create_qp_resp);
442         }
443
444         return ib_qp;
445
446 err_create_qp:
447         if (ib_mr)
448                 ibv_dereg_mr(ib_mr);
449 err_alloc_mr:
450         if( p_umv_buf && p_umv_buf->command ) 
451                 p_umv_buf->status = IB_ERROR;
452         HCA_EXIT(HCA_DBG_QP);
453         return ERR_PTR(err);
454 }
455
456 int ibv_modify_qp(struct ib_qp *qp,
457                  struct ib_qp_attr *qp_attr,
458                  int qp_attr_mask)
459 {
460         return qp->device->modify_qp(qp, qp_attr, qp_attr_mask);
461 }
462
463 int ibv_query_qp(struct ib_qp *qp,
464                 struct ib_qp_attr *qp_attr,
465                 int qp_attr_mask,
466                 struct ib_qp_init_attr *qp_init_attr)
467 {
468         return qp->device->query_qp ?
469                 qp->device->query_qp(qp, qp_attr, qp_attr_mask, qp_init_attr) :
470                 -ENOSYS;
471 }
472         
473 int ibv_destroy_qp(struct ib_qp *qp)
474 {
475         struct ib_pd *pd;
476         struct ib_cq *scq, *rcq;
477         struct ib_srq *srq;
478         int ret;
479         struct ib_ucontext      *ucontext;
480         struct ib_mr * ib_mr;
481
482         pd  = qp->pd;
483         scq = qp->send_cq;
484         rcq = qp->recv_cq;
485         srq = qp->srq;
486         ucontext = pd->ucontext;
487         ib_mr = qp->ib_mr;
488
489         ret = qp->device->destroy_qp(qp);
490         if (!ret) {
491                 atomic_dec(&pd->usecnt);
492                 atomic_dec(&scq->usecnt);
493                 atomic_dec(&rcq->usecnt);
494                 HCA_PRINT(TRACE_LEVEL_INFORMATION  ,HCA_DBG_SHIM        ,("PD%d use cnt %d, pd_handle %p, ctx %p \n", 
495                         ((struct mthca_pd*)pd)->pd_num, pd->usecnt, pd, pd->ucontext));
496                 if (srq)
497                         atomic_dec(&srq->usecnt);
498                 release_user_cq_qp_resources(ucontext, ib_mr);
499         }
500
501         return ret;
502 }
503
504 /* Completion queues */
505
506 struct ib_cq *ibv_create_cq(struct ib_device *device,
507                            ib_comp_handler comp_handler,
508                            void (*event_handler)(struct ib_event *, void *),
509                            void *cq_context, int cqe, 
510                            struct ib_ucontext *context, ci_umv_buf_t* const p_umv_buf)
511 {
512         int err;
513         struct ib_cq *cq;
514         struct ib_mr *ib_mr = NULL;
515         u64 user_handle = 0;
516
517         // for user call we need also allocate MR
518         if (context && p_umv_buf && p_umv_buf->p_inout_buf) {
519                 struct ibv_create_cq *create_cq = (struct ibv_create_cq *)(void*)p_umv_buf->p_inout_buf;
520                 
521                 // create region
522                 ib_mr   = ibv_reg_mr( 
523                         (struct ib_pd *)(ULONG_PTR)create_cq->mr.pd_handle, 
524                         create_cq->mr.access_flags, 
525                         (void*)(ULONG_PTR)create_cq->mr.start,
526                         create_cq->mr.length, create_cq->mr.hca_va, TRUE );
527                 if (IS_ERR(ib_mr)) {
528                         err = PTR_ERR(ib_mr);
529                         HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_LOW ,("ibv_reg_mr failed (%d)\n", err));
530                         goto err_alloc_mr;
531                 }
532                 user_handle = create_cq->user_handle;
533                 create_cq->lkey = ib_mr->lkey;
534                 cqe = create_cq->cqe;
535         }
536         
537         // create cq
538         cq = device->create_cq(device, cqe, context, p_umv_buf);
539         if (IS_ERR(cq)) {
540                 err = PTR_ERR(cq);
541                 HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_LOW ,("create_qp failed (%d)\n", err));
542                 goto err_create_cq;
543         }
544
545         cq->device        = device;
546         cq->ucontext = context;
547         cq->comp_handler  = comp_handler;
548         cq->event_handler = event_handler;
549         cq->cq_context    = cq_context;
550         atomic_set(&cq->usecnt, 0);
551         if (context)
552                 atomic_inc(&context->usecnt);
553
554         // fill results
555         if (context && p_umv_buf && p_umv_buf->p_inout_buf) {
556                 struct ibv_create_cq_resp *create_cq_resp = (struct ibv_create_cq_resp *)(void*)p_umv_buf->p_inout_buf;
557                 cq->ib_mr = ib_mr;
558                 create_cq_resp->user_handle = user_handle;
559                 create_cq_resp->mr.lkey = ib_mr->lkey;
560                 create_cq_resp->mr.rkey = ib_mr->rkey;
561                 create_cq_resp->mr.mr_handle = (u64)(ULONG_PTR)ib_mr;
562                 create_cq_resp->cq_handle = (u64)(ULONG_PTR)cq;
563                 create_cq_resp->cqe = cq->cqe;
564                 p_umv_buf->output_size = sizeof(struct ibv_create_cq_resp);
565         }
566         
567         return cq;
568
569 err_create_cq:
570         if (ib_mr)
571                 ibv_dereg_mr(ib_mr);
572 err_alloc_mr:
573         if( p_umv_buf && p_umv_buf->command ) 
574                 p_umv_buf->status = IB_ERROR;
575         return ERR_PTR(err);
576 }
577
578 int ibv_destroy_cq(struct ib_cq *cq)
579 {
580         int ret;
581         struct ib_ucontext      *ucontext = cq->ucontext;
582         struct ib_mr * ib_mr = cq->ib_mr;
583         
584         if (atomic_read(&cq->usecnt))
585                 return -EBUSY;
586
587         ret = cq->device->destroy_cq(cq);
588
589         release_user_cq_qp_resources(ucontext, ib_mr);
590         
591         return ret;
592 }
593
594 int ibv_resize_cq(struct ib_cq *cq,
595                  int           cqe)
596 {
597         int ret;
598
599         if (!cq->device->resize_cq)
600                 return -ENOSYS;
601
602         ret = cq->device->resize_cq(cq, &cqe);
603         if (!ret)
604                 cq->cqe = cqe;
605
606         return ret;
607 }
608
609 /* Memory regions */
610
611 struct ib_mr *ibv_reg_mr(struct ib_pd *pd, 
612         mthca_qp_access_t mr_access_flags,
613         void* __ptr64                   vaddr,
614         uint64_t                                length,
615         uint64_t                                hca_va,
616         boolean_t                       um_call
617         )
618 {
619         struct ib_mr *ib_mr;
620         int                          err;
621         HCA_ENTER(HCA_DBG_MEMORY);
622         /* sanity check */
623         if (!um_call) {
624                 err = -ENOSYS;
625                 HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_MEMORY ,("ibv_reg_mr for kernel mode is not supported (%d)\n", err));
626                 goto err_not_supported;
627         }
628
629         ib_mr = pd->device->reg_user_mr(pd, vaddr, length, hca_va, mr_access_flags);
630         if (IS_ERR(ib_mr)) {
631                 err = PTR_ERR(ib_mr);
632                 HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_MEMORY ,("mthca_reg_user_mr failed (%d)\n", err));
633                 goto err_reg_user_mr;
634         }
635
636         ib_mr->device  = pd->device;
637         ib_mr->pd      = pd;
638         atomic_inc(&pd->usecnt);
639         atomic_set(&ib_mr->usecnt, 0);
640         HCA_PRINT(TRACE_LEVEL_INFORMATION ,HCA_DBG_MEMORY ,("PD%d use cnt %d, pd_handle %p, ctx %p \n", 
641                 ((struct mthca_pd*)pd)->pd_num, pd->usecnt, pd, pd->ucontext));
642         HCA_EXIT(HCA_DBG_MEMORY);
643         return ib_mr;
644
645 err_reg_user_mr:
646 err_not_supported:
647         HCA_EXIT(HCA_DBG_MEMORY);
648         return ERR_PTR(err);
649 }
650
651 struct ib_mr *ibv_get_dma_mr(struct ib_pd *pd, mthca_qp_access_t mr_access_flags)
652 {
653         struct ib_mr *mr;
654
655         mr = pd->device->get_dma_mr(pd, mr_access_flags);
656
657         if (!IS_ERR(mr)) {
658                 mr->device  = pd->device;
659                 mr->pd      = pd;
660                 atomic_inc(&pd->usecnt);
661                 atomic_set(&mr->usecnt, 0);
662                 HCA_PRINT(TRACE_LEVEL_INFORMATION ,HCA_DBG_MEMORY ,("PD%d use cnt %d \n", 
663                         ((struct mthca_pd*)pd)->pd_num, pd->usecnt));
664         }
665
666         return mr;
667 }
668
669 struct ib_mr *ibv_reg_phys_mr(struct ib_pd *pd,
670                              struct ib_phys_buf *phys_buf_array,
671                              int num_phys_buf,
672                              mthca_qp_access_t mr_access_flags,
673                              u64 *iova_start)
674 {
675         struct ib_mr *mr;
676
677         mr = pd->device->reg_phys_mr(pd, phys_buf_array, num_phys_buf,
678                                      mr_access_flags, iova_start);
679
680         if (!IS_ERR(mr)) {
681                 mr->device  = pd->device;
682                 mr->pd      = pd;
683                 atomic_inc(&pd->usecnt);
684                 atomic_set(&mr->usecnt, 0);
685                 HCA_PRINT(TRACE_LEVEL_INFORMATION  ,HCA_DBG_SHIM        ,("PD%d use cnt %d \n", 
686                         ((struct mthca_pd*)pd)->pd_num, pd->usecnt));
687         }
688
689         return mr;
690 }
691
692 int ibv_rereg_phys_mr(struct ib_mr *mr,
693                      int mr_rereg_mask,
694                      struct ib_pd *pd,
695                      struct ib_phys_buf *phys_buf_array,
696                      int num_phys_buf,
697                      mthca_qp_access_t mr_access_flags,
698                      u64 *iova_start)
699 {
700         struct ib_pd *old_pd;
701         int ret;
702
703         if (!mr->device->rereg_phys_mr)
704                 return -ENOSYS;
705
706         if (atomic_read(&mr->usecnt))
707                 return -EBUSY;
708
709         old_pd = mr->pd;
710
711         ret = mr->device->rereg_phys_mr(mr, mr_rereg_mask, pd,
712                                         phys_buf_array, num_phys_buf,
713                                         mr_access_flags, iova_start);
714
715         if (!ret && (mr_rereg_mask & IB_MR_REREG_PD)) {
716                 atomic_dec(&old_pd->usecnt);
717                 atomic_inc(&pd->usecnt);
718                 HCA_PRINT(TRACE_LEVEL_INFORMATION  ,HCA_DBG_SHIM        ,("PD%d use cnt %d \n", 
719                         ((struct mthca_pd*)pd)->pd_num, pd->usecnt));
720         }
721
722         return ret;
723 }
724
725 int ibv_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
726 {
727         return mr->device->query_mr ?
728                 mr->device->query_mr(mr, mr_attr) : -ENOSYS;
729 }
730
731 int ibv_dereg_mr(struct ib_mr *mr)
732 {
733         int ret;
734         struct ib_pd *pd;
735
736         if (atomic_read(&mr->usecnt))
737                 return -EBUSY;
738
739         pd = mr->pd;
740         ret = mr->device->dereg_mr(mr);
741         if (!ret) {
742                 atomic_dec(&pd->usecnt);
743                 HCA_PRINT(TRACE_LEVEL_INFORMATION  ,HCA_DBG_SHIM        ,("PD%d use cnt %d, pd_handle %p, ctx %p \n", 
744                         ((struct mthca_pd*)pd)->pd_num, pd->usecnt, pd, pd->ucontext));
745         }
746
747         return ret;
748 }
749
750 /* Memory windows */
751
752 struct ib_mw *ibv_alloc_mw(struct ib_pd *pd)
753 {
754         struct ib_mw *mw;
755
756         if (!pd->device->alloc_mw)
757                 return ERR_PTR(-ENOSYS);
758
759         mw = pd->device->alloc_mw(pd);
760         if (!IS_ERR(mw)) {
761                 mw->device  = pd->device;
762                 mw->pd      = pd;
763                 atomic_inc(&pd->usecnt);
764                 HCA_PRINT(TRACE_LEVEL_INFORMATION  ,HCA_DBG_SHIM        ,("PD%d use cnt %d \n", 
765                         ((struct mthca_pd*)pd)->pd_num, pd->usecnt));
766         }
767
768         return mw;
769 }
770
771 int ibv_dealloc_mw(struct ib_mw *mw)
772 {
773         struct ib_pd *pd;
774         int ret;
775
776         pd = mw->pd;
777         ret = mw->device->dealloc_mw(mw);
778         if (!ret) {
779                 atomic_dec(&pd->usecnt);
780                 HCA_PRINT(TRACE_LEVEL_INFORMATION  ,HCA_DBG_SHIM        ,("PD%d use cnt %d \n", 
781                         ((struct mthca_pd*)pd)->pd_num, pd->usecnt));
782         }
783
784         return ret;
785 }
786
787 /* "Fast" memory regions */
788
789 struct ib_fmr *ibv_alloc_fmr(struct ib_pd *pd,
790                             mthca_qp_access_t mr_access_flags,
791                             struct ib_fmr_attr *fmr_attr)
792 {
793         struct ib_fmr *fmr;
794
795         if (!pd->device->alloc_fmr)
796                 return ERR_PTR(-ENOSYS);
797
798         fmr = pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr);
799         if (!IS_ERR(fmr)) {
800                 fmr->device = pd->device;
801                 fmr->pd     = pd;
802                 atomic_inc(&pd->usecnt);
803                 HCA_PRINT(TRACE_LEVEL_INFORMATION  ,HCA_DBG_SHIM        ,("PD%d use cnt %d \n", 
804                         ((struct mthca_pd*)pd)->pd_num, pd->usecnt));
805         }
806
807         return fmr;
808 }
809
810 int ibv_unmap_fmr(struct list_head *fmr_list)
811 {
812         struct ib_fmr *fmr;
813
814         if (list_empty(fmr_list))
815                 return 0;
816
817         fmr = list_entry(fmr_list->next, struct ib_fmr, list);
818         return fmr->device->unmap_fmr(fmr_list);
819 }
820
821 int ibv_dealloc_fmr(struct ib_fmr *fmr)
822 {
823         struct ib_pd *pd;
824         int ret;
825
826         pd = fmr->pd;
827         ret = fmr->device->dealloc_fmr(fmr);
828         if (!ret) {
829                 atomic_dec(&pd->usecnt);
830                 HCA_PRINT(TRACE_LEVEL_INFORMATION  ,HCA_DBG_SHIM        ,("PD%d use cnt %d \n", 
831                         ((struct mthca_pd*)pd)->pd_num, pd->usecnt));
832         }
833
834         return ret;
835 }
836
837 /* Multicast groups */
838
839 int ibv_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
840 {
841         if (!qp->device->attach_mcast)
842                 return -ENOSYS;
843         if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UNRELIABLE_DGRM)
844                 return -EINVAL;
845
846         return qp->device->attach_mcast(qp, gid, lid);
847 }
848
849 int ibv_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
850 {
851         if (!qp->device->detach_mcast)
852                 return -ENOSYS;
853         if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UNRELIABLE_DGRM)
854                 return -EINVAL;
855
856         return qp->device->detach_mcast(qp, gid, lid);
857 }