[MTHCA] 1. bugfix: gid lookup use wrong port number;
[mirror/winof/.git] / hw / mthca / kernel / mthca_provider.c
1 /* 
2  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
3  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4  * Copyright (c) 2005 Cisco Systems. All rights reserved.
5  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
6  * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
7  *
8  * This software is available to you under a choice of one of two
9  * licenses.  You may choose to be licensed under the terms of the GNU
10  * General Public License (GPL) Version 2, available from the file
11  * COPYING in the main directory of this source tree, or the
12  * OpenIB.org BSD license below:
13  *
14  *     Redistribution and use in source and binary forms, with or
15  *     without modification, are permitted provided that the following
16  *     conditions are met:
17  *
18  *      - Redistributions of source code must retain the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer.
21  *
22  *      - Redistributions in binary form must reproduce the above
23  *        copyright notice, this list of conditions and the following
24  *        disclaimer in the documentation and/or other materials
25  *        provided with the distribution.
26  *
27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34  * SOFTWARE.
35  *
36  * $Id: mthca_provider.c 3047 2005-08-10 03:59:35Z roland $
37  */
38
39 #include <ib_smi.h>
40
41 #include "mx_abi.h"
42 #include "mthca_dev.h"
43
44 #if defined(EVENT_TRACING)
45 #ifdef offsetof
46 #undef offsetof
47 #endif
48 #include "mthca_provider.tmh"
49 #endif
50 #include "mthca_cmd.h"
51 #include "mthca_memfree.h"
52
53  void ibv_umem_release(struct ib_device *dev, struct ib_umem *umem);
54  int ibv_umem_get(struct ib_device *dev, struct ib_umem *mem,
55                  void *addr, size_t size, int write);
56  
57  static void init_query_mad(struct ib_smp *mad)
58  {
59          mad->base_version      = 1;
60          mad->mgmt_class                = IB_MGMT_CLASS_SUBN_LID_ROUTED;
61          mad->class_version = 1;
62          mad->method                            = IB_MGMT_METHOD_GET;
63  }
64
65  int mthca_query_device(struct ib_device *ibdev,
66                               struct ib_device_attr *props)
67 {
68         struct ib_smp *in_mad  = NULL;
69         struct ib_smp *out_mad = NULL;
70         int err = -ENOMEM;
71         struct mthca_dev* mdev = to_mdev(ibdev);
72
73         u8 status;
74
75         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
76         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
77         if (!in_mad || !out_mad)
78                 goto out;
79
80         init_query_mad(in_mad);
81         in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
82
83         err = mthca_MAD_IFC(mdev, 1, 1,
84             1, NULL, NULL, in_mad, out_mad, &status);
85         if (err)
86                 goto out;
87         if (status) {
88                 err = -EINVAL;
89                 goto out;
90         }
91
92         RtlZeroMemory(props, sizeof *props);
93         props->fw_ver              = mdev->fw_ver;
94         props->device_cap_flags    = mdev->device_cap_flags;
95         props->vendor_id           = cl_ntoh32(*(__be32 *) (out_mad->data + 36)) &
96                 0xffffff;
97         props->vendor_part_id      = cl_ntoh16(*(__be16 *) (out_mad->data + 30));
98         props->hw_ver              = cl_ntoh32(*(__be32 *) (out_mad->data + 32));
99         memcpy(&props->sys_image_guid, out_mad->data +  4, 8);
100         props->max_mr_size         = ~0ull;
101         props->page_size_cap       = mdev->limits.page_size_cap;
102         props->max_qp              = mdev->limits.num_qps - mdev->limits.reserved_qps;
103         props->max_qp_wr           = mdev->limits.max_wqes;
104         props->max_sge             = mdev->limits.max_sg;
105         props->max_cq              = mdev->limits.num_cqs - mdev->limits.reserved_cqs;
106         props->max_cqe             = mdev->limits.max_cqes;
107         props->max_mr              = mdev->limits.num_mpts - mdev->limits.reserved_mrws;
108         props->max_pd              = mdev->limits.num_pds - mdev->limits.reserved_pds;
109         props->max_qp_rd_atom      = 1 << mdev->qp_table.rdb_shift;
110         props->max_qp_init_rd_atom = mdev->limits.max_qp_init_rdma;
111         props->max_res_rd_atom     = props->max_qp_rd_atom * props->max_qp;
112         props->max_srq             = mdev->limits.num_srqs - mdev->limits.reserved_srqs;
113         props->max_srq_wr          = mdev->limits.max_srq_wqes;
114         props->max_srq_sge         = mdev->limits.max_sg;
115         props->local_ca_ack_delay  = (u8)mdev->limits.local_ca_ack_delay;
116         props->atomic_cap          = mdev->limits.flags & DEV_LIM_FLAG_ATOMIC ? 
117                                         IB_ATOMIC_LOCAL : IB_ATOMIC_NONE;
118         props->max_pkeys           = (u16)mdev->limits.pkey_table_len;
119         props->max_mcast_grp       = mdev->limits.num_mgms + mdev->limits.num_amgms;
120         props->max_mcast_qp_attach = MTHCA_QP_PER_MGM;
121         props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * 
122                                            props->max_mcast_grp;
123
124         err = 0;
125  out:
126         kfree(in_mad);
127         kfree(out_mad);
128         return err;
129 }
130
131 int mthca_query_port(struct ib_device *ibdev,
132                             u8 port, struct ib_port_attr *props)
133 {
134         struct ib_smp *in_mad  = NULL;
135         struct ib_smp *out_mad = NULL;
136         int err = -ENOMEM;
137         u8 status;
138
139         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
140         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
141         if (!in_mad || !out_mad)
142                 goto out;
143
144         init_query_mad(in_mad);
145         in_mad->attr_id  = IB_SMP_ATTR_PORT_INFO;
146         in_mad->attr_mod = cl_hton32(port);
147
148         err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
149                             port, NULL, NULL, in_mad, out_mad,
150                             &status);
151         if (err)
152                 goto out;
153         if (status) {
154                 err = -EINVAL;
155                 goto out;
156         }
157
158         RtlZeroMemory(props, sizeof *props);
159         props->lid               = cl_ntoh16(*(__be16 *) (out_mad->data + 16));
160         props->lmc               = out_mad->data[34] & 0x7;
161         props->sm_lid            = cl_ntoh16(*(__be16 *) (out_mad->data + 18));
162         props->sm_sl             = out_mad->data[36] & 0xf;
163         props->state             = out_mad->data[32] & 0xf;
164         props->phys_state        = out_mad->data[33] >> 4;
165         props->port_cap_flags    = cl_ntoh32(*(__be32 *) (out_mad->data + 20));
166         props->gid_tbl_len       = to_mdev(ibdev)->limits.gid_table_len;
167         props->max_msg_sz        = 0x80000000;
168         props->pkey_tbl_len      = (u16)to_mdev(ibdev)->limits.pkey_table_len;
169         props->bad_pkey_cntr     = cl_ntoh16(*(__be16 *) (out_mad->data + 46));
170         props->qkey_viol_cntr    = cl_ntoh16(*(__be16 *) (out_mad->data + 48));
171         props->active_width      = out_mad->data[31] & 0xf;
172         props->active_speed      = out_mad->data[35] >> 4;
173         props->max_mtu           = out_mad->data[41] & 0xf;
174         props->active_mtu        = out_mad->data[36] >> 4;
175         props->subnet_timeout    = out_mad->data[51] & 0x1f;
176
177  out:
178         kfree(in_mad);
179         kfree(out_mad);
180         return err;
181 }
182
183 int mthca_modify_port(struct ib_device *ibdev,
184                              u8 port, int port_modify_mask,
185                              struct ib_port_modify *props)
186 {
187         struct mthca_set_ib_param set_ib;
188         struct ib_port_attr attr;
189         int err;
190         u8 status;
191
192         if (down_interruptible(&to_mdev(ibdev)->cap_mask_mutex))
193                 return -EFAULT;
194
195         err = mthca_query_port(ibdev, port, &attr);
196         if (err)
197                 goto out;
198
199         set_ib.set_si_guid     = 0;
200         set_ib.reset_qkey_viol = !!(port_modify_mask & IB_PORT_RESET_QKEY_CNTR);
201
202         set_ib.cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) &
203                 ~props->clr_port_cap_mask;
204
205         err = mthca_SET_IB(to_mdev(ibdev), &set_ib, port, &status);
206         if (err)
207                 goto out;
208         if (status) {
209                 err = -EINVAL;
210                 goto out;
211         }
212
213 out:
214         up(&to_mdev(ibdev)->cap_mask_mutex);
215         return err;
216 }
217
218 int mthca_query_pkey_chunk(struct ib_device *ibdev,
219                             u8 port, u16 index, u16 pkey[32])
220 {
221         struct ib_smp *in_mad  = NULL;
222         struct ib_smp *out_mad = NULL;
223         int err = -ENOMEM;
224         u8 status;
225
226         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
227         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
228         if (!in_mad || !out_mad)
229                 goto out;
230
231         init_query_mad(in_mad);
232         in_mad->attr_id  = IB_SMP_ATTR_PKEY_TABLE;
233         in_mad->attr_mod = cl_hton32(index / 32);
234
235         err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
236                             port, NULL, NULL, in_mad, out_mad,
237                             &status);
238         if (err)
239                 goto out;
240         if (status) {
241                 err = -EINVAL;
242                 goto out;
243         }
244
245         { // copy the results
246                 int i;
247                 __be16 *pkey_chunk = (__be16 *)out_mad->data;
248                 for (i=0; i<32; ++i) 
249                         pkey[i] = cl_ntoh16(pkey_chunk[i]);
250         }
251
252  out:
253         kfree(in_mad);
254         kfree(out_mad);
255         return err;
256 }
257
258 int mthca_query_gid_chunk(struct ib_device *ibdev, u8 port,
259                            int index, union ib_gid gid[8])
260 {
261         struct ib_smp *in_mad  = NULL;
262         struct ib_smp *out_mad = NULL;
263         int err = -ENOMEM;
264         u8 status;
265         __be64  subnet_prefix;
266
267         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
268         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
269         if (!in_mad || !out_mad)
270                 goto out;
271
272         init_query_mad(in_mad);
273         in_mad->attr_id  = IB_SMP_ATTR_PORT_INFO;
274         in_mad->attr_mod = cl_hton32(port);
275
276         err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
277                             port, NULL, NULL, in_mad, out_mad,
278                             &status);
279         if (err)
280                 goto out;
281         if (status) {
282                 err = -EINVAL;
283                 goto out;
284         }
285
286         memcpy(&subnet_prefix, out_mad->data + 8, 8);
287
288         init_query_mad(in_mad);
289         in_mad->attr_id  = IB_SMP_ATTR_GUID_INFO;
290         in_mad->attr_mod = cl_hton32(index / 8);
291
292         err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
293                             port, NULL, NULL, in_mad, out_mad,
294                             &status);
295         if (err)
296                 goto out;
297         if (status) {
298                 err = -EINVAL;
299                 goto out;
300         }
301
302         { // copy the results
303                 int i;
304                 __be64 *guid = (__be64 *)out_mad->data;
305                 for (i=0; i<8; ++i) {
306                         gid[i].global.subnet_prefix = subnet_prefix;
307                         gid[i].global.interface_id = guid[i];
308                 }
309         }
310
311  out:
312         kfree(in_mad);
313         kfree(out_mad);
314         return err;
315 }
316
317 struct ib_ucontext *mthca_alloc_ucontext(struct ib_device *ibdev,
318                                                 ci_umv_buf_t* const     p_umv_buf)
319 {
320         struct mthca_alloc_ucontext_resp uresp;
321         struct mthca_ucontext           *context;
322         int                              err;
323
324         RtlZeroMemory(&uresp, sizeof uresp);
325
326         uresp.qp_tab_size = to_mdev(ibdev)->limits.num_qps;
327         if (mthca_is_memfree(to_mdev(ibdev)))
328                 uresp.uarc_size = to_mdev(ibdev)->uar_table.uarc_size;
329         else
330                 uresp.uarc_size = 0;
331
332         context = kzalloc(sizeof *context, GFP_KERNEL);
333         if (!context) {
334                 err = -ENOMEM;
335                 goto err_nomem;
336         }
337         
338         err = mthca_uar_alloc(to_mdev(ibdev), &context->uar);
339         if (err) 
340                 goto err_uar_alloc;
341
342         /*
343         * map uar to user space
344         */
345
346         /* map UAR to kernel */
347         context->kva = ioremap(context->uar.pfn << PAGE_SHIFT, PAGE_SIZE,&context->uar_size);
348         if (!context->kva) {
349                 HCA_PRINT( TRACE_LEVEL_ERROR, HCA_DBG_LOW ,("Couldn't map kernel access region, aborting.\n") );
350                 err = -ENOMEM;
351                 goto err_ioremap;
352         }
353
354         /* build MDL */
355         context->mdl = IoAllocateMdl( context->kva, (ULONG)context->uar_size,
356                 FALSE, TRUE, NULL );
357         if( !context->mdl ) {
358                 err = -ENOMEM;
359                 goto err_alloc_mdl;
360         }
361         MmBuildMdlForNonPagedPool( context->mdl );
362
363         /* Map the memory into the calling process's address space. */
364         __try   {
365                 context->ibucontext.user_uar = MmMapLockedPagesSpecifyCache( context->mdl,
366                         UserMode, MmNonCached, NULL, FALSE, NormalPagePriority );
367         }
368         __except(EXCEPTION_EXECUTE_HANDLER) {
369                 err = -EACCES;
370                 goto err_map;
371         }
372
373         /* user_db_tab */
374         context->db_tab = mthca_init_user_db_tab(to_mdev(ibdev));
375         if (IS_ERR(context->db_tab)) {
376                 err = PTR_ERR(context->db_tab);
377                 goto err_init_user;
378         }
379
380         err = ib_copy_to_umv_buf(p_umv_buf, &uresp, sizeof uresp);
381         if (err) 
382                 goto err_copy_to_umv_buf;
383
384         context->ibucontext.device = ibdev;
385         
386         atomic_set(&context->ibucontext.usecnt, 0);
387         return &context->ibucontext;
388
389 err_copy_to_umv_buf:
390         mthca_cleanup_user_db_tab(to_mdev(ibdev), &context->uar,
391                 context->db_tab);
392 err_init_user:  
393         MmUnmapLockedPages( context->ibucontext.user_uar, context->mdl );
394 err_map:
395         IoFreeMdl(context->mdl);
396 err_alloc_mdl:  
397         iounmap(context->kva, PAGE_SIZE);
398 err_ioremap:    
399         mthca_uar_free(to_mdev(ibdev), &context->uar);
400 err_uar_alloc:
401         kfree(context);
402 err_nomem:      
403         return ERR_PTR(err);
404 }
405
406  int mthca_dealloc_ucontext(struct ib_ucontext *context)
407 {
408          struct mthca_ucontext                                   *mucontext = to_mucontext(context);
409
410         mthca_cleanup_user_db_tab(to_mdev(context->device), &mucontext->uar,
411                                   mucontext->db_tab);
412         MmUnmapLockedPages( mucontext->ibucontext.user_uar, mucontext->mdl );
413         IoFreeMdl(mucontext->mdl);
414         iounmap(mucontext->kva, PAGE_SIZE);
415         mthca_uar_free(to_mdev(context->device), &mucontext->uar);
416         kfree(mucontext);
417         
418         return 0;
419 }
420
421 struct ib_pd *mthca_alloc_pd(struct ib_device *ibdev,
422                                     struct ib_ucontext *context,
423                                     ci_umv_buf_t* const                 p_umv_buf)
424 {
425         int err;
426         struct mthca_pd *pd;
427         struct ibv_alloc_pd_resp resp;
428
429         /* sanity check */
430         if (p_umv_buf && p_umv_buf->command) {
431                 if (p_umv_buf->output_size < sizeof(struct ibv_alloc_pd_resp)) {
432                         err = -EINVAL;
433                         goto err_param;
434                 }
435         }
436         
437         pd = kmalloc(sizeof *pd, GFP_KERNEL);
438         if (!pd) {
439                 err = -ENOMEM;
440                 goto err_mem;
441         }
442
443         err = mthca_pd_alloc(to_mdev(ibdev), !context, pd);
444         if (err) {
445                 goto err_pd_alloc;
446         }
447
448         if (p_umv_buf && p_umv_buf->command) {
449                 resp.pd_handle = (u64)(UINT_PTR)pd;
450                 resp.pdn = pd->pd_num;
451                 if (ib_copy_to_umv_buf(p_umv_buf, &resp, sizeof(struct ibv_alloc_pd_resp))) {
452                         err = -EFAULT;
453                         goto err_copy;
454                 }
455         }
456
457         return &pd->ibpd;
458
459 err_copy:       
460         mthca_pd_free(to_mdev(ibdev), pd);
461 err_pd_alloc:
462         kfree(pd);
463 err_mem:
464 err_param:
465         return ERR_PTR(err);
466 }
467
468 int mthca_dealloc_pd(struct ib_pd *pd)
469 {
470         mthca_pd_free(to_mdev(pd->device), to_mpd(pd));
471         kfree(pd);
472
473         return 0;
474 }
475
476 struct ib_ah *mthca_ah_create(struct ib_pd *pd,
477                                      struct ib_ah_attr *ah_attr)
478 {
479         int err;
480         struct mthca_ah *ah;
481
482         ah = kzalloc(sizeof *ah, GFP_ATOMIC);
483         if (!ah)
484                 return ERR_PTR(-ENOMEM);
485
486         err = mthca_create_ah(to_mdev(pd->device), to_mpd(pd), ah_attr, ah);
487         if (err) {
488                 kfree(ah);
489                 return ERR_PTR(err);
490         }
491
492         return &ah->ibah;
493 }
494
495 int mthca_ah_destroy(struct ib_ah *ah)
496 {
497         mthca_destroy_ah(to_mdev(ah->device), to_mah(ah));
498         kfree(ah);
499
500         return 0;
501 }
502
503 struct ib_srq *mthca_create_srq(struct ib_pd *pd,
504                                        struct ib_srq_init_attr *init_attr,
505                                        ci_umv_buf_t* const                      p_umv_buf)
506 {
507 #ifdef WIN_TO_BE_CHANGED
508         struct mthca_create_srq ucmd;
509         struct mthca_ucontext *context = NULL;
510         struct mthca_srq *srq;
511         int err;
512
513         srq = kmalloc(sizeof *srq, GFP_KERNEL);
514         if (!srq)
515                 return ERR_PTR(-ENOMEM);
516
517         if (pd->ucontext) {
518                 context = to_mucontext(pd->ucontext);
519
520                 if (ib_copy_from_umv_buf(&ucmd, p_umv_buf, sizeof ucmd)) {
521                         err = -EFAULT;
522                         goto err_free;
523                 }
524                 err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
525                                         context->db_tab, ucmd.db_index,
526                                         ucmd.db_page, NULL);
527
528                 if (err)
529                         goto err_free;
530
531                 srq->mr.ibmr.lkey = ucmd.lkey;
532                 srq->db_index     = ucmd.db_index;
533         }
534
535         err = mthca_alloc_srq(to_mdev(pd->device), to_mpd(pd),
536                               &init_attr->attr, srq);
537
538         if (err && pd->ucontext)
539                 mthca_unmap_user_db(to_mdev(pd->device), &context->uar,
540                                     context->db_tab, ucmd.db_index);
541
542         if (err)
543                 goto err_free;
544
545         if (context && ib_copy_to_umv_buf(p_umv_buf, &srq->srqn, sizeof (u32))) {
546                 mthca_free_srq(to_mdev(pd->device), srq);
547                 err = -EFAULT;
548                 goto err_free;
549         }
550
551         return &srq->ibsrq;
552
553 err_free:
554         kfree(srq);
555
556         return ERR_PTR(err);
557 #else
558         UNREFERENCED_PARAMETER(p_umv_buf);
559         UNREFERENCED_PARAMETER(init_attr);
560         UNREFERENCED_PARAMETER(pd);
561         return NULL;
562 #endif
563 }
564
565 int mthca_destroy_srq(struct ib_srq *srq)
566 {
567         struct mthca_ucontext *context;
568
569         if (srq->uobject) {
570                 context = to_mucontext(srq->uobject->context);
571
572                 mthca_unmap_user_db(to_mdev(srq->device), &context->uar,
573                                     context->db_tab, to_msrq(srq)->db_index);
574         }
575
576         mthca_free_srq(to_mdev(srq->device), to_msrq(srq));
577         kfree(srq);
578
579         return 0;
580 }
581
582 struct ib_qp *mthca_create_qp(struct ib_pd *pd,
583                                      struct ib_qp_init_attr *init_attr,
584                                       ci_umv_buf_t* const                       p_umv_buf)
585 {
586         struct ibv_create_qp ucmd = {0};
587         struct mthca_qp *qp = NULL;
588         struct mthca_ucontext *context = NULL;
589         int err;
590
591         switch (init_attr->qp_type) {
592         case IB_QPT_RELIABLE_CONN:
593         case IB_QPT_UNRELIABLE_CONN:
594         case IB_QPT_UNRELIABLE_DGRM:
595         {
596
597                 qp = kmalloc(sizeof *qp, GFP_KERNEL);
598                 if (!qp) {
599                         err = -ENOMEM;
600                         goto err_mem;
601                 }
602
603                 if (pd->ucontext) {
604                         context = to_mucontext(pd->ucontext);
605
606                         if (ib_copy_from_umv_buf(&ucmd, p_umv_buf, sizeof ucmd)) {
607                                 err = -EFAULT;
608                                 goto err_copy;
609                         }
610
611                         err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
612                                                 context->db_tab,
613                                                 ucmd.sq_db_index, ucmd.sq_db_page, NULL);
614                         if (err) 
615                                 goto err_map1;
616
617                         err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
618                                                 context->db_tab,
619                                                 ucmd.rq_db_index, ucmd.rq_db_page, NULL);
620                         if (err) 
621                                 goto err_map2;
622
623                         qp->mr.ibmr.lkey = ucmd.lkey;
624                         qp->sq.db_index  = ucmd.sq_db_index;
625                         qp->rq.db_index  = ucmd.rq_db_index;
626                 }
627
628                 err = mthca_alloc_qp(to_mdev(pd->device), to_mpd(pd),
629                                      to_mcq(init_attr->send_cq),
630                                      to_mcq(init_attr->recv_cq),
631                                      init_attr->qp_type, init_attr->sq_sig_type,
632                                      &init_attr->cap, qp);
633
634                 if (err)
635                         if (pd->ucontext) 
636                                 goto err_alloc_qp_user;
637                         else 
638                                 goto err_copy;
639
640                 qp->ibqp.qp_num = qp->qpn;
641                 break;
642         }
643         case IB_QPT_QP0:
644         case IB_QPT_QP1:
645         {
646                 /* Don't allow userspace to create special QPs */
647                 if (pd->ucontext) {
648                         err = -EINVAL;
649                         goto err_inval;
650                 }
651
652                 qp = kmalloc(sizeof (struct mthca_sqp), GFP_KERNEL);
653                 if (!qp) {
654                         err = -ENOMEM;
655                         goto err_mem;
656                 }
657
658                 qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_QP0 ? 0 : 1;
659
660                 err = mthca_alloc_sqp(to_mdev(pd->device), to_mpd(pd),
661                                       to_mcq(init_attr->send_cq),
662                                       to_mcq(init_attr->recv_cq),
663                                       init_attr->sq_sig_type, &init_attr->cap,
664                                       qp->ibqp.qp_num, init_attr->port_num,
665                                       to_msqp(qp));
666                 if (err)
667                         goto err_alloc_sqp;
668                 
669                 break;
670         }
671         default:
672                 /* Don't support raw QPs */
673                 err = -ENOSYS;
674                 goto err_unsupported;
675         }
676
677         init_attr->cap.max_send_wr     = qp->sq.max;
678         init_attr->cap.max_recv_wr     = qp->rq.max;
679         init_attr->cap.max_send_sge    = qp->sq.max_gs;
680         init_attr->cap.max_recv_sge    = qp->rq.max_gs;
681         init_attr->cap.max_inline_data    = qp->max_inline_data;
682
683         return &qp->ibqp;
684
685                 
686 err_alloc_qp_user:
687         if (pd->ucontext) 
688                 mthca_unmap_user_db(to_mdev(pd->device),
689                         &context->uar, context->db_tab, ucmd.rq_db_index);
690 err_map2:
691         if (pd->ucontext) 
692                 mthca_unmap_user_db(to_mdev(pd->device),
693                         &context->uar, context->db_tab, ucmd.sq_db_index);
694 err_map1: err_copy: err_alloc_sqp:
695         if (qp)
696                 kfree(qp);
697 err_mem: err_inval:     err_unsupported:
698         return ERR_PTR(err);
699 }
700
701 int mthca_destroy_qp(struct ib_qp *qp)
702 {
703         if (qp->ucontext) {
704                 mthca_unmap_user_db(to_mdev(qp->device),
705                                     &to_mucontext(qp->ucontext)->uar,
706                                     to_mucontext(qp->ucontext)->db_tab,
707                                     to_mqp(qp)->sq.db_index);
708                 mthca_unmap_user_db(to_mdev(qp->device),
709                                     &to_mucontext(qp->ucontext)->uar,
710                                     to_mucontext(qp->ucontext)->db_tab,
711                                     to_mqp(qp)->rq.db_index);
712         }
713         mthca_free_qp(to_mdev(qp->device), to_mqp(qp));
714         kfree(qp);
715         return 0;
716 }
717
718 struct ib_cq *mthca_create_cq(struct ib_device *ibdev, int entries,
719                                      struct ib_ucontext *context,
720                                      ci_umv_buf_t* const                        p_umv_buf)
721 {
722         struct ibv_create_cq ucmd = {0};
723         struct mthca_cq *cq;
724         int nent;
725         int err;
726         void *u_arm_db_page = 0;
727
728         if (entries < 1 || entries > to_mdev(ibdev)->limits.max_cqes)   
729                 return ERR_PTR(-EINVAL);
730
731         if (context) {
732                 if (ib_copy_from_umv_buf(&ucmd, p_umv_buf, sizeof ucmd))
733                         return ERR_PTR(-EFAULT);
734
735                 err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
736                                         to_mucontext(context)->db_tab,
737                                         ucmd.set_db_index, ucmd.set_db_page, NULL);
738                 if (err)
739                         return ERR_PTR(err);
740
741                 err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
742                                         to_mucontext(context)->db_tab,
743                                         ucmd.arm_db_index, ucmd.arm_db_page, NULL);
744                 if (err)
745                         goto err_unmap_set;
746
747                 err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
748                                         to_mucontext(context)->db_tab,
749                                         ucmd.u_arm_db_index, 
750                                         (u64)(ULONG_PTR)PAGE_ALIGN(ucmd.u_arm_db_page), 
751                                         &u_arm_db_page);
752                 if (err)
753                         goto err_unmap_arm;
754         }
755
756         cq = kmalloc(sizeof *cq, GFP_KERNEL);
757         if (!cq) {
758                 err = -ENOMEM;
759                 goto err_unmap_ev;
760         }
761
762         if (context) {
763                 cq->mr.ibmr.lkey = ucmd.lkey;
764                 cq->set_ci_db_index = ucmd.set_db_index;
765                 cq->arm_db_index    = ucmd.arm_db_index;
766                 cq->u_arm_db_index    = ucmd.u_arm_db_index;
767                 cq->p_u_arm_sn = (int*)((char*)u_arm_db_page + BYTE_OFFSET(ucmd.u_arm_db_page));
768         }
769
770         for (nent = 1; nent <= entries; nent <<= 1)
771                 ; /* nothing */
772
773         err = mthca_init_cq(to_mdev(ibdev), nent, 
774                             context ? to_mucontext(context) : NULL,
775                             context ? ucmd.mr.pdn : to_mdev(ibdev)->driver_pd.pd_num,
776                             cq);
777         if (err)
778                 goto err_free;
779
780         if (context ) {
781                 struct ibv_create_cq_resp *create_cq_resp = (struct ibv_create_cq_resp *)(void*)p_umv_buf->p_inout_buf;
782                 create_cq_resp->cqn = cq->cqn;
783         }
784
785         HCA_PRINT( TRACE_LEVEL_INFORMATION, HCA_DBG_LOW ,
786                 ("uctx %p, cq_hndl %p, cq_num %#x, cqe  %#x\n",
787                 context, &cq->ibcq, cq->cqn, cq->ibcq.cqe ) );
788         
789         return &cq->ibcq;
790
791 err_free:
792         kfree(cq);
793
794 err_unmap_ev:
795         if (context)
796                 mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
797                                     to_mucontext(context)->db_tab, ucmd.u_arm_db_index);
798
799 err_unmap_arm:
800         if (context)
801                 mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
802                                     to_mucontext(context)->db_tab, ucmd.arm_db_index);
803
804 err_unmap_set:
805         if (context)
806                 mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
807                                     to_mucontext(context)->db_tab, ucmd.set_db_index);
808
809         return ERR_PTR(err);
810 }
811
812 int mthca_destroy_cq(struct ib_cq *cq)
813 {
814         if (cq->ucontext) {
815                 mthca_unmap_user_db(to_mdev(cq->device),
816                                     &to_mucontext(cq->ucontext)->uar,
817                                     to_mucontext(cq->ucontext)->db_tab,
818                                     to_mcq(cq)->u_arm_db_index);
819                 mthca_unmap_user_db(to_mdev(cq->device),
820                                     &to_mucontext(cq->ucontext)->uar,
821                                     to_mucontext(cq->ucontext)->db_tab,
822                                     to_mcq(cq)->arm_db_index);
823                 mthca_unmap_user_db(to_mdev(cq->device),
824                                     &to_mucontext(cq->ucontext)->uar,
825                                     to_mucontext(cq->ucontext)->db_tab,
826                                     to_mcq(cq)->set_ci_db_index);
827         }
828         mthca_free_cq(to_mdev(cq->device), to_mcq(cq));
829         kfree(cq);
830
831         return 0;
832 }
833
834 static
835 mthca_mpt_access_t
836 map_qp_mpt(
837         IN                              mthca_qp_access_t                               qp_acl)
838 {
839 #define ACL_MTHCA(mfl,ifl) if (qp_acl & mfl)   mpt_acl |= ifl
840         mthca_mpt_access_t mpt_acl = 0;
841
842         ACL_MTHCA(MTHCA_ACCESS_REMOTE_READ,MTHCA_MPT_FLAG_REMOTE_READ);
843         ACL_MTHCA(MTHCA_ACCESS_REMOTE_WRITE,MTHCA_MPT_FLAG_REMOTE_WRITE);
844         ACL_MTHCA(MTHCA_ACCESS_REMOTE_ATOMIC,MTHCA_MPT_FLAG_ATOMIC);
845         ACL_MTHCA(MTHCA_ACCESS_LOCAL_WRITE,MTHCA_MPT_FLAG_LOCAL_WRITE);
846
847         return (mpt_acl | MTHCA_MPT_FLAG_LOCAL_READ);
848 }
849
850 struct ib_mr *mthca_get_dma_mr(struct ib_pd *pd, mthca_qp_access_t acc)
851 {
852         struct mthca_mr *mr;
853         int err;
854
855         mr = kmalloc(sizeof *mr, GFP_KERNEL);
856         if (!mr)
857                 return ERR_PTR(-ENOMEM);
858         RtlZeroMemory(mr, sizeof *mr);
859
860         err = mthca_mr_alloc_notrans(to_mdev(pd->device),
861                                      to_mpd(pd)->pd_num,
862                                      map_qp_mpt(acc), mr);
863
864         if (err) {
865                 kfree(mr);
866                 return ERR_PTR(err);
867         }
868
869         return &mr->ibmr;
870 }
871
872 struct ib_mr *mthca_reg_phys_mr(struct ib_pd       *pd,
873                                        struct ib_phys_buf *buffer_list,
874                                        int                 num_phys_buf,
875                                        mthca_qp_access_t                 acc,
876                                        u64                *iova_start)
877 {
878         struct mthca_mr *mr;
879         u64 *page_list;
880         u64 total_size;
881         u64 mask;
882         int shift;
883         int npages;
884         int err;
885         int i, j, n;
886
887         /* First check that we have enough alignment */
888         if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK))
889                 return ERR_PTR(-EINVAL);
890
891         if (num_phys_buf > 1 &&
892             ((buffer_list[0].addr + buffer_list[0].size) & ~PAGE_MASK))
893                 return ERR_PTR(-EINVAL);
894
895         mask = 0;
896         total_size = 0;
897         for (i = 0; i < num_phys_buf; ++i) {
898                 if (i != 0)
899                         mask |= buffer_list[i].addr;
900                 if (i != num_phys_buf - 1)
901                         mask |= buffer_list[i].addr + buffer_list[i].size;
902
903                 total_size += buffer_list[i].size;
904         }
905
906         if (mask & ~PAGE_MASK)
907                 return ERR_PTR(-EINVAL);
908
909         /* Find largest page shift we can use to cover buffers */
910         for (shift = PAGE_SHIFT; shift < 31; ++shift)
911                 if (num_phys_buf > 1) {
912                         if ((1ULL << shift) & mask)
913                                 break;
914                 } else {
915                         if (1ULL << shift >=
916                             buffer_list[0].size +
917                             (buffer_list[0].addr & ((1ULL << shift) - 1)))
918                                 break;
919                 }
920
921         buffer_list[0].size += buffer_list[0].addr & ((1ULL << shift) - 1);
922         buffer_list[0].addr &= ~0ull << shift;
923
924         mr = kmalloc(sizeof *mr, GFP_KERNEL);
925         if (!mr)
926                 return ERR_PTR(-ENOMEM);
927         RtlZeroMemory(mr, sizeof *mr);
928
929         npages = 0;
930         for (i = 0; i < num_phys_buf; ++i)
931                 npages += (int)((buffer_list[i].size + (1ULL << shift) - 1) >> shift);
932
933         if (!npages)
934                 return &mr->ibmr;
935
936         page_list = kmalloc(npages * sizeof *page_list, GFP_KERNEL);
937         if (!page_list) {
938                 kfree(mr);
939                 return ERR_PTR(-ENOMEM);
940         }
941
942         n = 0;
943         for (i = 0; i < num_phys_buf; ++i)
944                 for (j = 0;
945                      j < (buffer_list[i].size + (1ULL << shift) - 1) >> shift;
946                      ++j)
947                         page_list[n++] = buffer_list[i].addr + ((u64) j << shift);
948
949         HCA_PRINT( TRACE_LEVEL_VERBOSE ,HCA_DBG_LOW ,("Registering memory at %I64x (iova %I64x) "
950                   "in PD %x; shift %d, npages %d.\n",
951                   (unsigned long long) buffer_list[0].addr,
952                   (unsigned long long) *iova_start,
953                   to_mpd(pd)->pd_num,
954                   shift, npages));
955
956         err = mthca_mr_alloc_phys(to_mdev(pd->device),
957                                   to_mpd(pd)->pd_num,
958                                   page_list, shift, npages,
959                                   *iova_start, total_size,
960                                   map_qp_mpt(acc), mr);
961
962         if (err) {
963                 kfree(page_list);
964                 kfree(mr);
965                 return ERR_PTR(err);
966         }
967
968         kfree(page_list);
969         return &mr->ibmr;
970 }
971
972 struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, 
973         void* __ptr64   vaddr, uint64_t length, uint64_t hca_va, mthca_qp_access_t acc)
974 {
975         struct mthca_dev *dev = to_mdev(pd->device);
976         struct ib_umem_chunk *chunk;
977         struct mthca_mr *mr;
978         struct ib_umem *region;
979         u64 *pages;
980         int shift, n, len;
981         int i, j, k;
982         int err = 0;
983
984         HCA_ENTER(HCA_DBG_MEMORY);
985         mr = kzalloc(sizeof *mr, GFP_KERNEL);
986         if (!mr) {
987                 err = -ENOMEM;
988                 goto err_nomem;
989         }
990         region = &mr->umem;
991
992         /*
993          * We ask for writable memory if any access flags other than
994          * "remote read" are set.  "Local write" and "remote write"
995          * obviously require write access.  "Remote atomic" can do
996          * things like fetch and add, which will modify memory, and
997          * "MW bind" can change permissions by binding a window.
998          */
999         err = ibv_umem_get(pd->device, region,
1000                           (void *)vaddr, (size_t)length,
1001                           !!(acc & ~MTHCA_ACCESS_REMOTE_READ));
1002         if (err)
1003                 goto err_umem_get;
1004
1005         region->virt_base = hca_va;     /* va in HCA */
1006
1007         n = 0;
1008         shift = ffs(region->page_size) - 1;
1009         list_for_each_entry(chunk, &region->chunk_list, list,struct ib_umem_chunk)
1010                 n += chunk->nents;
1011
1012         mr->mtt = mthca_alloc_mtt(dev, n);
1013         if (IS_ERR(mr->mtt)) {
1014                 err = PTR_ERR(mr->mtt);
1015                 goto err_alloc_mtt;
1016         }
1017
1018         pages = (u64 *) kmalloc(PAGE_SIZE,GFP_KERNEL);
1019         if (!pages) {
1020                 err = -ENOMEM;
1021                 goto err_pages;
1022         }
1023
1024         i = n = 0;
1025
1026         list_for_each_entry(chunk, &region->chunk_list, list,struct ib_umem_chunk)
1027                 for (j = 0; j < chunk->nmap; ++j) {
1028                         len = sg_dma_len(&chunk->page_list[j]) >> shift;
1029                         for (k = 0; k < len; ++k) {
1030                                 pages[i++] = sg_dma_address(&chunk->page_list[j]) +
1031                                         region->page_size * k;
1032                                 /*
1033                                  * Be friendly to WRITE_MTT command
1034                                  * and leave two empty slots for the
1035                                  * index and reserved fields of the
1036                                  * mailbox.
1037                                  */
1038                                 if (i == PAGE_SIZE / sizeof (u64) - 2) {
1039                                         err = mthca_write_mtt(dev, mr->mtt,
1040                                                               n, pages, i);
1041                                         if (err)
1042                                                 goto err_write_mtt;
1043                                         n += i;
1044                                         i = 0;
1045                                 }
1046                         }
1047                 }
1048
1049         if (i) {
1050                 err = mthca_write_mtt(dev, mr->mtt, n, pages, i);
1051                 if (err)
1052                         goto err_write_mtt;
1053         }       
1054
1055         err = mthca_mr_alloc(dev, to_mpd(pd)->pd_num, shift, region->virt_base,
1056                              region->length, map_qp_mpt(acc), mr);
1057         if (err)
1058                 goto err_mt_alloc;
1059
1060         free_page((void*) pages);
1061         HCA_EXIT(HCA_DBG_MEMORY);
1062         return &mr->ibmr;
1063
1064 err_mt_alloc:
1065 err_write_mtt:
1066         free_page((void*) pages);
1067 err_pages:
1068         mthca_free_mtt(dev, mr->mtt);
1069 err_alloc_mtt:
1070         ibv_umem_release(pd->device, region);
1071 err_umem_get:   
1072         kfree(mr);
1073 err_nomem:      
1074         HCA_EXIT(HCA_DBG_MEMORY);
1075         return ERR_PTR(err);
1076 }
1077
1078 int mthca_dereg_mr(struct ib_mr *mr)
1079 {
1080         struct mthca_mr *mmr = to_mmr(mr);
1081         mthca_free_mr(to_mdev(mr->device), mmr);
1082         if (mr->pd->ucontext)
1083                 ibv_umem_release(mr->pd->device, &mmr->umem);
1084         kfree(mmr);
1085         return 0;
1086 }
1087
1088 struct ib_fmr *mthca_alloc_fmr(struct ib_pd *pd, mthca_qp_access_t acc,
1089                                       struct ib_fmr_attr *fmr_attr)
1090 {
1091         struct mthca_fmr *fmr;
1092         int err;
1093
1094         fmr = kmalloc(sizeof *fmr, GFP_KERNEL);
1095         if (!fmr)
1096                 return ERR_PTR(-ENOMEM);
1097
1098         memcpy(&fmr->attr, fmr_attr, sizeof *fmr_attr);
1099         err = mthca_fmr_alloc(to_mdev(pd->device), to_mpd(pd)->pd_num,
1100                              map_qp_mpt(acc), fmr);
1101
1102         if (err) {
1103                 kfree(fmr);
1104                 return ERR_PTR(err);
1105         }
1106
1107         return &fmr->ibmr;
1108 }
1109
1110 int mthca_dealloc_fmr(struct ib_fmr *fmr)
1111 {
1112         struct mthca_fmr *mfmr = to_mfmr(fmr);
1113         int err;
1114
1115         err = mthca_free_fmr(to_mdev(fmr->device), mfmr);
1116         if (err)
1117                 return err;
1118
1119         kfree(mfmr);
1120         return 0;
1121 }
1122
1123 int mthca_unmap_fmr(struct list_head *fmr_list)
1124 {
1125         struct ib_fmr *fmr;
1126         int err;
1127         u8 status;
1128         struct mthca_dev *mdev = NULL;
1129
1130         list_for_each_entry(fmr, fmr_list, list,struct ib_fmr) {
1131                 if (mdev && to_mdev(fmr->device) != mdev)
1132                         return -EINVAL;
1133                 mdev = to_mdev(fmr->device);
1134         }
1135
1136         if (!mdev)
1137                 return 0;
1138
1139         if (mthca_is_memfree(mdev)) {
1140                 list_for_each_entry(fmr, fmr_list, list,struct ib_fmr)
1141                         mthca_arbel_fmr_unmap(mdev, to_mfmr(fmr));
1142
1143                 wmb();
1144         } else
1145                 list_for_each_entry(fmr, fmr_list, list,struct ib_fmr)
1146                         mthca_tavor_fmr_unmap(mdev, to_mfmr(fmr));
1147
1148         err = mthca_SYNC_TPT(mdev, &status);
1149         if (err)
1150                 return err;
1151         if (status)
1152                 return -EINVAL;
1153         return 0;
1154 }
1155
1156 static int mthca_init_node_data(struct mthca_dev *dev)
1157 {
1158         struct ib_smp *in_mad  = NULL;
1159         struct ib_smp *out_mad = NULL;
1160         int err = -ENOMEM;
1161         u8 status;
1162
1163         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
1164         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
1165         if (!in_mad || !out_mad)
1166                 goto out;
1167
1168         init_query_mad(in_mad);
1169         in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
1170
1171         err = mthca_MAD_IFC(dev, 1, 1,
1172                             1, NULL, NULL, in_mad, out_mad,
1173                             &status);
1174         if (err)
1175                 goto out;
1176         if (status) {
1177                 err = -EINVAL;
1178                 goto out;
1179         }
1180
1181         memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8);
1182
1183 out:
1184         kfree(in_mad);
1185         kfree(out_mad);
1186         return err;
1187 }
1188
1189 int mthca_register_device(struct mthca_dev *dev)
1190 {
1191         int ret;
1192
1193         ret = mthca_init_node_data(dev);        
1194         if (ret)
1195                 return ret;
1196
1197         strlcpy(dev->ib_dev.name, "mthca%d", IB_DEVICE_NAME_MAX);
1198         dev->ib_dev.node_type            = IB_NODE_CA;
1199         dev->ib_dev.phys_port_cnt        = (u8)dev->limits.num_ports;
1200         dev->ib_dev.mdev                                = dev;
1201         dev->ib_dev.query_device         = mthca_query_device;
1202         dev->ib_dev.query_port           = mthca_query_port;
1203         dev->ib_dev.modify_port          = mthca_modify_port;
1204         dev->ib_dev.query_pkey_chunk           = mthca_query_pkey_chunk;
1205         dev->ib_dev.query_gid_chunk            = mthca_query_gid_chunk;
1206         dev->ib_dev.alloc_ucontext       = mthca_alloc_ucontext;
1207         dev->ib_dev.dealloc_ucontext     = mthca_dealloc_ucontext;
1208         dev->ib_dev.alloc_pd             = mthca_alloc_pd;
1209         dev->ib_dev.dealloc_pd           = mthca_dealloc_pd;
1210         dev->ib_dev.create_ah            = mthca_ah_create;
1211         dev->ib_dev.destroy_ah           = mthca_ah_destroy;
1212
1213         if (dev->mthca_flags & MTHCA_FLAG_SRQ) {
1214                 dev->ib_dev.create_srq           = mthca_create_srq;
1215                 dev->ib_dev.modify_srq           = mthca_modify_srq;
1216                 dev->ib_dev.destroy_srq          = mthca_destroy_srq;
1217
1218                 if (mthca_is_memfree(dev))
1219                         dev->ib_dev.post_srq_recv = mthca_arbel_post_srq_recv;
1220                 else
1221                         dev->ib_dev.post_srq_recv = mthca_tavor_post_srq_recv;
1222         }
1223
1224         dev->ib_dev.create_qp            = mthca_create_qp;
1225         dev->ib_dev.modify_qp            = mthca_modify_qp;
1226         dev->ib_dev.destroy_qp           = mthca_destroy_qp;
1227         dev->ib_dev.create_cq            = mthca_create_cq;
1228         dev->ib_dev.destroy_cq           = mthca_destroy_cq;
1229         dev->ib_dev.poll_cq              = mthca_poll_cq;
1230         dev->ib_dev.get_dma_mr           = mthca_get_dma_mr;
1231         dev->ib_dev.reg_phys_mr          = mthca_reg_phys_mr;
1232         dev->ib_dev.reg_user_mr          = mthca_reg_user_mr;
1233         dev->ib_dev.dereg_mr             = mthca_dereg_mr;
1234
1235         if (dev->mthca_flags & MTHCA_FLAG_FMR) {
1236                 dev->ib_dev.alloc_fmr            = mthca_alloc_fmr;
1237                 dev->ib_dev.unmap_fmr            = mthca_unmap_fmr;
1238                 dev->ib_dev.dealloc_fmr          = mthca_dealloc_fmr;
1239                 if (mthca_is_memfree(dev))
1240                         dev->ib_dev.map_phys_fmr = mthca_arbel_map_phys_fmr;
1241                 else
1242                         dev->ib_dev.map_phys_fmr = mthca_tavor_map_phys_fmr;
1243         }
1244
1245         dev->ib_dev.attach_mcast         = mthca_multicast_attach;
1246         dev->ib_dev.detach_mcast         = mthca_multicast_detach;
1247         dev->ib_dev.process_mad          = mthca_process_mad;
1248
1249         if (mthca_is_memfree(dev)) {
1250                 dev->ib_dev.req_notify_cq = mthca_arbel_arm_cq;
1251                 dev->ib_dev.post_send     = mthca_arbel_post_send;
1252                 dev->ib_dev.post_recv     = mthca_arbel_post_receive;
1253         } else {
1254                 dev->ib_dev.req_notify_cq = mthca_tavor_arm_cq;
1255                 dev->ib_dev.post_send     = mthca_tavor_post_send;
1256                 dev->ib_dev.post_recv     = mthca_tavor_post_receive;
1257         }
1258
1259         KeInitializeMutex(&dev->cap_mask_mutex, 0);
1260
1261         ret = ib_register_device(&dev->ib_dev);
1262         if (ret)
1263                 return ret;
1264
1265         mthca_start_catas_poll(dev);
1266
1267         return 0;
1268 }
1269
1270 void mthca_unregister_device(struct mthca_dev *dev)
1271 {
1272         mthca_stop_catas_poll(dev);
1273         ib_unregister_device(&dev->ib_dev);
1274 }