81f811e7b7cd4215e2fb30aec1fe6a82acc53bc5
[mirror/winof/.git] / hw / mthca / kernel / mthca_provider.c
1 /* 
2  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
3  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4  * Copyright (c) 2005 Cisco Systems. All rights reserved.
5  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
6  * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
7  *
8  * This software is available to you under a choice of one of two
9  * licenses.  You may choose to be licensed under the terms of the GNU
10  * General Public License (GPL) Version 2, available from the file
11  * COPYING in the main directory of this source tree, or the
12  * OpenIB.org BSD license below:
13  *
14  *     Redistribution and use in source and binary forms, with or
15  *     without modification, are permitted provided that the following
16  *     conditions are met:
17  *
18  *      - Redistributions of source code must retain the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer.
21  *
22  *      - Redistributions in binary form must reproduce the above
23  *        copyright notice, this list of conditions and the following
24  *        disclaimer in the documentation and/or other materials
25  *        provided with the distribution.
26  *
27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34  * SOFTWARE.
35  *
36  * $Id$
37  */
38
39 #include <ib_smi.h>
40
41 #include "mx_abi.h"
42 #include "mthca_dev.h"
43 #include "mt_pa_cash.h"
44
45 #if defined(EVENT_TRACING)
46 #ifdef offsetof
47 #undef offsetof
48 #endif
49 #include "mthca_provider.tmh"
50 #endif
51 #include "mthca_cmd.h"
52 #include "mthca_memfree.h"
53
54 static void init_query_mad(struct ib_smp *mad)
55 {
56          mad->base_version      = 1;
57          mad->mgmt_class                = IB_MGMT_CLASS_SUBN_LID_ROUTED;
58          mad->class_version = 1;
59          mad->method                            = IB_MGMT_METHOD_GET;
60 }
61
62 int mthca_query_device(struct ib_device *ibdev,
63                               struct ib_device_attr *props)
64 {
65         struct ib_smp *in_mad  = NULL;
66         struct ib_smp *out_mad = NULL;
67         int err = -ENOMEM;
68         struct mthca_dev* mdev = to_mdev(ibdev);
69
70         u8 status;
71
72         RtlZeroMemory(props, sizeof *props);
73
74         if (mthca_is_livefish(mdev)) {
75                 props->max_pd = 1;
76                 props->vendor_id = mdev->ext->hcaConfig.VendorID;
77                 props->vendor_part_id = mdev->ext->hcaConfig.DeviceID;
78                 return 0;
79         }
80
81         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
82         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
83         if (!in_mad || !out_mad)
84                 goto out;
85
86         init_query_mad(in_mad);
87         in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
88
89         err = mthca_MAD_IFC(mdev, 1, 1,
90             1, NULL, NULL, in_mad, out_mad, &status);
91         if (err)
92                 goto out;
93         if (status) {
94                 err = -EINVAL;
95                 goto out;
96         }
97
98         props->fw_ver              = mdev->fw_ver;
99         props->device_cap_flags    = mdev->device_cap_flags;
100         props->vendor_id           = cl_ntoh32(*(__be32 *) (out_mad->data + 36)) &
101                 0xffffff;
102         props->vendor_part_id      = cl_ntoh16(*(__be16 *) (out_mad->data + 30));
103         props->hw_ver              = cl_ntoh32(*(__be32 *) (out_mad->data + 32));
104         memcpy(&props->sys_image_guid, out_mad->data +  4, 8);
105         props->max_mr_size         = ~0Ui64;
106         props->page_size_cap       = mdev->limits.page_size_cap;
107         props->max_qp              = mdev->limits.num_qps - mdev->limits.reserved_qps;
108         props->max_qp_wr           = mdev->limits.max_wqes;
109         props->max_sge             = mdev->limits.max_sg;
110         props->max_cq              = mdev->limits.num_cqs - mdev->limits.reserved_cqs;
111         props->max_cqe             = mdev->limits.max_cqes;
112         props->max_mr              = mdev->limits.num_mpts - mdev->limits.reserved_mrws;
113         props->max_pd              = mdev->limits.num_pds - mdev->limits.reserved_pds;
114         props->max_qp_rd_atom      = 1 << mdev->qp_table.rdb_shift;
115         props->max_qp_init_rd_atom = mdev->limits.max_qp_init_rdma;
116         props->max_res_rd_atom     = props->max_qp_rd_atom * props->max_qp;
117         props->max_srq             = mdev->limits.num_srqs - mdev->limits.reserved_srqs;
118         props->max_srq_wr          = mdev->limits.max_srq_wqes;
119         if (mthca_is_memfree(mdev))
120                 --props->max_srq_wr;
121         props->max_srq_sge         = mdev->limits.max_srq_sge;
122         props->local_ca_ack_delay  = (u8)mdev->limits.local_ca_ack_delay;
123         props->atomic_cap          = mdev->limits.flags & DEV_LIM_FLAG_ATOMIC ? 
124                                         IB_ATOMIC_LOCAL : IB_ATOMIC_NONE;
125         props->max_pkeys           = (u16)mdev->limits.pkey_table_len;
126         props->max_mcast_grp       = mdev->limits.num_mgms + mdev->limits.num_amgms;
127         props->max_mcast_qp_attach = MTHCA_QP_PER_MGM;
128         props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * 
129                                            props->max_mcast_grp;
130
131         err = 0;
132  out:
133         kfree(in_mad);
134         kfree(out_mad);
135         return err;
136 }
137
138 int mthca_query_port(struct ib_device *ibdev,
139                             u8 port, struct ib_port_attr *props)
140 {
141         struct ib_smp *in_mad  = NULL;
142         struct ib_smp *out_mad = NULL;
143         int err = -ENOMEM;
144         u8 status;
145
146         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
147         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
148         if (!in_mad || !out_mad)
149                 goto out;
150
151         init_query_mad(in_mad);
152         in_mad->attr_id  = IB_SMP_ATTR_PORT_INFO;
153         in_mad->attr_mod = cl_hton32(port);
154
155         err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
156                             port, NULL, NULL, in_mad, out_mad,
157                             &status);
158         if (err)
159                 goto out;
160         if (status) {
161                 err = -EINVAL;
162                 goto out;
163         }
164
165         RtlZeroMemory(props, sizeof *props);
166         props->lid               = cl_ntoh16(*(__be16 *) (out_mad->data + 16));
167         props->lmc               = out_mad->data[34] & 0x7;
168         props->sm_lid            = cl_ntoh16(*(__be16 *) (out_mad->data + 18));
169         props->sm_sl             = out_mad->data[36] & 0xf;
170         props->state             = out_mad->data[32] & 0xf;
171         props->phys_state        = out_mad->data[33] >> 4;
172         props->port_cap_flags    = cl_ntoh32(*(__be32 *) (out_mad->data + 20));
173         props->gid_tbl_len       = to_mdev(ibdev)->limits.gid_table_len;
174         props->max_msg_sz        = 0x80000000;
175         props->pkey_tbl_len      = (u16)to_mdev(ibdev)->limits.pkey_table_len;
176         props->bad_pkey_cntr     = cl_ntoh16(*(__be16 *) (out_mad->data + 46));
177         props->qkey_viol_cntr    = cl_ntoh16(*(__be16 *) (out_mad->data + 48));
178         props->active_width      = out_mad->data[31] & 0xf;
179         props->active_speed      = out_mad->data[35] >> 4;
180         props->max_mtu           = out_mad->data[41] & 0xf;
181         props->active_mtu        = out_mad->data[36] >> 4;
182         props->subnet_timeout    = out_mad->data[51] & 0x1f;
183
184  out:
185         kfree(in_mad);
186         kfree(out_mad);
187         return err;
188 }
189
190 int mthca_modify_port(struct ib_device *ibdev,
191                              u8 port, int port_modify_mask,
192                              struct ib_port_modify *props)
193 {
194         struct mthca_set_ib_param set_ib;
195         struct ib_port_attr attr;
196         int err;
197         u8 status;
198
199         if (down_interruptible(&to_mdev(ibdev)->cap_mask_mutex))
200                 return -EFAULT;
201
202         err = mthca_query_port(ibdev, port, &attr);
203         if (err)
204                 goto out;
205
206         set_ib.set_si_guid     = 0;
207         set_ib.reset_qkey_viol = !!(port_modify_mask & IB_PORT_RESET_QKEY_CNTR);
208
209         set_ib.cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) &
210                 ~props->clr_port_cap_mask;
211
212         err = mthca_SET_IB(to_mdev(ibdev), &set_ib, port, &status);
213         if (err)
214                 goto out;
215         if (status) {
216                 err = -EINVAL;
217                 goto out;
218         }
219
220 out:
221         up(&to_mdev(ibdev)->cap_mask_mutex);
222         return err;
223 }
224
225 static int mthca_query_pkey_chunk(struct ib_device *ibdev,
226                             u8 port, u16 index, u16 pkey[32])
227 {
228         struct ib_smp *in_mad  = NULL;
229         struct ib_smp *out_mad = NULL;
230         int err = -ENOMEM;
231         u8 status;
232
233         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
234         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
235         if (!in_mad || !out_mad)
236                 goto out;
237
238         init_query_mad(in_mad);
239         in_mad->attr_id  = IB_SMP_ATTR_PKEY_TABLE;
240         in_mad->attr_mod = cl_hton32(index / 32);
241
242         err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
243                             port, NULL, NULL, in_mad, out_mad,
244                             &status);
245         if (err)
246                 goto out;
247         if (status) {
248                 err = -EINVAL;
249                 goto out;
250         }
251
252         { // copy the results
253                 int i;
254                 __be16 *pkey_chunk = (__be16 *)out_mad->data;
255                 for (i=0; i<32; ++i) 
256                         pkey[i] = cl_ntoh16(pkey_chunk[i]);
257         }
258
259  out:
260         kfree(in_mad);
261         kfree(out_mad);
262         return err;
263 }
264
265 static int mthca_query_gid_chunk(struct ib_device *ibdev, u8 port,
266                            int index, union ib_gid gid[8])
267 {
268         struct ib_smp *in_mad  = NULL;
269         struct ib_smp *out_mad = NULL;
270         int err = -ENOMEM;
271         u8 status;
272         __be64  subnet_prefix;
273
274         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
275         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
276         if (!in_mad || !out_mad)
277                 goto out;
278
279         init_query_mad(in_mad);
280         in_mad->attr_id  = IB_SMP_ATTR_PORT_INFO;
281         in_mad->attr_mod = cl_hton32(port);
282
283         err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
284                             port, NULL, NULL, in_mad, out_mad,
285                             &status);
286         if (err)
287                 goto out;
288         if (status) {
289                 err = -EINVAL;
290                 goto out;
291         }
292
293         memcpy(&subnet_prefix, out_mad->data + 8, 8);
294
295         init_query_mad(in_mad);
296         in_mad->attr_id  = IB_SMP_ATTR_GUID_INFO;
297         in_mad->attr_mod = cl_hton32(index / 8);
298
299         err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
300                             port, NULL, NULL, in_mad, out_mad,
301                             &status);
302         if (err)
303                 goto out;
304         if (status) {
305                 err = -EINVAL;
306                 goto out;
307         }
308
309         { // copy the results
310                 int i;
311                 __be64 *guid = (__be64 *)out_mad->data;
312                 for (i=0; i<8; ++i) {
313                         gid[i].global.subnet_prefix = subnet_prefix;
314                         gid[i].global.interface_id = guid[i];
315                 }
316         }
317
318  out:
319         kfree(in_mad);
320         kfree(out_mad);
321         return err;
322 }
323
324 struct ib_ucontext *mthca_alloc_ucontext(struct ib_device *ibdev,
325                                                 ci_umv_buf_t* const     p_umv_buf)
326 {
327         struct ibv_get_context_resp uresp;
328         struct mthca_ucontext           *context;
329         int                              err;
330
331         RtlZeroMemory(&uresp, sizeof uresp);
332
333         uresp.qp_tab_size = to_mdev(ibdev)->limits.num_qps;
334         if (mthca_is_memfree(to_mdev(ibdev)))
335                 uresp.uarc_size = to_mdev(ibdev)->uar_table.uarc_size;
336         else
337                 uresp.uarc_size = 0;
338
339         context = kzalloc(sizeof *context, GFP_KERNEL);
340         if (!context) {
341                 err = -ENOMEM;
342                 goto err_nomem;
343         }
344
345         if (mthca_is_livefish(to_mdev(ibdev)))
346                 goto done;
347         
348         err = mthca_uar_alloc(to_mdev(ibdev), &context->uar);
349         if (err) 
350                 goto err_uar_alloc;
351
352         /*
353         * map uar to user space
354         */
355
356         /* map UAR to kernel */
357         context->kva = ioremap((io_addr_t)context->uar.pfn << PAGE_SHIFT, PAGE_SIZE,&context->uar_size);
358         if (!context->kva) {
359                 HCA_PRINT( TRACE_LEVEL_ERROR, HCA_DBG_LOW ,("Couldn't map kernel access region, aborting.\n") );
360                 err = -ENOMEM;
361                 goto err_ioremap;
362         }
363
364         /* build MDL */
365         context->mdl = IoAllocateMdl( context->kva, (ULONG)context->uar_size,
366                 FALSE, TRUE, NULL );
367         if( !context->mdl ) {
368                 err = -ENOMEM;
369                 goto err_alloc_mdl;
370         }
371         MmBuildMdlForNonPagedPool( context->mdl );
372
373         /* Map the memory into the calling process's address space. */
374         __try   {
375                 context->ibucontext.user_uar = MmMapLockedPagesSpecifyCache( context->mdl,
376                         UserMode, MmNonCached, NULL, FALSE, NormalPagePriority );
377         }
378         __except(EXCEPTION_EXECUTE_HANDLER) {
379                 err = -EACCES;
380                 goto err_map;
381         }
382
383         /* user_db_tab */
384         context->db_tab = mthca_init_user_db_tab(to_mdev(ibdev));
385         if (IS_ERR(context->db_tab)) {
386                 err = PTR_ERR(context->db_tab);
387                 goto err_init_user;
388         }
389
390 done:
391         err = ib_copy_to_umv_buf(p_umv_buf, &uresp, sizeof uresp);
392         if (err) 
393                 goto err_copy_to_umv_buf;
394
395         context->ibucontext.device = ibdev;
396         
397         atomic_set(&context->ibucontext.usecnt, 0);
398         return &context->ibucontext;
399
400 err_copy_to_umv_buf:
401         mthca_cleanup_user_db_tab(to_mdev(ibdev), &context->uar,
402                 context->db_tab);
403 err_init_user:  
404         MmUnmapLockedPages( context->ibucontext.user_uar, context->mdl );
405 err_map:
406         IoFreeMdl(context->mdl);
407 err_alloc_mdl:  
408         iounmap(context->kva, PAGE_SIZE);
409 err_ioremap:    
410         mthca_uar_free(to_mdev(ibdev), &context->uar);
411 err_uar_alloc:
412         kfree(context);
413 err_nomem:      
414         return ERR_PTR(err);
415 }
416
417  int mthca_dealloc_ucontext(struct ib_ucontext *context)
418 {
419          struct mthca_ucontext                                   *mucontext = to_mucontext(context);
420
421         if (mthca_is_livefish(to_mdev(context->device)))
422                 goto done;
423         mthca_cleanup_user_db_tab(to_mdev(context->device), &mucontext->uar,
424                                   mucontext->db_tab);
425         MmUnmapLockedPages( mucontext->ibucontext.user_uar, mucontext->mdl );
426         IoFreeMdl(mucontext->mdl);
427         iounmap(mucontext->kva, PAGE_SIZE);
428         mthca_uar_free(to_mdev(context->device), &mucontext->uar);
429 done:   
430         kfree(mucontext);
431         
432         return 0;
433 }
434
435 struct ib_pd *mthca_alloc_pd(struct ib_device *ibdev,
436                                     struct ib_ucontext *context,
437                                     ci_umv_buf_t* const                 p_umv_buf)
438 {
439         int err;
440         struct mthca_pd *pd;
441         struct ibv_alloc_pd_resp resp;
442
443         /* sanity check */
444         if (p_umv_buf && p_umv_buf->command) {
445                 if (p_umv_buf->output_size < sizeof(struct ibv_alloc_pd_resp)) {
446                         err = -EINVAL;
447                         goto err_param;
448                 }
449         }
450         
451         pd = kmalloc(sizeof *pd, GFP_KERNEL);
452         if (!pd) {
453                 err = -ENOMEM;
454                 goto err_mem;
455         }
456
457         if (mthca_is_livefish(to_mdev(ibdev))) 
458                 goto done;
459
460         err = mthca_pd_alloc(to_mdev(ibdev), !context, pd);
461         if (err) {
462                 goto err_pd_alloc;
463         }
464
465 done:
466         if (p_umv_buf && p_umv_buf->command) {
467                 resp.pd_handle = (u64)(UINT_PTR)pd;
468                 resp.pdn = pd->pd_num;
469                 if (ib_copy_to_umv_buf(p_umv_buf, &resp, sizeof(struct ibv_alloc_pd_resp))) {
470                         err = -EFAULT;
471                         goto err_copy;
472                 }
473         }
474
475         return &pd->ibpd;
476
477 err_copy:       
478         mthca_pd_free(to_mdev(ibdev), pd);
479 err_pd_alloc:
480         kfree(pd);
481 err_mem:
482 err_param:
483         return ERR_PTR(err);
484 }
485
486 int mthca_dealloc_pd(struct ib_pd *pd)
487 {
488         if (mthca_is_livefish(to_mdev(pd->device))) 
489                 goto done;
490
491         mthca_pd_free(to_mdev(pd->device), to_mpd(pd));
492
493 done:   
494         kfree(pd);
495         return 0;
496 }
497
498 static struct ib_ah *mthca_ah_create(struct ib_pd *pd,
499                                      struct ib_ah_attr *ah_attr)
500 {
501         int err;
502         struct mthca_ah *ah;
503
504         ah = kzalloc(sizeof *ah, GFP_ATOMIC);
505         if (!ah)
506                 return ERR_PTR(-ENOMEM);
507
508         err = mthca_create_ah(to_mdev(pd->device), to_mpd(pd), ah_attr, ah);
509         if (err) {
510                 kfree(ah);
511                 return ERR_PTR(err);
512         }
513
514         return &ah->ibah;
515 }
516
517 static int mthca_ah_destroy(struct ib_ah *ah)
518 {
519         mthca_destroy_ah(to_mdev(ah->device), to_mah(ah));
520         kfree(ah);
521
522         return 0;
523 }
524
525 static struct ib_srq *mthca_create_srq(struct ib_pd *pd,
526                                 struct ib_srq_init_attr *init_attr,
527                                 ci_umv_buf_t* const p_umv_buf)
528 {
529         struct ibv_create_srq ucmd = { 0 };
530         struct mthca_ucontext *context = NULL;
531         struct mthca_srq *srq;
532         int err;
533
534         srq = kzalloc(sizeof *srq, GFP_KERNEL);
535         if (!srq)
536                 return ERR_PTR(-ENOMEM);
537
538         if (pd->ucontext) {
539                 context = to_mucontext(pd->ucontext);
540
541                 if (ib_copy_from_umv_buf(&ucmd, p_umv_buf, sizeof ucmd)) {
542                         err = -EFAULT;
543                         goto err_free;
544                 }
545                 err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
546                                         context->db_tab, ucmd.db_index,
547                                         ucmd.db_page, NULL);
548
549                 if (err)
550                         goto err_free;
551
552                 srq->mr.ibmr.lkey = ucmd.lkey;
553                 srq->db_index     = ucmd.db_index;
554         }
555
556         err = mthca_alloc_srq(to_mdev(pd->device), to_mpd(pd),
557                 &init_attr->attr, srq);
558
559         if (err && pd->ucontext)
560                 mthca_unmap_user_db(to_mdev(pd->device), &context->uar,
561                         context->db_tab, ucmd.db_index);
562
563         if (err)
564                 goto err_free;
565
566         if (context && ib_copy_to_umv_buf(p_umv_buf, &srq->srqn, sizeof (u32))) {
567                 mthca_free_srq(to_mdev(pd->device), srq);
568                 err = -EFAULT;
569                 goto err_free;
570         }
571
572         return &srq->ibsrq;
573
574 err_free:
575         kfree(srq);
576
577         return ERR_PTR(err);
578 }
579
580 static int mthca_destroy_srq(struct ib_srq *srq)
581 {
582         struct mthca_ucontext *context;
583
584         if (srq->ucontext) {
585                 context = to_mucontext(srq->ucontext);
586
587                 mthca_unmap_user_db(to_mdev(srq->device), &context->uar,
588                         context->db_tab, to_msrq(srq)->db_index);
589         }
590
591         mthca_free_srq(to_mdev(srq->device), to_msrq(srq));
592         kfree(srq);
593
594         return 0;
595 }
596
597 static struct ib_qp *mthca_create_qp(struct ib_pd *pd,
598                                      struct ib_qp_init_attr *init_attr,
599                                       ci_umv_buf_t* const                       p_umv_buf)
600 {
601         struct ibv_create_qp ucmd = {0};
602         struct mthca_qp *qp = NULL;
603         struct mthca_ucontext *context = NULL;
604         int err;
605
606         switch (init_attr->qp_type) {
607         case IB_QPT_RELIABLE_CONN:
608         case IB_QPT_UNRELIABLE_CONN:
609         case IB_QPT_UNRELIABLE_DGRM:
610         {
611
612                 qp = kmalloc(sizeof *qp, GFP_KERNEL);
613                 if (!qp) {
614                         err = -ENOMEM;
615                         goto err_mem;
616                 }
617
618                 if (pd->ucontext) {
619                         context = to_mucontext(pd->ucontext);
620
621                         if (ib_copy_from_umv_buf(&ucmd, p_umv_buf, sizeof ucmd)) {
622                                 err = -EFAULT;
623                                 goto err_copy;
624                         }
625
626                         err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
627                                                 context->db_tab,
628                                                 ucmd.sq_db_index, ucmd.sq_db_page, NULL);
629                         if (err) 
630                                 goto err_map1;
631
632                         err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
633                                                 context->db_tab,
634                                                 ucmd.rq_db_index, ucmd.rq_db_page, NULL);
635                         if (err) 
636                                 goto err_map2;
637
638                         qp->mr.ibmr.lkey = ucmd.lkey;
639                         qp->sq.db_index  = ucmd.sq_db_index;
640                         qp->rq.db_index  = ucmd.rq_db_index;
641                 }
642
643                 err = mthca_alloc_qp(to_mdev(pd->device), to_mpd(pd),
644                                      to_mcq(init_attr->send_cq),
645                                      to_mcq(init_attr->recv_cq),
646                                      init_attr->qp_type, init_attr->sq_sig_type,
647                                      &init_attr->cap, qp);
648
649                 if (err)
650                         if (pd->ucontext) 
651                                 goto err_alloc_qp_user;
652                         else 
653                                 goto err_copy;
654
655                 qp->ibqp.qp_num = qp->qpn;
656                 break;
657         }
658         case IB_QPT_QP0:
659         case IB_QPT_QP1:
660         {
661                 /* Don't allow userspace to create special QPs */
662                 if (pd->ucontext) {
663                         err = -EINVAL;
664                         goto err_inval;
665                 }
666
667                 qp = kmalloc(sizeof (struct mthca_sqp), GFP_KERNEL);
668                 if (!qp) {
669                         err = -ENOMEM;
670                         goto err_mem;
671                 }
672
673                 qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_QP0 ? 0 : 1;
674
675                 err = mthca_alloc_sqp(to_mdev(pd->device), to_mpd(pd),
676                                       to_mcq(init_attr->send_cq),
677                                       to_mcq(init_attr->recv_cq),
678                                       init_attr->sq_sig_type, &init_attr->cap,
679                                       qp->ibqp.qp_num, init_attr->port_num,
680                                       to_msqp(qp));
681                 if (err)
682                         goto err_alloc_sqp;
683                 
684                 break;
685         }
686         default:
687                 /* Don't support raw QPs */
688                 err = -ENOSYS;
689                 goto err_unsupported;
690         }
691
692         init_attr->cap.max_send_wr     = qp->sq.max;
693         init_attr->cap.max_recv_wr     = qp->rq.max;
694         init_attr->cap.max_send_sge    = qp->sq.max_gs;
695         init_attr->cap.max_recv_sge    = qp->rq.max_gs;
696         init_attr->cap.max_inline_data    = qp->max_inline_data;
697
698         return &qp->ibqp;
699
700                 
701 err_alloc_qp_user:
702         if (pd->ucontext) 
703                 mthca_unmap_user_db(to_mdev(pd->device),
704                         &context->uar, context->db_tab, ucmd.rq_db_index);
705 err_map2:
706         if (pd->ucontext) 
707                 mthca_unmap_user_db(to_mdev(pd->device),
708                         &context->uar, context->db_tab, ucmd.sq_db_index);
709 err_map1: err_copy: err_alloc_sqp:
710         if (qp)
711                 kfree(qp);
712 err_mem: err_inval:     err_unsupported:
713         return ERR_PTR(err);
714 }
715
716 static int mthca_destroy_qp(struct ib_qp *qp)
717 {
718         if (qp->ucontext) {
719                 mthca_unmap_user_db(to_mdev(qp->device),
720                                     &to_mucontext(qp->ucontext)->uar,
721                                     to_mucontext(qp->ucontext)->db_tab,
722                                     to_mqp(qp)->sq.db_index);
723                 mthca_unmap_user_db(to_mdev(qp->device),
724                                     &to_mucontext(qp->ucontext)->uar,
725                                     to_mucontext(qp->ucontext)->db_tab,
726                                     to_mqp(qp)->rq.db_index);
727         }
728         mthca_free_qp(to_mdev(qp->device), to_mqp(qp));
729         kfree(qp);
730         return 0;
731 }
732
733 static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, int entries,
734                                      struct ib_ucontext *context,
735                                      ci_umv_buf_t* const                        p_umv_buf)
736 {
737         struct ibv_create_cq ucmd = {0};
738         struct mthca_cq *cq;
739         int nent;
740         int err;
741         void *u_arm_db_page = 0;
742
743         if (entries < 1 || entries > to_mdev(ibdev)->limits.max_cqes)   
744                 return ERR_PTR(-EINVAL);
745
746         if (context) {
747                 if (ib_copy_from_umv_buf(&ucmd, p_umv_buf, sizeof ucmd))
748                         return ERR_PTR(-EFAULT);
749
750                 err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
751                                         to_mucontext(context)->db_tab,
752                                         ucmd.set_db_index, ucmd.set_db_page, NULL);
753                 if (err)
754                         return ERR_PTR(err);
755
756                 err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
757                                         to_mucontext(context)->db_tab,
758                                         ucmd.arm_db_index, ucmd.arm_db_page, NULL);
759                 if (err)
760                         goto err_unmap_set;
761
762                 err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
763                                         to_mucontext(context)->db_tab,
764                                         ucmd.u_arm_db_index, 
765                                         (u64)(ULONG_PTR)PAGE_ALIGN(ucmd.u_arm_db_page), 
766                                         &u_arm_db_page);
767                 if (err)
768                         goto err_unmap_arm;
769         }
770
771         cq = kmalloc(sizeof *cq, GFP_KERNEL);
772         if (!cq) {
773                 err = -ENOMEM;
774                 goto err_unmap_ev;
775         }
776
777         if (context) {
778                 cq->mr.ibmr.lkey = ucmd.lkey;
779                 cq->set_ci_db_index = ucmd.set_db_index;
780                 cq->arm_db_index    = ucmd.arm_db_index;
781                 cq->u_arm_db_index    = ucmd.u_arm_db_index;
782                 cq->p_u_arm_sn = (int*)((char*)u_arm_db_page + BYTE_OFFSET(ucmd.u_arm_db_page));
783         }
784
785         for (nent = 1; nent <= entries; nent <<= 1)
786                 ; /* nothing */
787
788         err = mthca_init_cq(to_mdev(ibdev), nent, 
789                             context ? to_mucontext(context) : NULL,
790                             context ? ucmd.mr.pdn : to_mdev(ibdev)->driver_pd.pd_num,
791                             cq);
792         if (err)
793                 goto err_free;
794
795         if (context ) {
796                 struct ibv_create_cq_resp *create_cq_resp = (struct ibv_create_cq_resp *)(void*)p_umv_buf->p_inout_buf;
797                 create_cq_resp->cqn = cq->cqn;
798         }
799
800         HCA_PRINT( TRACE_LEVEL_INFORMATION, HCA_DBG_LOW ,
801                 ("uctx %p, cq_hndl %p, cq_num %#x, cqe  %#x\n",
802                 context, &cq->ibcq, cq->cqn, cq->ibcq.cqe ) );
803         
804         return &cq->ibcq;
805
806 err_free:
807         kfree(cq);
808
809 err_unmap_ev:
810         if (context)
811                 mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
812                                     to_mucontext(context)->db_tab, ucmd.u_arm_db_index);
813
814 err_unmap_arm:
815         if (context)
816                 mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
817                                     to_mucontext(context)->db_tab, ucmd.arm_db_index);
818
819 err_unmap_set:
820         if (context)
821                 mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
822                                     to_mucontext(context)->db_tab, ucmd.set_db_index);
823
824         return ERR_PTR(err);
825 }
826
827 static int mthca_destroy_cq(struct ib_cq *cq)
828 {
829         if (cq->ucontext) {
830                 mthca_unmap_user_db(to_mdev(cq->device),
831                                     &to_mucontext(cq->ucontext)->uar,
832                                     to_mucontext(cq->ucontext)->db_tab,
833                                     to_mcq(cq)->u_arm_db_index);
834                 mthca_unmap_user_db(to_mdev(cq->device),
835                                     &to_mucontext(cq->ucontext)->uar,
836                                     to_mucontext(cq->ucontext)->db_tab,
837                                     to_mcq(cq)->arm_db_index);
838                 mthca_unmap_user_db(to_mdev(cq->device),
839                                     &to_mucontext(cq->ucontext)->uar,
840                                     to_mucontext(cq->ucontext)->db_tab,
841                                     to_mcq(cq)->set_ci_db_index);
842         }
843         mthca_free_cq(to_mdev(cq->device), to_mcq(cq));
844         kfree(cq);
845
846         return 0;
847 }
848
849 static
850 mthca_mpt_access_t
851 map_qp_mpt(
852         IN                              mthca_qp_access_t                               qp_acl)
853 {
854 #define ACL_MTHCA(mfl,ifl) if (qp_acl & mfl)   mpt_acl |= ifl
855         mthca_mpt_access_t mpt_acl = 0;
856
857         ACL_MTHCA(MTHCA_ACCESS_REMOTE_READ,MTHCA_MPT_FLAG_REMOTE_READ);
858         ACL_MTHCA(MTHCA_ACCESS_REMOTE_WRITE,MTHCA_MPT_FLAG_REMOTE_WRITE);
859         ACL_MTHCA(MTHCA_ACCESS_REMOTE_ATOMIC,MTHCA_MPT_FLAG_ATOMIC);
860         ACL_MTHCA(MTHCA_ACCESS_LOCAL_WRITE,MTHCA_MPT_FLAG_LOCAL_WRITE);
861
862         return (mpt_acl | MTHCA_MPT_FLAG_LOCAL_READ);
863 }
864
865 struct ib_mr *mthca_get_dma_mr(struct ib_pd *pd, mthca_qp_access_t acc)
866 {
867         struct mthca_mr *mr;
868         int err;
869
870         mr = kzalloc(sizeof *mr, GFP_KERNEL);
871         if (!mr)
872                 return ERR_PTR(-ENOMEM);
873
874         err = mthca_mr_alloc_notrans(to_mdev(pd->device),
875                                      to_mpd(pd)->pd_num,
876                                      map_qp_mpt(acc), mr);
877
878         if (err) {
879                 kfree(mr);
880                 return ERR_PTR(err);
881         }
882
883         return &mr->ibmr;
884 }
885
886 static struct ib_mr *mthca_reg_phys_mr(struct ib_pd       *pd,
887                                        struct ib_phys_buf *buffer_list,
888                                        int                 num_phys_buf,
889                                        mthca_qp_access_t                 acc,
890                                        u64                *iova_start)
891 {
892         struct mthca_mr *mr;
893         u64 *page_list;
894         u64 total_size;
895         u64 mask;
896         int shift;
897         int npages;
898         int err;
899         int i, j, n;
900
901         /* First check that we have enough alignment */
902         if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK))
903                 return ERR_PTR(-EINVAL);
904
905         if (num_phys_buf > 1 &&
906             ((buffer_list[0].addr + buffer_list[0].size) & ~PAGE_MASK))
907                 return ERR_PTR(-EINVAL);
908
909         mask = 0;
910         total_size = 0;
911         for (i = 0; i < num_phys_buf; ++i) {
912                 if (i != 0)
913                         mask |= buffer_list[i].addr;
914                 if (i != num_phys_buf - 1)
915                         mask |= buffer_list[i].addr + buffer_list[i].size;
916
917                 total_size += buffer_list[i].size;
918         }
919
920         if (mask & ~PAGE_MASK)
921                 return ERR_PTR(-EINVAL);
922
923         /* Find largest page shift we can use to cover buffers */
924         for (shift = PAGE_SHIFT; shift < 31; ++shift)
925                 if (num_phys_buf > 1) {
926                         if ((1Ui64 << shift) & mask)
927                                 break;
928                 } else {
929                         if (1Ui64 << shift >=
930                             buffer_list[0].size +
931                             (buffer_list[0].addr & ((1Ui64 << shift) - 1)))
932                                 break;
933                 }
934
935         buffer_list[0].size += buffer_list[0].addr & ((1Ui64 << shift) - 1);
936         buffer_list[0].addr &= ~0Ui64 << shift;
937
938         mr = kzalloc(sizeof *mr, GFP_KERNEL);
939         if (!mr)
940                 return ERR_PTR(-ENOMEM);
941
942         npages = 0;
943         for (i = 0; i < num_phys_buf; ++i)
944                 npages += (int)((buffer_list[i].size + (1Ui64 << shift) - 1) >> shift);
945
946         if (!npages)
947                 return &mr->ibmr;
948
949         page_list = kmalloc(npages * sizeof *page_list, GFP_KERNEL);
950         if (!page_list) {
951                 kfree(mr);
952                 return ERR_PTR(-ENOMEM);
953         }
954
955         n = 0;
956         for (i = 0; i < num_phys_buf; ++i)
957                 for (j = 0;
958                      j < (buffer_list[i].size + (1Ui64 << shift) - 1) >> shift;
959                      ++j)
960                         page_list[n++] = buffer_list[i].addr + ((u64) j << shift);
961
962         HCA_PRINT( TRACE_LEVEL_VERBOSE ,HCA_DBG_LOW ,("Registering memory at %I64x (iova %I64x) "
963                   "in PD %x; shift %d, npages %d.\n",
964                   (u64) buffer_list[0].addr,
965                   (u64) *iova_start,
966                   to_mpd(pd)->pd_num,
967                   shift, npages));
968
969         err = mthca_mr_alloc_phys(to_mdev(pd->device),
970                                   to_mpd(pd)->pd_num,
971                                   page_list, shift, npages,
972                                   *iova_start, total_size,
973                                   map_qp_mpt(acc), mr);
974
975         if (err) {
976                 kfree(page_list);
977                 kfree(mr);
978                 return ERR_PTR(err);
979         }
980
981         kfree(page_list);
982         return &mr->ibmr;
983 }
984
985 static struct ib_mr *mthca_reg_virt_mr(struct ib_pd *pd, 
986         void* __ptr64   vaddr, uint64_t length, uint64_t hca_va,
987         mthca_qp_access_t acc, boolean_t um_call)
988 {
989         struct mthca_dev *dev = to_mdev(pd->device);
990         struct mthca_mr *mr;
991         u64 *pages;
992         int err = 0;
993         uint32_t i, n;
994         mt_iobuf_t *iobuf_p;
995         mt_iobuf_iter_t iobuf_iter;
996         ib_access_t ib_acc;
997
998         /*
999          * Be friendly to WRITE_MTT command and leave two 
1000          * empty slots for the  index and reserved fields of the mailbox.
1001          */
1002         int max_buf_list_size = PAGE_SIZE / sizeof (u64) - 2;
1003
1004         HCA_ENTER(HCA_DBG_MEMORY);
1005
1006         mr = kzalloc(sizeof *mr, GFP_KERNEL);
1007         if (!mr) {
1008                 err = -ENOMEM;
1009                 goto err_nomem;
1010         }
1011
1012         /*
1013          * We ask for writable memory if any access flags other than
1014          * "remote read" are set.  "Local write" and "remote write"
1015          * obviously require write access.  "Remote atomic" can do
1016          * things like fetch and add, which will modify memory, and
1017          * "MW bind" can change permissions by binding a window.
1018          */
1019
1020         // try register the buffer
1021         iobuf_p = &mr->iobuf;
1022         iobuf_init( (u64)vaddr, length, um_call, iobuf_p);
1023         ib_acc = (acc & ~MTHCA_ACCESS_REMOTE_READ) ? IB_AC_LOCAL_WRITE : 0;
1024         err =  iobuf_register_with_cash( (u64)vaddr, length, um_call, 
1025                 &ib_acc, iobuf_p );
1026         if (err)
1027                 goto err_reg_mem;
1028         mr->iobuf_used = TRUE;
1029
1030         // allocate MTT's
1031         mr->mtt = mthca_alloc_mtt(dev, iobuf_p->nr_pages);
1032         if (IS_ERR(mr->mtt)) {
1033                 err = PTR_ERR(mr->mtt);
1034                 goto err_alloc_mtt;
1035         }
1036
1037         // allocate buffer_list for writing MTT's
1038         pages = (u64 *) kmalloc(PAGE_SIZE,GFP_KERNEL);
1039         if (!pages) {
1040                 err = -ENOMEM;
1041                 goto err_pages;
1042         }
1043
1044         // write MTT's
1045         iobuf_iter_init( iobuf_p, &iobuf_iter );
1046         n = 0;
1047         for (;;) {
1048                 // get up to  max_buf_list_size page physical addresses
1049                 i = iobuf_get_tpt_seg( iobuf_p, &iobuf_iter, max_buf_list_size, pages );
1050                 if (!i)
1051                         break;
1052
1053                 //TODO: convert physical adresses to dma one's
1054
1055                 // write 'i' dma addresses
1056                 err = mthca_write_mtt(dev, mr->mtt, n, pages, i);
1057                 if (err)
1058                         goto err_write_mtt;
1059                 n += i;
1060                 if (n >= iobuf_p->nr_pages)
1061                         break;
1062         }
1063
1064         CL_ASSERT(n == iobuf_p->nr_pages);
1065         
1066         // write MPT
1067         err = mthca_mr_alloc(dev, to_mpd(pd)->pd_num, PAGE_SHIFT, hca_va,
1068                 length, map_qp_mpt(acc), mr);
1069         if (err)
1070                 goto err_mt_alloc;
1071
1072         // secure memory
1073         if (!pd->ucontext)
1074                 goto done;
1075         __try {
1076                 mr->secure_handle = MmSecureVirtualMemory ( vaddr, (SIZE_T)length,
1077                         (ib_acc & IB_AC_LOCAL_WRITE) ? PAGE_READWRITE : PAGE_READONLY );
1078                 if (mr->secure_handle == NULL) {
1079                         err = -EFAULT;
1080                         goto err_secure;
1081                 }
1082         }
1083         __except (EXCEPTION_EXECUTE_HANDLER) {
1084                 NTSTATUS Status = GetExceptionCode();
1085                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_MEMORY ,
1086                         ("Exception 0x%x on MmSecureVirtualMemory(), addr %p, size %I64d, access %#x\n", 
1087                         Status, vaddr, length, acc ));
1088                 err = -EFAULT;
1089                 goto err_secure;
1090         }
1091
1092 done:   
1093         free_page((void*) pages);
1094
1095         HCA_EXIT(HCA_DBG_MEMORY);
1096         return &mr->ibmr;
1097
1098 err_secure:
1099 err_mt_alloc:
1100 err_write_mtt:
1101         free_page((void*) pages);
1102 err_pages:
1103         mthca_free_mtt(dev, mr->mtt);
1104 err_alloc_mtt:
1105         iobuf_deregister(iobuf_p);
1106 err_reg_mem:    
1107         kfree(mr);
1108 err_nomem:      
1109
1110         HCA_EXIT(HCA_DBG_MEMORY);
1111         return ERR_PTR(err);
1112 }
1113
1114 int mthca_dereg_mr(struct ib_mr *mr)
1115 {
1116         struct mthca_mr *mmr = to_mmr(mr);
1117         struct mthca_dev* dev = to_mdev(mr->device);
1118
1119         if (mmr->secure_handle)
1120                 MmUnsecureVirtualMemory ( mmr->secure_handle );
1121         mthca_free_mr(dev, mmr);
1122         if (mmr->iobuf_used)
1123                 iobuf_deregister_with_cash(&mmr->iobuf);
1124         kfree(mmr);
1125         return 0;
1126 }
1127
1128 static struct ib_fmr *mthca_alloc_fmr(struct ib_pd *pd, mthca_qp_access_t acc,
1129                                       struct ib_fmr_attr *fmr_attr)
1130 {
1131         struct mthca_fmr *fmr;
1132         int err;
1133
1134         fmr = kzalloc(sizeof *fmr, GFP_KERNEL);
1135         if (!fmr)
1136                 return ERR_PTR(-ENOMEM);
1137
1138         RtlCopyMemory(&fmr->attr, fmr_attr, sizeof *fmr_attr);
1139         err = mthca_fmr_alloc(to_mdev(pd->device), to_mpd(pd)->pd_num,
1140                              map_qp_mpt(acc), fmr);
1141
1142         if (err) {
1143                 kfree(fmr);
1144                 return ERR_PTR(err);
1145         }
1146
1147         return &fmr->ibmr;
1148 }
1149
1150 static int mthca_dealloc_fmr(struct ib_fmr *fmr)
1151 {
1152         struct mthca_fmr *mfmr = to_mfmr(fmr);
1153         int err;
1154
1155         err = mthca_free_fmr(to_mdev(fmr->device), mfmr);
1156         if (err)
1157                 return err;
1158
1159         kfree(mfmr);
1160         return 0;
1161 }
1162
1163 static int mthca_unmap_fmr(struct list_head *fmr_list)
1164 {
1165         struct ib_fmr *fmr;
1166         int err;
1167         u8 status;
1168         struct mthca_dev *mdev = NULL;
1169
1170         list_for_each_entry(fmr, fmr_list, list,struct ib_fmr) {
1171                 if (mdev && to_mdev(fmr->device) != mdev)
1172                         return -EINVAL;
1173                 mdev = to_mdev(fmr->device);
1174         }
1175
1176         if (!mdev)
1177                 return 0;
1178
1179         if (mthca_is_memfree(mdev)) {
1180                 list_for_each_entry(fmr, fmr_list, list,struct ib_fmr)
1181                         mthca_arbel_fmr_unmap(mdev, to_mfmr(fmr));
1182
1183                 wmb();
1184         } else
1185                 list_for_each_entry(fmr, fmr_list, list,struct ib_fmr)
1186                         mthca_tavor_fmr_unmap(mdev, to_mfmr(fmr));
1187
1188         err = mthca_SYNC_TPT(mdev, &status);
1189         if (err)
1190                 return err;
1191         if (status)
1192                 return -EINVAL;
1193         return 0;
1194 }
1195
1196 static int mthca_init_node_data(struct mthca_dev *dev)
1197 {
1198         struct ib_smp *in_mad  = NULL;
1199         struct ib_smp *out_mad = NULL;
1200         int err = -ENOMEM;
1201         u8 status;
1202
1203         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
1204         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
1205         if (!in_mad || !out_mad)
1206                 goto out;
1207
1208         init_query_mad(in_mad);
1209         in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
1210
1211         err = mthca_MAD_IFC(dev, 1, 1,
1212                             1, NULL, NULL, in_mad, out_mad,
1213                             &status);
1214         if (err)
1215                 goto out;
1216         if (status) {
1217                 err = -EINVAL;
1218                 goto out;
1219         }
1220
1221         memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8);
1222
1223 out:
1224         kfree(in_mad);
1225         kfree(out_mad);
1226         return err;
1227 }
1228
1229 int mthca_register_device(struct mthca_dev *dev)
1230 {
1231         int ret;
1232
1233         ret = mthca_init_node_data(dev);        
1234         if (ret)
1235                 return ret;
1236
1237         strlcpy(dev->ib_dev.name, "mthca%d", IB_DEVICE_NAME_MAX);
1238         dev->ib_dev.node_type            = IB_NODE_CA;
1239         dev->ib_dev.phys_port_cnt        = (u8)dev->limits.num_ports;
1240         dev->ib_dev.mdev                                = dev;
1241         dev->ib_dev.query_device         = mthca_query_device;
1242         dev->ib_dev.query_port           = mthca_query_port;
1243         dev->ib_dev.modify_port          = mthca_modify_port;
1244         dev->ib_dev.query_pkey_chunk           = mthca_query_pkey_chunk;
1245         dev->ib_dev.query_gid_chunk            = mthca_query_gid_chunk;
1246         dev->ib_dev.alloc_ucontext       = mthca_alloc_ucontext;
1247         dev->ib_dev.dealloc_ucontext     = mthca_dealloc_ucontext;
1248         dev->ib_dev.alloc_pd             = mthca_alloc_pd;
1249         dev->ib_dev.dealloc_pd           = mthca_dealloc_pd;
1250         dev->ib_dev.create_ah            = mthca_ah_create;
1251         dev->ib_dev.destroy_ah           = mthca_ah_destroy;
1252
1253         if (dev->mthca_flags & MTHCA_FLAG_SRQ) {
1254                 dev->ib_dev.create_srq           = mthca_create_srq;
1255                 dev->ib_dev.modify_srq           = mthca_modify_srq;
1256                 dev->ib_dev.query_srq            = mthca_query_srq;
1257                 dev->ib_dev.destroy_srq          = mthca_destroy_srq;
1258
1259                 if (mthca_is_memfree(dev))
1260                         dev->ib_dev.post_srq_recv = mthca_arbel_post_srq_recv;
1261                 else
1262                         dev->ib_dev.post_srq_recv = mthca_tavor_post_srq_recv;
1263         }
1264
1265         dev->ib_dev.create_qp            = mthca_create_qp;
1266         dev->ib_dev.modify_qp            = mthca_modify_qp;
1267         dev->ib_dev.destroy_qp           = mthca_destroy_qp;
1268         dev->ib_dev.create_cq            = mthca_create_cq;
1269         dev->ib_dev.destroy_cq           = mthca_destroy_cq;
1270         dev->ib_dev.poll_cq              = mthca_poll_cq;
1271         dev->ib_dev.get_dma_mr           = mthca_get_dma_mr;
1272         dev->ib_dev.reg_phys_mr          = mthca_reg_phys_mr;
1273         dev->ib_dev.reg_virt_mr                  = mthca_reg_virt_mr;
1274         dev->ib_dev.dereg_mr             = mthca_dereg_mr;
1275
1276         if (dev->mthca_flags & MTHCA_FLAG_FMR) {
1277                 dev->ib_dev.alloc_fmr            = mthca_alloc_fmr;
1278                 dev->ib_dev.unmap_fmr            = mthca_unmap_fmr;
1279                 dev->ib_dev.dealloc_fmr          = mthca_dealloc_fmr;
1280                 if (mthca_is_memfree(dev))
1281                         dev->ib_dev.map_phys_fmr = mthca_arbel_map_phys_fmr;
1282                 else
1283                         dev->ib_dev.map_phys_fmr = mthca_tavor_map_phys_fmr;
1284         }
1285
1286         dev->ib_dev.attach_mcast         = mthca_multicast_attach;
1287         dev->ib_dev.detach_mcast         = mthca_multicast_detach;
1288         dev->ib_dev.process_mad          = mthca_process_mad;
1289
1290         if (mthca_is_memfree(dev)) {
1291                 dev->ib_dev.req_notify_cq = mthca_arbel_arm_cq;
1292                 dev->ib_dev.post_send     = mthca_arbel_post_send;
1293                 dev->ib_dev.post_recv     = mthca_arbel_post_receive;
1294         } else {
1295                 dev->ib_dev.req_notify_cq = mthca_tavor_arm_cq;
1296                 dev->ib_dev.post_send     = mthca_tavor_post_send;
1297                 dev->ib_dev.post_recv     = mthca_tavor_post_receive;
1298         }
1299
1300         KeInitializeMutex(&dev->cap_mask_mutex, 0);
1301
1302         ret = ib_register_device(&dev->ib_dev);
1303         if (ret)
1304                 return ret;
1305
1306         mthca_start_catas_poll(dev);
1307
1308         return 0;
1309 }
1310
1311 void mthca_unregister_device(struct mthca_dev *dev)
1312 {
1313         mthca_stop_catas_poll(dev);
1314         ib_unregister_device(&dev->ib_dev);
1315 }