01a6b3e8c785d9ae0f1a27678164a4dc7b1f7bbc
[mirror/winof/.git] / hw / mthca / kernel / mthca_provider.c
1 /* 
2  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
3  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4  * Copyright (c) 2005 Cisco Systems. All rights reserved.
5  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
6  * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
7  *
8  * This software is available to you under a choice of one of two
9  * licenses.  You may choose to be licensed under the terms of the GNU
10  * General Public License (GPL) Version 2, available from the file
11  * COPYING in the main directory of this source tree, or the
12  * OpenIB.org BSD license below:
13  *
14  *     Redistribution and use in source and binary forms, with or
15  *     without modification, are permitted provided that the following
16  *     conditions are met:
17  *
18  *      - Redistributions of source code must retain the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer.
21  *
22  *      - Redistributions in binary form must reproduce the above
23  *        copyright notice, this list of conditions and the following
24  *        disclaimer in the documentation and/or other materials
25  *        provided with the distribution.
26  *
27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34  * SOFTWARE.
35  *
36  * $Id: mthca_provider.c 3047 2005-08-10 03:59:35Z roland $
37  */
38
39 #include <ib_smi.h>
40
41 #include "mx_abi.h"
42 #include "mthca_dev.h"
43
44 #if defined(EVENT_TRACING)
45 #ifdef offsetof
46 #undef offsetof
47 #endif
48 #include "mthca_provider.tmh"
49 #endif
50 #include "mthca_cmd.h"
51 #include "mthca_memfree.h"
52
53  static void init_query_mad(struct ib_smp *mad)
54  {
55          mad->base_version      = 1;
56          mad->mgmt_class                = IB_MGMT_CLASS_SUBN_LID_ROUTED;
57          mad->class_version = 1;
58          mad->method                            = IB_MGMT_METHOD_GET;
59  }
60
61  int mthca_query_device(struct ib_device *ibdev,
62                               struct ib_device_attr *props)
63 {
64         struct ib_smp *in_mad  = NULL;
65         struct ib_smp *out_mad = NULL;
66         int err = -ENOMEM;
67         struct mthca_dev* mdev = to_mdev(ibdev);
68
69         u8 status;
70
71         RtlZeroMemory(props, sizeof *props);
72
73         if (mthca_is_livefish(mdev)) {
74                 props->max_pd = 1;
75                 props->vendor_id = mdev->ext->hcaConfig.VendorID;
76                 props->vendor_part_id = mdev->ext->hcaConfig.DeviceID;
77                 return 0;
78         }
79
80         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
81         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
82         if (!in_mad || !out_mad)
83                 goto out;
84
85         init_query_mad(in_mad);
86         in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
87
88         err = mthca_MAD_IFC(mdev, 1, 1,
89             1, NULL, NULL, in_mad, out_mad, &status);
90         if (err)
91                 goto out;
92         if (status) {
93                 err = -EINVAL;
94                 goto out;
95         }
96
97         props->fw_ver              = mdev->fw_ver;
98         props->device_cap_flags    = mdev->device_cap_flags;
99         props->vendor_id           = cl_ntoh32(*(__be32 *) (out_mad->data + 36)) &
100                 0xffffff;
101         props->vendor_part_id      = cl_ntoh16(*(__be16 *) (out_mad->data + 30));
102         props->hw_ver              = cl_ntoh32(*(__be32 *) (out_mad->data + 32));
103         memcpy(&props->sys_image_guid, out_mad->data +  4, 8);
104         props->max_mr_size         = ~0Ui64;
105         props->page_size_cap       = mdev->limits.page_size_cap;
106         props->max_qp              = mdev->limits.num_qps - mdev->limits.reserved_qps;
107         props->max_qp_wr           = mdev->limits.max_wqes;
108         props->max_sge             = mdev->limits.max_sg;
109         props->max_cq              = mdev->limits.num_cqs - mdev->limits.reserved_cqs;
110         props->max_cqe             = mdev->limits.max_cqes;
111         props->max_mr              = mdev->limits.num_mpts - mdev->limits.reserved_mrws;
112         props->max_pd              = mdev->limits.num_pds - mdev->limits.reserved_pds;
113         props->max_qp_rd_atom      = 1 << mdev->qp_table.rdb_shift;
114         props->max_qp_init_rd_atom = mdev->limits.max_qp_init_rdma;
115         props->max_res_rd_atom     = props->max_qp_rd_atom * props->max_qp;
116         props->max_srq             = mdev->limits.num_srqs - mdev->limits.reserved_srqs;
117         props->max_srq_wr          = mdev->limits.max_srq_wqes;
118         props->max_srq_sge         = mdev->limits.max_sg;
119         props->local_ca_ack_delay  = (u8)mdev->limits.local_ca_ack_delay;
120         props->atomic_cap          = mdev->limits.flags & DEV_LIM_FLAG_ATOMIC ? 
121                                         IB_ATOMIC_LOCAL : IB_ATOMIC_NONE;
122         props->max_pkeys           = (u16)mdev->limits.pkey_table_len;
123         props->max_mcast_grp       = mdev->limits.num_mgms + mdev->limits.num_amgms;
124         props->max_mcast_qp_attach = MTHCA_QP_PER_MGM;
125         props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * 
126                                            props->max_mcast_grp;
127
128         err = 0;
129  out:
130         kfree(in_mad);
131         kfree(out_mad);
132         return err;
133 }
134
135 int mthca_query_port(struct ib_device *ibdev,
136                             u8 port, struct ib_port_attr *props)
137 {
138         struct ib_smp *in_mad  = NULL;
139         struct ib_smp *out_mad = NULL;
140         int err = -ENOMEM;
141         u8 status;
142
143         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
144         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
145         if (!in_mad || !out_mad)
146                 goto out;
147
148         init_query_mad(in_mad);
149         in_mad->attr_id  = IB_SMP_ATTR_PORT_INFO;
150         in_mad->attr_mod = cl_hton32(port);
151
152         err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
153                             port, NULL, NULL, in_mad, out_mad,
154                             &status);
155         if (err)
156                 goto out;
157         if (status) {
158                 err = -EINVAL;
159                 goto out;
160         }
161
162         RtlZeroMemory(props, sizeof *props);
163         props->lid               = cl_ntoh16(*(__be16 *) (out_mad->data + 16));
164         props->lmc               = out_mad->data[34] & 0x7;
165         props->sm_lid            = cl_ntoh16(*(__be16 *) (out_mad->data + 18));
166         props->sm_sl             = out_mad->data[36] & 0xf;
167         props->state             = out_mad->data[32] & 0xf;
168         props->phys_state        = out_mad->data[33] >> 4;
169         props->port_cap_flags    = cl_ntoh32(*(__be32 *) (out_mad->data + 20));
170         props->gid_tbl_len       = to_mdev(ibdev)->limits.gid_table_len;
171         props->max_msg_sz        = 0x80000000;
172         props->pkey_tbl_len      = (u16)to_mdev(ibdev)->limits.pkey_table_len;
173         props->bad_pkey_cntr     = cl_ntoh16(*(__be16 *) (out_mad->data + 46));
174         props->qkey_viol_cntr    = cl_ntoh16(*(__be16 *) (out_mad->data + 48));
175         props->active_width      = out_mad->data[31] & 0xf;
176         props->active_speed      = out_mad->data[35] >> 4;
177         props->max_mtu           = out_mad->data[41] & 0xf;
178         props->active_mtu        = out_mad->data[36] >> 4;
179         props->subnet_timeout    = out_mad->data[51] & 0x1f;
180
181  out:
182         kfree(in_mad);
183         kfree(out_mad);
184         return err;
185 }
186
187 int mthca_modify_port(struct ib_device *ibdev,
188                              u8 port, int port_modify_mask,
189                              struct ib_port_modify *props)
190 {
191         struct mthca_set_ib_param set_ib;
192         struct ib_port_attr attr;
193         int err;
194         u8 status;
195
196         if (down_interruptible(&to_mdev(ibdev)->cap_mask_mutex))
197                 return -EFAULT;
198
199         err = mthca_query_port(ibdev, port, &attr);
200         if (err)
201                 goto out;
202
203         set_ib.set_si_guid     = 0;
204         set_ib.reset_qkey_viol = !!(port_modify_mask & IB_PORT_RESET_QKEY_CNTR);
205
206         set_ib.cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) &
207                 ~props->clr_port_cap_mask;
208
209         err = mthca_SET_IB(to_mdev(ibdev), &set_ib, port, &status);
210         if (err)
211                 goto out;
212         if (status) {
213                 err = -EINVAL;
214                 goto out;
215         }
216
217 out:
218         up(&to_mdev(ibdev)->cap_mask_mutex);
219         return err;
220 }
221
222 int mthca_query_pkey_chunk(struct ib_device *ibdev,
223                             u8 port, u16 index, u16 pkey[32])
224 {
225         struct ib_smp *in_mad  = NULL;
226         struct ib_smp *out_mad = NULL;
227         int err = -ENOMEM;
228         u8 status;
229
230         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
231         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
232         if (!in_mad || !out_mad)
233                 goto out;
234
235         init_query_mad(in_mad);
236         in_mad->attr_id  = IB_SMP_ATTR_PKEY_TABLE;
237         in_mad->attr_mod = cl_hton32(index / 32);
238
239         err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
240                             port, NULL, NULL, in_mad, out_mad,
241                             &status);
242         if (err)
243                 goto out;
244         if (status) {
245                 err = -EINVAL;
246                 goto out;
247         }
248
249         { // copy the results
250                 int i;
251                 __be16 *pkey_chunk = (__be16 *)out_mad->data;
252                 for (i=0; i<32; ++i) 
253                         pkey[i] = cl_ntoh16(pkey_chunk[i]);
254         }
255
256  out:
257         kfree(in_mad);
258         kfree(out_mad);
259         return err;
260 }
261
262 int mthca_query_gid_chunk(struct ib_device *ibdev, u8 port,
263                            int index, union ib_gid gid[8])
264 {
265         struct ib_smp *in_mad  = NULL;
266         struct ib_smp *out_mad = NULL;
267         int err = -ENOMEM;
268         u8 status;
269         __be64  subnet_prefix;
270
271         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
272         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
273         if (!in_mad || !out_mad)
274                 goto out;
275
276         init_query_mad(in_mad);
277         in_mad->attr_id  = IB_SMP_ATTR_PORT_INFO;
278         in_mad->attr_mod = cl_hton32(port);
279
280         err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
281                             port, NULL, NULL, in_mad, out_mad,
282                             &status);
283         if (err)
284                 goto out;
285         if (status) {
286                 err = -EINVAL;
287                 goto out;
288         }
289
290         memcpy(&subnet_prefix, out_mad->data + 8, 8);
291
292         init_query_mad(in_mad);
293         in_mad->attr_id  = IB_SMP_ATTR_GUID_INFO;
294         in_mad->attr_mod = cl_hton32(index / 8);
295
296         err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
297                             port, NULL, NULL, in_mad, out_mad,
298                             &status);
299         if (err)
300                 goto out;
301         if (status) {
302                 err = -EINVAL;
303                 goto out;
304         }
305
306         { // copy the results
307                 int i;
308                 __be64 *guid = (__be64 *)out_mad->data;
309                 for (i=0; i<8; ++i) {
310                         gid[i].global.subnet_prefix = subnet_prefix;
311                         gid[i].global.interface_id = guid[i];
312                 }
313         }
314
315  out:
316         kfree(in_mad);
317         kfree(out_mad);
318         return err;
319 }
320
321 struct ib_ucontext *mthca_alloc_ucontext(struct ib_device *ibdev,
322                                                 ci_umv_buf_t* const     p_umv_buf)
323 {
324         struct mthca_alloc_ucontext_resp uresp;
325         struct mthca_ucontext           *context;
326         int                              err;
327
328         RtlZeroMemory(&uresp, sizeof uresp);
329
330         uresp.qp_tab_size = to_mdev(ibdev)->limits.num_qps;
331         if (mthca_is_memfree(to_mdev(ibdev)))
332                 uresp.uarc_size = to_mdev(ibdev)->uar_table.uarc_size;
333         else
334                 uresp.uarc_size = 0;
335
336         context = kzalloc(sizeof *context, GFP_KERNEL);
337         if (!context) {
338                 err = -ENOMEM;
339                 goto err_nomem;
340         }
341
342         if (mthca_is_livefish(to_mdev(ibdev)))
343                 goto done;
344         
345         err = mthca_uar_alloc(to_mdev(ibdev), &context->uar);
346         if (err) 
347                 goto err_uar_alloc;
348
349         /*
350         * map uar to user space
351         */
352
353         /* map UAR to kernel */
354         context->kva = ioremap((io_addr_t)context->uar.pfn << PAGE_SHIFT, PAGE_SIZE,&context->uar_size);
355         if (!context->kva) {
356                 HCA_PRINT( TRACE_LEVEL_ERROR, HCA_DBG_LOW ,("Couldn't map kernel access region, aborting.\n") );
357                 err = -ENOMEM;
358                 goto err_ioremap;
359         }
360
361         /* build MDL */
362         context->mdl = IoAllocateMdl( context->kva, (ULONG)context->uar_size,
363                 FALSE, TRUE, NULL );
364         if( !context->mdl ) {
365                 err = -ENOMEM;
366                 goto err_alloc_mdl;
367         }
368         MmBuildMdlForNonPagedPool( context->mdl );
369
370         /* Map the memory into the calling process's address space. */
371         __try   {
372                 context->ibucontext.user_uar = MmMapLockedPagesSpecifyCache( context->mdl,
373                         UserMode, MmNonCached, NULL, FALSE, NormalPagePriority );
374         }
375         __except(EXCEPTION_EXECUTE_HANDLER) {
376                 err = -EACCES;
377                 goto err_map;
378         }
379
380         /* user_db_tab */
381         context->db_tab = mthca_init_user_db_tab(to_mdev(ibdev));
382         if (IS_ERR(context->db_tab)) {
383                 err = PTR_ERR(context->db_tab);
384                 goto err_init_user;
385         }
386
387 done:
388         err = ib_copy_to_umv_buf(p_umv_buf, &uresp, sizeof uresp);
389         if (err) 
390                 goto err_copy_to_umv_buf;
391
392         context->ibucontext.device = ibdev;
393         
394         atomic_set(&context->ibucontext.usecnt, 0);
395         return &context->ibucontext;
396
397 err_copy_to_umv_buf:
398         mthca_cleanup_user_db_tab(to_mdev(ibdev), &context->uar,
399                 context->db_tab);
400 err_init_user:  
401         MmUnmapLockedPages( context->ibucontext.user_uar, context->mdl );
402 err_map:
403         IoFreeMdl(context->mdl);
404 err_alloc_mdl:  
405         iounmap(context->kva, PAGE_SIZE);
406 err_ioremap:    
407         mthca_uar_free(to_mdev(ibdev), &context->uar);
408 err_uar_alloc:
409         kfree(context);
410 err_nomem:      
411         return ERR_PTR(err);
412 }
413
414  int mthca_dealloc_ucontext(struct ib_ucontext *context)
415 {
416          struct mthca_ucontext                                   *mucontext = to_mucontext(context);
417
418         if (mthca_is_livefish(to_mdev(context->device)))
419                 goto done;
420         mthca_cleanup_user_db_tab(to_mdev(context->device), &mucontext->uar,
421                                   mucontext->db_tab);
422         MmUnmapLockedPages( mucontext->ibucontext.user_uar, mucontext->mdl );
423         IoFreeMdl(mucontext->mdl);
424         iounmap(mucontext->kva, PAGE_SIZE);
425         mthca_uar_free(to_mdev(context->device), &mucontext->uar);
426 done:   
427         kfree(mucontext);
428         
429         return 0;
430 }
431
432 struct ib_pd *mthca_alloc_pd(struct ib_device *ibdev,
433                                     struct ib_ucontext *context,
434                                     ci_umv_buf_t* const                 p_umv_buf)
435 {
436         int err;
437         struct mthca_pd *pd;
438         struct ibv_alloc_pd_resp resp;
439
440         /* sanity check */
441         if (p_umv_buf && p_umv_buf->command) {
442                 if (p_umv_buf->output_size < sizeof(struct ibv_alloc_pd_resp)) {
443                         err = -EINVAL;
444                         goto err_param;
445                 }
446         }
447         
448         pd = kmalloc(sizeof *pd, GFP_KERNEL);
449         if (!pd) {
450                 err = -ENOMEM;
451                 goto err_mem;
452         }
453
454         if (mthca_is_livefish(to_mdev(ibdev))) 
455                 goto done;
456
457         err = mthca_pd_alloc(to_mdev(ibdev), !context, pd);
458         if (err) {
459                 goto err_pd_alloc;
460         }
461
462 done:
463         if (p_umv_buf && p_umv_buf->command) {
464                 resp.pd_handle = (u64)(UINT_PTR)pd;
465                 resp.pdn = pd->pd_num;
466                 if (ib_copy_to_umv_buf(p_umv_buf, &resp, sizeof(struct ibv_alloc_pd_resp))) {
467                         err = -EFAULT;
468                         goto err_copy;
469                 }
470         }
471
472         return &pd->ibpd;
473
474 err_copy:       
475         mthca_pd_free(to_mdev(ibdev), pd);
476 err_pd_alloc:
477         kfree(pd);
478 err_mem:
479 err_param:
480         return ERR_PTR(err);
481 }
482
483 int mthca_dealloc_pd(struct ib_pd *pd)
484 {
485         if (mthca_is_livefish(to_mdev(pd->device))) 
486                 goto done;
487
488         mthca_pd_free(to_mdev(pd->device), to_mpd(pd));
489
490 done:   
491         kfree(pd);
492         return 0;
493 }
494
495 struct ib_ah *mthca_ah_create(struct ib_pd *pd,
496                                      struct ib_ah_attr *ah_attr)
497 {
498         int err;
499         struct mthca_ah *ah;
500
501         ah = kzalloc(sizeof *ah, GFP_ATOMIC);
502         if (!ah)
503                 return ERR_PTR(-ENOMEM);
504
505         err = mthca_create_ah(to_mdev(pd->device), to_mpd(pd), ah_attr, ah);
506         if (err) {
507                 kfree(ah);
508                 return ERR_PTR(err);
509         }
510
511         return &ah->ibah;
512 }
513
514 int mthca_ah_destroy(struct ib_ah *ah)
515 {
516         mthca_destroy_ah(to_mdev(ah->device), to_mah(ah));
517         kfree(ah);
518
519         return 0;
520 }
521
522 struct ib_srq *mthca_create_srq(struct ib_pd *pd,
523                                        struct ib_srq_init_attr *init_attr,
524                                        ci_umv_buf_t* const                      p_umv_buf)
525 {
526 #ifdef WIN_TO_BE_CHANGED
527         struct mthca_create_srq ucmd;
528         struct mthca_ucontext *context = NULL;
529         struct mthca_srq *srq;
530         int err;
531
532         srq = kmalloc(sizeof *srq, GFP_KERNEL);
533         if (!srq)
534                 return ERR_PTR(-ENOMEM);
535
536         if (pd->ucontext) {
537                 context = to_mucontext(pd->ucontext);
538
539                 if (ib_copy_from_umv_buf(&ucmd, p_umv_buf, sizeof ucmd)) {
540                         err = -EFAULT;
541                         goto err_free;
542                 }
543                 err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
544                                         context->db_tab, ucmd.db_index,
545                                         ucmd.db_page, NULL);
546
547                 if (err)
548                         goto err_free;
549
550                 srq->mr.ibmr.lkey = ucmd.lkey;
551                 srq->db_index     = ucmd.db_index;
552         }
553
554         err = mthca_alloc_srq(to_mdev(pd->device), to_mpd(pd),
555                               &init_attr->attr, srq);
556
557         if (err && pd->ucontext)
558                 mthca_unmap_user_db(to_mdev(pd->device), &context->uar,
559                                     context->db_tab, ucmd.db_index);
560
561         if (err)
562                 goto err_free;
563
564         if (context && ib_copy_to_umv_buf(p_umv_buf, &srq->srqn, sizeof (u32))) {
565                 mthca_free_srq(to_mdev(pd->device), srq);
566                 err = -EFAULT;
567                 goto err_free;
568         }
569
570         return &srq->ibsrq;
571
572 err_free:
573         kfree(srq);
574
575         return ERR_PTR(err);
576 #else
577         UNREFERENCED_PARAMETER(p_umv_buf);
578         UNREFERENCED_PARAMETER(init_attr);
579         UNREFERENCED_PARAMETER(pd);
580         return NULL;
581 #endif
582 }
583
584 int mthca_destroy_srq(struct ib_srq *srq)
585 {
586         struct mthca_ucontext *context;
587
588         if (srq->uobject) {
589                 context = to_mucontext(srq->uobject->context);
590
591                 mthca_unmap_user_db(to_mdev(srq->device), &context->uar,
592                                     context->db_tab, to_msrq(srq)->db_index);
593         }
594
595         mthca_free_srq(to_mdev(srq->device), to_msrq(srq));
596         kfree(srq);
597
598         return 0;
599 }
600
601 struct ib_qp *mthca_create_qp(struct ib_pd *pd,
602                                      struct ib_qp_init_attr *init_attr,
603                                       ci_umv_buf_t* const                       p_umv_buf)
604 {
605         struct ibv_create_qp ucmd = {0};
606         struct mthca_qp *qp = NULL;
607         struct mthca_ucontext *context = NULL;
608         int err;
609
610         switch (init_attr->qp_type) {
611         case IB_QPT_RELIABLE_CONN:
612         case IB_QPT_UNRELIABLE_CONN:
613         case IB_QPT_UNRELIABLE_DGRM:
614         {
615
616                 qp = kmalloc(sizeof *qp, GFP_KERNEL);
617                 if (!qp) {
618                         err = -ENOMEM;
619                         goto err_mem;
620                 }
621
622                 if (pd->ucontext) {
623                         context = to_mucontext(pd->ucontext);
624
625                         if (ib_copy_from_umv_buf(&ucmd, p_umv_buf, sizeof ucmd)) {
626                                 err = -EFAULT;
627                                 goto err_copy;
628                         }
629
630                         err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
631                                                 context->db_tab,
632                                                 ucmd.sq_db_index, ucmd.sq_db_page, NULL);
633                         if (err) 
634                                 goto err_map1;
635
636                         err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
637                                                 context->db_tab,
638                                                 ucmd.rq_db_index, ucmd.rq_db_page, NULL);
639                         if (err) 
640                                 goto err_map2;
641
642                         qp->mr.ibmr.lkey = ucmd.lkey;
643                         qp->sq.db_index  = ucmd.sq_db_index;
644                         qp->rq.db_index  = ucmd.rq_db_index;
645                 }
646
647                 err = mthca_alloc_qp(to_mdev(pd->device), to_mpd(pd),
648                                      to_mcq(init_attr->send_cq),
649                                      to_mcq(init_attr->recv_cq),
650                                      init_attr->qp_type, init_attr->sq_sig_type,
651                                      &init_attr->cap, qp);
652
653                 if (err)
654                         if (pd->ucontext) 
655                                 goto err_alloc_qp_user;
656                         else 
657                                 goto err_copy;
658
659                 qp->ibqp.qp_num = qp->qpn;
660                 break;
661         }
662         case IB_QPT_QP0:
663         case IB_QPT_QP1:
664         {
665                 /* Don't allow userspace to create special QPs */
666                 if (pd->ucontext) {
667                         err = -EINVAL;
668                         goto err_inval;
669                 }
670
671                 qp = kmalloc(sizeof (struct mthca_sqp), GFP_KERNEL);
672                 if (!qp) {
673                         err = -ENOMEM;
674                         goto err_mem;
675                 }
676
677                 qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_QP0 ? 0 : 1;
678
679                 err = mthca_alloc_sqp(to_mdev(pd->device), to_mpd(pd),
680                                       to_mcq(init_attr->send_cq),
681                                       to_mcq(init_attr->recv_cq),
682                                       init_attr->sq_sig_type, &init_attr->cap,
683                                       qp->ibqp.qp_num, init_attr->port_num,
684                                       to_msqp(qp));
685                 if (err)
686                         goto err_alloc_sqp;
687                 
688                 break;
689         }
690         default:
691                 /* Don't support raw QPs */
692                 err = -ENOSYS;
693                 goto err_unsupported;
694         }
695
696         init_attr->cap.max_send_wr     = qp->sq.max;
697         init_attr->cap.max_recv_wr     = qp->rq.max;
698         init_attr->cap.max_send_sge    = qp->sq.max_gs;
699         init_attr->cap.max_recv_sge    = qp->rq.max_gs;
700         init_attr->cap.max_inline_data    = qp->max_inline_data;
701
702         return &qp->ibqp;
703
704                 
705 err_alloc_qp_user:
706         if (pd->ucontext) 
707                 mthca_unmap_user_db(to_mdev(pd->device),
708                         &context->uar, context->db_tab, ucmd.rq_db_index);
709 err_map2:
710         if (pd->ucontext) 
711                 mthca_unmap_user_db(to_mdev(pd->device),
712                         &context->uar, context->db_tab, ucmd.sq_db_index);
713 err_map1: err_copy: err_alloc_sqp:
714         if (qp)
715                 kfree(qp);
716 err_mem: err_inval:     err_unsupported:
717         return ERR_PTR(err);
718 }
719
720 int mthca_destroy_qp(struct ib_qp *qp)
721 {
722         if (qp->ucontext) {
723                 mthca_unmap_user_db(to_mdev(qp->device),
724                                     &to_mucontext(qp->ucontext)->uar,
725                                     to_mucontext(qp->ucontext)->db_tab,
726                                     to_mqp(qp)->sq.db_index);
727                 mthca_unmap_user_db(to_mdev(qp->device),
728                                     &to_mucontext(qp->ucontext)->uar,
729                                     to_mucontext(qp->ucontext)->db_tab,
730                                     to_mqp(qp)->rq.db_index);
731         }
732         mthca_free_qp(to_mdev(qp->device), to_mqp(qp));
733         kfree(qp);
734         return 0;
735 }
736
737 struct ib_cq *mthca_create_cq(struct ib_device *ibdev, int entries,
738                                      struct ib_ucontext *context,
739                                      ci_umv_buf_t* const                        p_umv_buf)
740 {
741         struct ibv_create_cq ucmd = {0};
742         struct mthca_cq *cq;
743         int nent;
744         int err;
745         void *u_arm_db_page = 0;
746
747         if (entries < 1 || entries > to_mdev(ibdev)->limits.max_cqes)   
748                 return ERR_PTR(-EINVAL);
749
750         if (context) {
751                 if (ib_copy_from_umv_buf(&ucmd, p_umv_buf, sizeof ucmd))
752                         return ERR_PTR(-EFAULT);
753
754                 err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
755                                         to_mucontext(context)->db_tab,
756                                         ucmd.set_db_index, ucmd.set_db_page, NULL);
757                 if (err)
758                         return ERR_PTR(err);
759
760                 err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
761                                         to_mucontext(context)->db_tab,
762                                         ucmd.arm_db_index, ucmd.arm_db_page, NULL);
763                 if (err)
764                         goto err_unmap_set;
765
766                 err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
767                                         to_mucontext(context)->db_tab,
768                                         ucmd.u_arm_db_index, 
769                                         (u64)(ULONG_PTR)PAGE_ALIGN(ucmd.u_arm_db_page), 
770                                         &u_arm_db_page);
771                 if (err)
772                         goto err_unmap_arm;
773         }
774
775         cq = kmalloc(sizeof *cq, GFP_KERNEL);
776         if (!cq) {
777                 err = -ENOMEM;
778                 goto err_unmap_ev;
779         }
780
781         if (context) {
782                 cq->mr.ibmr.lkey = ucmd.lkey;
783                 cq->set_ci_db_index = ucmd.set_db_index;
784                 cq->arm_db_index    = ucmd.arm_db_index;
785                 cq->u_arm_db_index    = ucmd.u_arm_db_index;
786                 cq->p_u_arm_sn = (int*)((char*)u_arm_db_page + BYTE_OFFSET(ucmd.u_arm_db_page));
787         }
788
789         for (nent = 1; nent <= entries; nent <<= 1)
790                 ; /* nothing */
791
792         err = mthca_init_cq(to_mdev(ibdev), nent, 
793                             context ? to_mucontext(context) : NULL,
794                             context ? ucmd.mr.pdn : to_mdev(ibdev)->driver_pd.pd_num,
795                             cq);
796         if (err)
797                 goto err_free;
798
799         if (context ) {
800                 struct ibv_create_cq_resp *create_cq_resp = (struct ibv_create_cq_resp *)(void*)p_umv_buf->p_inout_buf;
801                 create_cq_resp->cqn = cq->cqn;
802         }
803
804         HCA_PRINT( TRACE_LEVEL_INFORMATION, HCA_DBG_LOW ,
805                 ("uctx %p, cq_hndl %p, cq_num %#x, cqe  %#x\n",
806                 context, &cq->ibcq, cq->cqn, cq->ibcq.cqe ) );
807         
808         return &cq->ibcq;
809
810 err_free:
811         kfree(cq);
812
813 err_unmap_ev:
814         if (context)
815                 mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
816                                     to_mucontext(context)->db_tab, ucmd.u_arm_db_index);
817
818 err_unmap_arm:
819         if (context)
820                 mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
821                                     to_mucontext(context)->db_tab, ucmd.arm_db_index);
822
823 err_unmap_set:
824         if (context)
825                 mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
826                                     to_mucontext(context)->db_tab, ucmd.set_db_index);
827
828         return ERR_PTR(err);
829 }
830
831 int mthca_destroy_cq(struct ib_cq *cq)
832 {
833         if (cq->ucontext) {
834                 mthca_unmap_user_db(to_mdev(cq->device),
835                                     &to_mucontext(cq->ucontext)->uar,
836                                     to_mucontext(cq->ucontext)->db_tab,
837                                     to_mcq(cq)->u_arm_db_index);
838                 mthca_unmap_user_db(to_mdev(cq->device),
839                                     &to_mucontext(cq->ucontext)->uar,
840                                     to_mucontext(cq->ucontext)->db_tab,
841                                     to_mcq(cq)->arm_db_index);
842                 mthca_unmap_user_db(to_mdev(cq->device),
843                                     &to_mucontext(cq->ucontext)->uar,
844                                     to_mucontext(cq->ucontext)->db_tab,
845                                     to_mcq(cq)->set_ci_db_index);
846         }
847         mthca_free_cq(to_mdev(cq->device), to_mcq(cq));
848         kfree(cq);
849
850         return 0;
851 }
852
853 static
854 mthca_mpt_access_t
855 map_qp_mpt(
856         IN                              mthca_qp_access_t                               qp_acl)
857 {
858 #define ACL_MTHCA(mfl,ifl) if (qp_acl & mfl)   mpt_acl |= ifl
859         mthca_mpt_access_t mpt_acl = 0;
860
861         ACL_MTHCA(MTHCA_ACCESS_REMOTE_READ,MTHCA_MPT_FLAG_REMOTE_READ);
862         ACL_MTHCA(MTHCA_ACCESS_REMOTE_WRITE,MTHCA_MPT_FLAG_REMOTE_WRITE);
863         ACL_MTHCA(MTHCA_ACCESS_REMOTE_ATOMIC,MTHCA_MPT_FLAG_ATOMIC);
864         ACL_MTHCA(MTHCA_ACCESS_LOCAL_WRITE,MTHCA_MPT_FLAG_LOCAL_WRITE);
865
866         return (mpt_acl | MTHCA_MPT_FLAG_LOCAL_READ);
867 }
868
869 struct ib_mr *mthca_get_dma_mr(struct ib_pd *pd, mthca_qp_access_t acc)
870 {
871         struct mthca_mr *mr;
872         int err;
873
874         mr = kzalloc(sizeof *mr, GFP_KERNEL);
875         if (!mr)
876                 return ERR_PTR(-ENOMEM);
877
878         err = mthca_mr_alloc_notrans(to_mdev(pd->device),
879                                      to_mpd(pd)->pd_num,
880                                      map_qp_mpt(acc), mr);
881
882         if (err) {
883                 kfree(mr);
884                 return ERR_PTR(err);
885         }
886
887         return &mr->ibmr;
888 }
889
890 struct ib_mr *mthca_reg_phys_mr(struct ib_pd       *pd,
891                                        struct ib_phys_buf *buffer_list,
892                                        int                 num_phys_buf,
893                                        mthca_qp_access_t                 acc,
894                                        u64                *iova_start)
895 {
896         struct mthca_mr *mr;
897         u64 *page_list;
898         u64 total_size;
899         u64 mask;
900         int shift;
901         int npages;
902         int err;
903         int i, j, n;
904
905         /* First check that we have enough alignment */
906         if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK))
907                 return ERR_PTR(-EINVAL);
908
909         if (num_phys_buf > 1 &&
910             ((buffer_list[0].addr + buffer_list[0].size) & ~PAGE_MASK))
911                 return ERR_PTR(-EINVAL);
912
913         mask = 0;
914         total_size = 0;
915         for (i = 0; i < num_phys_buf; ++i) {
916                 if (i != 0)
917                         mask |= buffer_list[i].addr;
918                 if (i != num_phys_buf - 1)
919                         mask |= buffer_list[i].addr + buffer_list[i].size;
920
921                 total_size += buffer_list[i].size;
922         }
923
924         if (mask & ~PAGE_MASK)
925                 return ERR_PTR(-EINVAL);
926
927         /* Find largest page shift we can use to cover buffers */
928         for (shift = PAGE_SHIFT; shift < 31; ++shift)
929                 if (num_phys_buf > 1) {
930                         if ((1Ui64 << shift) & mask)
931                                 break;
932                 } else {
933                         if (1Ui64 << shift >=
934                             buffer_list[0].size +
935                             (buffer_list[0].addr & ((1Ui64 << shift) - 1)))
936                                 break;
937                 }
938
939         buffer_list[0].size += buffer_list[0].addr & ((1Ui64 << shift) - 1);
940         buffer_list[0].addr &= ~0Ui64 << shift;
941
942         mr = kzalloc(sizeof *mr, GFP_KERNEL);
943         if (!mr)
944                 return ERR_PTR(-ENOMEM);
945
946         npages = 0;
947         for (i = 0; i < num_phys_buf; ++i)
948                 npages += (int)((buffer_list[i].size + (1Ui64 << shift) - 1) >> shift);
949
950         if (!npages)
951                 return &mr->ibmr;
952
953         page_list = kmalloc(npages * sizeof *page_list, GFP_KERNEL);
954         if (!page_list) {
955                 kfree(mr);
956                 return ERR_PTR(-ENOMEM);
957         }
958
959         n = 0;
960         for (i = 0; i < num_phys_buf; ++i)
961                 for (j = 0;
962                      j < (buffer_list[i].size + (1Ui64 << shift) - 1) >> shift;
963                      ++j)
964                         page_list[n++] = buffer_list[i].addr + ((u64) j << shift);
965
966         HCA_PRINT( TRACE_LEVEL_VERBOSE ,HCA_DBG_LOW ,("Registering memory at %I64x (iova %I64x) "
967                   "in PD %x; shift %d, npages %d.\n",
968                   (u64) buffer_list[0].addr,
969                   (u64) *iova_start,
970                   to_mpd(pd)->pd_num,
971                   shift, npages));
972
973         err = mthca_mr_alloc_phys(to_mdev(pd->device),
974                                   to_mpd(pd)->pd_num,
975                                   page_list, shift, npages,
976                                   *iova_start, total_size,
977                                   map_qp_mpt(acc), mr);
978
979         if (err) {
980                 kfree(page_list);
981                 kfree(mr);
982                 return ERR_PTR(err);
983         }
984
985         kfree(page_list);
986         return &mr->ibmr;
987 }
988
989 struct ib_mr *mthca_reg_virt_mr(struct ib_pd *pd, 
990         void* __ptr64   vaddr, uint64_t length, uint64_t hca_va,
991         mthca_qp_access_t acc, boolean_t um_call)
992 {
993         struct mthca_dev *dev = to_mdev(pd->device);
994         struct mthca_mr *mr;
995         u64 *pages;
996         int err = 0;
997         uint32_t i, n;
998         mt_iobuf_t *iobuf_p;
999         mt_iobuf_iter_t iobuf_iter;
1000
1001         /*
1002          * Be friendly to WRITE_MTT command and leave two 
1003          * empty slots for the  index and reserved fields of the mailbox.
1004          */
1005         int max_buf_list_size = PAGE_SIZE / sizeof (u64) - 2;
1006
1007         HCA_ENTER(HCA_DBG_MEMORY);
1008
1009
1010         mr = kzalloc(sizeof *mr, GFP_KERNEL);
1011         if (!mr) {
1012                 err = -ENOMEM;
1013                 goto err_nomem;
1014         }
1015
1016         /*
1017          * We ask for writable memory if any access flags other than
1018          * "remote read" are set.  "Local write" and "remote write"
1019          * obviously require write access.  "Remote atomic" can do
1020          * things like fetch and add, which will modify memory, and
1021          * "MW bind" can change permissions by binding a window.
1022          */
1023
1024         // try register the buffer
1025         iobuf_p = &mr->iobuf;
1026         err =  iobuf_register( (u64)vaddr, length, um_call, 
1027                 (acc & ~MTHCA_ACCESS_REMOTE_READ) ? IB_AC_LOCAL_WRITE : 0, iobuf_p );
1028         if (err)
1029                 goto err_reg_mem;
1030         mr->iobuf_used = TRUE;
1031
1032         // allocate MTT's
1033         mr->mtt = mthca_alloc_mtt(dev, iobuf_p->nr_pages);
1034         if (IS_ERR(mr->mtt)) {
1035                 err = PTR_ERR(mr->mtt);
1036                 goto err_alloc_mtt;
1037         }
1038
1039         // allocate buffer_list for writing MTT's
1040         pages = (u64 *) kmalloc(PAGE_SIZE,GFP_KERNEL);
1041         if (!pages) {
1042                 err = -ENOMEM;
1043                 goto err_pages;
1044         }
1045
1046         // write MTT's
1047         iobuf_iter_init( iobuf_p, &iobuf_iter );
1048         n = 0;
1049         for (;;) {
1050                 // get up to  max_buf_list_size page physical addresses
1051                 i = iobuf_get_tpt_seg( iobuf_p, &iobuf_iter, max_buf_list_size, pages );
1052                 if (!i)
1053                         break;
1054
1055                 //TODO: convert physical adresses to dma one's
1056
1057                 // write 'i' dma addresses
1058                 err = mthca_write_mtt(dev, mr->mtt, n, pages, i);
1059                 if (err)
1060                         goto err_write_mtt;
1061                 n += i;
1062                 if (n >= iobuf_p->nr_pages)
1063                         break;
1064         }
1065
1066         CL_ASSERT(n == iobuf_p->nr_pages);
1067         
1068         // write MPT
1069         err = mthca_mr_alloc(dev, to_mpd(pd)->pd_num, PAGE_SHIFT, hca_va,
1070                 length, map_qp_mpt(acc), mr);
1071         if (err)
1072                 goto err_mt_alloc;
1073
1074         // secure memory
1075         if (!pd->ucontext)
1076                 goto done;
1077         __try {
1078                 mr->secure_handle = MmSecureVirtualMemory ( vaddr, (SIZE_T)length,
1079                         (acc & ~MTHCA_ACCESS_REMOTE_READ) ? PAGE_READWRITE : PAGE_READONLY );
1080                 if (mr->secure_handle == NULL) {
1081                         err = -EFAULT;
1082                         goto err_secure;
1083                 }
1084         }
1085         __except (EXCEPTION_EXECUTE_HANDLER) {
1086                 NTSTATUS Status = GetExceptionCode();
1087                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_MEMORY ,
1088                         ("Exception 0x%x on MmSecureVirtualMemory(), addr %p, size %I64d, access %#x\n", 
1089                         Status, vaddr, length, acc ));
1090                 err = -EFAULT;
1091                 goto err_secure;
1092         }
1093 done:   
1094         free_page((void*) pages);
1095
1096         HCA_EXIT(HCA_DBG_MEMORY);
1097         return &mr->ibmr;
1098
1099 err_secure:
1100 err_mt_alloc:
1101 err_write_mtt:
1102         free_page((void*) pages);
1103 err_pages:
1104         mthca_free_mtt(dev, mr->mtt);
1105 err_alloc_mtt:
1106         iobuf_deregister(iobuf_p);
1107 err_reg_mem:    
1108         kfree(mr);
1109 err_nomem:      
1110
1111         HCA_EXIT(HCA_DBG_MEMORY);
1112         return ERR_PTR(err);
1113 }
1114
1115 int mthca_dereg_mr(struct ib_mr *mr)
1116 {
1117         struct mthca_mr *mmr = to_mmr(mr);
1118         struct mthca_dev* dev = to_mdev(mr->device);
1119
1120         if (mr->pd->ucontext)
1121                 MmUnsecureVirtualMemory ( mmr->secure_handle );
1122         mthca_free_mr(dev, mmr);
1123         if (mmr->iobuf_used)
1124                 iobuf_deregister(&mmr->iobuf);
1125         kfree(mmr);
1126         return 0;
1127 }
1128
1129 struct ib_fmr *mthca_alloc_fmr(struct ib_pd *pd, mthca_qp_access_t acc,
1130                                       struct ib_fmr_attr *fmr_attr)
1131 {
1132         struct mthca_fmr *fmr;
1133         int err;
1134
1135         fmr = kzalloc(sizeof *fmr, GFP_KERNEL);
1136         if (!fmr)
1137                 return ERR_PTR(-ENOMEM);
1138
1139         RtlCopyMemory(&fmr->attr, fmr_attr, sizeof *fmr_attr);
1140         err = mthca_fmr_alloc(to_mdev(pd->device), to_mpd(pd)->pd_num,
1141                              map_qp_mpt(acc), fmr);
1142
1143         if (err) {
1144                 kfree(fmr);
1145                 return ERR_PTR(err);
1146         }
1147
1148         return &fmr->ibmr;
1149 }
1150
1151 int mthca_dealloc_fmr(struct ib_fmr *fmr)
1152 {
1153         struct mthca_fmr *mfmr = to_mfmr(fmr);
1154         int err;
1155
1156         err = mthca_free_fmr(to_mdev(fmr->device), mfmr);
1157         if (err)
1158                 return err;
1159
1160         kfree(mfmr);
1161         return 0;
1162 }
1163
1164 int mthca_unmap_fmr(struct list_head *fmr_list)
1165 {
1166         struct ib_fmr *fmr;
1167         int err;
1168         u8 status;
1169         struct mthca_dev *mdev = NULL;
1170
1171         list_for_each_entry(fmr, fmr_list, list,struct ib_fmr) {
1172                 if (mdev && to_mdev(fmr->device) != mdev)
1173                         return -EINVAL;
1174                 mdev = to_mdev(fmr->device);
1175         }
1176
1177         if (!mdev)
1178                 return 0;
1179
1180         if (mthca_is_memfree(mdev)) {
1181                 list_for_each_entry(fmr, fmr_list, list,struct ib_fmr)
1182                         mthca_arbel_fmr_unmap(mdev, to_mfmr(fmr));
1183
1184                 wmb();
1185         } else
1186                 list_for_each_entry(fmr, fmr_list, list,struct ib_fmr)
1187                         mthca_tavor_fmr_unmap(mdev, to_mfmr(fmr));
1188
1189         err = mthca_SYNC_TPT(mdev, &status);
1190         if (err)
1191                 return err;
1192         if (status)
1193                 return -EINVAL;
1194         return 0;
1195 }
1196
1197 static int mthca_init_node_data(struct mthca_dev *dev)
1198 {
1199         struct ib_smp *in_mad  = NULL;
1200         struct ib_smp *out_mad = NULL;
1201         int err = -ENOMEM;
1202         u8 status;
1203
1204         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
1205         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
1206         if (!in_mad || !out_mad)
1207                 goto out;
1208
1209         init_query_mad(in_mad);
1210         in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
1211
1212         err = mthca_MAD_IFC(dev, 1, 1,
1213                             1, NULL, NULL, in_mad, out_mad,
1214                             &status);
1215         if (err)
1216                 goto out;
1217         if (status) {
1218                 err = -EINVAL;
1219                 goto out;
1220         }
1221
1222         memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8);
1223
1224 out:
1225         kfree(in_mad);
1226         kfree(out_mad);
1227         return err;
1228 }
1229
1230 int mthca_register_device(struct mthca_dev *dev)
1231 {
1232         int ret;
1233
1234         ret = mthca_init_node_data(dev);        
1235         if (ret)
1236                 return ret;
1237
1238         strlcpy(dev->ib_dev.name, "mthca%d", IB_DEVICE_NAME_MAX);
1239         dev->ib_dev.node_type            = IB_NODE_CA;
1240         dev->ib_dev.phys_port_cnt        = (u8)dev->limits.num_ports;
1241         dev->ib_dev.mdev                                = dev;
1242         dev->ib_dev.query_device         = mthca_query_device;
1243         dev->ib_dev.query_port           = mthca_query_port;
1244         dev->ib_dev.modify_port          = mthca_modify_port;
1245         dev->ib_dev.query_pkey_chunk           = mthca_query_pkey_chunk;
1246         dev->ib_dev.query_gid_chunk            = mthca_query_gid_chunk;
1247         dev->ib_dev.alloc_ucontext       = mthca_alloc_ucontext;
1248         dev->ib_dev.dealloc_ucontext     = mthca_dealloc_ucontext;
1249         dev->ib_dev.alloc_pd             = mthca_alloc_pd;
1250         dev->ib_dev.dealloc_pd           = mthca_dealloc_pd;
1251         dev->ib_dev.create_ah            = mthca_ah_create;
1252         dev->ib_dev.destroy_ah           = mthca_ah_destroy;
1253
1254         if (dev->mthca_flags & MTHCA_FLAG_SRQ) {
1255                 dev->ib_dev.create_srq           = mthca_create_srq;
1256                 dev->ib_dev.modify_srq           = mthca_modify_srq;
1257                 dev->ib_dev.destroy_srq          = mthca_destroy_srq;
1258
1259                 if (mthca_is_memfree(dev))
1260                         dev->ib_dev.post_srq_recv = mthca_arbel_post_srq_recv;
1261                 else
1262                         dev->ib_dev.post_srq_recv = mthca_tavor_post_srq_recv;
1263         }
1264
1265         dev->ib_dev.create_qp            = mthca_create_qp;
1266         dev->ib_dev.modify_qp            = mthca_modify_qp;
1267         dev->ib_dev.destroy_qp           = mthca_destroy_qp;
1268         dev->ib_dev.create_cq            = mthca_create_cq;
1269         dev->ib_dev.destroy_cq           = mthca_destroy_cq;
1270         dev->ib_dev.poll_cq              = mthca_poll_cq;
1271         dev->ib_dev.get_dma_mr           = mthca_get_dma_mr;
1272         dev->ib_dev.reg_phys_mr          = mthca_reg_phys_mr;
1273         dev->ib_dev.reg_virt_mr                  = mthca_reg_virt_mr;
1274         dev->ib_dev.dereg_mr             = mthca_dereg_mr;
1275
1276         if (dev->mthca_flags & MTHCA_FLAG_FMR) {
1277                 dev->ib_dev.alloc_fmr            = mthca_alloc_fmr;
1278                 dev->ib_dev.unmap_fmr            = mthca_unmap_fmr;
1279                 dev->ib_dev.dealloc_fmr          = mthca_dealloc_fmr;
1280                 if (mthca_is_memfree(dev))
1281                         dev->ib_dev.map_phys_fmr = mthca_arbel_map_phys_fmr;
1282                 else
1283                         dev->ib_dev.map_phys_fmr = mthca_tavor_map_phys_fmr;
1284         }
1285
1286         dev->ib_dev.attach_mcast         = mthca_multicast_attach;
1287         dev->ib_dev.detach_mcast         = mthca_multicast_detach;
1288         dev->ib_dev.process_mad          = mthca_process_mad;
1289
1290         if (mthca_is_memfree(dev)) {
1291                 dev->ib_dev.req_notify_cq = mthca_arbel_arm_cq;
1292                 dev->ib_dev.post_send     = mthca_arbel_post_send;
1293                 dev->ib_dev.post_recv     = mthca_arbel_post_receive;
1294         } else {
1295                 dev->ib_dev.req_notify_cq = mthca_tavor_arm_cq;
1296                 dev->ib_dev.post_send     = mthca_tavor_post_send;
1297                 dev->ib_dev.post_recv     = mthca_tavor_post_receive;
1298         }
1299
1300         KeInitializeMutex(&dev->cap_mask_mutex, 0);
1301
1302         ret = ib_register_device(&dev->ib_dev);
1303         if (ret)
1304                 return ret;
1305
1306         mthca_start_catas_poll(dev);
1307
1308         return 0;
1309 }
1310
1311 void mthca_unregister_device(struct mthca_dev *dev)
1312 {
1313         mthca_stop_catas_poll(dev);
1314         ib_unregister_device(&dev->ib_dev);
1315 }