2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
\r
3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
\r
4 * Copyright (c) 2005 Cisco Systems. All rights reserved.
\r
5 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
\r
6 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
\r
7 * Portions Copyright (c) 2008 Microsoft Corporation. All rights reserved.
\r
9 * This software is available to you under the OpenIB.org BSD license
\r
12 * Redistribution and use in source and binary forms, with or
\r
13 * without modification, are permitted provided that the following
\r
14 * conditions are met:
\r
16 * - Redistributions of source code must retain the above
\r
17 * copyright notice, this list of conditions and the following
\r
20 * - Redistributions in binary form must reproduce the above
\r
21 * copyright notice, this list of conditions and the following
\r
22 * disclaimer in the documentation and/or other materials
\r
23 * provided with the distribution.
\r
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
\r
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
\r
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
\r
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
\r
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
\r
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
\r
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
\r
40 #include "mthca_dev.h"
\r
41 #include "mt_pa_cash.h"
\r
43 #if defined(EVENT_TRACING)
\r
47 #include "mthca_provider.tmh"
\r
49 #include "mthca_cmd.h"
\r
50 #include "mthca_memfree.h"
\r
52 static void init_query_mad(struct ib_smp *mad)
\r
54 mad->base_version = 1;
\r
55 mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
\r
56 mad->class_version = 1;
\r
57 mad->method = IB_MGMT_METHOD_GET;
\r
60 int mthca_query_device(struct ib_device *ibdev,
\r
61 struct ib_device_attr *props)
\r
63 struct ib_smp *in_mad = NULL;
\r
64 struct ib_smp *out_mad = NULL;
\r
66 struct mthca_dev* mdev = to_mdev(ibdev);
\r
70 RtlZeroMemory(props, sizeof *props);
\r
72 if (mthca_is_livefish(mdev)) {
\r
74 if ( !mdev || !mdev->ext )
\r
76 props->vendor_id = mdev->ext->hcaConfig.VendorID;
\r
77 props->vendor_part_id = mdev->ext->hcaConfig.DeviceID;
\r
81 in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL);
\r
82 out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
\r
83 if (!in_mad || !out_mad)
\r
86 init_query_mad(in_mad);
\r
87 in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
\r
89 err = mthca_MAD_IFC(mdev, 1, 1,
\r
90 1, NULL, NULL, in_mad, out_mad, &status);
\r
98 props->fw_ver = mdev->fw_ver;
\r
99 props->device_cap_flags = mdev->device_cap_flags;
\r
100 props->vendor_id = cl_ntoh32(*(__be32 *) (out_mad->data + 36)) &
\r
102 props->vendor_part_id = cl_ntoh16(*(__be16 *) (out_mad->data + 30));
\r
103 props->hw_ver = cl_ntoh32(*(__be32 *) (out_mad->data + 32));
\r
104 memcpy(&props->sys_image_guid, out_mad->data + 4, 8);
\r
105 props->max_mr_size = ~0Ui64;
\r
106 props->page_size_cap = mdev->limits.page_size_cap;
\r
107 props->max_qp = mdev->limits.num_qps - mdev->limits.reserved_qps;
\r
108 props->max_qp_wr = mdev->limits.max_wqes;
\r
109 props->max_sge = mdev->limits.max_sg;
\r
110 props->max_cq = mdev->limits.num_cqs - mdev->limits.reserved_cqs;
\r
111 props->max_cqe = mdev->limits.max_cqes;
\r
112 props->max_mr = mdev->limits.num_mpts - mdev->limits.reserved_mrws;
\r
113 props->max_pd = mdev->limits.num_pds - mdev->limits.reserved_pds;
\r
114 props->max_qp_rd_atom = 1 << mdev->qp_table.rdb_shift;
\r
115 props->max_qp_init_rd_atom = mdev->limits.max_qp_init_rdma;
\r
116 props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp;
\r
117 props->max_srq = mdev->limits.num_srqs - mdev->limits.reserved_srqs;
\r
118 props->max_srq_wr = mdev->limits.max_srq_wqes;
\r
119 if (mthca_is_memfree(mdev))
\r
120 --props->max_srq_wr;
\r
121 props->max_srq_sge = mdev->limits.max_srq_sge;
\r
122 props->local_ca_ack_delay = (u8)mdev->limits.local_ca_ack_delay;
\r
123 props->atomic_cap = mdev->limits.flags & DEV_LIM_FLAG_ATOMIC ?
\r
124 IB_ATOMIC_LOCAL : IB_ATOMIC_NONE;
\r
125 props->max_pkeys = (u16)mdev->limits.pkey_table_len;
\r
126 props->max_mcast_grp = mdev->limits.num_mgms + mdev->limits.num_amgms;
\r
127 props->max_mcast_qp_attach = MTHCA_QP_PER_MGM;
\r
128 props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
\r
129 props->max_mcast_grp;
\r
130 props->max_ah = mdev->limits.num_avs;
\r
133 * If Sinai memory key optimization is being used, then only
\r
134 * the 8-bit key portion will change. For other HCAs, the
\r
135 * unused index bits will also be used for FMR remapping.
\r
137 if (mdev->mthca_flags & MTHCA_FLAG_SINAI_OPT)
\r
138 props->max_map_per_fmr = 255;
\r
140 props->max_map_per_fmr =
\r
141 (1 << (32 - long_log2(mdev->limits.num_mpts))) - 1;
\r
150 int mthca_query_port(struct ib_device *ibdev,
\r
151 u8 port, struct ib_port_attr *props)
\r
153 struct ib_smp *in_mad = NULL;
\r
154 struct ib_smp *out_mad = NULL;
\r
158 in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL);
\r
159 out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
\r
160 if (!in_mad || !out_mad)
\r
163 init_query_mad(in_mad);
\r
164 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;
\r
165 in_mad->attr_mod = cl_hton32(port);
\r
167 err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
\r
168 port, NULL, NULL, in_mad, out_mad,
\r
177 RtlZeroMemory(props, sizeof *props);
\r
178 props->lid = cl_ntoh16(*(__be16 *) (out_mad->data + 16));
\r
179 props->lmc = out_mad->data[34] & 0x7;
\r
180 props->sm_lid = cl_ntoh16(*(__be16 *) (out_mad->data + 18));
\r
181 props->sm_sl = out_mad->data[36] & 0xf;
\r
182 props->state = out_mad->data[32] & 0xf;
\r
183 props->phys_state = out_mad->data[33] >> 4;
\r
184 props->port_cap_flags = cl_ntoh32(*(__be32 *) (out_mad->data + 20));
\r
185 props->gid_tbl_len = to_mdev(ibdev)->limits.gid_table_len;
\r
186 props->max_msg_sz = 0x80000000;
\r
187 props->pkey_tbl_len = (u16)to_mdev(ibdev)->limits.pkey_table_len;
\r
188 props->bad_pkey_cntr = cl_ntoh16(*(__be16 *) (out_mad->data + 46));
\r
189 props->qkey_viol_cntr = cl_ntoh16(*(__be16 *) (out_mad->data + 48));
\r
190 props->active_width = out_mad->data[31] & 0xf;
\r
191 props->active_speed = out_mad->data[35] >> 4;
\r
192 props->max_mtu = out_mad->data[41] & 0xf;
\r
193 props->active_mtu = out_mad->data[36] >> 4;
\r
194 props->subnet_timeout = out_mad->data[51] & 0x1f;
\r
202 int mthca_modify_port(struct ib_device *ibdev,
\r
203 u8 port, int port_modify_mask,
\r
204 struct ib_port_modify *props)
\r
206 struct mthca_set_ib_param set_ib;
\r
207 struct ib_port_attr attr;
\r
211 if (down_interruptible(&to_mdev(ibdev)->cap_mask_mutex))
\r
214 err = mthca_query_port(ibdev, port, &attr);
\r
218 set_ib.set_si_guid = 0;
\r
219 set_ib.reset_qkey_viol = !!(port_modify_mask & IB_PORT_RESET_QKEY_CNTR);
\r
221 set_ib.cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) &
\r
222 ~props->clr_port_cap_mask;
\r
224 err = mthca_SET_IB(to_mdev(ibdev), &set_ib, port, &status);
\r
233 up(&to_mdev(ibdev)->cap_mask_mutex);
\r
237 static int mthca_query_pkey_chunk(struct ib_device *ibdev,
\r
238 u8 port, u16 index, __be16 pkey[32])
\r
240 struct ib_smp *in_mad = NULL;
\r
241 struct ib_smp *out_mad = NULL;
\r
245 in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL);
\r
246 out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
\r
247 if (!in_mad || !out_mad)
\r
250 init_query_mad(in_mad);
\r
251 in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE;
\r
252 in_mad->attr_mod = cl_hton32(index / 32);
\r
254 err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
\r
255 port, NULL, NULL, in_mad, out_mad,
\r
264 { // copy the results
\r
266 __be16 *pkey_chunk = (__be16 *)out_mad->data;
\r
267 for (i=0; i<32; ++i)
\r
268 pkey[i] = pkey_chunk[i];
\r
277 static int mthca_query_gid_chunk(struct ib_device *ibdev, u8 port,
\r
278 int index, union ib_gid gid[8])
\r
280 struct ib_smp *in_mad = NULL;
\r
281 struct ib_smp *out_mad = NULL;
\r
284 __be64 subnet_prefix;
\r
286 in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL);
\r
287 out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
\r
288 if (!in_mad || !out_mad)
\r
291 init_query_mad(in_mad);
\r
292 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;
\r
293 in_mad->attr_mod = cl_hton32(port);
\r
295 err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
\r
296 port, NULL, NULL, in_mad, out_mad,
\r
305 memcpy(&subnet_prefix, out_mad->data + 8, 8);
\r
307 init_query_mad(in_mad);
\r
308 in_mad->attr_id = IB_SMP_ATTR_GUID_INFO;
\r
309 in_mad->attr_mod = cl_hton32(index / 8);
\r
311 err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
\r
312 port, NULL, NULL, in_mad, out_mad,
\r
321 { // copy the results
\r
323 __be64 *guid = (__be64 *)out_mad->data;
\r
324 for (i=0; i<8; ++i) {
\r
325 gid[i].global.subnet_prefix = subnet_prefix;
\r
326 gid[i].global.interface_id = guid[i];
\r
336 struct ib_ucontext *mthca_alloc_ucontext(struct ib_device *ibdev,
\r
337 ci_umv_buf_t* const p_umv_buf)
\r
339 struct ibv_get_context_resp uresp;
\r
340 struct mthca_ucontext *context;
\r
343 RtlZeroMemory(&uresp, sizeof uresp);
\r
345 uresp.qp_tab_size = to_mdev(ibdev)->limits.num_qps;
\r
346 if (mthca_is_memfree(to_mdev(ibdev)))
\r
347 uresp.uarc_size = to_mdev(ibdev)->uar_table.uarc_size;
\r
349 uresp.uarc_size = 0;
\r
351 context = kzalloc(sizeof *context, GFP_KERNEL);
\r
357 if (mthca_is_livefish(to_mdev(ibdev)))
\r
360 err = mthca_uar_alloc(to_mdev(ibdev), &context->uar);
\r
362 goto err_uar_alloc;
\r
365 * map uar to user space
\r
368 /* map UAR to kernel */
\r
369 context->kva = ioremap((io_addr_t)context->uar.pfn << PAGE_SHIFT, PAGE_SIZE,&context->uar_size);
\r
370 if (!context->kva) {
\r
371 HCA_PRINT( TRACE_LEVEL_ERROR, HCA_DBG_LOW ,("Couldn't map kernel access region, aborting.\n") );
\r
377 context->mdl = IoAllocateMdl( context->kva, (ULONG)context->uar_size,
\r
378 FALSE, TRUE, NULL );
\r
379 if( !context->mdl ) {
\r
381 goto err_alloc_mdl;
\r
383 MmBuildMdlForNonPagedPool( context->mdl );
\r
385 /* Map the memory into the calling process's address space. */
\r
387 context->ibucontext.user_uar = MmMapLockedPagesSpecifyCache( context->mdl,
\r
388 UserMode, MmNonCached, NULL, FALSE, NormalPagePriority );
\r
390 __except(EXCEPTION_EXECUTE_HANDLER) {
\r
396 context->db_tab = mthca_init_user_db_tab(to_mdev(ibdev));
\r
397 if (IS_ERR(context->db_tab)) {
\r
398 err = PTR_ERR(context->db_tab);
\r
399 goto err_init_user;
\r
403 err = ib_copy_to_umv_buf(p_umv_buf, &uresp, sizeof uresp);
\r
405 goto err_copy_to_umv_buf;
\r
407 context->ibucontext.device = ibdev;
\r
409 atomic_set(&context->ibucontext.usecnt, 0);
\r
410 return &context->ibucontext;
\r
412 err_copy_to_umv_buf:
\r
413 mthca_cleanup_user_db_tab(to_mdev(ibdev), &context->uar,
\r
416 MmUnmapLockedPages( context->ibucontext.user_uar, context->mdl );
\r
418 IoFreeMdl(context->mdl);
\r
420 iounmap(context->kva, PAGE_SIZE);
\r
422 mthca_uar_free(to_mdev(ibdev), &context->uar);
\r
426 return ERR_PTR(err);
\r
429 int mthca_dealloc_ucontext(struct ib_ucontext *context)
\r
431 struct mthca_ucontext *mucontext = to_mucontext(context);
\r
433 if (mthca_is_livefish(to_mdev(context->device)))
\r
435 mthca_cleanup_user_db_tab(to_mdev(context->device), &mucontext->uar,
\r
436 mucontext->db_tab);
\r
437 MmUnmapLockedPages( mucontext->ibucontext.user_uar, mucontext->mdl );
\r
438 IoFreeMdl(mucontext->mdl);
\r
439 iounmap(mucontext->kva, PAGE_SIZE);
\r
440 mthca_uar_free(to_mdev(context->device), &mucontext->uar);
\r
447 struct ib_pd *mthca_alloc_pd(struct ib_device *ibdev,
\r
448 struct ib_ucontext *context,
\r
449 ci_umv_buf_t* const p_umv_buf)
\r
452 struct mthca_pd *pd;
\r
453 struct ibv_alloc_pd_resp resp;
\r
456 if (p_umv_buf && p_umv_buf->command) {
\r
457 if (p_umv_buf->output_size < sizeof(struct ibv_alloc_pd_resp)) {
\r
463 pd = kmalloc(sizeof *pd, GFP_KERNEL);
\r
469 if (mthca_is_livefish(to_mdev(ibdev)))
\r
472 err = mthca_pd_alloc(to_mdev(ibdev), !context, pd);
\r
478 if (p_umv_buf && p_umv_buf->command) {
\r
479 resp.pd_handle = (u64)(UINT_PTR)pd;
\r
480 resp.pdn = pd->pd_num;
\r
481 if (ib_copy_to_umv_buf(p_umv_buf, &resp, sizeof(struct ibv_alloc_pd_resp))) {
\r
490 mthca_pd_free(to_mdev(ibdev), pd);
\r
495 return ERR_PTR(err);
\r
498 int mthca_dealloc_pd(struct ib_pd *pd)
\r
500 if (mthca_is_livefish(to_mdev(pd->device)))
\r
503 mthca_pd_free(to_mdev(pd->device), to_mpd(pd));
\r
510 static struct ib_ah *mthca_ah_create(struct ib_pd *pd,
\r
511 struct ib_ah_attr *ah_attr)
\r
514 struct mthca_ah *ah;
\r
516 ah = kzalloc(sizeof *ah, GFP_ATOMIC);
\r
518 return ERR_PTR(-ENOMEM);
\r
520 err = mthca_create_ah(to_mdev(pd->device), to_mpd(pd), ah_attr, ah);
\r
523 return ERR_PTR(err);
\r
529 static int mthca_ah_destroy(struct ib_ah *ah)
\r
531 mthca_destroy_ah(to_mdev(ah->device), to_mah(ah));
\r
537 static struct ib_srq *mthca_create_srq(struct ib_pd *pd,
\r
538 struct ib_srq_init_attr *init_attr,
\r
539 ci_umv_buf_t* const p_umv_buf)
\r
541 struct ibv_create_srq ucmd = { 0 };
\r
542 struct mthca_ucontext *context = NULL;
\r
543 struct mthca_srq *srq;
\r
546 srq = kzalloc(sizeof *srq, GFP_KERNEL);
\r
548 return ERR_PTR(-ENOMEM);
\r
550 if (pd->ucontext) {
\r
551 context = to_mucontext(pd->ucontext);
\r
553 if (ib_copy_from_umv_buf(&ucmd, p_umv_buf, sizeof ucmd)) {
\r
557 err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
\r
558 context->db_tab, ucmd.db_index,
\r
559 ucmd.db_page, NULL);
\r
564 srq->mr.ibmr.lkey = ucmd.lkey;
\r
565 srq->db_index = ucmd.db_index;
\r
568 err = mthca_alloc_srq(to_mdev(pd->device), to_mpd(pd),
\r
569 &init_attr->attr, srq);
\r
571 if (err && pd->ucontext)
\r
572 mthca_unmap_user_db(to_mdev(pd->device), &context->uar,
\r
573 context->db_tab, ucmd.db_index);
\r
578 if (context && ib_copy_to_umv_buf(p_umv_buf, &srq->srqn, sizeof (u32))) {
\r
579 mthca_free_srq(to_mdev(pd->device), srq);
\r
584 return &srq->ibsrq;
\r
589 return ERR_PTR(err);
\r
592 static int mthca_destroy_srq(struct ib_srq *srq)
\r
594 struct mthca_ucontext *context;
\r
596 if (srq->ucontext) {
\r
597 context = to_mucontext(srq->ucontext);
\r
599 mthca_unmap_user_db(to_mdev(srq->device), &context->uar,
\r
600 context->db_tab, to_msrq(srq)->db_index);
\r
603 mthca_free_srq(to_mdev(srq->device), to_msrq(srq));
\r
609 static struct ib_qp *mthca_create_qp(struct ib_pd *pd,
\r
610 struct ib_qp_init_attr *init_attr,
\r
611 ci_umv_buf_t* const p_umv_buf)
\r
613 struct ibv_create_qp ucmd = {0};
\r
614 struct mthca_qp *qp = NULL;
\r
615 struct mthca_ucontext *context = NULL;
\r
618 switch (init_attr->qp_type) {
\r
619 case IB_QPT_RELIABLE_CONN:
\r
620 case IB_QPT_UNRELIABLE_CONN:
\r
621 case IB_QPT_UNRELIABLE_DGRM:
\r
624 qp = kmalloc(sizeof *qp, GFP_KERNEL);
\r
630 if (pd->ucontext) {
\r
631 context = to_mucontext(pd->ucontext);
\r
633 if (ib_copy_from_umv_buf(&ucmd, p_umv_buf, sizeof ucmd)) {
\r
638 err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
\r
640 ucmd.sq_db_index, ucmd.sq_db_page, NULL);
\r
644 err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
\r
646 ucmd.rq_db_index, ucmd.rq_db_page, NULL);
\r
650 qp->mr.ibmr.lkey = ucmd.lkey;
\r
651 qp->sq.db_index = ucmd.sq_db_index;
\r
652 qp->rq.db_index = ucmd.rq_db_index;
\r
655 err = mthca_alloc_qp(to_mdev(pd->device), to_mpd(pd),
\r
656 to_mcq(init_attr->send_cq),
\r
657 to_mcq(init_attr->recv_cq),
\r
658 init_attr->qp_type, init_attr->sq_sig_type,
\r
659 &init_attr->cap, qp);
\r
663 goto err_alloc_qp_user;
\r
667 qp->ibqp.qp_num = qp->qpn;
\r
673 /* Don't allow userspace to create special QPs */
\r
674 if (pd->ucontext) {
\r
679 qp = kmalloc(sizeof (struct mthca_sqp), GFP_KERNEL);
\r
685 qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_QP0 ? 0 : 1;
\r
687 err = mthca_alloc_sqp(to_mdev(pd->device), to_mpd(pd),
\r
688 to_mcq(init_attr->send_cq),
\r
689 to_mcq(init_attr->recv_cq),
\r
690 init_attr->sq_sig_type, &init_attr->cap,
\r
691 qp->ibqp.qp_num, init_attr->port_num,
\r
694 goto err_alloc_sqp;
\r
699 /* Don't support raw QPs */
\r
701 goto err_unsupported;
\r
704 init_attr->cap.max_send_wr = qp->sq.max;
\r
705 init_attr->cap.max_recv_wr = qp->rq.max;
\r
706 init_attr->cap.max_send_sge = qp->sq.max_gs;
\r
707 init_attr->cap.max_recv_sge = qp->rq.max_gs;
\r
708 init_attr->cap.max_inline_data = qp->max_inline_data;
\r
715 mthca_unmap_user_db(to_mdev(pd->device),
\r
716 &context->uar, context->db_tab, ucmd.rq_db_index);
\r
719 mthca_unmap_user_db(to_mdev(pd->device),
\r
720 &context->uar, context->db_tab, ucmd.sq_db_index);
\r
721 err_map1: err_copy: err_alloc_sqp:
\r
724 err_mem: err_inval: err_unsupported:
\r
725 return ERR_PTR(err);
\r
728 static int mthca_destroy_qp(struct ib_qp *qp)
\r
730 if (qp->ucontext) {
\r
731 mthca_unmap_user_db(to_mdev(qp->device),
\r
732 &to_mucontext(qp->ucontext)->uar,
\r
733 to_mucontext(qp->ucontext)->db_tab,
\r
734 to_mqp(qp)->sq.db_index);
\r
735 mthca_unmap_user_db(to_mdev(qp->device),
\r
736 &to_mucontext(qp->ucontext)->uar,
\r
737 to_mucontext(qp->ucontext)->db_tab,
\r
738 to_mqp(qp)->rq.db_index);
\r
740 mthca_free_qp(to_mdev(qp->device), to_mqp(qp));
\r
745 static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, int entries,
\r
746 struct ib_ucontext *context,
\r
747 ci_umv_buf_t* const p_umv_buf)
\r
749 struct ibv_create_cq ucmd = {0};
\r
750 struct mthca_cq *cq;
\r
753 void *u_arm_db_page = 0;
\r
755 if (entries < 1 || entries > to_mdev(ibdev)->limits.max_cqes)
\r
756 return ERR_PTR(-EINVAL);
\r
759 if (ib_copy_from_umv_buf(&ucmd, p_umv_buf, sizeof ucmd))
\r
760 return ERR_PTR(-EFAULT);
\r
762 err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
\r
763 to_mucontext(context)->db_tab,
\r
764 ucmd.set_db_index, ucmd.set_db_page, NULL);
\r
766 return ERR_PTR(err);
\r
768 err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
\r
769 to_mucontext(context)->db_tab,
\r
770 ucmd.arm_db_index, ucmd.arm_db_page, NULL);
\r
772 goto err_unmap_set;
\r
774 err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
\r
775 to_mucontext(context)->db_tab,
\r
776 ucmd.u_arm_db_index,
\r
777 (u64)(ULONG_PTR)PAGE_ALIGN(ucmd.u_arm_db_page),
\r
780 goto err_unmap_arm;
\r
783 cq = kmalloc(sizeof *cq, GFP_KERNEL);
\r
790 cq->mr.ibmr.lkey = ucmd.lkey;
\r
791 cq->set_ci_db_index = ucmd.set_db_index;
\r
792 cq->arm_db_index = ucmd.arm_db_index;
\r
793 cq->u_arm_db_index = ucmd.u_arm_db_index;
\r
794 cq->p_u_arm_sn = (int*)((char*)u_arm_db_page + BYTE_OFFSET(ucmd.u_arm_db_page));
\r
797 for (nent = 1; nent <= entries; nent <<= 1)
\r
800 err = mthca_init_cq(to_mdev(ibdev), nent,
\r
801 context ? to_mucontext(context) : NULL,
\r
802 context ? ucmd.mr.pdn : to_mdev(ibdev)->driver_pd.pd_num,
\r
808 struct ibv_create_cq_resp *create_cq_resp = (struct ibv_create_cq_resp *)(ULONG_PTR)p_umv_buf->p_inout_buf;
\r
809 create_cq_resp->cqn = cq->cqn;
\r
812 HCA_PRINT( TRACE_LEVEL_INFORMATION, HCA_DBG_LOW ,
\r
813 ("uctx %p, cq_hndl %p, cq_num %#x, cqe %#x\n",
\r
814 context, &cq->ibcq, cq->cqn, cq->ibcq.cqe ) );
\r
823 mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
\r
824 to_mucontext(context)->db_tab, ucmd.u_arm_db_index);
\r
828 mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
\r
829 to_mucontext(context)->db_tab, ucmd.arm_db_index);
\r
833 mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
\r
834 to_mucontext(context)->db_tab, ucmd.set_db_index);
\r
836 return ERR_PTR(err);
\r
839 static int mthca_destroy_cq(struct ib_cq *cq)
\r
841 if (cq->ucontext) {
\r
842 mthca_unmap_user_db(to_mdev(cq->device),
\r
843 &to_mucontext(cq->ucontext)->uar,
\r
844 to_mucontext(cq->ucontext)->db_tab,
\r
845 to_mcq(cq)->u_arm_db_index);
\r
846 mthca_unmap_user_db(to_mdev(cq->device),
\r
847 &to_mucontext(cq->ucontext)->uar,
\r
848 to_mucontext(cq->ucontext)->db_tab,
\r
849 to_mcq(cq)->arm_db_index);
\r
850 mthca_unmap_user_db(to_mdev(cq->device),
\r
851 &to_mucontext(cq->ucontext)->uar,
\r
852 to_mucontext(cq->ucontext)->db_tab,
\r
853 to_mcq(cq)->set_ci_db_index);
\r
855 mthca_free_cq(to_mdev(cq->device), to_mcq(cq));
\r
864 IN mthca_qp_access_t qp_acl)
\r
866 #define ACL_MTHCA(mfl,ifl) if (qp_acl & mfl) mpt_acl |= ifl
\r
867 mthca_mpt_access_t mpt_acl = 0;
\r
869 ACL_MTHCA(MTHCA_ACCESS_REMOTE_READ,MTHCA_MPT_FLAG_REMOTE_READ);
\r
870 ACL_MTHCA(MTHCA_ACCESS_REMOTE_WRITE,MTHCA_MPT_FLAG_REMOTE_WRITE);
\r
871 ACL_MTHCA(MTHCA_ACCESS_REMOTE_ATOMIC,MTHCA_MPT_FLAG_ATOMIC);
\r
872 ACL_MTHCA(MTHCA_ACCESS_LOCAL_WRITE,MTHCA_MPT_FLAG_LOCAL_WRITE);
\r
874 return (mpt_acl | MTHCA_MPT_FLAG_LOCAL_READ);
\r
877 struct ib_mr *mthca_get_dma_mr(struct ib_pd *pd, mthca_qp_access_t acc)
\r
879 struct mthca_mr *mr;
\r
882 mr = kzalloc(sizeof *mr, GFP_KERNEL);
\r
884 return ERR_PTR(-ENOMEM);
\r
886 err = mthca_mr_alloc_notrans(to_mdev(pd->device),
\r
887 to_mpd(pd)->pd_num,
\r
888 map_qp_mpt(acc), mr);
\r
892 return ERR_PTR(err);
\r
898 static struct ib_mr *mthca_reg_phys_mr(struct ib_pd *pd,
\r
899 struct ib_phys_buf *buffer_list,
\r
901 mthca_qp_access_t acc,
\r
904 struct mthca_mr *mr;
\r
913 /* First check that we have enough alignment */
\r
914 if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK))
\r
915 return ERR_PTR(-EINVAL);
\r
917 if (num_phys_buf > 1 &&
\r
918 ((buffer_list[0].addr + buffer_list[0].size) & ~PAGE_MASK))
\r
919 return ERR_PTR(-EINVAL);
\r
923 for (i = 0; i < num_phys_buf; ++i) {
\r
925 mask |= buffer_list[i].addr;
\r
926 if (i != num_phys_buf - 1)
\r
927 mask |= buffer_list[i].addr + buffer_list[i].size;
\r
929 total_size += buffer_list[i].size;
\r
932 if (mask & ~PAGE_MASK)
\r
933 return ERR_PTR(-EINVAL);
\r
935 /* Find largest page shift we can use to cover buffers */
\r
936 for (shift = PAGE_SHIFT; shift < 31; ++shift)
\r
937 if (num_phys_buf > 1) {
\r
938 if ((1Ui64 << shift) & mask)
\r
941 if (1Ui64 << shift >=
\r
942 buffer_list[0].size +
\r
943 (buffer_list[0].addr & ((1Ui64 << shift) - 1)))
\r
947 buffer_list[0].size += buffer_list[0].addr & ((1Ui64 << shift) - 1);
\r
948 buffer_list[0].addr &= ~0Ui64 << shift;
\r
950 mr = kzalloc(sizeof *mr, GFP_KERNEL);
\r
952 return ERR_PTR(-ENOMEM);
\r
955 for (i = 0; i < num_phys_buf; ++i)
\r
956 npages += (int)((buffer_list[i].size + (1Ui64 << shift) - 1) >> shift);
\r
961 page_list = kmalloc(npages * sizeof *page_list, GFP_KERNEL);
\r
964 return ERR_PTR(-ENOMEM);
\r
968 for (i = 0; i < num_phys_buf; ++i)
\r
970 j < (buffer_list[i].size + (1Ui64 << shift) - 1) >> shift;
\r
972 page_list[n++] = buffer_list[i].addr + ((u64) j << shift);
\r
974 HCA_PRINT( TRACE_LEVEL_VERBOSE ,HCA_DBG_LOW ,("Registering memory at %I64x (iova %I64x) "
\r
975 "in PD %x; shift %d, npages %d.\n",
\r
976 (u64) buffer_list[0].addr,
\r
978 to_mpd(pd)->pd_num,
\r
981 err = mthca_mr_alloc_phys(to_mdev(pd->device),
\r
982 to_mpd(pd)->pd_num,
\r
983 page_list, shift, npages,
\r
984 *iova_start, total_size,
\r
985 map_qp_mpt(acc), mr);
\r
990 return ERR_PTR(err);
\r
997 static struct ib_mr *mthca_reg_virt_mr(struct ib_pd *pd,
\r
998 void* vaddr, uint64_t length, uint64_t hca_va,
\r
999 mthca_qp_access_t acc, boolean_t um_call, boolean_t secure)
\r
1001 struct mthca_dev *dev = to_mdev(pd->device);
\r
1002 struct mthca_mr *mr;
\r
1006 mt_iobuf_t *iobuf_p;
\r
1007 mt_iobuf_iter_t iobuf_iter;
\r
1008 ib_access_t ib_acc;
\r
1011 * Be friendly to WRITE_MTT command and leave two
\r
1012 * empty slots for the index and reserved fields of the mailbox.
\r
1014 int max_buf_list_size = PAGE_SIZE / sizeof (u64) - 2;
\r
1016 HCA_ENTER(HCA_DBG_MEMORY);
\r
1018 mr = kzalloc(sizeof *mr, GFP_KERNEL);
\r
1025 * We ask for writable memory if any access flags other than
\r
1026 * "remote read" are set. "Local write" and "remote write"
\r
1027 * obviously require write access. "Remote atomic" can do
\r
1028 * things like fetch and add, which will modify memory, and
\r
1029 * "MW bind" can change permissions by binding a window.
\r
1032 // try register the buffer
\r
1033 iobuf_p = &mr->iobuf;
\r
1034 iobuf_init( (ULONG_PTR)vaddr, length, um_call, iobuf_p);
\r
1035 ib_acc = (acc & ~MTHCA_ACCESS_REMOTE_READ) ? IB_AC_LOCAL_WRITE : 0;
\r
1036 err = iobuf_register_with_cash( (ULONG_PTR)vaddr, length, um_call,
\r
1037 &ib_acc, iobuf_p );
\r
1040 mr->iobuf_used = TRUE;
\r
1043 mr->mtt = mthca_alloc_mtt(dev, iobuf_p->nr_pages);
\r
1044 if (IS_ERR(mr->mtt)) {
\r
1045 err = PTR_ERR(mr->mtt);
\r
1046 goto err_alloc_mtt;
\r
1049 // allocate buffer_list for writing MTT's
\r
1050 pages = (u64 *) kmalloc(PAGE_SIZE,GFP_KERNEL);
\r
1057 iobuf_iter_init( iobuf_p, &iobuf_iter );
\r
1060 // get up to max_buf_list_size page physical addresses
\r
1061 i = iobuf_get_tpt_seg( iobuf_p, &iobuf_iter, max_buf_list_size, pages );
\r
1065 //TODO: convert physical adresses to dma one's
\r
1067 // write 'i' dma addresses
\r
1068 err = mthca_write_mtt(dev, mr->mtt, n, pages, i);
\r
1070 goto err_write_mtt;
\r
1072 if (n >= iobuf_p->nr_pages)
\r
1076 CL_ASSERT(n == iobuf_p->nr_pages);
\r
1079 err = mthca_mr_alloc(dev, to_mpd(pd)->pd_num, PAGE_SHIFT, hca_va,
\r
1080 length, map_qp_mpt(acc), mr);
\r
1082 goto err_mt_alloc;
\r
1085 if (!pd->ucontext || !secure)
\r
1088 mr->secure_handle = MmSecureVirtualMemory ( vaddr, (SIZE_T)length,
\r
1089 (ib_acc & IB_AC_LOCAL_WRITE) ? PAGE_READWRITE : PAGE_READONLY );
\r
1090 if (mr->secure_handle == NULL) {
\r
1095 __except (EXCEPTION_EXECUTE_HANDLER) {
\r
1096 NTSTATUS Status = GetExceptionCode();
\r
1097 UNUSED_PARAM_WOWPP(Status);
\r
1098 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_MEMORY ,
\r
1099 ("Exception 0x%x on MmSecureVirtualMemory(), addr %p, size %I64d, access %#x\n",
\r
1100 Status, vaddr, length, acc ));
\r
1106 free_page((void*) pages);
\r
1108 HCA_EXIT(HCA_DBG_MEMORY);
\r
1114 free_page((void*) pages);
\r
1116 mthca_free_mtt(dev, mr->mtt);
\r
1118 iobuf_deregister(iobuf_p);
\r
1123 HCA_EXIT(HCA_DBG_MEMORY);
\r
1124 return ERR_PTR(err);
\r
1127 int mthca_dereg_mr(struct ib_mr *mr)
\r
1129 struct mthca_mr *mmr = to_mmr(mr);
\r
1130 struct mthca_dev* dev = to_mdev(mr->device);
\r
1132 if (mmr->secure_handle) {
\r
1134 MmUnsecureVirtualMemory( mmr->secure_handle );
\r
1135 mmr->secure_handle = NULL;
\r
1137 __except (EXCEPTION_EXECUTE_HANDLER) {
\r
1138 NTSTATUS Status = GetExceptionCode();
\r
1139 UNUSED_PARAM_WOWPP(Status);
\r
1140 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_MEMORY ,
\r
1141 ("Exception 0x%x on MmUnsecureVirtualMemory(), addr %I64x, size %I64x, seg_num %d, nr_pages %d\n",
\r
1142 Status, mmr->iobuf.va, (u64)mmr->iobuf.size, mmr->iobuf.seg_num, mmr->iobuf.nr_pages ));
\r
1145 mthca_free_mr(dev, mmr);
\r
1146 if (mmr->iobuf_used)
\r
1147 iobuf_deregister_with_cash(&mmr->iobuf);
\r
1152 static struct ib_fmr *mthca_alloc_fmr(struct ib_pd *pd, mthca_qp_access_t acc,
\r
1153 struct ib_fmr_attr *fmr_attr)
\r
1155 struct mthca_fmr *fmr;
\r
1158 fmr = kzalloc(sizeof *fmr, GFP_KERNEL);
\r
1160 return ERR_PTR(-ENOMEM);
\r
1162 RtlCopyMemory(&fmr->attr, fmr_attr, sizeof *fmr_attr);
\r
1163 err = mthca_fmr_alloc(to_mdev(pd->device), to_mpd(pd)->pd_num,
\r
1164 map_qp_mpt(acc), fmr);
\r
1168 return ERR_PTR(err);
\r
1171 return &fmr->ibfmr;
\r
1174 static int mthca_dealloc_fmr(struct ib_fmr *fmr)
\r
1176 struct mthca_fmr *mfmr = to_mfmr(fmr);
\r
1179 err = mthca_free_fmr(to_mdev(fmr->device), mfmr);
\r
1187 static int mthca_unmap_fmr(struct list_head *fmr_list)
\r
1189 struct ib_fmr *fmr;
\r
1192 struct mthca_dev *mdev = NULL;
\r
1194 list_for_each_entry(fmr, fmr_list, list,struct ib_fmr) {
\r
1195 if (mdev && to_mdev(fmr->device) != mdev)
\r
1197 mdev = to_mdev(fmr->device);
\r
1203 if (mthca_is_memfree(mdev)) {
\r
1204 list_for_each_entry(fmr, fmr_list, list,struct ib_fmr)
\r
1205 mthca_arbel_fmr_unmap(mdev, to_mfmr(fmr));
\r
1209 list_for_each_entry(fmr, fmr_list, list,struct ib_fmr)
\r
1210 mthca_tavor_fmr_unmap(mdev, to_mfmr(fmr));
\r
1212 err = mthca_SYNC_TPT(mdev, &status);
\r
1220 static int mthca_init_node_data(struct mthca_dev *dev)
\r
1222 struct ib_smp *in_mad = NULL;
\r
1223 struct ib_smp *out_mad = NULL;
\r
1224 int err = -ENOMEM;
\r
1227 in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL);
\r
1228 out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
\r
1229 if (!in_mad || !out_mad)
\r
1232 init_query_mad(in_mad);
\r
1233 in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
\r
1235 err = mthca_MAD_IFC(dev, 1, 1,
\r
1236 1, NULL, NULL, in_mad, out_mad,
\r
1245 memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8);
\r
1253 int mthca_register_device(struct mthca_dev *dev)
\r
1257 ret = mthca_init_node_data(dev);
\r
1261 strlcpy(dev->ib_dev.name, "mthca%d", IB_DEVICE_NAME_MAX);
\r
1262 dev->ib_dev.node_type = IB_NODE_CA;
\r
1263 dev->ib_dev.phys_port_cnt = (u8)dev->limits.num_ports;
\r
1264 dev->ib_dev.mdev = dev;
\r
1265 dev->ib_dev.query_device = mthca_query_device;
\r
1266 dev->ib_dev.query_port = mthca_query_port;
\r
1267 dev->ib_dev.modify_port = mthca_modify_port;
\r
1268 dev->ib_dev.query_pkey_chunk = mthca_query_pkey_chunk;
\r
1269 dev->ib_dev.query_gid_chunk = mthca_query_gid_chunk;
\r
1270 dev->ib_dev.alloc_ucontext = mthca_alloc_ucontext;
\r
1271 dev->ib_dev.dealloc_ucontext = mthca_dealloc_ucontext;
\r
1272 dev->ib_dev.alloc_pd = mthca_alloc_pd;
\r
1273 dev->ib_dev.dealloc_pd = mthca_dealloc_pd;
\r
1274 dev->ib_dev.create_ah = mthca_ah_create;
\r
1275 dev->ib_dev.destroy_ah = mthca_ah_destroy;
\r
1277 if (dev->mthca_flags & MTHCA_FLAG_SRQ) {
\r
1278 dev->ib_dev.create_srq = mthca_create_srq;
\r
1279 dev->ib_dev.modify_srq = mthca_modify_srq;
\r
1280 dev->ib_dev.query_srq = mthca_query_srq;
\r
1281 dev->ib_dev.destroy_srq = mthca_destroy_srq;
\r
1283 if (mthca_is_memfree(dev))
\r
1284 dev->ib_dev.post_srq_recv = mthca_arbel_post_srq_recv;
\r
1286 dev->ib_dev.post_srq_recv = mthca_tavor_post_srq_recv;
\r
1289 dev->ib_dev.create_qp = mthca_create_qp;
\r
1290 dev->ib_dev.modify_qp = mthca_modify_qp;
\r
1291 dev->ib_dev.query_qp = mthca_query_qp;
\r
1292 dev->ib_dev.destroy_qp = mthca_destroy_qp;
\r
1293 dev->ib_dev.create_cq = mthca_create_cq;
\r
1294 dev->ib_dev.destroy_cq = mthca_destroy_cq;
\r
1295 dev->ib_dev.poll_cq = mthca_poll_cq;
\r
1296 dev->ib_dev.get_dma_mr = mthca_get_dma_mr;
\r
1297 dev->ib_dev.reg_phys_mr = mthca_reg_phys_mr;
\r
1298 dev->ib_dev.reg_virt_mr = mthca_reg_virt_mr;
\r
1299 dev->ib_dev.dereg_mr = mthca_dereg_mr;
\r
1301 if (dev->mthca_flags & MTHCA_FLAG_FMR) {
\r
1302 dev->ib_dev.alloc_fmr = mthca_alloc_fmr;
\r
1303 dev->ib_dev.unmap_fmr = mthca_unmap_fmr;
\r
1304 dev->ib_dev.dealloc_fmr = mthca_dealloc_fmr;
\r
1305 if (mthca_is_memfree(dev))
\r
1306 dev->ib_dev.map_phys_fmr = mthca_arbel_map_phys_fmr;
\r
1308 dev->ib_dev.map_phys_fmr = mthca_tavor_map_phys_fmr;
\r
1311 dev->ib_dev.attach_mcast = mthca_multicast_attach;
\r
1312 dev->ib_dev.detach_mcast = mthca_multicast_detach;
\r
1313 dev->ib_dev.process_mad = mthca_process_mad;
\r
1315 if (mthca_is_memfree(dev)) {
\r
1316 dev->ib_dev.req_notify_cq = mthca_arbel_arm_cq;
\r
1317 dev->ib_dev.post_send = mthca_arbel_post_send;
\r
1318 dev->ib_dev.post_recv = mthca_arbel_post_recv;
\r
1320 dev->ib_dev.req_notify_cq = mthca_tavor_arm_cq;
\r
1321 dev->ib_dev.post_send = mthca_tavor_post_send;
\r
1322 dev->ib_dev.post_recv = mthca_tavor_post_recv;
\r
1325 KeInitializeMutex(&dev->cap_mask_mutex, 0);
\r
1327 ret = ib_register_device(&dev->ib_dev);
\r
1331 mthca_start_catas_poll(dev);
\r
1336 void mthca_unregister_device(struct mthca_dev *dev)
\r
1338 mthca_stop_catas_poll(dev);
\r
1339 ib_unregister_device(&dev->ib_dev);
\r