2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
\r
3 * Copyright (c) 2005 Cisco Systems. All rights reserved.
\r
4 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
\r
5 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
\r
7 * This software is available to you under a choice of one of two
\r
8 * licenses. You may choose to be licensed under the terms of the GNU
\r
9 * General Public License (GPL) Version 2, available from the file
\r
10 * COPYING in the main directory of this source tree, or the
\r
11 * OpenIB.org BSD license below:
\r
13 * Redistribution and use in source and binary forms, with or
\r
14 * without modification, are permitted provided that the following
\r
15 * conditions are met:
\r
17 * - Redistributions of source code must retain the above
\r
18 * copyright notice, this list of conditions and the following
\r
21 * - Redistributions in binary form must reproduce the above
\r
22 * copyright notice, this list of conditions and the following
\r
23 * disclaimer in the documentation and/or other materials
\r
24 * provided with the distribution.
\r
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
\r
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
\r
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
\r
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
\r
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
\r
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
\r
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
\r
38 #include <ib_verbs.h>
\r
39 #include <ib_cache.h>
\r
40 #include <ib_pack.h>
\r
42 #include "mthca_dev.h"
\r
43 #if defined(EVENT_TRACING)
\r
47 #include "mthca_qp.tmh"
\r
49 #include "mthca_cmd.h"
\r
50 #include "mthca_memfree.h"
\r
51 #include "mthca_wqe.h"
\r
55 MTHCA_MAX_DIRECT_QP_SIZE = 4 * PAGE_SIZE,
\r
56 MTHCA_ACK_REQ_FREQ = 10,
\r
57 MTHCA_FLIGHT_LIMIT = 9,
\r
58 MTHCA_UD_HEADER_SIZE = 72, /* largest UD header possible */
\r
59 MTHCA_INLINE_HEADER_SIZE = 4, /* data segment overhead for inline */
\r
60 MTHCA_INLINE_CHUNK_SIZE = 16 /* inline data segment chunk */
\r
64 MTHCA_QP_STATE_RST = 0,
\r
65 MTHCA_QP_STATE_INIT = 1,
\r
66 MTHCA_QP_STATE_RTR = 2,
\r
67 MTHCA_QP_STATE_RTS = 3,
\r
68 MTHCA_QP_STATE_SQE = 4,
\r
69 MTHCA_QP_STATE_SQD = 5,
\r
70 MTHCA_QP_STATE_ERR = 6,
\r
71 MTHCA_QP_STATE_DRAINING = 7
\r
75 MTHCA_QP_ST_RC = 0x0,
\r
76 MTHCA_QP_ST_UC = 0x1,
\r
77 MTHCA_QP_ST_RD = 0x2,
\r
78 MTHCA_QP_ST_UD = 0x3,
\r
79 MTHCA_QP_ST_MLX = 0x7
\r
83 MTHCA_QP_PM_MIGRATED = 0x3,
\r
84 MTHCA_QP_PM_ARMED = 0x0,
\r
85 MTHCA_QP_PM_REARM = 0x1
\r
89 /* qp_context flags */
\r
90 MTHCA_QP_BIT_DE = 1 << 8,
\r
92 MTHCA_QP_BIT_SRE = 1 << 15,
\r
93 MTHCA_QP_BIT_SWE = 1 << 14,
\r
94 MTHCA_QP_BIT_SAE = 1 << 13,
\r
95 MTHCA_QP_BIT_SIC = 1 << 4,
\r
96 MTHCA_QP_BIT_SSC = 1 << 3,
\r
98 MTHCA_QP_BIT_RRE = 1 << 15,
\r
99 MTHCA_QP_BIT_RWE = 1 << 14,
\r
100 MTHCA_QP_BIT_RAE = 1 << 13,
\r
101 MTHCA_QP_BIT_RIC = 1 << 4,
\r
102 MTHCA_QP_BIT_RSC = 1 << 3
\r
105 #pragma pack(push,1)
\r
106 struct mthca_qp_path {
\r
115 __be32 sl_tclass_flowlabel;
\r
119 struct mthca_qp_context {
\r
121 __be32 tavor_sched_queue; /* Reserved on Arbel */
\r
123 u8 rq_size_stride; /* Reserved on Tavor */
\r
124 u8 sq_size_stride; /* Reserved on Tavor */
\r
125 u8 rlkey_arbel_sched_queue; /* Reserved on Tavor */
\r
130 struct mthca_qp_path pri_path;
\r
131 struct mthca_qp_path alt_path;
\r
138 __be32 next_send_psn;
\r
140 __be32 snd_wqe_base_l; /* Next send WQE on Tavor */
\r
141 __be32 snd_db_index; /* (debugging only entries) */
\r
142 __be32 last_acked_psn;
\r
145 __be32 rnr_nextrecvpsn;
\r
146 __be32 ra_buff_indx;
\r
148 __be32 rcv_wqe_base_l; /* Next recv WQE on Tavor */
\r
149 __be32 rcv_db_index; /* (debugging only entries) */
\r
153 __be16 rq_wqe_counter; /* reserved on Tavor */
\r
154 __be16 sq_wqe_counter; /* reserved on Tavor */
\r
158 struct mthca_qp_param {
\r
159 __be32 opt_param_mask;
\r
161 struct mthca_qp_context context;
\r
167 MTHCA_QP_OPTPAR_ALT_ADDR_PATH = 1 << 0,
\r
168 MTHCA_QP_OPTPAR_RRE = 1 << 1,
\r
169 MTHCA_QP_OPTPAR_RAE = 1 << 2,
\r
170 MTHCA_QP_OPTPAR_RWE = 1 << 3,
\r
171 MTHCA_QP_OPTPAR_PKEY_INDEX = 1 << 4,
\r
172 MTHCA_QP_OPTPAR_Q_KEY = 1 << 5,
\r
173 MTHCA_QP_OPTPAR_RNR_TIMEOUT = 1 << 6,
\r
174 MTHCA_QP_OPTPAR_PRIMARY_ADDR_PATH = 1 << 7,
\r
175 MTHCA_QP_OPTPAR_SRA_MAX = 1 << 8,
\r
176 MTHCA_QP_OPTPAR_RRA_MAX = 1 << 9,
\r
177 MTHCA_QP_OPTPAR_PM_STATE = 1 << 10,
\r
178 MTHCA_QP_OPTPAR_PORT_NUM = 1 << 11,
\r
179 MTHCA_QP_OPTPAR_RETRY_COUNT = 1 << 12,
\r
180 MTHCA_QP_OPTPAR_ALT_RNR_RETRY = 1 << 13,
\r
181 MTHCA_QP_OPTPAR_ACK_TIMEOUT = 1 << 14,
\r
182 MTHCA_QP_OPTPAR_RNR_RETRY = 1 << 15,
\r
183 MTHCA_QP_OPTPAR_SCHED_QUEUE = 1 << 16
\r
186 static const u8 mthca_opcode[] = {
\r
187 MTHCA_OPCODE_RDMA_WRITE,
\r
188 MTHCA_OPCODE_RDMA_WRITE_IMM,
\r
190 MTHCA_OPCODE_SEND_IMM,
\r
191 MTHCA_OPCODE_RDMA_READ,
\r
192 MTHCA_OPCODE_ATOMIC_CS,
\r
193 MTHCA_OPCODE_ATOMIC_FA
\r
197 enum { RC, UC, UD, RD, RDEE, MLX, NUM_TRANS };
\r
199 static struct _state_table {
\r
201 u32 req_param[NUM_TRANS];
\r
202 u32 opt_param[NUM_TRANS];
\r
203 } state_table[IBQPS_ERR + 1][IBQPS_ERR + 1]= {0};
\r
205 static void fill_state_table()
\r
207 struct _state_table *t;
\r
208 RtlZeroMemory( state_table, sizeof(state_table) );
\r
211 t = &state_table[IBQPS_RESET][0];
\r
212 t[IBQPS_RESET].trans = MTHCA_TRANS_ANY2RST;
\r
213 t[IBQPS_ERR].trans = MTHCA_TRANS_ANY2ERR;
\r
215 t[IBQPS_INIT].trans = MTHCA_TRANS_RST2INIT;
\r
216 t[IBQPS_INIT].req_param[UD] = IB_QP_PKEY_INDEX |IB_QP_PORT |IB_QP_QKEY;
\r
217 t[IBQPS_INIT].req_param[UC] = IB_QP_PKEY_INDEX |IB_QP_PORT |IB_QP_ACCESS_FLAGS;
\r
218 t[IBQPS_INIT].req_param[RC] = IB_QP_PKEY_INDEX |IB_QP_PORT |IB_QP_ACCESS_FLAGS;
\r
219 t[IBQPS_INIT].req_param[MLX] = IB_QP_PKEY_INDEX |IB_QP_QKEY;
\r
220 t[IBQPS_INIT].opt_param[MLX] = IB_QP_PORT;
\r
223 t = &state_table[IBQPS_INIT][0];
\r
224 t[IBQPS_RESET].trans = MTHCA_TRANS_ANY2RST;
\r
225 t[IBQPS_ERR].trans = MTHCA_TRANS_ANY2ERR;
\r
227 t[IBQPS_INIT].trans = MTHCA_TRANS_INIT2INIT;
\r
228 t[IBQPS_INIT].opt_param[UD] = IB_QP_PKEY_INDEX |IB_QP_PORT |IB_QP_QKEY;
\r
229 t[IBQPS_INIT].opt_param[UC] = IB_QP_PKEY_INDEX |IB_QP_PORT |IB_QP_ACCESS_FLAGS;
\r
230 t[IBQPS_INIT].opt_param[RC] = IB_QP_PKEY_INDEX |IB_QP_PORT |IB_QP_ACCESS_FLAGS;
\r
231 t[IBQPS_INIT].opt_param[MLX] = IB_QP_PKEY_INDEX |IB_QP_QKEY;
\r
233 t[IBQPS_RTR].trans = MTHCA_TRANS_INIT2RTR;
\r
234 t[IBQPS_RTR].req_param[UC] =
\r
235 IB_QP_AV |IB_QP_PATH_MTU |IB_QP_DEST_QPN |IB_QP_RQ_PSN;
\r
236 t[IBQPS_RTR].req_param[RC] =
\r
237 IB_QP_AV |IB_QP_PATH_MTU |IB_QP_DEST_QPN |IB_QP_RQ_PSN |IB_QP_MAX_DEST_RD_ATOMIC |IB_QP_MIN_RNR_TIMER;
\r
238 t[IBQPS_RTR].opt_param[UD] = IB_QP_PKEY_INDEX |IB_QP_QKEY;
\r
239 t[IBQPS_RTR].opt_param[UC] = IB_QP_PKEY_INDEX |IB_QP_ALT_PATH |IB_QP_ACCESS_FLAGS;
\r
240 t[IBQPS_RTR].opt_param[RC] = IB_QP_PKEY_INDEX |IB_QP_ALT_PATH |IB_QP_ACCESS_FLAGS;
\r
241 t[IBQPS_RTR].opt_param[MLX] = IB_QP_PKEY_INDEX |IB_QP_QKEY;
\r
244 t = &state_table[IBQPS_RTR][0];
\r
245 t[IBQPS_RESET].trans = MTHCA_TRANS_ANY2RST;
\r
246 t[IBQPS_ERR].trans = MTHCA_TRANS_ANY2ERR;
\r
248 t[IBQPS_RTS].trans = MTHCA_TRANS_RTR2RTS;
\r
249 t[IBQPS_RTS].req_param[UD] = IB_QP_SQ_PSN;
\r
250 t[IBQPS_RTS].req_param[UC] = IB_QP_SQ_PSN;
\r
251 t[IBQPS_RTS].req_param[RC] =
\r
252 IB_QP_TIMEOUT |IB_QP_RETRY_CNT |IB_QP_RNR_RETRY |IB_QP_SQ_PSN |IB_QP_MAX_QP_RD_ATOMIC;
\r
253 t[IBQPS_RTS].req_param[MLX] = IB_QP_SQ_PSN;
\r
254 t[IBQPS_RTS].opt_param[UD] = IB_QP_CUR_STATE |IB_QP_QKEY;
\r
255 t[IBQPS_RTS].opt_param[UC] =
\r
256 IB_QP_CUR_STATE |IB_QP_ALT_PATH |IB_QP_ACCESS_FLAGS |IB_QP_PATH_MIG_STATE;
\r
257 t[IBQPS_RTS].opt_param[RC] = IB_QP_CUR_STATE |IB_QP_ALT_PATH |
\r
258 IB_QP_ACCESS_FLAGS |IB_QP_MIN_RNR_TIMER |IB_QP_PATH_MIG_STATE;
\r
259 t[IBQPS_RTS].opt_param[MLX] = IB_QP_CUR_STATE |IB_QP_QKEY;
\r
262 t = &state_table[IBQPS_RTS][0];
\r
263 t[IBQPS_RESET].trans = MTHCA_TRANS_ANY2RST;
\r
264 t[IBQPS_ERR].trans = MTHCA_TRANS_ANY2ERR;
\r
266 t[IBQPS_RTS].trans = MTHCA_TRANS_RTS2RTS;
\r
267 t[IBQPS_RTS].opt_param[UD] = IB_QP_CUR_STATE |IB_QP_QKEY;
\r
268 t[IBQPS_RTS].opt_param[UC] = IB_QP_ACCESS_FLAGS |IB_QP_ALT_PATH |IB_QP_PATH_MIG_STATE;
\r
269 t[IBQPS_RTS].opt_param[RC] = IB_QP_ACCESS_FLAGS |
\r
270 IB_QP_ALT_PATH |IB_QP_PATH_MIG_STATE |IB_QP_MIN_RNR_TIMER;
\r
271 t[IBQPS_RTS].opt_param[MLX] = IB_QP_CUR_STATE |IB_QP_QKEY;
\r
273 t[IBQPS_SQD].trans = MTHCA_TRANS_RTS2SQD;
\r
274 t[IBQPS_SQD].opt_param[UD] = IB_QP_EN_SQD_ASYNC_NOTIFY;
\r
275 t[IBQPS_SQD].opt_param[UC] = IB_QP_EN_SQD_ASYNC_NOTIFY;
\r
276 t[IBQPS_SQD].opt_param[RC] = IB_QP_EN_SQD_ASYNC_NOTIFY;
\r
277 t[IBQPS_SQD].opt_param[MLX] = IB_QP_EN_SQD_ASYNC_NOTIFY;
\r
280 t = &state_table[IBQPS_SQD][0];
\r
281 t[IBQPS_RESET].trans = MTHCA_TRANS_ANY2RST;
\r
282 t[IBQPS_ERR].trans = MTHCA_TRANS_ANY2ERR;
\r
284 t[IBQPS_RTS].trans = MTHCA_TRANS_SQD2RTS;
\r
285 t[IBQPS_RTS].opt_param[UD] = IB_QP_CUR_STATE |IB_QP_QKEY;
\r
286 t[IBQPS_RTS].opt_param[UC] = IB_QP_CUR_STATE |
\r
287 IB_QP_ALT_PATH |IB_QP_ACCESS_FLAGS |IB_QP_PATH_MIG_STATE;
\r
288 t[IBQPS_RTS].opt_param[RC] = IB_QP_CUR_STATE |IB_QP_ALT_PATH |
\r
289 IB_QP_ACCESS_FLAGS |IB_QP_MIN_RNR_TIMER |IB_QP_PATH_MIG_STATE;
\r
290 t[IBQPS_RTS].opt_param[MLX] = IB_QP_CUR_STATE |IB_QP_QKEY;
\r
292 t[IBQPS_SQD].trans = MTHCA_TRANS_SQD2SQD;
\r
293 t[IBQPS_SQD].opt_param[UD] = IB_QP_PKEY_INDEX |IB_QP_QKEY;
\r
294 t[IBQPS_SQD].opt_param[UC] = IB_QP_AV | IB_QP_CUR_STATE |
\r
295 IB_QP_ALT_PATH |IB_QP_ACCESS_FLAGS |IB_QP_PKEY_INDEX |IB_QP_PATH_MIG_STATE;
\r
296 t[IBQPS_SQD].opt_param[RC] = IB_QP_AV |IB_QP_TIMEOUT |IB_QP_RETRY_CNT |IB_QP_RNR_RETRY |
\r
297 IB_QP_MAX_QP_RD_ATOMIC |IB_QP_MAX_DEST_RD_ATOMIC |IB_QP_CUR_STATE |IB_QP_ALT_PATH |
\r
298 IB_QP_ACCESS_FLAGS |IB_QP_PKEY_INDEX |IB_QP_MIN_RNR_TIMER |IB_QP_PATH_MIG_STATE;
\r
299 t[IBQPS_SQD].opt_param[MLX] = IB_QP_PKEY_INDEX |IB_QP_QKEY;
\r
302 t = &state_table[IBQPS_SQE][0];
\r
303 t[IBQPS_RESET].trans = MTHCA_TRANS_ANY2RST;
\r
304 t[IBQPS_ERR].trans = MTHCA_TRANS_ANY2ERR;
\r
306 t[IBQPS_RTS].trans = MTHCA_TRANS_SQERR2RTS;
\r
307 t[IBQPS_RTS].opt_param[UD] = IB_QP_CUR_STATE |IB_QP_QKEY;
\r
308 t[IBQPS_RTS].opt_param[UC] = IB_QP_CUR_STATE | IB_QP_ACCESS_FLAGS;
\r
309 // t[IBQPS_RTS].opt_param[RC] = IB_QP_CUR_STATE |IB_QP_MIN_RNR_TIMER;
\r
310 t[IBQPS_RTS].opt_param[MLX] = IB_QP_CUR_STATE |IB_QP_QKEY;
\r
313 t = &state_table[IBQPS_ERR][0];
\r
314 t[IBQPS_RESET].trans = MTHCA_TRANS_ANY2RST;
\r
315 t[IBQPS_ERR].trans = MTHCA_TRANS_ANY2ERR;
\r
320 static int is_sqp(struct mthca_dev *dev, struct mthca_qp *qp)
\r
322 return qp->qpn >= (u32)dev->qp_table.sqp_start &&
\r
323 qp->qpn <= (u32)dev->qp_table.sqp_start + 3;
\r
326 static int is_qp0(struct mthca_dev *dev, struct mthca_qp *qp)
\r
328 return qp->qpn >= (u32)dev->qp_table.sqp_start &&
\r
329 qp->qpn <= (u32)(dev->qp_table.sqp_start + 1);
\r
333 static void dump_wqe(u32 print_lvl, u32 *wqe_ptr , struct mthca_qp *qp_ptr)
\r
335 __be32 *wqe = wqe_ptr;
\r
337 UNUSED_PARAM_WOWPP(qp_ptr);
\r
338 UNUSED_PARAM_WOWPP(print_lvl);
\r
340 (void) wqe; /* avoid warning if mthca_dbg compiled away... */
\r
341 HCA_PRINT(print_lvl,HCA_DBG_QP,("WQE contents QPN 0x%06x \n",qp_ptr->qpn));
\r
342 HCA_PRINT(print_lvl,HCA_DBG_QP,("WQE contents [%02x] %08x %08x %08x %08x \n",0
\r
343 , cl_ntoh32(wqe[0]), cl_ntoh32(wqe[1]), cl_ntoh32(wqe[2]), cl_ntoh32(wqe[3])));
\r
344 HCA_PRINT(print_lvl,HCA_DBG_QP,("WQE contents [%02x] %08x %08x %08x %08x \n",4
\r
345 , cl_ntoh32(wqe[4]), cl_ntoh32(wqe[5]), cl_ntoh32(wqe[6]), cl_ntoh32(wqe[7])));
\r
346 HCA_PRINT(print_lvl,HCA_DBG_QP,("WQE contents [%02x] %08x %08x %08x %08x \n",8
\r
347 , cl_ntoh32(wqe[8]), cl_ntoh32(wqe[9]), cl_ntoh32(wqe[10]), cl_ntoh32(wqe[11])));
\r
348 HCA_PRINT(print_lvl,HCA_DBG_QP,("WQE contents [%02x] %08x %08x %08x %08x \n",12
\r
349 , cl_ntoh32(wqe[12]), cl_ntoh32(wqe[13]), cl_ntoh32(wqe[14]), cl_ntoh32(wqe[15])));
\r
354 static void *get_recv_wqe(struct mthca_qp *qp, int n)
\r
357 return (u8*)qp->queue.direct.page + (n << qp->rq.wqe_shift);
\r
359 return (u8*)qp->queue.page_list[(n << qp->rq.wqe_shift) >> PAGE_SHIFT].page +
\r
360 ((n << qp->rq.wqe_shift) & (PAGE_SIZE - 1));
\r
363 static void *get_send_wqe(struct mthca_qp *qp, int n)
\r
366 return (u8*)qp->queue.direct.page + qp->send_wqe_offset +
\r
367 (n << qp->sq.wqe_shift);
\r
369 return (u8*)qp->queue.page_list[(qp->send_wqe_offset +
\r
370 (n << qp->sq.wqe_shift)) >>
\r
372 ((qp->send_wqe_offset + (n << qp->sq.wqe_shift)) &
\r
376 static void mthca_wq_init(struct mthca_wq *wq)
\r
378 spin_lock_init(&wq->lock);
\r
380 wq->last_comp = wq->max - 1;
\r
385 void mthca_qp_event(struct mthca_dev *dev, u32 qpn,
\r
386 enum ib_event_type event_type, u8 vendor_code)
\r
388 struct mthca_qp *qp;
\r
389 ib_event_rec_t event;
\r
390 SPIN_LOCK_PREP(lh);
\r
392 spin_lock(&dev->qp_table.lock, &lh);
\r
393 qp = mthca_array_get(&dev->qp_table.qp, qpn & (dev->limits.num_qps - 1));
\r
395 atomic_inc(&qp->refcount);
\r
399 HCA_PRINT(TRACE_LEVEL_WARNING,HCA_DBG_QP,("QP %06x Async event for bogus \n", qpn));
\r
403 event.type = event_type;
\r
404 event.context = qp->ibqp.qp_context;
\r
405 event.vendor_specific = vendor_code;
\r
406 HCA_PRINT(TRACE_LEVEL_WARNING,HCA_DBG_QP,("QP %06x Async event event_type 0x%x vendor_code 0x%x\n",
\r
407 qpn,event_type,vendor_code));
\r
408 qp->ibqp.event_handler(&event);
\r
410 if (atomic_dec_and_test(&qp->refcount))
\r
411 wake_up(&qp->wait);
\r
414 static int to_mthca_state(enum ib_qp_state ib_state)
\r
416 switch (ib_state) {
\r
417 case IBQPS_RESET: return MTHCA_QP_STATE_RST;
\r
418 case IBQPS_INIT: return MTHCA_QP_STATE_INIT;
\r
419 case IBQPS_RTR: return MTHCA_QP_STATE_RTR;
\r
420 case IBQPS_RTS: return MTHCA_QP_STATE_RTS;
\r
421 case IBQPS_SQD: return MTHCA_QP_STATE_SQD;
\r
422 case IBQPS_SQE: return MTHCA_QP_STATE_SQE;
\r
423 case IBQPS_ERR: return MTHCA_QP_STATE_ERR;
\r
424 default: return -1;
\r
428 static int to_mthca_st(int transport)
\r
430 switch (transport) {
\r
431 case RC: return MTHCA_QP_ST_RC;
\r
432 case UC: return MTHCA_QP_ST_UC;
\r
433 case UD: return MTHCA_QP_ST_UD;
\r
434 case RD: return MTHCA_QP_ST_RD;
\r
435 case MLX: return MTHCA_QP_ST_MLX;
\r
436 default: return -1;
\r
440 static inline enum ib_qp_state to_ib_qp_state(int mthca_state)
\r
442 switch (mthca_state) {
\r
443 case MTHCA_QP_STATE_RST: return IBQPS_RESET;
\r
444 case MTHCA_QP_STATE_INIT: return IBQPS_INIT;
\r
445 case MTHCA_QP_STATE_RTR: return IBQPS_RTR;
\r
446 case MTHCA_QP_STATE_RTS: return IBQPS_RTS;
\r
447 case MTHCA_QP_STATE_SQD: return IBQPS_SQD;
\r
448 case MTHCA_QP_STATE_DRAINING: return IBQPS_SQD;
\r
449 case MTHCA_QP_STATE_SQE: return IBQPS_SQE;
\r
450 case MTHCA_QP_STATE_ERR: return IBQPS_ERR;
\r
451 default: return -1;
\r
455 static inline enum ib_mig_state to_ib_mig_state(int mthca_mig_state)
\r
457 switch (mthca_mig_state) {
\r
458 case 0: return IB_MIG_ARMED;
\r
459 case 1: return IB_MIG_REARM;
\r
460 case 3: return IB_MIG_MIGRATED;
\r
461 default: return -1;
\r
465 static int to_ib_qp_access_flags(int mthca_flags)
\r
469 if (mthca_flags & MTHCA_QP_BIT_RRE)
\r
470 ib_flags |= MTHCA_ACCESS_REMOTE_READ;
\r
471 if (mthca_flags & MTHCA_QP_BIT_RWE)
\r
472 ib_flags |= MTHCA_ACCESS_REMOTE_WRITE;
\r
473 if (mthca_flags & MTHCA_QP_BIT_RAE)
\r
474 ib_flags |= MTHCA_ACCESS_REMOTE_ATOMIC;
\r
479 static void to_ib_ah_attr(struct mthca_dev *dev, struct ib_ah_attr *ib_ah_attr,
\r
480 struct mthca_qp_path *path)
\r
482 memset(ib_ah_attr, 0, sizeof *ib_ah_attr);
\r
483 ib_ah_attr->port_num = (u8)((cl_ntoh32(path->port_pkey) >> 24) & 0x3);
\r
485 if (ib_ah_attr->port_num == 0 || ib_ah_attr->port_num > dev->limits.num_ports)
\r
488 ib_ah_attr->dlid = cl_ntoh16(path->rlid);
\r
489 ib_ah_attr->sl = (u8)(cl_ntoh32(path->sl_tclass_flowlabel) >> 28);
\r
490 ib_ah_attr->src_path_bits = path->g_mylmc & 0x7f;
\r
491 //TODO: work around: set always full speed - really, it's much more complicate
\r
492 ib_ah_attr->static_rate = 0;
\r
493 ib_ah_attr->ah_flags = (path->g_mylmc & (1 << 7)) ? IB_AH_GRH : 0;
\r
494 if (ib_ah_attr->ah_flags) {
\r
495 ib_ah_attr->grh.sgid_index = (u8)(path->mgid_index & (dev->limits.gid_table_len - 1));
\r
496 ib_ah_attr->grh.hop_limit = path->hop_limit;
\r
497 ib_ah_attr->grh.traffic_class =
\r
498 (u8)((cl_ntoh32(path->sl_tclass_flowlabel) >> 20) & 0xff);
\r
499 ib_ah_attr->grh.flow_label =
\r
500 cl_ntoh32(path->sl_tclass_flowlabel) & 0xfffff;
\r
501 memcpy(ib_ah_attr->grh.dgid.raw,
\r
502 path->rgid, sizeof ib_ah_attr->grh.dgid.raw);
\r
506 int mthca_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
\r
507 struct ib_qp_init_attr *qp_init_attr)
\r
509 struct mthca_dev *dev = to_mdev(ibqp->device);
\r
510 struct mthca_qp *qp = to_mqp(ibqp);
\r
512 struct mthca_mailbox *mailbox = NULL;
\r
513 struct mthca_qp_param *qp_param;
\r
514 struct mthca_qp_context *context;
\r
518 UNUSED_PARAM(qp_attr_mask);
\r
520 down( &qp->mutex );
\r
522 if (qp->state == IBQPS_RESET) {
\r
523 qp_attr->qp_state = IBQPS_RESET;
\r
527 mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
\r
528 if (IS_ERR(mailbox)) {
\r
529 err = PTR_ERR(mailbox);
\r
533 err = mthca_QUERY_QP(dev, qp->qpn, 0, mailbox, &status);
\r
537 HCA_PRINT(TRACE_LEVEL_WARNING ,HCA_DBG_QP,
\r
538 ("QUERY_QP returned status %02x\n", status));
\r
543 qp_param = mailbox->buf;
\r
544 context = &qp_param->context;
\r
545 mthca_state = cl_ntoh32(context->flags) >> 28;
\r
547 qp->state = to_ib_qp_state(mthca_state);
\r
548 qp_attr->qp_state = qp->state;
\r
549 qp_attr->path_mtu = context->mtu_msgmax >> 5;
\r
550 qp_attr->path_mig_state =
\r
551 to_ib_mig_state((cl_ntoh32(context->flags) >> 11) & 0x3);
\r
552 qp_attr->qkey = cl_ntoh32(context->qkey);
\r
553 qp_attr->rq_psn = cl_ntoh32(context->rnr_nextrecvpsn) & 0xffffff;
\r
554 qp_attr->sq_psn = cl_ntoh32(context->next_send_psn) & 0xffffff;
\r
555 qp_attr->dest_qp_num = cl_ntoh32(context->remote_qpn) & 0xffffff;
\r
556 qp_attr->qp_access_flags =
\r
557 to_ib_qp_access_flags(cl_ntoh32(context->params2));
\r
559 if (qp->transport == RC || qp->transport == UC) {
\r
560 to_ib_ah_attr(dev, &qp_attr->ah_attr, &context->pri_path);
\r
561 to_ib_ah_attr(dev, &qp_attr->alt_ah_attr, &context->alt_path);
\r
562 qp_attr->alt_pkey_index =
\r
563 (u16)(cl_ntoh32(context->alt_path.port_pkey) & 0x7f);
\r
564 qp_attr->alt_port_num = qp_attr->alt_ah_attr.port_num;
\r
567 qp_attr->pkey_index = (u16)(cl_ntoh32(context->pri_path.port_pkey) & 0x7f);
\r
568 qp_attr->port_num =
\r
569 (u8)((cl_ntoh32(context->pri_path.port_pkey) >> 24) & 0x3);
\r
571 /* qp_attr->en_sqd_async_notify is only applicable in modify qp */
\r
572 qp_attr->sq_draining = (u8)(mthca_state == MTHCA_QP_STATE_DRAINING);
\r
574 qp_attr->max_rd_atomic = (u8)(1 << ((cl_ntoh32(context->params1) >> 21) & 0x7));
\r
576 qp_attr->max_dest_rd_atomic =
\r
577 (u8)(1 << ((cl_ntoh32(context->params2) >> 21) & 0x7));
\r
578 qp_attr->min_rnr_timer =
\r
579 (u8)((cl_ntoh32(context->rnr_nextrecvpsn) >> 24) & 0x1f);
\r
580 qp_attr->timeout = context->pri_path.ackto >> 3;
\r
581 qp_attr->retry_cnt = (u8)((cl_ntoh32(context->params1) >> 16) & 0x7);
\r
582 qp_attr->rnr_retry = context->pri_path.rnr_retry >> 5;
\r
583 qp_attr->alt_timeout = context->alt_path.ackto >> 3;
\r
586 qp_attr->cur_qp_state = qp_attr->qp_state;
\r
587 qp_attr->cap.max_send_wr = qp->sq.max;
\r
588 qp_attr->cap.max_recv_wr = qp->rq.max;
\r
589 qp_attr->cap.max_send_sge = qp->sq.max_gs;
\r
590 qp_attr->cap.max_recv_sge = qp->rq.max_gs;
\r
591 qp_attr->cap.max_inline_data = qp->max_inline_data;
\r
593 qp_init_attr->cap = qp_attr->cap;
\r
596 mthca_free_mailbox(dev, mailbox);
\r
603 static void store_attrs(struct mthca_sqp *sqp, struct ib_qp_attr *attr,
\r
606 if (attr_mask & IB_QP_PKEY_INDEX)
\r
607 sqp->pkey_index = attr->pkey_index;
\r
608 if (attr_mask & IB_QP_QKEY)
\r
609 sqp->qkey = attr->qkey;
\r
610 if (attr_mask & IB_QP_SQ_PSN)
\r
611 sqp->send_psn = attr->sq_psn;
\r
614 static void init_port(struct mthca_dev *dev, int port)
\r
618 struct mthca_init_ib_param param;
\r
620 RtlZeroMemory(¶m, sizeof param);
\r
622 param.port_width = dev->limits.port_width_cap;
\r
623 param.vl_cap = dev->limits.vl_cap;
\r
624 param.mtu_cap = dev->limits.mtu_cap;
\r
625 param.gid_cap = (u16)dev->limits.gid_table_len;
\r
626 param.pkey_cap = (u16)dev->limits.pkey_table_len;
\r
628 err = mthca_INIT_IB(dev, ¶m, port, &status);
\r
630 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP ,("INIT_IB failed, return code %d.\n", err));
\r
632 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP ,("INIT_IB returned status %02x.\n", status));
\r
636 static __be32 get_hw_access_flags(struct mthca_qp *qp, struct ib_qp_attr *attr,
\r
641 u32 hw_access_flags = 0;
\r
643 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
\r
644 dest_rd_atomic = attr->max_dest_rd_atomic;
\r
646 dest_rd_atomic = qp->resp_depth;
\r
648 if (attr_mask & IB_QP_ACCESS_FLAGS)
\r
649 access_flags = attr->qp_access_flags;
\r
651 access_flags = qp->atomic_rd_en;
\r
653 if (!dest_rd_atomic)
\r
654 access_flags &= MTHCA_ACCESS_REMOTE_WRITE;
\r
656 if (access_flags & MTHCA_ACCESS_REMOTE_READ)
\r
657 hw_access_flags |= MTHCA_QP_BIT_RRE;
\r
658 if (access_flags & MTHCA_ACCESS_REMOTE_ATOMIC)
\r
659 hw_access_flags |= MTHCA_QP_BIT_RAE;
\r
660 if (access_flags & MTHCA_ACCESS_REMOTE_WRITE)
\r
661 hw_access_flags |= MTHCA_QP_BIT_RWE;
\r
663 return cl_hton32(hw_access_flags);
\r
666 int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask)
\r
668 struct mthca_dev *dev = to_mdev(ibqp->device);
\r
669 struct mthca_qp *qp = to_mqp(ibqp);
\r
670 enum ib_qp_state cur_state, new_state;
\r
671 struct mthca_mailbox *mailbox;
\r
672 struct mthca_qp_param *qp_param;
\r
673 struct mthca_qp_context *qp_context;
\r
674 u32 req_param, opt_param;
\r
678 SPIN_LOCK_PREP(lhs);
\r
679 SPIN_LOCK_PREP(lhr);
\r
681 down( &qp->mutex );
\r
683 if (attr_mask & IB_QP_CUR_STATE) {
\r
684 if (attr->cur_qp_state != IBQPS_RTR &&
\r
685 attr->cur_qp_state != IBQPS_RTS &&
\r
686 attr->cur_qp_state != IBQPS_SQD &&
\r
687 attr->cur_qp_state != IBQPS_SQE)
\r
690 cur_state = attr->cur_qp_state;
\r
692 spin_lock_irq(&qp->sq.lock, &lhs);
\r
693 spin_lock(&qp->rq.lock, &lhr);
\r
694 cur_state = qp->state;
\r
696 spin_unlock_irq(&lhs);
\r
699 if (attr_mask & IB_QP_STATE) {
\r
700 if (attr->qp_state < 0 || attr->qp_state > IBQPS_ERR)
\r
702 new_state = attr->qp_state;
\r
704 new_state = cur_state;
\r
706 if (state_table[cur_state][new_state].trans == MTHCA_TRANS_INVALID) {
\r
707 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("Illegal QP transition "
\r
708 "%d->%d\n", cur_state, new_state));
\r
712 req_param = state_table[cur_state][new_state].req_param[qp->transport];
\r
713 opt_param = state_table[cur_state][new_state].opt_param[qp->transport];
\r
715 if ((req_param & attr_mask) != req_param) {
\r
716 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("QP transition "
\r
717 "%d->%d missing req attr 0x%08x\n",
\r
718 cur_state, new_state,
\r
719 req_param & ~attr_mask));
\r
720 //NB: IBAL doesn't use all the fields, so we can miss some mandatory flags
\r
724 if (attr_mask & ~(req_param | opt_param | IB_QP_STATE)) {
\r
725 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("QP transition (transport %d) "
\r
726 "%d->%d has extra attr 0x%08x\n",
\r
728 cur_state, new_state,
\r
729 attr_mask & ~(req_param | opt_param |
\r
731 //NB: The old code sometimes uses optional flags that are not so in this code
\r
735 if ((attr_mask & IB_QP_PKEY_INDEX) &&
\r
736 attr->pkey_index >= dev->limits.pkey_table_len) {
\r
737 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("PKey index (%u) too large. max is %d\n",
\r
738 attr->pkey_index,dev->limits.pkey_table_len-1));
\r
742 if ((attr_mask & IB_QP_PORT) &&
\r
743 (attr->port_num == 0 || attr->port_num > dev->limits.num_ports)) {
\r
744 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("Port number (%u) is invalid\n", attr->port_num));
\r
748 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
\r
749 attr->max_rd_atomic > dev->limits.max_qp_init_rdma) {
\r
750 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("Max rdma_atomic as initiator %u too large (max is %d)\n",
\r
751 attr->max_rd_atomic, dev->limits.max_qp_init_rdma));
\r
755 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
\r
756 attr->max_dest_rd_atomic > 1 << dev->qp_table.rdb_shift) {
\r
757 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("Max rdma_atomic as responder %u too large (max %d)\n",
\r
758 attr->max_dest_rd_atomic, 1 << dev->qp_table.rdb_shift));
\r
762 mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
\r
763 if (IS_ERR(mailbox)) {
\r
764 err = PTR_ERR(mailbox);
\r
767 qp_param = mailbox->buf;
\r
768 qp_context = &qp_param->context;
\r
769 RtlZeroMemory(qp_param, sizeof *qp_param);
\r
771 qp_context->flags = cl_hton32((to_mthca_state(new_state) << 28) |
\r
772 (to_mthca_st(qp->transport) << 16));
\r
773 qp_context->flags |= cl_hton32(MTHCA_QP_BIT_DE);
\r
774 if (!(attr_mask & IB_QP_PATH_MIG_STATE))
\r
775 qp_context->flags |= cl_hton32(MTHCA_QP_PM_MIGRATED << 11);
\r
777 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_PM_STATE);
\r
778 switch (attr->path_mig_state) {
\r
779 case IB_APM_MIGRATED:
\r
780 qp_context->flags |= cl_hton32(MTHCA_QP_PM_MIGRATED << 11);
\r
783 qp_context->flags |= cl_hton32(MTHCA_QP_PM_REARM << 11);
\r
786 qp_context->flags |= cl_hton32(MTHCA_QP_PM_ARMED << 11);
\r
791 /* leave tavor_sched_queue as 0 */
\r
793 if (qp->transport == MLX || qp->transport == UD)
\r
794 qp_context->mtu_msgmax = (IB_MTU_LEN_2048 << 5) | 11;
\r
795 else if (attr_mask & IB_QP_PATH_MTU) {
\r
796 if (attr->path_mtu < IB_MTU_LEN_256 || attr->path_mtu > IB_MTU_LEN_2048) {
\r
797 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,
\r
798 ("path MTU (%u) is invalid\n", attr->path_mtu));
\r
801 qp_context->mtu_msgmax = (u8)((attr->path_mtu << 5) | 31);
\r
804 if (mthca_is_memfree(dev)) {
\r
806 qp_context->rq_size_stride = (u8)(long_log2(qp->rq.max) << 3);
\r
807 qp_context->rq_size_stride |= qp->rq.wqe_shift - 4;
\r
810 qp_context->sq_size_stride = (u8)(long_log2(qp->sq.max) << 3);
\r
811 qp_context->sq_size_stride |= qp->sq.wqe_shift - 4;
\r
814 /* leave arbel_sched_queue as 0 */
\r
816 if (qp->ibqp.ucontext)
\r
817 qp_context->usr_page =
\r
818 cl_hton32(to_mucontext(qp->ibqp.ucontext)->uar.index);
\r
820 qp_context->usr_page = cl_hton32(dev->driver_uar.index);
\r
821 qp_context->local_qpn = cl_hton32(qp->qpn);
\r
822 if (attr_mask & IB_QP_DEST_QPN) {
\r
823 qp_context->remote_qpn = cl_hton32(attr->dest_qp_num);
\r
826 if (qp->transport == MLX)
\r
827 qp_context->pri_path.port_pkey |=
\r
828 cl_hton32(to_msqp(qp)->port << 24);
\r
830 if (attr_mask & IB_QP_PORT) {
\r
831 qp_context->pri_path.port_pkey |=
\r
832 cl_hton32(attr->port_num << 24);
\r
833 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_PORT_NUM);
\r
837 if (attr_mask & IB_QP_PKEY_INDEX) {
\r
838 qp_context->pri_path.port_pkey |=
\r
839 cl_hton32(attr->pkey_index);
\r
840 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_PKEY_INDEX);
\r
843 if (attr_mask & IB_QP_RNR_RETRY) {
\r
844 qp_context->pri_path.rnr_retry = attr->rnr_retry << 5;
\r
845 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_RNR_RETRY);
\r
848 if (attr_mask & IB_QP_AV) {
\r
849 qp_context->pri_path.g_mylmc = attr->ah_attr.src_path_bits & 0x7f;
\r
850 qp_context->pri_path.rlid = cl_hton16(attr->ah_attr.dlid);
\r
851 //TODO: work around: set always full speed - really, it's much more complicate
\r
852 qp_context->pri_path.static_rate = 0;
\r
853 if (attr->ah_attr.ah_flags & IB_AH_GRH) {
\r
854 qp_context->pri_path.g_mylmc |= 1 << 7;
\r
855 qp_context->pri_path.mgid_index = attr->ah_attr.grh.sgid_index;
\r
856 qp_context->pri_path.hop_limit = attr->ah_attr.grh.hop_limit;
\r
857 qp_context->pri_path.sl_tclass_flowlabel =
\r
858 cl_hton32((attr->ah_attr.sl << 28) |
\r
859 (attr->ah_attr.grh.traffic_class << 20) |
\r
860 (attr->ah_attr.grh.flow_label));
\r
861 memcpy(qp_context->pri_path.rgid,
\r
862 attr->ah_attr.grh.dgid.raw, 16);
\r
864 qp_context->pri_path.sl_tclass_flowlabel =
\r
865 cl_hton32(attr->ah_attr.sl << 28);
\r
867 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_PRIMARY_ADDR_PATH);
\r
870 if (attr_mask & IB_QP_TIMEOUT) {
\r
871 qp_context->pri_path.ackto = attr->timeout << 3;
\r
872 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_ACK_TIMEOUT);
\r
877 /* leave rdd as 0 */
\r
878 qp_context->pd = cl_hton32(to_mpd(ibqp->pd)->pd_num);
\r
879 /* leave wqe_base as 0 (we always create an MR based at 0 for WQs) */
\r
880 qp_context->wqe_lkey = cl_hton32(qp->mr.ibmr.lkey);
\r
881 qp_context->params1 = cl_hton32((unsigned long)(
\r
882 (MTHCA_ACK_REQ_FREQ << 28) |
\r
883 (MTHCA_FLIGHT_LIMIT << 24) |
\r
884 MTHCA_QP_BIT_SWE));
\r
885 if (qp->sq_policy == IB_SIGNAL_ALL_WR)
\r
886 qp_context->params1 |= cl_hton32(MTHCA_QP_BIT_SSC);
\r
887 if (attr_mask & IB_QP_RETRY_CNT) {
\r
888 qp_context->params1 |= cl_hton32(attr->retry_cnt << 16);
\r
889 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_RETRY_COUNT);
\r
892 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
\r
893 if (attr->max_rd_atomic) {
\r
894 qp_context->params1 |=
\r
895 cl_hton32(MTHCA_QP_BIT_SRE |
\r
897 qp_context->params1 |=
\r
898 cl_hton32(fls(attr->max_rd_atomic - 1) << 21);
\r
900 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_SRA_MAX);
\r
903 if (attr_mask & IB_QP_SQ_PSN)
\r
904 qp_context->next_send_psn = cl_hton32(attr->sq_psn);
\r
905 qp_context->cqn_snd = cl_hton32(to_mcq(ibqp->send_cq)->cqn);
\r
907 if (mthca_is_memfree(dev)) {
\r
908 qp_context->snd_wqe_base_l = cl_hton32(qp->send_wqe_offset);
\r
909 qp_context->snd_db_index = cl_hton32(qp->sq.db_index);
\r
912 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
\r
914 if (attr->max_dest_rd_atomic)
\r
915 qp_context->params2 |=
\r
916 cl_hton32(fls(attr->max_dest_rd_atomic - 1) << 21);
\r
918 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_RRA_MAX);
\r
922 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {
\r
923 qp_context->params2 |= get_hw_access_flags(qp, attr, attr_mask);
\r
924 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_RWE |
\r
925 MTHCA_QP_OPTPAR_RRE |
\r
926 MTHCA_QP_OPTPAR_RAE);
\r
929 qp_context->params2 |= cl_hton32(MTHCA_QP_BIT_RSC);
\r
932 qp_context->params2 |= cl_hton32(MTHCA_QP_BIT_RIC);
\r
934 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
\r
935 qp_context->rnr_nextrecvpsn |= cl_hton32(attr->min_rnr_timer << 24);
\r
936 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_RNR_TIMEOUT);
\r
938 if (attr_mask & IB_QP_RQ_PSN)
\r
939 qp_context->rnr_nextrecvpsn |= cl_hton32(attr->rq_psn);
\r
941 qp_context->ra_buff_indx =
\r
942 cl_hton32(dev->qp_table.rdb_base +
\r
943 ((qp->qpn & (dev->limits.num_qps - 1)) * MTHCA_RDB_ENTRY_SIZE <<
\r
944 dev->qp_table.rdb_shift));
\r
946 qp_context->cqn_rcv = cl_hton32(to_mcq(ibqp->recv_cq)->cqn);
\r
948 if (mthca_is_memfree(dev))
\r
949 qp_context->rcv_db_index = cl_hton32(qp->rq.db_index);
\r
951 if (attr_mask & IB_QP_QKEY) {
\r
952 qp_context->qkey = cl_hton32(attr->qkey);
\r
953 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_Q_KEY);
\r
957 qp_context->srqn = cl_hton32(1 << 24 |
\r
958 to_msrq(ibqp->srq)->srqn);
\r
960 if (cur_state == IBQPS_RTS && new_state == IBQPS_SQD &&
\r
961 attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY &&
\r
962 attr->en_sqd_async_notify)
\r
963 sqd_event = (u32)(1 << 31);
\r
965 err = mthca_MODIFY_QP(dev, state_table[cur_state][new_state].trans,
\r
966 qp->qpn, 0, mailbox, sqd_event, &status);
\r
968 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("mthca_MODIFY_QP returned error (qp-num = 0x%x) returned status %02x "
\r
969 "cur_state = %d new_state = %d attr_mask = %d req_param = %d opt_param = %d\n",
\r
970 ibqp->qp_num, status, cur_state, new_state,
\r
971 attr_mask, req_param, opt_param));
\r
975 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("mthca_MODIFY_QP bad status(qp-num = 0x%x) returned status %02x "
\r
976 "cur_state = %d new_state = %d attr_mask = %d req_param = %d opt_param = %d\n",
\r
977 ibqp->qp_num, status, cur_state, new_state,
\r
978 attr_mask, req_param, opt_param));
\r
983 qp->state = new_state;
\r
984 if (attr_mask & IB_QP_ACCESS_FLAGS)
\r
985 qp->atomic_rd_en = (u8)attr->qp_access_flags;
\r
986 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
\r
987 qp->resp_depth = attr->max_dest_rd_atomic;
\r
989 if (is_sqp(dev, qp))
\r
990 store_attrs(to_msqp(qp), attr, attr_mask);
\r
993 * If we moved QP0 to RTR, bring the IB link up; if we moved
\r
994 * QP0 to RESET or ERROR, bring the link back down.
\r
996 if (is_qp0(dev, qp)) {
\r
997 if (cur_state != IBQPS_RTR &&
\r
998 new_state == IBQPS_RTR)
\r
999 init_port(dev, to_msqp(qp)->port);
\r
1001 if (cur_state != IBQPS_RESET &&
\r
1002 cur_state != IBQPS_ERR &&
\r
1003 (new_state == IBQPS_RESET ||
\r
1004 new_state == IBQPS_ERR))
\r
1005 mthca_CLOSE_IB(dev, to_msqp(qp)->port, &status);
\r
1009 * If we moved a kernel QP to RESET, clean up all old CQ
\r
1010 * entries and reinitialize the QP.
\r
1012 if (new_state == IBQPS_RESET && !qp->ibqp.ucontext) {
\r
1013 mthca_cq_clean(dev, to_mcq(qp->ibqp.send_cq)->cqn, qp->qpn,
\r
1014 qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);
\r
1015 if (qp->ibqp.send_cq != qp->ibqp.recv_cq)
\r
1016 mthca_cq_clean(dev, to_mcq(qp->ibqp.recv_cq)->cqn, qp->qpn,
\r
1017 qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);
\r
1019 mthca_wq_init(&qp->sq);
\r
1020 qp->sq.last = get_send_wqe(qp, qp->sq.max - 1);
\r
1021 mthca_wq_init(&qp->rq);
\r
1022 qp->rq.last = get_recv_wqe(qp, qp->rq.max - 1);
\r
1024 if (mthca_is_memfree(dev)) {
\r
1031 mthca_free_mailbox(dev, mailbox);
\r
1038 static int mthca_max_data_size(struct mthca_dev *dev, struct mthca_qp *qp, int desc_sz)
\r
1042 * Calculate the maximum size of WQE s/g segments, excluding
\r
1043 * the next segment and other non-data segments.
\r
1045 int max_data_size = desc_sz - sizeof (struct mthca_next_seg);
\r
1047 switch (qp->transport) {
\r
1049 max_data_size -= 2 * sizeof (struct mthca_data_seg);
\r
1053 if (mthca_is_memfree(dev))
\r
1054 max_data_size -= sizeof (struct mthca_arbel_ud_seg);
\r
1056 max_data_size -= sizeof (struct mthca_tavor_ud_seg);
\r
1060 max_data_size -= sizeof (struct mthca_raddr_seg);
\r
1063 return max_data_size;
\r
1066 static inline int mthca_max_inline_data(int max_data_size)
\r
1068 return max_data_size - MTHCA_INLINE_HEADER_SIZE ;
\r
1071 static void mthca_adjust_qp_caps(struct mthca_dev *dev,
\r
1072 struct mthca_qp *qp)
\r
1074 int max_data_size = mthca_max_data_size(dev, qp,
\r
1075 min(dev->limits.max_desc_sz, 1 << qp->sq.wqe_shift));
\r
1077 qp->max_inline_data = mthca_max_inline_data( max_data_size);
\r
1079 qp->sq.max_gs = min(dev->limits.max_sg,
\r
1080 (int)(max_data_size / sizeof (struct mthca_data_seg)));
\r
1081 qp->rq.max_gs = min(dev->limits.max_sg,
\r
1082 (int)((min(dev->limits.max_desc_sz, 1 << qp->rq.wqe_shift) -
\r
1083 sizeof (struct mthca_next_seg)) / sizeof (struct mthca_data_seg)));
\r
1087 * Allocate and register buffer for WQEs. qp->rq.max, sq.max,
\r
1088 * rq.max_gs and sq.max_gs must all be assigned.
\r
1089 * mthca_alloc_wqe_buf will calculate rq.wqe_shift and
\r
1090 * sq.wqe_shift (as well as send_wqe_offset, is_direct, and
\r
1093 static int mthca_alloc_wqe_buf(struct mthca_dev *dev,
\r
1094 struct mthca_pd *pd,
\r
1095 struct mthca_qp *qp)
\r
1098 int err = -ENOMEM;
\r
1100 HCA_ENTER(HCA_DBG_QP);
\r
1101 size = sizeof (struct mthca_next_seg) +
\r
1102 qp->rq.max_gs * sizeof (struct mthca_data_seg);
\r
1104 if (size > dev->limits.max_desc_sz)
\r
1107 for (qp->rq.wqe_shift = 6; 1 << qp->rq.wqe_shift < size;
\r
1108 qp->rq.wqe_shift++)
\r
1111 size = qp->sq.max_gs * sizeof (struct mthca_data_seg);
\r
1112 switch (qp->transport) {
\r
1114 size += 2 * sizeof (struct mthca_data_seg);
\r
1118 size += mthca_is_memfree(dev) ?
\r
1119 sizeof (struct mthca_arbel_ud_seg) :
\r
1120 sizeof (struct mthca_tavor_ud_seg);
\r
1124 size += sizeof (struct mthca_raddr_seg);
\r
1128 size += sizeof (struct mthca_raddr_seg);
\r
1130 * An atomic op will require an atomic segment, a
\r
1131 * remote address segment and one scatter entry.
\r
1134 sizeof (struct mthca_atomic_seg) +
\r
1135 sizeof (struct mthca_raddr_seg) +
\r
1136 sizeof (struct mthca_data_seg));
\r
1143 /* Make sure that we have enough space for a bind request */
\r
1144 size = max(size, sizeof (struct mthca_bind_seg));
\r
1146 size += sizeof (struct mthca_next_seg);
\r
1148 if (size > dev->limits.max_desc_sz)
\r
1151 for (qp->sq.wqe_shift = 6; 1 << qp->sq.wqe_shift < size;
\r
1152 qp->sq.wqe_shift++)
\r
1155 qp->send_wqe_offset = ALIGN(qp->rq.max << qp->rq.wqe_shift,
\r
1156 1 << qp->sq.wqe_shift);
\r
1159 * If this is a userspace QP, we don't actually have to
\r
1160 * allocate anything. All we need is to calculate the WQE
\r
1161 * sizes and the send_wqe_offset, so we're done now.
\r
1163 if (pd->ibpd.ucontext)
\r
1166 size = (int)(LONG_PTR)NEXT_PAGE_ALIGN(qp->send_wqe_offset +
\r
1167 (qp->sq.max << qp->sq.wqe_shift));
\r
1169 qp->wrid = kmalloc((qp->rq.max + qp->sq.max) * sizeof (u64),
\r
1174 err = mthca_buf_alloc(dev, size, MTHCA_MAX_DIRECT_QP_SIZE,
\r
1175 &qp->queue, &qp->is_direct, pd, 0, &qp->mr);
\r
1179 HCA_EXIT(HCA_DBG_QP);
\r
1187 static void mthca_free_wqe_buf(struct mthca_dev *dev,
\r
1188 struct mthca_qp *qp)
\r
1190 mthca_buf_free(dev, (int)(LONG_PTR)NEXT_PAGE_ALIGN(qp->send_wqe_offset +
\r
1191 (qp->sq.max << qp->sq.wqe_shift)),
\r
1192 &qp->queue, qp->is_direct, &qp->mr);
\r
1196 static int mthca_map_memfree(struct mthca_dev *dev,
\r
1197 struct mthca_qp *qp)
\r
1201 if (mthca_is_memfree(dev)) {
\r
1202 ret = mthca_table_get(dev, dev->qp_table.qp_table, qp->qpn);
\r
1206 ret = mthca_table_get(dev, dev->qp_table.eqp_table, qp->qpn);
\r
1210 ret = mthca_table_get(dev, dev->qp_table.rdb_table,
\r
1211 qp->qpn << dev->qp_table.rdb_shift);
\r
1220 mthca_table_put(dev, dev->qp_table.eqp_table, qp->qpn);
\r
1223 mthca_table_put(dev, dev->qp_table.qp_table, qp->qpn);
\r
1228 static void mthca_unmap_memfree(struct mthca_dev *dev,
\r
1229 struct mthca_qp *qp)
\r
1231 mthca_table_put(dev, dev->qp_table.rdb_table,
\r
1232 qp->qpn << dev->qp_table.rdb_shift);
\r
1233 mthca_table_put(dev, dev->qp_table.eqp_table, qp->qpn);
\r
1234 mthca_table_put(dev, dev->qp_table.qp_table, qp->qpn);
\r
1237 static int mthca_alloc_memfree(struct mthca_dev *dev,
\r
1238 struct mthca_qp *qp)
\r
1242 if (mthca_is_memfree(dev)) {
\r
1243 qp->rq.db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_RQ,
\r
1244 qp->qpn, &qp->rq.db);
\r
1245 if (qp->rq.db_index < 0)
\r
1246 return qp->rq.db_index;
\r
1248 qp->sq.db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_SQ,
\r
1249 qp->qpn, &qp->sq.db);
\r
1250 if (qp->sq.db_index < 0){
\r
1251 mthca_free_db(dev, MTHCA_DB_TYPE_RQ, qp->rq.db_index);
\r
1252 return qp->sq.db_index;
\r
1260 static void mthca_free_memfree(struct mthca_dev *dev,
\r
1261 struct mthca_qp *qp)
\r
1263 if (mthca_is_memfree(dev)) {
\r
1264 mthca_free_db(dev, MTHCA_DB_TYPE_SQ, qp->sq.db_index);
\r
1265 mthca_free_db(dev, MTHCA_DB_TYPE_RQ, qp->rq.db_index);
\r
1269 static int mthca_alloc_qp_common(struct mthca_dev *dev,
\r
1270 struct mthca_pd *pd,
\r
1271 struct mthca_cq *send_cq,
\r
1272 struct mthca_cq *recv_cq,
\r
1273 enum ib_sig_type send_policy,
\r
1274 struct mthca_qp *qp)
\r
1279 atomic_set(&qp->refcount, 1);
\r
1280 init_waitqueue_head(&qp->wait);
\r
1281 KeInitializeMutex(&qp->mutex, 0);
\r
1283 qp->state = IBQPS_RESET;
\r
1284 qp->atomic_rd_en = 0;
\r
1285 qp->resp_depth = 0;
\r
1286 qp->sq_policy = send_policy;
\r
1287 mthca_wq_init(&qp->sq);
\r
1288 mthca_wq_init(&qp->rq);
\r
1290 UNREFERENCED_PARAMETER(send_cq);
\r
1291 UNREFERENCED_PARAMETER(recv_cq);
\r
1293 ret = mthca_map_memfree(dev, qp);
\r
1297 ret = mthca_alloc_wqe_buf(dev, pd, qp);
\r
1299 mthca_unmap_memfree(dev, qp);
\r
1303 mthca_adjust_qp_caps(dev, qp);
\r
1306 * If this is a userspace QP, we're done now. The doorbells
\r
1307 * will be allocated and buffers will be initialized in
\r
1310 if (pd->ibpd.ucontext)
\r
1313 ret = mthca_alloc_memfree(dev, qp);
\r
1315 mthca_free_wqe_buf(dev, qp);
\r
1316 mthca_unmap_memfree(dev, qp);
\r
1320 if (mthca_is_memfree(dev)) {
\r
1321 struct mthca_next_seg *next;
\r
1322 struct mthca_data_seg *scatter;
\r
1323 int size = (sizeof (struct mthca_next_seg) +
\r
1324 qp->rq.max_gs * sizeof (struct mthca_data_seg)) / 16;
\r
1326 for (i = 0; i < qp->rq.max; ++i) {
\r
1327 next = get_recv_wqe(qp, i);
\r
1328 next->nda_op = cl_hton32(((i + 1) & (qp->rq.max - 1)) <<
\r
1329 qp->rq.wqe_shift);
\r
1330 next->ee_nds = cl_hton32(size);
\r
1332 for (scatter = (void *) (next + 1);
\r
1333 (void *) scatter < (void *) ((u8*)next + (u32)(1 << qp->rq.wqe_shift));
\r
1335 scatter->lkey = cl_hton32(MTHCA_INVAL_LKEY);
\r
1338 for (i = 0; i < qp->sq.max; ++i) {
\r
1339 next = get_send_wqe(qp, i);
\r
1340 next->nda_op = cl_hton32((((i + 1) & (qp->sq.max - 1)) <<
\r
1341 qp->sq.wqe_shift) +
\r
1342 qp->send_wqe_offset);
\r
1346 qp->sq.last = get_send_wqe(qp, qp->sq.max - 1);
\r
1347 qp->rq.last = get_recv_wqe(qp, qp->rq.max - 1);
\r
1352 static int mthca_set_qp_size(struct mthca_dev *dev, struct ib_qp_cap *cap,
\r
1353 struct mthca_qp *qp)
\r
1355 int max_data_size = mthca_max_data_size(dev, qp, dev->limits.max_desc_sz);
\r
1357 /* Sanity check QP size before proceeding */
\r
1358 if (cap->max_send_wr > (u32)dev->limits.max_wqes ||
\r
1359 cap->max_recv_wr > (u32)dev->limits.max_wqes ||
\r
1360 cap->max_send_sge > (u32)dev->limits.max_sg ||
\r
1361 cap->max_recv_sge > (u32)dev->limits.max_sg ||
\r
1362 cap->max_inline_data > (u32)mthca_max_inline_data(max_data_size))
\r
1366 * For MLX transport we need 2 extra S/G entries:
\r
1367 * one for the header and one for the checksum at the end
\r
1369 if (qp->transport == MLX && cap->max_recv_sge + 2 > (u32)dev->limits.max_sg)
\r
1372 /* Enable creating zero-sized QPs */
\r
1373 if (!cap->max_recv_wr)
\r
1374 cap->max_recv_wr = 1;
\r
1375 if (!cap->max_send_wr)
\r
1376 cap->max_send_wr = 1;
\r
1378 if (mthca_is_memfree(dev)) {
\r
1379 qp->rq.max = cap->max_recv_wr ?
\r
1380 roundup_pow_of_two(cap->max_recv_wr) : 0;
\r
1381 qp->sq.max = cap->max_send_wr ?
\r
1382 roundup_pow_of_two(cap->max_send_wr) : 0;
\r
1384 qp->rq.max = cap->max_recv_wr;
\r
1385 qp->sq.max = cap->max_send_wr;
\r
1388 qp->rq.max_gs = cap->max_recv_sge;
\r
1389 qp->sq.max_gs = MAX(cap->max_send_sge,
\r
1390 ALIGN(cap->max_inline_data + MTHCA_INLINE_HEADER_SIZE,
\r
1391 MTHCA_INLINE_CHUNK_SIZE) /
\r
1392 (int)sizeof (struct mthca_data_seg));
\r
1397 int mthca_alloc_qp(struct mthca_dev *dev,
\r
1398 struct mthca_pd *pd,
\r
1399 struct mthca_cq *send_cq,
\r
1400 struct mthca_cq *recv_cq,
\r
1401 enum ib_qp_type_t type,
\r
1402 enum ib_sig_type send_policy,
\r
1403 struct ib_qp_cap *cap,
\r
1404 struct mthca_qp *qp)
\r
1407 SPIN_LOCK_PREP(lh);
\r
1410 case IB_QPT_RELIABLE_CONN: qp->transport = RC; break;
\r
1411 case IB_QPT_UNRELIABLE_CONN: qp->transport = UC; break;
\r
1412 case IB_QPT_UNRELIABLE_DGRM: qp->transport = UD; break;
\r
1413 default: return -EINVAL;
\r
1416 err = mthca_set_qp_size(dev, cap, qp);
\r
1420 qp->qpn = mthca_alloc(&dev->qp_table.alloc);
\r
1421 if (qp->qpn == -1)
\r
1424 err = mthca_alloc_qp_common(dev, pd, send_cq, recv_cq,
\r
1427 mthca_free(&dev->qp_table.alloc, qp->qpn);
\r
1431 spin_lock_irq(&dev->qp_table.lock, &lh);
\r
1432 mthca_array_set(&dev->qp_table.qp,
\r
1433 qp->qpn & (dev->limits.num_qps - 1), qp);
\r
1434 spin_unlock_irq(&lh);
\r
1439 int mthca_alloc_sqp(struct mthca_dev *dev,
\r
1440 struct mthca_pd *pd,
\r
1441 struct mthca_cq *send_cq,
\r
1442 struct mthca_cq *recv_cq,
\r
1443 enum ib_sig_type send_policy,
\r
1444 struct ib_qp_cap *cap,
\r
1447 struct mthca_sqp *sqp)
\r
1449 u32 mqpn = qpn * 2 + dev->qp_table.sqp_start + port - 1;
\r
1451 SPIN_LOCK_PREP(lhs);
\r
1452 SPIN_LOCK_PREP(lhr);
\r
1453 SPIN_LOCK_PREP(lht);
\r
1455 err = mthca_set_qp_size(dev, cap, &sqp->qp);
\r
1459 alloc_dma_zmem_map(dev,
\r
1460 sqp->qp.sq.max * MTHCA_UD_HEADER_SIZE,
\r
1461 PCI_DMA_BIDIRECTIONAL,
\r
1463 if (!sqp->sg.page)
\r
1466 spin_lock_irq(&dev->qp_table.lock, &lht);
\r
1467 if (mthca_array_get(&dev->qp_table.qp, mqpn))
\r
1470 mthca_array_set(&dev->qp_table.qp, mqpn, sqp);
\r
1471 spin_unlock_irq(&lht);
\r
1477 sqp->qp.qpn = mqpn;
\r
1478 sqp->qp.transport = MLX;
\r
1480 err = mthca_alloc_qp_common(dev, pd, send_cq, recv_cq,
\r
1481 send_policy, &sqp->qp);
\r
1483 goto err_out_free;
\r
1485 atomic_inc(&pd->sqp_count);
\r
1491 * Lock CQs here, so that CQ polling code can do QP lookup
\r
1492 * without taking a lock.
\r
1494 spin_lock_irq(&send_cq->lock, &lhs);
\r
1495 if (send_cq != recv_cq)
\r
1496 spin_lock(&recv_cq->lock, &lhr);
\r
1498 spin_lock(&dev->qp_table.lock, &lht);
\r
1499 mthca_array_clear(&dev->qp_table.qp, mqpn);
\r
1500 spin_unlock(&lht);
\r
1502 if (send_cq != recv_cq)
\r
1503 spin_unlock(&lhr);
\r
1504 spin_unlock_irq(&lhs);
\r
1507 free_dma_mem_map(dev, &sqp->sg, PCI_DMA_BIDIRECTIONAL);
\r
1512 void mthca_free_qp(struct mthca_dev *dev,
\r
1513 struct mthca_qp *qp)
\r
1516 struct mthca_cq *send_cq;
\r
1517 struct mthca_cq *recv_cq;
\r
1518 SPIN_LOCK_PREP(lhs);
\r
1519 SPIN_LOCK_PREP(lhr);
\r
1520 SPIN_LOCK_PREP(lht);
\r
1522 send_cq = to_mcq(qp->ibqp.send_cq);
\r
1523 recv_cq = to_mcq(qp->ibqp.recv_cq);
\r
1526 * Lock CQs here, so that CQ polling code can do QP lookup
\r
1527 * without taking a lock.
\r
1529 spin_lock_irq(&send_cq->lock, &lhs);
\r
1530 if (send_cq != recv_cq)
\r
1531 spin_lock(&recv_cq->lock, &lhr);
\r
1533 spin_lock(&dev->qp_table.lock, &lht);
\r
1534 mthca_array_clear(&dev->qp_table.qp,
\r
1535 qp->qpn & (dev->limits.num_qps - 1));
\r
1536 spin_unlock(&lht);
\r
1538 if (send_cq != recv_cq)
\r
1539 spin_unlock(&lhr);
\r
1540 spin_unlock_irq(&lhs);
\r
1542 atomic_dec(&qp->refcount);
\r
1543 wait_event(&qp->wait, !atomic_read(&qp->refcount));
\r
1545 if (qp->state != IBQPS_RESET) {
\r
1546 mthca_MODIFY_QP(dev, MTHCA_TRANS_ANY2RST, qp->qpn, 0, NULL, 0, &status);
\r
1550 * If this is a userspace QP, the buffers, MR, CQs and so on
\r
1551 * will be cleaned up in userspace, so all we have to do is
\r
1552 * unref the mem-free tables and free the QPN in our table.
\r
1554 if (!qp->ibqp.ucontext) {
\r
1555 mthca_cq_clean(dev, to_mcq(qp->ibqp.send_cq)->cqn, qp->qpn,
\r
1556 qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);
\r
1557 if (qp->ibqp.send_cq != qp->ibqp.recv_cq)
\r
1558 mthca_cq_clean(dev, to_mcq(qp->ibqp.recv_cq)->cqn, qp->qpn,
\r
1559 qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);
\r
1561 mthca_free_memfree(dev, qp);
\r
1562 mthca_free_wqe_buf(dev, qp);
\r
1565 mthca_unmap_memfree(dev, qp);
\r
1567 if (is_sqp(dev, qp)) {
\r
1568 atomic_dec(&(to_mpd(qp->ibqp.pd)->sqp_count));
\r
1569 free_dma_mem_map(dev, &to_msqp(qp)->sg, PCI_DMA_BIDIRECTIONAL);
\r
1571 mthca_free(&dev->qp_table.alloc, qp->qpn);
\r
1574 static enum mthca_wr_opcode conv_ibal_wr_opcode(struct _ib_send_wr *wr)
\r
1577 enum mthca_wr_opcode opcode = -1; //= wr->wr_type;
\r
1579 switch (wr->wr_type) {
\r
1581 opcode = (wr->send_opt & IB_SEND_OPT_IMMEDIATE) ? MTHCA_OPCODE_SEND_IMM : MTHCA_OPCODE_SEND;
\r
1583 case WR_RDMA_WRITE:
\r
1584 opcode = (wr->send_opt & IB_SEND_OPT_IMMEDIATE) ? MTHCA_OPCODE_RDMA_WRITE_IMM : MTHCA_OPCODE_RDMA_WRITE;
\r
1586 case WR_RDMA_READ: opcode = MTHCA_OPCODE_RDMA_READ; break;
\r
1587 case WR_COMPARE_SWAP: opcode = MTHCA_OPCODE_ATOMIC_CS; break;
\r
1588 case WR_FETCH_ADD: opcode = MTHCA_OPCODE_ATOMIC_FA; break;
\r
1589 default: opcode = MTHCA_OPCODE_INVALID;break;
\r
1594 /* Create UD header for an MLX send and build a data segment for it */
\r
1595 static int build_mlx_header(struct mthca_dev *dev, struct mthca_sqp *sqp,
\r
1596 int ind, struct _ib_send_wr *wr,
\r
1597 struct mthca_mlx_seg *mlx,
\r
1598 struct mthca_data_seg *data)
\r
1600 enum ib_wr_opcode opcode = conv_ibal_wr_opcode(wr);
\r
1606 if (!wr->dgrm.ud.h_av) {
\r
1607 HCA_PRINT(TRACE_LEVEL_ERROR , HCA_DBG_AV,
\r
1608 ("absent AV in send wr %p\n", wr));
\r
1612 ib_ud_header_init(256, /* assume a MAD */
\r
1613 mthca_ah_grh_present(to_mah((struct ib_ah *)wr->dgrm.ud.h_av)),
\r
1616 err = mthca_read_ah(dev, to_mah((struct ib_ah *)wr->dgrm.ud.h_av), &sqp->ud_header);
\r
1618 HCA_PRINT(TRACE_LEVEL_ERROR , HCA_DBG_AV, ("read av error%p\n",
\r
1619 to_mah((struct ib_ah *)wr->dgrm.ud.h_av)->av));
\r
1622 mlx->flags &= ~cl_hton32(MTHCA_NEXT_SOLICIT | 1);
\r
1623 mlx->flags |= cl_hton32((!sqp->qp.ibqp.qp_num ? MTHCA_MLX_VL15 : 0) |
\r
1624 (sqp->ud_header.lrh.destination_lid ==
\r
1625 IB_LID_PERMISSIVE ? MTHCA_MLX_SLR : 0) |
\r
1626 (sqp->ud_header.lrh.service_level << 8));
\r
1627 mlx->rlid = sqp->ud_header.lrh.destination_lid;
\r
1631 case MTHCA_OPCODE_SEND:
\r
1632 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
\r
1633 sqp->ud_header.immediate_present = 0;
\r
1635 case MTHCA_OPCODE_SEND_IMM:
\r
1636 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
\r
1637 sqp->ud_header.immediate_present = 1;
\r
1638 sqp->ud_header.immediate_data = wr->immediate_data;
\r
1644 sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 : 0;
\r
1645 if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE)
\r
1646 sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE;
\r
1647 sqp->ud_header.bth.solicited_event = (u8)!!(wr->send_opt & IB_SEND_OPT_SOLICITED);
\r
1648 if (!sqp->qp.ibqp.qp_num)
\r
1649 ib_get_cached_pkey(&dev->ib_dev, (u8)sqp->port,
\r
1650 sqp->pkey_index, &pkey);
\r
1652 ib_get_cached_pkey(&dev->ib_dev, (u8)sqp->port,
\r
1653 wr->dgrm.ud.pkey_index, &pkey);
\r
1654 sqp->ud_header.bth.pkey = pkey;
\r
1655 sqp->ud_header.bth.destination_qpn = wr->dgrm.ud.remote_qp;
\r
1656 sqp->ud_header.bth.psn = cl_hton32((sqp->send_psn++) & ((1 << 24) - 1));
\r
1657 sqp->ud_header.deth.qkey = wr->dgrm.ud.remote_qkey & 0x00000080 ?
\r
1658 cl_hton32(sqp->qkey) : wr->dgrm.ud.remote_qkey;
\r
1659 sqp->ud_header.deth.source_qpn = cl_hton32(sqp->qp.ibqp.qp_num);
\r
1661 header_size = ib_ud_header_pack(&sqp->ud_header,
\r
1662 (u8*)sqp->sg.page +
\r
1663 ind * MTHCA_UD_HEADER_SIZE);
\r
1665 data->byte_count = cl_hton32(header_size);
\r
1666 data->lkey = cl_hton32(to_mpd(sqp->qp.ibqp.pd)->ntmr.ibmr.lkey);
\r
1667 data->addr = CPU_2_BE64(sqp->sg.dma_address +
\r
1668 ind * MTHCA_UD_HEADER_SIZE);
\r
1673 static inline int mthca_wq_overflow(struct mthca_wq *wq, int nreq,
\r
1674 struct ib_cq *ib_cq)
\r
1677 struct mthca_cq *cq;
\r
1678 SPIN_LOCK_PREP(lh);
\r
1680 cur = wq->head - wq->tail;
\r
1681 if (likely((int)cur + nreq < wq->max))
\r
1684 cq = to_mcq(ib_cq);
\r
1685 spin_lock_dpc(&cq->lock, &lh);
\r
1686 cur = wq->head - wq->tail;
\r
1687 spin_unlock_dpc(&lh);
\r
1689 return (int)cur + nreq >= wq->max;
\r
1692 int mthca_tavor_post_send(struct ib_qp *ibqp, struct _ib_send_wr *wr,
\r
1693 struct _ib_send_wr **bad_wr)
\r
1695 struct mthca_dev *dev = to_mdev(ibqp->device);
\r
1696 struct mthca_qp *qp = to_mqp(ibqp);
\r
1704 u32 f0 = unlikely(wr->send_opt & IB_SEND_OPT_FENCE) ? MTHCA_SEND_DOORBELL_FENCE : 0;
\r
1707 enum ib_wr_opcode opcode;
\r
1708 SPIN_LOCK_PREP(lh);
\r
1710 spin_lock_irqsave(&qp->sq.lock, &lh);
\r
1712 /* XXX check that state is OK to post send */
\r
1714 ind = qp->sq.next_ind;
\r
1716 for (nreq = 0; wr; ++nreq, wr = wr->p_next) {
\r
1717 if (mthca_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
\r
1718 HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_QP,("SQ %06x full (%u head, %u tail,"
\r
1719 " %d max, %d nreq)\n", qp->qpn,
\r
1720 qp->sq.head, qp->sq.tail,
\r
1721 qp->sq.max, nreq));
\r
1728 wqe = get_send_wqe(qp, ind);
\r
1729 prev_wqe = qp->sq.last;
\r
1730 qp->sq.last = wqe;
\r
1731 opcode = conv_ibal_wr_opcode(wr);
\r
1733 ((struct mthca_next_seg *) wqe)->nda_op = 0;
\r
1734 ((struct mthca_next_seg *) wqe)->ee_nds = 0;
\r
1735 ((struct mthca_next_seg *) wqe)->flags =
\r
1736 ((wr->send_opt & IB_SEND_OPT_SIGNALED) ?
\r
1737 cl_hton32(MTHCA_NEXT_CQ_UPDATE) : 0) |
\r
1738 ((wr->send_opt & IB_SEND_OPT_SOLICITED) ?
\r
1739 cl_hton32(MTHCA_NEXT_SOLICIT) : 0) |
\r
1741 if (opcode == MTHCA_OPCODE_SEND_IMM||
\r
1742 opcode == MTHCA_OPCODE_RDMA_WRITE_IMM)
\r
1743 ((struct mthca_next_seg *) wqe)->imm = wr->immediate_data;
\r
1745 wqe += sizeof (struct mthca_next_seg);
\r
1746 size = sizeof (struct mthca_next_seg) / 16;
\r
1748 switch (qp->transport) {
\r
1751 case MTHCA_OPCODE_ATOMIC_CS:
\r
1752 case MTHCA_OPCODE_ATOMIC_FA:
\r
1753 ((struct mthca_raddr_seg *) wqe)->raddr =
\r
1754 cl_hton64(wr->remote_ops.vaddr);
\r
1755 ((struct mthca_raddr_seg *) wqe)->rkey =
\r
1756 wr->remote_ops.rkey;
\r
1757 ((struct mthca_raddr_seg *) wqe)->reserved = 0;
\r
1759 wqe += sizeof (struct mthca_raddr_seg);
\r
1761 if (opcode == MTHCA_OPCODE_ATOMIC_CS) {
\r
1762 ((struct mthca_atomic_seg *) wqe)->swap_add =
\r
1763 (wr->remote_ops.atomic2);
\r
1764 ((struct mthca_atomic_seg *) wqe)->compare =
\r
1765 (wr->remote_ops.atomic1);
\r
1767 ((struct mthca_atomic_seg *) wqe)->swap_add =
\r
1768 (wr->remote_ops.atomic1);
\r
1769 ((struct mthca_atomic_seg *) wqe)->compare = 0;
\r
1772 wqe += sizeof (struct mthca_atomic_seg);
\r
1773 size += (sizeof (struct mthca_raddr_seg) +
\r
1774 sizeof (struct mthca_atomic_seg)) / 16 ;
\r
1777 case MTHCA_OPCODE_RDMA_READ:
\r
1778 case MTHCA_OPCODE_RDMA_WRITE:
\r
1779 case MTHCA_OPCODE_RDMA_WRITE_IMM:
\r
1780 ((struct mthca_raddr_seg *) wqe)->raddr =
\r
1781 cl_hton64(wr->remote_ops.vaddr);
\r
1782 ((struct mthca_raddr_seg *) wqe)->rkey =
\r
1783 wr->remote_ops.rkey;
\r
1784 ((struct mthca_raddr_seg *) wqe)->reserved = 0;
\r
1785 wqe += sizeof (struct mthca_raddr_seg);
\r
1786 size += sizeof (struct mthca_raddr_seg) / 16;
\r
1790 /* No extra segments required for sends */
\r
1798 case MTHCA_OPCODE_RDMA_WRITE:
\r
1799 case MTHCA_OPCODE_RDMA_WRITE_IMM:
\r
1800 ((struct mthca_raddr_seg *) wqe)->raddr =
\r
1801 cl_hton64(wr->remote_ops.vaddr);
\r
1802 ((struct mthca_raddr_seg *) wqe)->rkey =
\r
1803 wr->remote_ops.rkey;
\r
1804 ((struct mthca_raddr_seg *) wqe)->reserved = 0;
\r
1805 wqe += sizeof (struct mthca_raddr_seg);
\r
1806 size += sizeof (struct mthca_raddr_seg) / 16;
\r
1810 /* No extra segments required for sends */
\r
1817 ((struct mthca_tavor_ud_seg *) wqe)->lkey =
\r
1818 cl_hton32(to_mah((struct ib_ah *)wr->dgrm.ud.h_av)->key);
\r
1819 ((struct mthca_tavor_ud_seg *) wqe)->av_addr =
\r
1820 cl_hton64(to_mah((struct ib_ah *)wr->dgrm.ud.h_av)->avdma);
\r
1821 ((struct mthca_tavor_ud_seg *) wqe)->dqpn = wr->dgrm.ud.remote_qp;
\r
1822 ((struct mthca_tavor_ud_seg *) wqe)->qkey = wr->dgrm.ud.remote_qkey;
\r
1824 wqe += sizeof (struct mthca_tavor_ud_seg);
\r
1825 size += sizeof (struct mthca_tavor_ud_seg) / 16;
\r
1829 err = build_mlx_header(dev, to_msqp(qp), ind, wr,
\r
1830 (void*)(wqe - sizeof (struct mthca_next_seg)),
\r
1837 wqe += sizeof (struct mthca_data_seg);
\r
1838 size += sizeof (struct mthca_data_seg) / 16;
\r
1842 if ((int)(int)wr->num_ds > qp->sq.max_gs) {
\r
1843 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP ,("SQ %06x too many gathers\n",qp->qpn));
\r
1849 if (wr->send_opt & IB_SEND_OPT_INLINE) {
\r
1851 struct mthca_inline_seg *seg = (struct mthca_inline_seg *)wqe;
\r
1854 wqe += sizeof *seg;
\r
1855 for (i = 0; i < (int)wr->num_ds; ++i) {
\r
1856 struct _ib_local_ds *sge = &wr->ds_array[i];
\r
1860 if (s > (uint32_t)qp->max_inline_data) {
\r
1867 memcpy(wqe, (void *) (ULONG_PTR) sge->vaddr,
\r
1869 wqe += sge->length;
\r
1872 seg->byte_count = cl_hton32(MTHCA_INLINE_SEG | s);
\r
1873 size += align(s + sizeof *seg, 16) / 16;
\r
1877 for (i = 0; i < (int)wr->num_ds; ++i) {
\r
1878 ((struct mthca_data_seg *) wqe)->byte_count =
\r
1879 cl_hton32(wr->ds_array[i].length);
\r
1880 ((struct mthca_data_seg *) wqe)->lkey =
\r
1881 cl_hton32(wr->ds_array[i].lkey);
\r
1882 ((struct mthca_data_seg *) wqe)->addr =
\r
1883 cl_hton64(wr->ds_array[i].vaddr);
\r
1884 wqe += sizeof (struct mthca_data_seg);
\r
1885 size += sizeof (struct mthca_data_seg) / 16;
\r
1886 HCA_PRINT(TRACE_LEVEL_VERBOSE ,HCA_DBG_QP ,("SQ %06x [%02x] lkey 0x%08x vaddr 0x%I64x 0x%x\n",qp->qpn,i,
\r
1887 (wr->ds_array[i].lkey),(wr->ds_array[i].vaddr),wr->ds_array[i].length));
\r
1891 /* Add one more inline data segment for ICRC */
\r
1892 if (qp->transport == MLX) {
\r
1893 ((struct mthca_data_seg *) wqe)->byte_count =
\r
1894 cl_hton32((unsigned long)((1 << 31) | 4));
\r
1895 ((u32 *) wqe)[1] = 0;
\r
1896 wqe += sizeof (struct mthca_data_seg);
\r
1897 size += sizeof (struct mthca_data_seg) / 16;
\r
1900 qp->wrid[ind + qp->rq.max] = wr->wr_id;
\r
1902 if (opcode == MTHCA_OPCODE_INVALID) {
\r
1903 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP ,("SQ %06x opcode invalid\n",qp->qpn));
\r
1910 ((struct mthca_next_seg *) prev_wqe)->nda_op =
\r
1911 cl_hton32(((ind << qp->sq.wqe_shift) +
\r
1912 qp->send_wqe_offset) |opcode);
\r
1914 ((struct mthca_next_seg *) prev_wqe)->ee_nds =
\r
1915 cl_hton32((size0 ? 0 : MTHCA_NEXT_DBD) | size |
\r
1916 ((wr->send_opt & IB_SEND_OPT_FENCE) ?
\r
1917 MTHCA_NEXT_FENCE : 0));
\r
1924 dump_wqe( TRACE_LEVEL_VERBOSE, (u32*)qp->sq.last,qp);
\r
1927 if (unlikely(ind >= qp->sq.max))
\r
1928 ind -= qp->sq.max;
\r
1932 if (likely(nreq)) {
\r
1933 __be32 doorbell[2];
\r
1935 doorbell[0] = cl_hton32(((qp->sq.next_ind << qp->sq.wqe_shift) +
\r
1936 qp->send_wqe_offset) | f0 | op0);
\r
1937 doorbell[1] = cl_hton32((qp->qpn << 8) | size0);
\r
1941 mthca_write64(doorbell,
\r
1942 dev->kar + MTHCA_SEND_DOORBELL,
\r
1943 MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
\r
1946 qp->sq.next_ind = ind;
\r
1947 qp->sq.head += nreq;
\r
1949 spin_unlock_irqrestore(&lh);
\r
1953 int mthca_tavor_post_recv(struct ib_qp *ibqp, struct _ib_recv_wr *wr,
\r
1954 struct _ib_recv_wr **bad_wr)
\r
1956 struct mthca_dev *dev = to_mdev(ibqp->device);
\r
1957 struct mthca_qp *qp = to_mqp(ibqp);
\r
1958 __be32 doorbell[2];
\r
1967 SPIN_LOCK_PREP(lh);
\r
1969 spin_lock_irqsave(&qp->rq.lock, &lh);
\r
1971 /* XXX check that state is OK to post receive */
\r
1973 ind = qp->rq.next_ind;
\r
1975 for (nreq = 0; wr; ++nreq, wr = wr->p_next) {
\r
1976 if (unlikely(nreq == MTHCA_TAVOR_MAX_WQES_PER_RECV_DB)) {
\r
1979 doorbell[0] = cl_hton32((qp->rq.next_ind << qp->rq.wqe_shift) | size0);
\r
1980 doorbell[1] = cl_hton32(qp->qpn << 8);
\r
1984 mthca_write64(doorbell, dev->kar + MTHCA_RECV_DOORBELL,
\r
1985 MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
\r
1987 qp->rq.head += MTHCA_TAVOR_MAX_WQES_PER_RECV_DB;
\r
1990 if (mthca_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
\r
1991 HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_QP,("RQ %06x full (%u head, %u tail,"
\r
1992 " %d max, %d nreq)\n", qp->qpn,
\r
1993 qp->rq.head, qp->rq.tail,
\r
1994 qp->rq.max, nreq));
\r
2001 wqe = get_recv_wqe(qp, ind);
\r
2002 prev_wqe = qp->rq.last;
\r
2003 qp->rq.last = wqe;
\r
2005 ((struct mthca_next_seg *) wqe)->nda_op = 0;
\r
2006 ((struct mthca_next_seg *) wqe)->ee_nds =
\r
2007 cl_hton32(MTHCA_NEXT_DBD);
\r
2008 ((struct mthca_next_seg *) wqe)->flags = 0;
\r
2010 wqe += sizeof (struct mthca_next_seg);
\r
2011 size = sizeof (struct mthca_next_seg) / 16;
\r
2013 if (unlikely((int)wr->num_ds > qp->rq.max_gs)) {
\r
2014 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP ,("RQ %06x too many gathers\n",qp->qpn));
\r
2021 for (i = 0; i < (int)wr->num_ds; ++i) {
\r
2022 ((struct mthca_data_seg *) wqe)->byte_count =
\r
2023 cl_hton32(wr->ds_array[i].length);
\r
2024 ((struct mthca_data_seg *) wqe)->lkey =
\r
2025 cl_hton32(wr->ds_array[i].lkey);
\r
2026 ((struct mthca_data_seg *) wqe)->addr =
\r
2027 cl_hton64(wr->ds_array[i].vaddr);
\r
2028 wqe += sizeof (struct mthca_data_seg);
\r
2029 size += sizeof (struct mthca_data_seg) / 16;
\r
2030 // HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP ,("RQ %06x [%02x] lkey 0x%08x vaddr 0x%I64x 0x %x 0x%08x\n",i,qp->qpn,
\r
2031 // (wr->ds_array[i].lkey),(wr->ds_array[i].vaddr),wr->ds_array[i].length, wr->wr_id));
\r
2034 qp->wrid[ind] = wr->wr_id;
\r
2036 ((struct mthca_next_seg *) prev_wqe)->nda_op =
\r
2037 cl_hton32((ind << qp->rq.wqe_shift) | 1);
\r
2039 ((struct mthca_next_seg *) prev_wqe)->ee_nds =
\r
2040 cl_hton32(MTHCA_NEXT_DBD | size);
\r
2045 dump_wqe(TRACE_LEVEL_VERBOSE, (u32*)wqe ,qp);
\r
2048 if (unlikely(ind >= qp->rq.max))
\r
2049 ind -= qp->rq.max;
\r
2053 if (likely(nreq)) {
\r
2054 doorbell[0] = cl_hton32((qp->rq.next_ind << qp->rq.wqe_shift) | size0);
\r
2055 doorbell[1] = cl_hton32((qp->qpn << 8) | (nreq & 255));
\r
2059 mthca_write64(doorbell, dev->kar + MTHCA_RECV_DOORBELL,
\r
2060 MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
\r
2063 qp->rq.next_ind = ind;
\r
2064 qp->rq.head += nreq;
\r
2066 spin_unlock_irqrestore(&lh);
\r
2070 int mthca_arbel_post_send(struct ib_qp *ibqp, struct _ib_send_wr *wr,
\r
2071 struct _ib_send_wr **bad_wr)
\r
2073 struct mthca_dev *dev = to_mdev(ibqp->device);
\r
2074 struct mthca_qp *qp = to_mqp(ibqp);
\r
2075 __be32 doorbell[2];
\r
2083 u32 f0 = unlikely(wr->send_opt & IB_SEND_OPT_FENCE) ? MTHCA_SEND_DOORBELL_FENCE : 0;
\r
2086 enum ib_wr_opcode opcode;
\r
2087 SPIN_LOCK_PREP(lh);
\r
2089 spin_lock_irqsave(&qp->sq.lock, &lh);
\r
2091 /* XXX check that state is OK to post send */
\r
2093 ind = qp->sq.head & (qp->sq.max - 1);
\r
2095 for (nreq = 0; wr; ++nreq, wr = wr->p_next) {
\r
2096 if (unlikely(nreq == MTHCA_ARBEL_MAX_WQES_PER_SEND_DB)) {
\r
2098 doorbell[0] = cl_hton32((MTHCA_ARBEL_MAX_WQES_PER_SEND_DB << 24) |
\r
2099 ((qp->sq.head & 0xffff) << 8) |f0 | op0);
\r
2100 doorbell[1] = cl_hton32((qp->qpn << 8) | size0);
\r
2101 qp->sq.head += MTHCA_ARBEL_MAX_WQES_PER_SEND_DB;
\r
2103 f0 = unlikely(wr->send_opt & IB_SEND_OPT_FENCE) ? MTHCA_SEND_DOORBELL_FENCE : 0;
\r
2106 * Make sure that descriptors are written before
\r
2107 * doorbell record.
\r
2110 *qp->sq.db = cl_hton32(qp->sq.head & 0xffff);
\r
2113 * Make sure doorbell record is written before we
\r
2114 * write MMIO send doorbell.
\r
2117 mthca_write64(doorbell, dev->kar + MTHCA_SEND_DOORBELL,
\r
2118 MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
\r
2121 if (mthca_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
\r
2122 HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_QP,("SQ %06x full (%u head, %u tail,"
\r
2123 " %d max, %d nreq)\n", qp->qpn,
\r
2124 qp->sq.head, qp->sq.tail,
\r
2125 qp->sq.max, nreq));
\r
2132 wqe = get_send_wqe(qp, ind);
\r
2133 prev_wqe = qp->sq.last;
\r
2134 qp->sq.last = wqe;
\r
2135 opcode = conv_ibal_wr_opcode(wr);
\r
2137 ((struct mthca_next_seg *) wqe)->flags =
\r
2138 ((wr->send_opt & IB_SEND_OPT_SIGNALED) ?
\r
2139 cl_hton32(MTHCA_NEXT_CQ_UPDATE) : 0) |
\r
2140 ((wr->send_opt & IB_SEND_OPT_SOLICITED) ?
\r
2141 cl_hton32(MTHCA_NEXT_SOLICIT) : 0) |
\r
2142 ((wr->send_opt & IB_SEND_OPT_TX_IP_CSUM) ?
\r
2143 cl_hton32(MTHCA_NEXT_IP_CSUM) : 0) |
\r
2144 ((wr->send_opt & IB_SEND_OPT_TX_TCP_UDP_CSUM) ?
\r
2145 cl_hton32(MTHCA_NEXT_TCP_UDP_CSUM) : 0) |
\r
2147 if (opcode == MTHCA_OPCODE_SEND_IMM||
\r
2148 opcode == MTHCA_OPCODE_RDMA_WRITE_IMM)
\r
2149 ((struct mthca_next_seg *) wqe)->imm = wr->immediate_data;
\r
2151 wqe += sizeof (struct mthca_next_seg);
\r
2152 size = sizeof (struct mthca_next_seg) / 16;
\r
2154 switch (qp->transport) {
\r
2157 case MTHCA_OPCODE_ATOMIC_CS:
\r
2158 case MTHCA_OPCODE_ATOMIC_FA:
\r
2159 ((struct mthca_raddr_seg *) wqe)->raddr =
\r
2160 cl_hton64(wr->remote_ops.vaddr);
\r
2161 ((struct mthca_raddr_seg *) wqe)->rkey =
\r
2162 wr->remote_ops.rkey;
\r
2163 ((struct mthca_raddr_seg *) wqe)->reserved = 0;
\r
2165 wqe += sizeof (struct mthca_raddr_seg);
\r
2167 if (opcode == MTHCA_OPCODE_ATOMIC_CS) {
\r
2168 ((struct mthca_atomic_seg *) wqe)->swap_add =
\r
2169 (wr->remote_ops.atomic2);
\r
2170 ((struct mthca_atomic_seg *) wqe)->compare =
\r
2171 (wr->remote_ops.atomic1);
\r
2173 ((struct mthca_atomic_seg *) wqe)->swap_add =
\r
2174 (wr->remote_ops.atomic1);
\r
2175 ((struct mthca_atomic_seg *) wqe)->compare = 0;
\r
2178 wqe += sizeof (struct mthca_atomic_seg);
\r
2179 size += (sizeof (struct mthca_raddr_seg) +
\r
2180 sizeof (struct mthca_atomic_seg)) / 16 ;
\r
2183 case MTHCA_OPCODE_RDMA_READ:
\r
2184 case MTHCA_OPCODE_RDMA_WRITE:
\r
2185 case MTHCA_OPCODE_RDMA_WRITE_IMM:
\r
2186 ((struct mthca_raddr_seg *) wqe)->raddr =
\r
2187 cl_hton64(wr->remote_ops.vaddr);
\r
2188 ((struct mthca_raddr_seg *) wqe)->rkey =
\r
2189 wr->remote_ops.rkey;
\r
2190 ((struct mthca_raddr_seg *) wqe)->reserved = 0;
\r
2191 wqe += sizeof (struct mthca_raddr_seg);
\r
2192 size += sizeof (struct mthca_raddr_seg) / 16;
\r
2196 /* No extra segments required for sends */
\r
2204 case MTHCA_OPCODE_RDMA_WRITE:
\r
2205 case MTHCA_OPCODE_RDMA_WRITE_IMM:
\r
2206 ((struct mthca_raddr_seg *) wqe)->raddr =
\r
2207 cl_hton64(wr->remote_ops.vaddr);
\r
2208 ((struct mthca_raddr_seg *) wqe)->rkey =
\r
2209 wr->remote_ops.rkey;
\r
2210 ((struct mthca_raddr_seg *) wqe)->reserved = 0;
\r
2211 wqe += sizeof (struct mthca_raddr_seg);
\r
2212 size += sizeof (struct mthca_raddr_seg) / 16;
\r
2216 /* No extra segments required for sends */
\r
2223 memcpy(((struct mthca_arbel_ud_seg *) wqe)->av,
\r
2224 to_mah((struct ib_ah *)wr->dgrm.ud.h_av)->av, MTHCA_AV_SIZE);
\r
2225 ((struct mthca_arbel_ud_seg *) wqe)->dqpn = wr->dgrm.ud.remote_qp;
\r
2226 ((struct mthca_arbel_ud_seg *) wqe)->qkey = wr->dgrm.ud.remote_qkey;
\r
2228 wqe += sizeof (struct mthca_arbel_ud_seg);
\r
2229 size += sizeof (struct mthca_arbel_ud_seg) / 16;
\r
2233 err = build_mlx_header(dev, to_msqp(qp), ind, wr,
\r
2234 (void*)(wqe - sizeof (struct mthca_next_seg)),
\r
2241 wqe += sizeof (struct mthca_data_seg);
\r
2242 size += sizeof (struct mthca_data_seg) / 16;
\r
2246 if ((int)wr->num_ds > qp->sq.max_gs) {
\r
2247 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP ,("SQ %06x full too many gathers\n",qp->qpn));
\r
2253 if (wr->send_opt & IB_SEND_OPT_INLINE) {
\r
2255 struct mthca_inline_seg *seg = (struct mthca_inline_seg *)wqe;
\r
2258 wqe += sizeof *seg;
\r
2259 for (i = 0; i < (int)wr->num_ds; ++i) {
\r
2260 struct _ib_local_ds *sge = &wr->ds_array[i];
\r
2264 if (s > (uint32_t)qp->max_inline_data) {
\r
2271 memcpy(wqe, (void *) (uintptr_t) sge->vaddr,
\r
2273 wqe += sge->length;
\r
2276 seg->byte_count = cl_hton32(MTHCA_INLINE_SEG | s);
\r
2277 size += align(s + sizeof *seg, 16) / 16;
\r
2280 for (i = 0; i < (int)wr->num_ds; ++i) {
\r
2281 ((struct mthca_data_seg *) wqe)->byte_count =
\r
2282 cl_hton32(wr->ds_array[i].length);
\r
2283 ((struct mthca_data_seg *) wqe)->lkey =
\r
2284 cl_hton32(wr->ds_array[i].lkey);
\r
2285 ((struct mthca_data_seg *) wqe)->addr =
\r
2286 cl_hton64(wr->ds_array[i].vaddr);
\r
2287 wqe += sizeof (struct mthca_data_seg);
\r
2288 size += sizeof (struct mthca_data_seg) / 16;
\r
2292 /* Add one more inline data segment for ICRC */
\r
2293 if (qp->transport == MLX) {
\r
2294 ((struct mthca_data_seg *) wqe)->byte_count =
\r
2295 cl_hton32((unsigned long)((1 << 31) | 4));
\r
2296 ((u32 *) wqe)[1] = 0;
\r
2297 wqe += sizeof (struct mthca_data_seg);
\r
2298 size += sizeof (struct mthca_data_seg) / 16;
\r
2301 qp->wrid[ind + qp->rq.max] = wr->wr_id;
\r
2303 if (opcode == MTHCA_OPCODE_INVALID) {
\r
2304 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP ,("SQ %06x opcode invalid\n",qp->qpn));
\r
2311 ((struct mthca_next_seg *) prev_wqe)->nda_op =
\r
2312 cl_hton32(((ind << qp->sq.wqe_shift) +
\r
2313 qp->send_wqe_offset) |opcode);
\r
2315 ((struct mthca_next_seg *) prev_wqe)->ee_nds =
\r
2316 cl_hton32(MTHCA_NEXT_DBD | size |
\r
2317 ((wr->send_opt & IB_SEND_OPT_FENCE) ?
\r
2318 MTHCA_NEXT_FENCE : 0));
\r
2326 if (unlikely(ind >= qp->sq.max))
\r
2327 ind -= qp->sq.max;
\r
2331 if (likely(nreq)) {
\r
2332 doorbell[0] = cl_hton32((nreq << 24) |
\r
2333 ((qp->sq.head & 0xffff) << 8) |f0 | op0);
\r
2334 doorbell[1] = cl_hton32((qp->qpn << 8) | size0);
\r
2335 qp->sq.head += nreq;
\r
2338 * Make sure that descriptors are written before
\r
2339 * doorbell record.
\r
2342 *qp->sq.db = cl_hton32(qp->sq.head & 0xffff);
\r
2345 * Make sure doorbell record is written before we
\r
2346 * write MMIO send doorbell.
\r
2349 mthca_write64(doorbell,
\r
2350 dev->kar + MTHCA_SEND_DOORBELL,
\r
2351 MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
\r
2354 spin_unlock_irqrestore(&lh);
\r
2358 int mthca_arbel_post_recv(struct ib_qp *ibqp, struct _ib_recv_wr *wr,
\r
2359 struct _ib_recv_wr **bad_wr)
\r
2361 struct mthca_qp *qp = to_mqp(ibqp);
\r
2367 SPIN_LOCK_PREP(lh);
\r
2369 spin_lock_irqsave(&qp->rq.lock, &lh);
\r
2371 /* XXX check that state is OK to post receive */
\r
2373 ind = qp->rq.head & (qp->rq.max - 1);
\r
2375 for (nreq = 0; wr; ++nreq, wr = wr->p_next) {
\r
2376 if (mthca_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
\r
2377 HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_QP,("RQ %06x full (%u head, %u tail,"
\r
2378 " %d max, %d nreq)\n", qp->qpn,
\r
2379 qp->rq.head, qp->rq.tail,
\r
2380 qp->rq.max, nreq));
\r
2387 wqe = get_recv_wqe(qp, ind);
\r
2389 ((struct mthca_next_seg *) wqe)->flags = 0;
\r
2391 wqe += sizeof (struct mthca_next_seg);
\r
2393 if (unlikely((int)wr->num_ds > qp->rq.max_gs)) {
\r
2394 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP ,("RQ %06x full too many scatter\n",qp->qpn));
\r
2401 for (i = 0; i < (int)wr->num_ds; ++i) {
\r
2402 ((struct mthca_data_seg *) wqe)->byte_count =
\r
2403 cl_hton32(wr->ds_array[i].length);
\r
2404 ((struct mthca_data_seg *) wqe)->lkey =
\r
2405 cl_hton32(wr->ds_array[i].lkey);
\r
2406 ((struct mthca_data_seg *) wqe)->addr =
\r
2407 cl_hton64(wr->ds_array[i].vaddr);
\r
2408 wqe += sizeof (struct mthca_data_seg);
\r
2411 if (i < qp->rq.max_gs) {
\r
2412 ((struct mthca_data_seg *) wqe)->byte_count = 0;
\r
2413 ((struct mthca_data_seg *) wqe)->lkey = cl_hton32(MTHCA_INVAL_LKEY);
\r
2414 ((struct mthca_data_seg *) wqe)->addr = 0;
\r
2417 qp->wrid[ind] = wr->wr_id;
\r
2420 if (unlikely(ind >= qp->rq.max))
\r
2421 ind -= qp->rq.max;
\r
2424 if (likely(nreq)) {
\r
2425 qp->rq.head += nreq;
\r
2428 * Make sure that descriptors are written before
\r
2429 * doorbell record.
\r
2432 *qp->rq.db = cl_hton32(qp->rq.head & 0xffff);
\r
2435 spin_unlock_irqrestore(&lh);
\r
2439 void mthca_free_err_wqe(struct mthca_dev *dev, struct mthca_qp *qp, int is_send,
\r
2440 int index, int *dbd, __be32 *new_wqe)
\r
2442 struct mthca_next_seg *next;
\r
2444 UNREFERENCED_PARAMETER(dev);
\r
2447 * For SRQs, all WQEs generate a CQE, so we're always at the
\r
2448 * end of the doorbell chain.
\r
2450 if (qp->ibqp.srq) {
\r
2456 next = get_send_wqe(qp, index);
\r
2458 next = get_recv_wqe(qp, index);
\r
2460 *dbd = !!(next->ee_nds & cl_hton32(MTHCA_NEXT_DBD));
\r
2461 if (next->ee_nds & cl_hton32(0x3f))
\r
2462 *new_wqe = (next->nda_op & cl_hton32((unsigned long)~0x3f)) |
\r
2463 (next->ee_nds & cl_hton32(0x3f));
\r
2468 int mthca_init_qp_table(struct mthca_dev *dev)
\r
2474 spin_lock_init(&dev->qp_table.lock);
\r
2475 fill_state_table();
\r
2478 * We reserve 2 extra QPs per port for the special QPs. The
\r
2479 * special QP for port 1 has to be even, so round up.
\r
2481 dev->qp_table.sqp_start = (dev->limits.reserved_qps + 1) & ~1UL;
\r
2482 err = mthca_alloc_init(&dev->qp_table.alloc,
\r
2483 dev->limits.num_qps,
\r
2485 dev->qp_table.sqp_start +
\r
2486 MTHCA_MAX_PORTS * 2);
\r
2490 err = mthca_array_init(&dev->qp_table.qp,
\r
2491 dev->limits.num_qps);
\r
2493 mthca_alloc_cleanup(&dev->qp_table.alloc);
\r
2497 for (i = 0; i < 2; ++i) {
\r
2498 err = mthca_CONF_SPECIAL_QP(dev, i ? IB_QPT_QP1 : IB_QPT_QP0,
\r
2499 dev->qp_table.sqp_start + i * 2,
\r
2504 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("CONF_SPECIAL_QP returned "
\r
2505 "status %02x, aborting.\n",
\r
2514 mthca_CONF_SPECIAL_QP(dev, IB_QPT_QP1, 0, &status);
\r
2515 mthca_CONF_SPECIAL_QP(dev, IB_QPT_QP0, 0, &status);
\r
2517 mthca_array_cleanup(&dev->qp_table.qp, dev->limits.num_qps);
\r
2518 mthca_alloc_cleanup(&dev->qp_table.alloc);
\r
2523 void mthca_cleanup_qp_table(struct mthca_dev *dev)
\r
2527 mthca_CONF_SPECIAL_QP(dev, IB_QPT_QP1, 0, &status);
\r
2528 mthca_CONF_SPECIAL_QP(dev, IB_QPT_QP0, 0, &status);
\r
2530 mthca_array_cleanup(&dev->qp_table.qp, dev->limits.num_qps);
\r
2531 mthca_alloc_cleanup(&dev->qp_table.alloc);
\r