[MTHCA] bug fix: removed unnecessary conversion to network order in atomic operations...
[mirror/winof/.git] / hw / mthca / kernel / mthca_qp.c
1 /*\r
2  * Copyright (c) 2004 Topspin Communications.  All rights reserved.\r
3  * Copyright (c) 2005 Cisco Systems. All rights reserved.\r
4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.\r
5  * Copyright (c) 2004 Voltaire, Inc. All rights reserved. \r
6  *\r
7  * This software is available to you under a choice of one of two\r
8  * licenses.  You may choose to be licensed under the terms of the GNU\r
9  * General Public License (GPL) Version 2, available from the file\r
10  * COPYING in the main directory of this source tree, or the\r
11  * OpenIB.org BSD license below:\r
12  *\r
13  *     Redistribution and use in source and binary forms, with or\r
14  *     without modification, are permitted provided that the following\r
15  *     conditions are met:\r
16  *\r
17  *      - Redistributions of source code must retain the above\r
18  *        copyright notice, this list of conditions and the following\r
19  *        disclaimer.\r
20  *\r
21  *      - Redistributions in binary form must reproduce the above\r
22  *        copyright notice, this list of conditions and the following\r
23  *        disclaimer in the documentation and/or other materials\r
24  *        provided with the distribution.\r
25  *\r
26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
33  * SOFTWARE.\r
34  *\r
35  * $Id$\r
36  */\r
37 \r
38 #include <ib_verbs.h>\r
39 #include <ib_cache.h>\r
40 #include <ib_pack.h>\r
41 \r
42 #include "mthca_dev.h"\r
43 #if defined(EVENT_TRACING)\r
44 #ifdef offsetof\r
45 #undef offsetof\r
46 #endif\r
47 #include "mthca_qp.tmh"\r
48 #endif\r
49 #include "mthca_cmd.h"\r
50 #include "mthca_memfree.h"\r
51 #include "mthca_wqe.h"\r
52 \r
53 \r
54 enum {\r
55         MTHCA_MAX_DIRECT_QP_SIZE = 4 * PAGE_SIZE,\r
56         MTHCA_ACK_REQ_FREQ       = 10,\r
57         MTHCA_FLIGHT_LIMIT       = 9,\r
58         MTHCA_UD_HEADER_SIZE     = 72, /* largest UD header possible */\r
59         MTHCA_INLINE_HEADER_SIZE = 4,  /* data segment overhead for inline */\r
60         MTHCA_INLINE_CHUNK_SIZE  = 16  /* inline data segment chunk */\r
61 };\r
62 \r
63 enum {\r
64         MTHCA_QP_STATE_RST  = 0,\r
65         MTHCA_QP_STATE_INIT = 1,\r
66         MTHCA_QP_STATE_RTR  = 2,\r
67         MTHCA_QP_STATE_RTS  = 3,\r
68         MTHCA_QP_STATE_SQE  = 4,\r
69         MTHCA_QP_STATE_SQD  = 5,\r
70         MTHCA_QP_STATE_ERR  = 6,\r
71         MTHCA_QP_STATE_DRAINING = 7\r
72 };\r
73 \r
74 enum {\r
75         MTHCA_QP_ST_RC  = 0x0,\r
76         MTHCA_QP_ST_UC  = 0x1,\r
77         MTHCA_QP_ST_RD  = 0x2,\r
78         MTHCA_QP_ST_UD  = 0x3,\r
79         MTHCA_QP_ST_MLX = 0x7\r
80 };\r
81 \r
82 enum {\r
83         MTHCA_QP_PM_MIGRATED = 0x3,\r
84         MTHCA_QP_PM_ARMED    = 0x0,\r
85         MTHCA_QP_PM_REARM    = 0x1\r
86 };\r
87 \r
88 enum {\r
89         /* qp_context flags */\r
90         MTHCA_QP_BIT_DE  = 1 <<  8,\r
91         /* params1 */\r
92         MTHCA_QP_BIT_SRE = 1 << 15,\r
93         MTHCA_QP_BIT_SWE = 1 << 14,\r
94         MTHCA_QP_BIT_SAE = 1 << 13,\r
95         MTHCA_QP_BIT_SIC = 1 <<  4,\r
96         MTHCA_QP_BIT_SSC = 1 <<  3,\r
97         /* params2 */\r
98         MTHCA_QP_BIT_RRE = 1 << 15,\r
99         MTHCA_QP_BIT_RWE = 1 << 14,\r
100         MTHCA_QP_BIT_RAE = 1 << 13,\r
101         MTHCA_QP_BIT_RIC = 1 <<  4,\r
102         MTHCA_QP_BIT_RSC = 1 <<  3\r
103 };\r
104 \r
105 #pragma pack(push,1)\r
106 struct mthca_qp_path {\r
107         __be32 port_pkey;\r
108         u8     rnr_retry;\r
109         u8     g_mylmc;\r
110         __be16 rlid;\r
111         u8     ackto;\r
112         u8     mgid_index;\r
113         u8     static_rate;\r
114         u8     hop_limit;\r
115         __be32 sl_tclass_flowlabel;\r
116         u8     rgid[16];\r
117 } ;\r
118 \r
119 struct mthca_qp_context {\r
120         __be32 flags;\r
121         __be32 tavor_sched_queue; /* Reserved on Arbel */\r
122         u8     mtu_msgmax;\r
123         u8     rq_size_stride;  /* Reserved on Tavor */\r
124         u8     sq_size_stride;  /* Reserved on Tavor */\r
125         u8     rlkey_arbel_sched_queue; /* Reserved on Tavor */\r
126         __be32 usr_page;\r
127         __be32 local_qpn;\r
128         __be32 remote_qpn;\r
129         u32    reserved1[2];\r
130         struct mthca_qp_path pri_path;\r
131         struct mthca_qp_path alt_path;\r
132         __be32 rdd;\r
133         __be32 pd;\r
134         __be32 wqe_base;\r
135         __be32 wqe_lkey;\r
136         __be32 params1;\r
137         __be32 reserved2;\r
138         __be32 next_send_psn;\r
139         __be32 cqn_snd;\r
140         __be32 snd_wqe_base_l;  /* Next send WQE on Tavor */\r
141         __be32 snd_db_index;    /* (debugging only entries) */\r
142         __be32 last_acked_psn;\r
143         __be32 ssn;\r
144         __be32 params2;\r
145         __be32 rnr_nextrecvpsn;\r
146         __be32 ra_buff_indx;\r
147         __be32 cqn_rcv;\r
148         __be32 rcv_wqe_base_l;  /* Next recv WQE on Tavor */\r
149         __be32 rcv_db_index;    /* (debugging only entries) */\r
150         __be32 qkey;\r
151         __be32 srqn;\r
152         __be32 rmsn;\r
153         __be16 rq_wqe_counter;  /* reserved on Tavor */\r
154         __be16 sq_wqe_counter;  /* reserved on Tavor */\r
155         u32    reserved3[18];\r
156 } ;\r
157 \r
158 struct mthca_qp_param {\r
159         __be32 opt_param_mask;\r
160         u32    reserved1;\r
161         struct mthca_qp_context context;\r
162         u32    reserved2[62];\r
163 } ;\r
164 #pragma pack(pop)\r
165 \r
166 enum {\r
167         MTHCA_QP_OPTPAR_ALT_ADDR_PATH     = 1 << 0,\r
168         MTHCA_QP_OPTPAR_RRE               = 1 << 1,\r
169         MTHCA_QP_OPTPAR_RAE               = 1 << 2,\r
170         MTHCA_QP_OPTPAR_RWE               = 1 << 3,\r
171         MTHCA_QP_OPTPAR_PKEY_INDEX        = 1 << 4,\r
172         MTHCA_QP_OPTPAR_Q_KEY             = 1 << 5,\r
173         MTHCA_QP_OPTPAR_RNR_TIMEOUT       = 1 << 6,\r
174         MTHCA_QP_OPTPAR_PRIMARY_ADDR_PATH = 1 << 7,\r
175         MTHCA_QP_OPTPAR_SRA_MAX           = 1 << 8,\r
176         MTHCA_QP_OPTPAR_RRA_MAX           = 1 << 9,\r
177         MTHCA_QP_OPTPAR_PM_STATE          = 1 << 10,\r
178         MTHCA_QP_OPTPAR_PORT_NUM          = 1 << 11,\r
179         MTHCA_QP_OPTPAR_RETRY_COUNT       = 1 << 12,\r
180         MTHCA_QP_OPTPAR_ALT_RNR_RETRY     = 1 << 13,\r
181         MTHCA_QP_OPTPAR_ACK_TIMEOUT       = 1 << 14,\r
182         MTHCA_QP_OPTPAR_RNR_RETRY         = 1 << 15,\r
183         MTHCA_QP_OPTPAR_SCHED_QUEUE       = 1 << 16\r
184 };\r
185 \r
186 static const u8 mthca_opcode[] = {\r
187         MTHCA_OPCODE_RDMA_WRITE,\r
188         MTHCA_OPCODE_RDMA_WRITE_IMM,\r
189         MTHCA_OPCODE_SEND,\r
190         MTHCA_OPCODE_SEND_IMM,\r
191         MTHCA_OPCODE_RDMA_READ,\r
192         MTHCA_OPCODE_ATOMIC_CS,\r
193         MTHCA_OPCODE_ATOMIC_FA\r
194 };\r
195 \r
196 \r
197 enum { RC, UC, UD, RD, RDEE, MLX, NUM_TRANS };\r
198 \r
199 static struct _state_table {\r
200         int trans;\r
201         u32 req_param[NUM_TRANS];\r
202         u32 opt_param[NUM_TRANS];\r
203 } state_table[IBQPS_ERR + 1][IBQPS_ERR + 1]= {0};\r
204 \r
205 static void fill_state_table()\r
206 {\r
207         struct _state_table *t;\r
208         RtlZeroMemory( state_table, sizeof(state_table) );\r
209 \r
210         /* IBQPS_RESET */       \r
211         t = &state_table[IBQPS_RESET][0];\r
212         t[IBQPS_RESET].trans                                    = MTHCA_TRANS_ANY2RST;\r
213         t[IBQPS_ERR].trans                                              = MTHCA_TRANS_ANY2ERR;\r
214 \r
215         t[IBQPS_INIT].trans                                             = MTHCA_TRANS_RST2INIT;\r
216         t[IBQPS_INIT].req_param[UD]     = IB_QP_PKEY_INDEX |IB_QP_PORT |IB_QP_QKEY;\r
217         t[IBQPS_INIT].req_param[UC]     = IB_QP_PKEY_INDEX |IB_QP_PORT |IB_QP_ACCESS_FLAGS;\r
218         t[IBQPS_INIT].req_param[RC]     = IB_QP_PKEY_INDEX |IB_QP_PORT |IB_QP_ACCESS_FLAGS;\r
219         t[IBQPS_INIT].req_param[MLX]    = IB_QP_PKEY_INDEX |IB_QP_QKEY;\r
220         t[IBQPS_INIT].opt_param[MLX]    = IB_QP_PORT;\r
221 \r
222         /* IBQPS_INIT */        \r
223         t = &state_table[IBQPS_INIT][0];\r
224         t[IBQPS_RESET].trans                                    = MTHCA_TRANS_ANY2RST;\r
225         t[IBQPS_ERR].trans                                              = MTHCA_TRANS_ANY2ERR;\r
226 \r
227         t[IBQPS_INIT].trans                                             = MTHCA_TRANS_INIT2INIT;\r
228         t[IBQPS_INIT].opt_param[UD]     = IB_QP_PKEY_INDEX |IB_QP_PORT |IB_QP_QKEY;\r
229         t[IBQPS_INIT].opt_param[UC]     = IB_QP_PKEY_INDEX |IB_QP_PORT |IB_QP_ACCESS_FLAGS;\r
230         t[IBQPS_INIT].opt_param[RC]     = IB_QP_PKEY_INDEX |IB_QP_PORT |IB_QP_ACCESS_FLAGS;\r
231         t[IBQPS_INIT].opt_param[MLX]    = IB_QP_PKEY_INDEX |IB_QP_QKEY;\r
232 \r
233         t[IBQPS_RTR].trans                                              = MTHCA_TRANS_INIT2RTR;\r
234         t[IBQPS_RTR].req_param[UC]      = \r
235                 IB_QP_AV |IB_QP_PATH_MTU |IB_QP_DEST_QPN |IB_QP_RQ_PSN;\r
236         t[IBQPS_RTR].req_param[RC]      = \r
237                 IB_QP_AV |IB_QP_PATH_MTU |IB_QP_DEST_QPN |IB_QP_RQ_PSN |IB_QP_MAX_DEST_RD_ATOMIC |IB_QP_MIN_RNR_TIMER;\r
238         t[IBQPS_RTR].opt_param[UD]      = IB_QP_PKEY_INDEX |IB_QP_QKEY;\r
239         t[IBQPS_RTR].opt_param[UC]      = IB_QP_PKEY_INDEX |IB_QP_ALT_PATH |IB_QP_ACCESS_FLAGS;\r
240         t[IBQPS_RTR].opt_param[RC]      = IB_QP_PKEY_INDEX |IB_QP_ALT_PATH |IB_QP_ACCESS_FLAGS;\r
241         t[IBQPS_RTR].opt_param[MLX]     = IB_QP_PKEY_INDEX |IB_QP_QKEY;\r
242 \r
243 /* IBQPS_RTR */ \r
244         t = &state_table[IBQPS_RTR][0];\r
245         t[IBQPS_RESET].trans                                    = MTHCA_TRANS_ANY2RST;\r
246         t[IBQPS_ERR].trans                                              = MTHCA_TRANS_ANY2ERR;\r
247 \r
248         t[IBQPS_RTS].trans                                              = MTHCA_TRANS_RTR2RTS;\r
249         t[IBQPS_RTS].req_param[UD]      = IB_QP_SQ_PSN;\r
250         t[IBQPS_RTS].req_param[UC]      = IB_QP_SQ_PSN;\r
251         t[IBQPS_RTS].req_param[RC]      = \r
252                 IB_QP_TIMEOUT |IB_QP_RETRY_CNT |IB_QP_RNR_RETRY |IB_QP_SQ_PSN |IB_QP_MAX_QP_RD_ATOMIC;\r
253         t[IBQPS_RTS].req_param[MLX]     = IB_QP_SQ_PSN;\r
254         t[IBQPS_RTS].opt_param[UD]      = IB_QP_CUR_STATE |IB_QP_QKEY;\r
255         t[IBQPS_RTS].opt_param[UC]      = \r
256                 IB_QP_CUR_STATE |IB_QP_ALT_PATH |IB_QP_ACCESS_FLAGS |IB_QP_PATH_MIG_STATE;\r
257         t[IBQPS_RTS].opt_param[RC]      =       IB_QP_CUR_STATE |IB_QP_ALT_PATH |\r
258                 IB_QP_ACCESS_FLAGS |IB_QP_MIN_RNR_TIMER |IB_QP_PATH_MIG_STATE;\r
259         t[IBQPS_RTS].opt_param[MLX]     = IB_QP_CUR_STATE |IB_QP_QKEY;\r
260 \r
261         /* IBQPS_RTS */ \r
262         t = &state_table[IBQPS_RTS][0];\r
263         t[IBQPS_RESET].trans                                    = MTHCA_TRANS_ANY2RST;\r
264         t[IBQPS_ERR].trans                                              = MTHCA_TRANS_ANY2ERR;\r
265 \r
266         t[IBQPS_RTS].trans                                              = MTHCA_TRANS_RTS2RTS;\r
267         t[IBQPS_RTS].opt_param[UD]      = IB_QP_CUR_STATE |IB_QP_QKEY;\r
268         t[IBQPS_RTS].opt_param[UC]      = IB_QP_ACCESS_FLAGS |IB_QP_ALT_PATH |IB_QP_PATH_MIG_STATE;\r
269         t[IBQPS_RTS].opt_param[RC]      =       IB_QP_ACCESS_FLAGS |\r
270                 IB_QP_ALT_PATH |IB_QP_PATH_MIG_STATE |IB_QP_MIN_RNR_TIMER;\r
271         t[IBQPS_RTS].opt_param[MLX]     = IB_QP_CUR_STATE |IB_QP_QKEY;\r
272 \r
273         t[IBQPS_SQD].trans                                              = MTHCA_TRANS_RTS2SQD;\r
274         t[IBQPS_SQD].opt_param[UD]      = IB_QP_EN_SQD_ASYNC_NOTIFY;\r
275         t[IBQPS_SQD].opt_param[UC]      = IB_QP_EN_SQD_ASYNC_NOTIFY;\r
276         t[IBQPS_SQD].opt_param[RC]      =       IB_QP_EN_SQD_ASYNC_NOTIFY;\r
277         t[IBQPS_SQD].opt_param[MLX]     = IB_QP_EN_SQD_ASYNC_NOTIFY;\r
278 \r
279         /* IBQPS_SQD */ \r
280         t = &state_table[IBQPS_SQD][0];\r
281         t[IBQPS_RESET].trans                                    = MTHCA_TRANS_ANY2RST;\r
282         t[IBQPS_ERR].trans                                              = MTHCA_TRANS_ANY2ERR;\r
283 \r
284         t[IBQPS_RTS].trans                                              = MTHCA_TRANS_SQD2RTS;\r
285         t[IBQPS_RTS].opt_param[UD]      = IB_QP_CUR_STATE |IB_QP_QKEY;\r
286         t[IBQPS_RTS].opt_param[UC]      = IB_QP_CUR_STATE |\r
287                 IB_QP_ALT_PATH |IB_QP_ACCESS_FLAGS |IB_QP_PATH_MIG_STATE;\r
288         t[IBQPS_RTS].opt_param[RC]      =       IB_QP_CUR_STATE |IB_QP_ALT_PATH |\r
289                 IB_QP_ACCESS_FLAGS |IB_QP_MIN_RNR_TIMER |IB_QP_PATH_MIG_STATE;\r
290         t[IBQPS_RTS].opt_param[MLX]     = IB_QP_CUR_STATE |IB_QP_QKEY;\r
291 \r
292         t[IBQPS_SQD].trans                                              = MTHCA_TRANS_SQD2SQD;\r
293         t[IBQPS_SQD].opt_param[UD]      = IB_QP_PKEY_INDEX |IB_QP_QKEY;\r
294         t[IBQPS_SQD].opt_param[UC]      = IB_QP_AV |    IB_QP_CUR_STATE |\r
295                 IB_QP_ALT_PATH |IB_QP_ACCESS_FLAGS |IB_QP_PKEY_INDEX |IB_QP_PATH_MIG_STATE;\r
296         t[IBQPS_SQD].opt_param[RC]      =       IB_QP_AV |IB_QP_TIMEOUT |IB_QP_RETRY_CNT |IB_QP_RNR_RETRY |\r
297                 IB_QP_MAX_QP_RD_ATOMIC |IB_QP_MAX_DEST_RD_ATOMIC |IB_QP_CUR_STATE |IB_QP_ALT_PATH |\r
298                 IB_QP_ACCESS_FLAGS |IB_QP_PKEY_INDEX |IB_QP_MIN_RNR_TIMER |IB_QP_PATH_MIG_STATE;\r
299         t[IBQPS_SQD].opt_param[MLX]     = IB_QP_PKEY_INDEX |IB_QP_QKEY;\r
300 \r
301         /* IBQPS_SQE */ \r
302         t = &state_table[IBQPS_SQE][0];\r
303         t[IBQPS_RESET].trans                                    = MTHCA_TRANS_ANY2RST;\r
304         t[IBQPS_ERR].trans                                              = MTHCA_TRANS_ANY2ERR;\r
305 \r
306         t[IBQPS_RTS].trans                                              = MTHCA_TRANS_SQERR2RTS;\r
307         t[IBQPS_RTS].opt_param[UD]      = IB_QP_CUR_STATE |IB_QP_QKEY;\r
308         t[IBQPS_RTS].opt_param[UC]      = IB_QP_CUR_STATE | IB_QP_ACCESS_FLAGS;\r
309 //      t[IBQPS_RTS].opt_param[RC]      =       IB_QP_CUR_STATE |IB_QP_MIN_RNR_TIMER;\r
310         t[IBQPS_RTS].opt_param[MLX]     = IB_QP_CUR_STATE |IB_QP_QKEY;\r
311 \r
312         /* IBQPS_ERR */ \r
313         t = &state_table[IBQPS_ERR][0];\r
314         t[IBQPS_RESET].trans                                    = MTHCA_TRANS_ANY2RST;\r
315         t[IBQPS_ERR].trans                                              = MTHCA_TRANS_ANY2ERR;\r
316 \r
317 };\r
318 \r
319 \r
320 static int is_sqp(struct mthca_dev *dev, struct mthca_qp *qp)\r
321 {\r
322         return qp->qpn >= (u32)dev->qp_table.sqp_start &&\r
323                 qp->qpn <= (u32)dev->qp_table.sqp_start + 3;\r
324 }\r
325 \r
326 static int is_qp0(struct mthca_dev *dev, struct mthca_qp *qp)\r
327 {\r
328         return qp->qpn >= (u32)dev->qp_table.sqp_start &&\r
329                 qp->qpn <= (u32)(dev->qp_table.sqp_start + 1);\r
330 }\r
331 \r
332 \r
333 static void dump_wqe(u32 print_lvl, u32 *wqe_ptr , struct mthca_qp *qp_ptr)\r
334 {\r
335         __be32 *wqe = wqe_ptr;\r
336 \r
337         UNUSED_PARAM_WOWPP(qp_ptr);\r
338         UNUSED_PARAM_WOWPP(print_lvl);\r
339 \r
340         (void) wqe;     /* avoid warning if mthca_dbg compiled away... */\r
341         HCA_PRINT(print_lvl,HCA_DBG_QP,("WQE contents  QPN 0x%06x \n",qp_ptr->qpn));\r
342         HCA_PRINT(print_lvl,HCA_DBG_QP,("WQE contents [%02x] %08x %08x %08x %08x \n",0\r
343                 , cl_ntoh32(wqe[0]), cl_ntoh32(wqe[1]), cl_ntoh32(wqe[2]), cl_ntoh32(wqe[3])));\r
344         HCA_PRINT(print_lvl,HCA_DBG_QP,("WQE contents [%02x] %08x %08x %08x %08x \n",4\r
345                 , cl_ntoh32(wqe[4]), cl_ntoh32(wqe[5]), cl_ntoh32(wqe[6]), cl_ntoh32(wqe[7])));\r
346         HCA_PRINT(print_lvl,HCA_DBG_QP,("WQE contents [%02x] %08x %08x %08x %08x \n",8\r
347                 , cl_ntoh32(wqe[8]), cl_ntoh32(wqe[9]), cl_ntoh32(wqe[10]), cl_ntoh32(wqe[11])));\r
348         HCA_PRINT(print_lvl,HCA_DBG_QP,("WQE contents [%02x] %08x %08x %08x %08x \n",12\r
349                 , cl_ntoh32(wqe[12]), cl_ntoh32(wqe[13]), cl_ntoh32(wqe[14]), cl_ntoh32(wqe[15])));\r
350 \r
351 }\r
352 \r
353 \r
354 static void *get_recv_wqe(struct mthca_qp *qp, int n)\r
355 {\r
356         if (qp->is_direct)\r
357                 return (u8*)qp->queue.direct.page + (n << qp->rq.wqe_shift);\r
358         else\r
359                 return (u8*)qp->queue.page_list[(n << qp->rq.wqe_shift) >> PAGE_SHIFT].page +\r
360                         ((n << qp->rq.wqe_shift) & (PAGE_SIZE - 1));\r
361 }\r
362 \r
363 static void *get_send_wqe(struct mthca_qp *qp, int n)\r
364 {\r
365         if (qp->is_direct)\r
366                 return (u8*)qp->queue.direct.page + qp->send_wqe_offset +\r
367                         (n << qp->sq.wqe_shift);\r
368         else\r
369                 return (u8*)qp->queue.page_list[(qp->send_wqe_offset +\r
370                                             (n << qp->sq.wqe_shift)) >>\r
371                                            PAGE_SHIFT].page +\r
372                         ((qp->send_wqe_offset + (n << qp->sq.wqe_shift)) &\r
373                          (PAGE_SIZE - 1));\r
374 }\r
375 \r
376 static void mthca_wq_init(struct mthca_wq *wq)\r
377 {       \r
378         spin_lock_init(&wq->lock);      \r
379         wq->next_ind  = 0;      \r
380         wq->last_comp = wq->max - 1;    \r
381         wq->head      = 0;      \r
382         wq->tail      = 0;      \r
383 }\r
384 \r
385 void mthca_qp_event(struct mthca_dev *dev, u32 qpn,\r
386                     enum ib_event_type event_type, u8 vendor_code)\r
387 {\r
388         struct mthca_qp *qp;\r
389         ib_event_rec_t event;\r
390         SPIN_LOCK_PREP(lh);\r
391 \r
392         spin_lock(&dev->qp_table.lock, &lh);\r
393         qp = mthca_array_get(&dev->qp_table.qp, qpn & (dev->limits.num_qps - 1));\r
394         if (qp)\r
395                 atomic_inc(&qp->refcount);\r
396         spin_unlock(&lh);\r
397 \r
398         if (!qp) {\r
399                 HCA_PRINT(TRACE_LEVEL_WARNING,HCA_DBG_QP,("QP %06x Async event for bogus \n", qpn));\r
400                 return;\r
401         }\r
402 \r
403         event.type = event_type;\r
404         event.context = qp->ibqp.qp_context;\r
405         event.vendor_specific = vendor_code;\r
406         HCA_PRINT(TRACE_LEVEL_WARNING,HCA_DBG_QP,("QP %06x Async event  event_type 0x%x vendor_code 0x%x\n",\r
407                 qpn,event_type,vendor_code));\r
408         qp->ibqp.event_handler(&event);\r
409 \r
410         if (atomic_dec_and_test(&qp->refcount))\r
411                 wake_up(&qp->wait);\r
412 }\r
413 \r
414 static int to_mthca_state(enum ib_qp_state ib_state)\r
415 {\r
416         switch (ib_state) {\r
417         case IBQPS_RESET: return MTHCA_QP_STATE_RST;\r
418         case IBQPS_INIT:  return MTHCA_QP_STATE_INIT;\r
419         case IBQPS_RTR:   return MTHCA_QP_STATE_RTR;\r
420         case IBQPS_RTS:   return MTHCA_QP_STATE_RTS;\r
421         case IBQPS_SQD:   return MTHCA_QP_STATE_SQD;\r
422         case IBQPS_SQE:   return MTHCA_QP_STATE_SQE;\r
423         case IBQPS_ERR:   return MTHCA_QP_STATE_ERR;\r
424         default:                return -1;\r
425         }\r
426 }\r
427 \r
428 static int to_mthca_st(int transport)\r
429 {\r
430         switch (transport) {\r
431         case RC:  return MTHCA_QP_ST_RC;\r
432         case UC:  return MTHCA_QP_ST_UC;\r
433         case UD:  return MTHCA_QP_ST_UD;\r
434         case RD:  return MTHCA_QP_ST_RD;\r
435         case MLX: return MTHCA_QP_ST_MLX;\r
436         default:  return -1;\r
437         }\r
438 }\r
439 \r
440 static inline enum ib_qp_state to_ib_qp_state(int mthca_state)\r
441 {\r
442         switch (mthca_state) {\r
443         case MTHCA_QP_STATE_RST: return IBQPS_RESET;\r
444         case MTHCA_QP_STATE_INIT:  return IBQPS_INIT;\r
445         case MTHCA_QP_STATE_RTR:   return IBQPS_RTR;\r
446         case MTHCA_QP_STATE_RTS:   return IBQPS_RTS;\r
447         case MTHCA_QP_STATE_SQD:   return IBQPS_SQD;\r
448         case MTHCA_QP_STATE_DRAINING:   return IBQPS_SQD;\r
449         case MTHCA_QP_STATE_SQE:   return IBQPS_SQE;\r
450         case MTHCA_QP_STATE_ERR:   return IBQPS_ERR;\r
451         default:                return -1;\r
452         }\r
453 }\r
454 \r
455 static inline enum ib_mig_state to_ib_mig_state(int mthca_mig_state)\r
456 {\r
457         switch (mthca_mig_state) {\r
458         case 0:  return IB_MIG_ARMED;\r
459         case 1:  return IB_MIG_REARM;\r
460         case 3:  return IB_MIG_MIGRATED;\r
461         default: return -1;\r
462         }\r
463 }\r
464 \r
465 static int to_ib_qp_access_flags(int mthca_flags)\r
466 {\r
467         int ib_flags = 0;\r
468 \r
469         if (mthca_flags & MTHCA_QP_BIT_RRE)\r
470                 ib_flags |= MTHCA_ACCESS_REMOTE_READ;\r
471         if (mthca_flags & MTHCA_QP_BIT_RWE)\r
472                 ib_flags |= MTHCA_ACCESS_REMOTE_WRITE;\r
473         if (mthca_flags & MTHCA_QP_BIT_RAE)\r
474                 ib_flags |= MTHCA_ACCESS_REMOTE_ATOMIC;\r
475 \r
476         return ib_flags;\r
477 }\r
478 \r
479 static void to_ib_ah_attr(struct mthca_dev *dev, struct ib_ah_attr *ib_ah_attr,\r
480                                 struct mthca_qp_path *path)\r
481 {\r
482         memset(ib_ah_attr, 0, sizeof *ib_ah_attr);\r
483         ib_ah_attr->port_num      = (u8)((cl_ntoh32(path->port_pkey) >> 24) & 0x3);\r
484 \r
485         if (ib_ah_attr->port_num == 0 || ib_ah_attr->port_num > dev->limits.num_ports)\r
486                 return;\r
487 \r
488         ib_ah_attr->dlid          = cl_ntoh16(path->rlid);\r
489         ib_ah_attr->sl            = (u8)(cl_ntoh32(path->sl_tclass_flowlabel) >> 28);\r
490         ib_ah_attr->src_path_bits = path->g_mylmc & 0x7f;\r
491         //TODO: work around: set always full speed      - really, it's much more complicate\r
492         ib_ah_attr->static_rate   = 0;\r
493         ib_ah_attr->ah_flags      = (path->g_mylmc & (1 << 7)) ? IB_AH_GRH : 0;\r
494         if (ib_ah_attr->ah_flags) {\r
495                 ib_ah_attr->grh.sgid_index = (u8)(path->mgid_index & (dev->limits.gid_table_len - 1));\r
496                 ib_ah_attr->grh.hop_limit  = path->hop_limit;\r
497                 ib_ah_attr->grh.traffic_class =\r
498                         (u8)((cl_ntoh32(path->sl_tclass_flowlabel) >> 20) & 0xff);\r
499                 ib_ah_attr->grh.flow_label =\r
500                         cl_ntoh32(path->sl_tclass_flowlabel) & 0xfffff;\r
501                 memcpy(ib_ah_attr->grh.dgid.raw,\r
502                         path->rgid, sizeof ib_ah_attr->grh.dgid.raw);\r
503         }\r
504 }\r
505 \r
506 int mthca_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,\r
507                    struct ib_qp_init_attr *qp_init_attr)\r
508 {\r
509         struct mthca_dev *dev = to_mdev(ibqp->device);\r
510         struct mthca_qp *qp = to_mqp(ibqp);\r
511         int err = 0;\r
512         struct mthca_mailbox *mailbox = NULL;\r
513         struct mthca_qp_param *qp_param;\r
514         struct mthca_qp_context *context;\r
515         int mthca_state;\r
516         u8 status;\r
517 \r
518         UNUSED_PARAM(qp_attr_mask);\r
519         \r
520         down( &qp->mutex );\r
521 \r
522         if (qp->state == IBQPS_RESET) {\r
523                 qp_attr->qp_state = IBQPS_RESET;\r
524                 goto done;\r
525         }\r
526 \r
527         mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);\r
528         if (IS_ERR(mailbox)) {\r
529                 err = PTR_ERR(mailbox);\r
530                 goto out;\r
531         }\r
532 \r
533         err = mthca_QUERY_QP(dev, qp->qpn, 0, mailbox, &status);\r
534         if (err)\r
535                 goto out_mailbox;\r
536         if (status) {\r
537                 HCA_PRINT(TRACE_LEVEL_WARNING ,HCA_DBG_QP,\r
538                         ("QUERY_QP returned status %02x\n", status));\r
539                 err = -EINVAL;\r
540                 goto out_mailbox;\r
541         }\r
542 \r
543         qp_param    = mailbox->buf;\r
544         context     = &qp_param->context;\r
545         mthca_state = cl_ntoh32(context->flags) >> 28;\r
546 \r
547         qp->state                    = to_ib_qp_state(mthca_state);\r
548         qp_attr->qp_state            = qp->state;\r
549         qp_attr->path_mtu            = context->mtu_msgmax >> 5;\r
550         qp_attr->path_mig_state      =\r
551                 to_ib_mig_state((cl_ntoh32(context->flags) >> 11) & 0x3);\r
552         qp_attr->qkey                = cl_ntoh32(context->qkey);\r
553         qp_attr->rq_psn              = cl_ntoh32(context->rnr_nextrecvpsn) & 0xffffff;\r
554         qp_attr->sq_psn              = cl_ntoh32(context->next_send_psn) & 0xffffff;\r
555         qp_attr->dest_qp_num         = cl_ntoh32(context->remote_qpn) & 0xffffff;\r
556         qp_attr->qp_access_flags     =\r
557                 to_ib_qp_access_flags(cl_ntoh32(context->params2));\r
558 \r
559         if (qp->transport == RC || qp->transport == UC) {\r
560                 to_ib_ah_attr(dev, &qp_attr->ah_attr, &context->pri_path);\r
561                 to_ib_ah_attr(dev, &qp_attr->alt_ah_attr, &context->alt_path);\r
562                 qp_attr->alt_pkey_index =\r
563                         (u16)(cl_ntoh32(context->alt_path.port_pkey) & 0x7f);\r
564                 qp_attr->alt_port_num   = qp_attr->alt_ah_attr.port_num;\r
565         }\r
566 \r
567         qp_attr->pkey_index = (u16)(cl_ntoh32(context->pri_path.port_pkey) & 0x7f);\r
568         qp_attr->port_num   =\r
569                 (u8)((cl_ntoh32(context->pri_path.port_pkey) >> 24) & 0x3);\r
570 \r
571         /* qp_attr->en_sqd_async_notify is only applicable in modify qp */\r
572         qp_attr->sq_draining = (u8)(mthca_state == MTHCA_QP_STATE_DRAINING);\r
573 \r
574         qp_attr->max_rd_atomic = (u8)(1 << ((cl_ntoh32(context->params1) >> 21) & 0x7));\r
575 \r
576         qp_attr->max_dest_rd_atomic =\r
577                 (u8)(1 << ((cl_ntoh32(context->params2) >> 21) & 0x7));\r
578         qp_attr->min_rnr_timer      =\r
579                 (u8)((cl_ntoh32(context->rnr_nextrecvpsn) >> 24) & 0x1f);\r
580         qp_attr->timeout            = context->pri_path.ackto >> 3;\r
581         qp_attr->retry_cnt          = (u8)((cl_ntoh32(context->params1) >> 16) & 0x7);\r
582         qp_attr->rnr_retry          = context->pri_path.rnr_retry >> 5;\r
583         qp_attr->alt_timeout        = context->alt_path.ackto >> 3;\r
584 \r
585 done:\r
586         qp_attr->cur_qp_state        = qp_attr->qp_state;\r
587         qp_attr->cap.max_send_wr     = qp->sq.max;\r
588         qp_attr->cap.max_recv_wr     = qp->rq.max;\r
589         qp_attr->cap.max_send_sge    = qp->sq.max_gs;\r
590         qp_attr->cap.max_recv_sge    = qp->rq.max_gs;\r
591         qp_attr->cap.max_inline_data = qp->max_inline_data;\r
592 \r
593         qp_init_attr->cap            = qp_attr->cap;\r
594 \r
595 out_mailbox:\r
596         mthca_free_mailbox(dev, mailbox);\r
597 \r
598 out:\r
599         up(&qp->mutex);\r
600         return err;\r
601 }\r
602 \r
603 static void store_attrs(struct mthca_sqp *sqp, struct ib_qp_attr *attr,\r
604                         int attr_mask)\r
605 {\r
606         if (attr_mask & IB_QP_PKEY_INDEX)\r
607                 sqp->pkey_index = attr->pkey_index;\r
608         if (attr_mask & IB_QP_QKEY)\r
609                 sqp->qkey = attr->qkey;\r
610         if (attr_mask & IB_QP_SQ_PSN)\r
611                 sqp->send_psn = attr->sq_psn;\r
612 }\r
613 \r
614 static void init_port(struct mthca_dev *dev, int port)\r
615 {\r
616         int err;\r
617         u8 status;\r
618         struct mthca_init_ib_param param;\r
619 \r
620         RtlZeroMemory(&param, sizeof param);\r
621 \r
622         param.port_width    = dev->limits.port_width_cap;\r
623         param.vl_cap    = dev->limits.vl_cap;\r
624         param.mtu_cap   = dev->limits.mtu_cap;\r
625         param.gid_cap   = (u16)dev->limits.gid_table_len;\r
626         param.pkey_cap  = (u16)dev->limits.pkey_table_len;\r
627 \r
628         err = mthca_INIT_IB(dev, &param, port, &status);\r
629         if (err)\r
630                 HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_QP  ,("INIT_IB failed, return code %d.\n", err));\r
631         if (status)\r
632                 HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_QP  ,("INIT_IB returned status %02x.\n", status));\r
633 }\r
634 \r
635 \r
636 static __be32 get_hw_access_flags(struct mthca_qp *qp, struct ib_qp_attr *attr,\r
637                                   int attr_mask)\r
638 {\r
639         u8 dest_rd_atomic;\r
640         u32 access_flags;\r
641         u32 hw_access_flags = 0;\r
642 \r
643         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)\r
644                 dest_rd_atomic = attr->max_dest_rd_atomic;\r
645         else\r
646                 dest_rd_atomic = qp->resp_depth;\r
647 \r
648         if (attr_mask & IB_QP_ACCESS_FLAGS)\r
649                 access_flags = attr->qp_access_flags;\r
650         else\r
651                 access_flags = qp->atomic_rd_en;\r
652 \r
653         if (!dest_rd_atomic)\r
654                 access_flags &= MTHCA_ACCESS_REMOTE_WRITE;\r
655 \r
656         if (access_flags & MTHCA_ACCESS_REMOTE_READ)\r
657                 hw_access_flags |= MTHCA_QP_BIT_RRE;\r
658         if (access_flags & MTHCA_ACCESS_REMOTE_ATOMIC)\r
659                 hw_access_flags |= MTHCA_QP_BIT_RAE;\r
660         if (access_flags & MTHCA_ACCESS_REMOTE_WRITE)\r
661                 hw_access_flags |= MTHCA_QP_BIT_RWE;\r
662 \r
663         return cl_hton32(hw_access_flags);\r
664 }\r
665 \r
666 int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask)\r
667 {\r
668         struct mthca_dev *dev = to_mdev(ibqp->device);\r
669         struct mthca_qp *qp = to_mqp(ibqp);\r
670         enum ib_qp_state cur_state, new_state;\r
671         struct mthca_mailbox *mailbox;\r
672         struct mthca_qp_param *qp_param;\r
673         struct mthca_qp_context *qp_context;\r
674         u32 req_param, opt_param;\r
675         u32 sqd_event = 0;\r
676         u8 status;\r
677         int err = -EINVAL;\r
678         SPIN_LOCK_PREP(lhs);\r
679         SPIN_LOCK_PREP(lhr);\r
680 \r
681         down( &qp->mutex );\r
682 \r
683         if (attr_mask & IB_QP_CUR_STATE) {\r
684                 if (attr->cur_qp_state != IBQPS_RTR &&\r
685                         attr->cur_qp_state != IBQPS_RTS &&\r
686                         attr->cur_qp_state != IBQPS_SQD &&\r
687                         attr->cur_qp_state != IBQPS_SQE)\r
688                         goto out;\r
689                 else\r
690                         cur_state = attr->cur_qp_state;\r
691         } else {\r
692                 spin_lock_irq(&qp->sq.lock, &lhs);\r
693                 spin_lock(&qp->rq.lock, &lhr);\r
694                 cur_state = qp->state;\r
695                 spin_unlock(&lhr);\r
696                 spin_unlock_irq(&lhs);\r
697         }\r
698 \r
699         if (attr_mask & IB_QP_STATE) {\r
700                 if (attr->qp_state < 0 || attr->qp_state > IBQPS_ERR)\r
701                         goto out;\r
702                 new_state = attr->qp_state;\r
703         } else\r
704                 new_state = cur_state;\r
705 \r
706         if (state_table[cur_state][new_state].trans == MTHCA_TRANS_INVALID) {\r
707                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("Illegal QP transition "\r
708                           "%d->%d\n", cur_state, new_state));\r
709                 goto out;\r
710         }\r
711 \r
712         req_param = state_table[cur_state][new_state].req_param[qp->transport];\r
713         opt_param = state_table[cur_state][new_state].opt_param[qp->transport];\r
714 \r
715         if ((req_param & attr_mask) != req_param) {\r
716                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("QP transition "\r
717                           "%d->%d missing req attr 0x%08x\n",\r
718                           cur_state, new_state,\r
719                           req_param & ~attr_mask));\r
720                 //NB: IBAL doesn't use all the fields, so we can miss some mandatory flags\r
721                 goto out;\r
722         }\r
723 \r
724         if (attr_mask & ~(req_param | opt_param | IB_QP_STATE)) {\r
725                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("QP transition (transport %d) "\r
726                           "%d->%d has extra attr 0x%08x\n",\r
727                           qp->transport,\r
728                           cur_state, new_state,\r
729                           attr_mask & ~(req_param | opt_param |\r
730                                                  IB_QP_STATE)));\r
731                 //NB: The old code sometimes uses optional flags that are not so in this code\r
732                 goto out;\r
733         }\r
734 \r
735         if ((attr_mask & IB_QP_PKEY_INDEX) && \r
736                 attr->pkey_index >= dev->limits.pkey_table_len) {\r
737                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("PKey index (%u) too large. max is %d\n",\r
738                           attr->pkey_index,dev->limits.pkey_table_len-1)); \r
739                 goto out;\r
740         }\r
741 \r
742         if ((attr_mask & IB_QP_PORT) &&\r
743                 (attr->port_num == 0 || attr->port_num > dev->limits.num_ports)) {\r
744                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("Port number (%u) is invalid\n", attr->port_num));\r
745                 goto out;\r
746         }\r
747 \r
748         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&\r
749                 attr->max_rd_atomic > dev->limits.max_qp_init_rdma) {\r
750                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("Max rdma_atomic as initiator %u too large (max is %d)\n",\r
751                           attr->max_rd_atomic, dev->limits.max_qp_init_rdma));\r
752                 goto out;\r
753         }\r
754 \r
755         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&\r
756             attr->max_dest_rd_atomic > 1 << dev->qp_table.rdb_shift) {\r
757                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("Max rdma_atomic as responder %u too large (max %d)\n",\r
758                           attr->max_dest_rd_atomic, 1 << dev->qp_table.rdb_shift));\r
759                 goto out;\r
760         }\r
761 \r
762         mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);\r
763         if (IS_ERR(mailbox)) {\r
764                 err = PTR_ERR(mailbox);\r
765                 goto out;\r
766         }\r
767         qp_param = mailbox->buf;\r
768         qp_context = &qp_param->context;\r
769         RtlZeroMemory(qp_param, sizeof *qp_param);\r
770 \r
771         qp_context->flags      = cl_hton32((to_mthca_state(new_state) << 28) |\r
772                                              (to_mthca_st(qp->transport) << 16));\r
773         qp_context->flags     |= cl_hton32(MTHCA_QP_BIT_DE);\r
774         if (!(attr_mask & IB_QP_PATH_MIG_STATE))\r
775                 qp_context->flags |= cl_hton32(MTHCA_QP_PM_MIGRATED << 11);\r
776         else {\r
777                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_PM_STATE);\r
778                 switch (attr->path_mig_state) {\r
779                 case IB_APM_MIGRATED:\r
780                         qp_context->flags |= cl_hton32(MTHCA_QP_PM_MIGRATED << 11);\r
781                         break;\r
782                 case IB_APM_REARM:\r
783                         qp_context->flags |= cl_hton32(MTHCA_QP_PM_REARM << 11);\r
784                         break;\r
785                 case IB_APM_ARMED:\r
786                         qp_context->flags |= cl_hton32(MTHCA_QP_PM_ARMED << 11);\r
787                         break;\r
788                 }\r
789         }\r
790 \r
791         /* leave tavor_sched_queue as 0 */\r
792 \r
793         if (qp->transport == MLX || qp->transport == UD)\r
794                 qp_context->mtu_msgmax = (IB_MTU_LEN_2048 << 5) | 11;\r
795         else if (attr_mask & IB_QP_PATH_MTU) {\r
796                 if (attr->path_mtu < IB_MTU_LEN_256 || attr->path_mtu > IB_MTU_LEN_2048) {\r
797                         HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,\r
798                                 ("path MTU (%u) is invalid\n", attr->path_mtu));\r
799                         goto out_mailbox;\r
800                 }\r
801                 qp_context->mtu_msgmax = (u8)((attr->path_mtu << 5) | 31);\r
802         }\r
803 \r
804         if (mthca_is_memfree(dev)) {\r
805                 if (qp->rq.max)\r
806                         qp_context->rq_size_stride = (u8)(long_log2(qp->rq.max) << 3);\r
807                 qp_context->rq_size_stride |= qp->rq.wqe_shift - 4;\r
808 \r
809                 if (qp->sq.max)\r
810                         qp_context->sq_size_stride = (u8)(long_log2(qp->sq.max) << 3);\r
811                 qp_context->sq_size_stride |= qp->sq.wqe_shift - 4;\r
812         }\r
813 \r
814         /* leave arbel_sched_queue as 0 */\r
815 \r
816         if (qp->ibqp.ucontext)\r
817                 qp_context->usr_page =\r
818                         cl_hton32(to_mucontext(qp->ibqp.ucontext)->uar.index);\r
819         else\r
820                 qp_context->usr_page = cl_hton32(dev->driver_uar.index);\r
821         qp_context->local_qpn  = cl_hton32(qp->qpn);\r
822         if (attr_mask & IB_QP_DEST_QPN) {\r
823                 qp_context->remote_qpn = cl_hton32(attr->dest_qp_num);\r
824         }\r
825 \r
826         if (qp->transport == MLX)\r
827                 qp_context->pri_path.port_pkey |=\r
828                         cl_hton32(to_msqp(qp)->port << 24);\r
829         else {\r
830                 if (attr_mask & IB_QP_PORT) {\r
831                         qp_context->pri_path.port_pkey |=\r
832                                 cl_hton32(attr->port_num << 24);\r
833                         qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_PORT_NUM);\r
834                 }\r
835         }\r
836 \r
837         if (attr_mask & IB_QP_PKEY_INDEX) {\r
838                 qp_context->pri_path.port_pkey |=\r
839                         cl_hton32(attr->pkey_index);\r
840                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_PKEY_INDEX);\r
841         }\r
842 \r
843         if (attr_mask & IB_QP_RNR_RETRY) {\r
844                 qp_context->pri_path.rnr_retry = attr->rnr_retry << 5;\r
845                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_RNR_RETRY);\r
846         }\r
847 \r
848         if (attr_mask & IB_QP_AV) {\r
849                 qp_context->pri_path.g_mylmc     = attr->ah_attr.src_path_bits & 0x7f;\r
850                 qp_context->pri_path.rlid        = cl_hton16(attr->ah_attr.dlid);\r
851                 //TODO: work around: set always full speed  - really, it's much more complicate\r
852                 qp_context->pri_path.static_rate = 0;\r
853                 if (attr->ah_attr.ah_flags & IB_AH_GRH) {\r
854                         qp_context->pri_path.g_mylmc |= 1 << 7;\r
855                         qp_context->pri_path.mgid_index = attr->ah_attr.grh.sgid_index;\r
856                         qp_context->pri_path.hop_limit = attr->ah_attr.grh.hop_limit;\r
857                         qp_context->pri_path.sl_tclass_flowlabel =\r
858                                 cl_hton32((attr->ah_attr.sl << 28)                |\r
859                                             (attr->ah_attr.grh.traffic_class << 20) |\r
860                                             (attr->ah_attr.grh.flow_label));\r
861                         memcpy(qp_context->pri_path.rgid,\r
862                                attr->ah_attr.grh.dgid.raw, 16);\r
863                 } else {\r
864                         qp_context->pri_path.sl_tclass_flowlabel =\r
865                                 cl_hton32(attr->ah_attr.sl << 28);\r
866                 }\r
867                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_PRIMARY_ADDR_PATH);\r
868         }\r
869 \r
870         if (attr_mask & IB_QP_TIMEOUT) {\r
871                 qp_context->pri_path.ackto = attr->timeout << 3;\r
872                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_ACK_TIMEOUT);\r
873         }\r
874 \r
875         /* XXX alt_path */\r
876 \r
877         /* leave rdd as 0 */\r
878         qp_context->pd         = cl_hton32(to_mpd(ibqp->pd)->pd_num);\r
879         /* leave wqe_base as 0 (we always create an MR based at 0 for WQs) */\r
880         qp_context->wqe_lkey   = cl_hton32(qp->mr.ibmr.lkey);\r
881         qp_context->params1    = cl_hton32((unsigned long)(\r
882                 (MTHCA_ACK_REQ_FREQ << 28) |\r
883                 (MTHCA_FLIGHT_LIMIT << 24) |\r
884                 MTHCA_QP_BIT_SWE));\r
885         if (qp->sq_policy == IB_SIGNAL_ALL_WR)\r
886                 qp_context->params1 |= cl_hton32(MTHCA_QP_BIT_SSC);\r
887         if (attr_mask & IB_QP_RETRY_CNT) {\r
888                 qp_context->params1 |= cl_hton32(attr->retry_cnt << 16);\r
889                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_RETRY_COUNT);\r
890         }\r
891 \r
892         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {\r
893                 if (attr->max_rd_atomic) {\r
894                         qp_context->params1 |=\r
895                                 cl_hton32(MTHCA_QP_BIT_SRE |\r
896                                             MTHCA_QP_BIT_SAE);\r
897                         qp_context->params1 |=\r
898                                 cl_hton32(fls(attr->max_rd_atomic - 1) << 21);\r
899                 }\r
900                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_SRA_MAX);\r
901         }\r
902 \r
903         if (attr_mask & IB_QP_SQ_PSN)\r
904                 qp_context->next_send_psn = cl_hton32(attr->sq_psn);\r
905         qp_context->cqn_snd = cl_hton32(to_mcq(ibqp->send_cq)->cqn);\r
906 \r
907         if (mthca_is_memfree(dev)) {\r
908                 qp_context->snd_wqe_base_l = cl_hton32(qp->send_wqe_offset);\r
909                 qp_context->snd_db_index   = cl_hton32(qp->sq.db_index);\r
910         }\r
911 \r
912         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {\r
913 \r
914                 if (attr->max_dest_rd_atomic)\r
915                         qp_context->params2 |=\r
916                                 cl_hton32(fls(attr->max_dest_rd_atomic - 1) << 21);\r
917 \r
918                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_RRA_MAX);\r
919 \r
920         }\r
921 \r
922         if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {\r
923                 qp_context->params2      |= get_hw_access_flags(qp, attr, attr_mask);\r
924                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_RWE |\r
925                                                         MTHCA_QP_OPTPAR_RRE |\r
926                                                         MTHCA_QP_OPTPAR_RAE);\r
927         }\r
928 \r
929         qp_context->params2 |= cl_hton32(MTHCA_QP_BIT_RSC);\r
930 \r
931         if (ibqp->srq)\r
932                 qp_context->params2 |= cl_hton32(MTHCA_QP_BIT_RIC);\r
933 \r
934         if (attr_mask & IB_QP_MIN_RNR_TIMER) {\r
935                 qp_context->rnr_nextrecvpsn |= cl_hton32(attr->min_rnr_timer << 24);\r
936                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_RNR_TIMEOUT);\r
937         }\r
938         if (attr_mask & IB_QP_RQ_PSN)\r
939                 qp_context->rnr_nextrecvpsn |= cl_hton32(attr->rq_psn);\r
940 \r
941         qp_context->ra_buff_indx =\r
942                 cl_hton32(dev->qp_table.rdb_base +\r
943                             ((qp->qpn & (dev->limits.num_qps - 1)) * MTHCA_RDB_ENTRY_SIZE <<\r
944                              dev->qp_table.rdb_shift));\r
945 \r
946         qp_context->cqn_rcv = cl_hton32(to_mcq(ibqp->recv_cq)->cqn);\r
947 \r
948         if (mthca_is_memfree(dev))\r
949                 qp_context->rcv_db_index   = cl_hton32(qp->rq.db_index);\r
950 \r
951         if (attr_mask & IB_QP_QKEY) {\r
952                 qp_context->qkey = cl_hton32(attr->qkey);\r
953                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_Q_KEY);\r
954         }\r
955 \r
956         if (ibqp->srq)\r
957                 qp_context->srqn = cl_hton32(1 << 24 |\r
958                                                to_msrq(ibqp->srq)->srqn);\r
959 \r
960         if (cur_state == IBQPS_RTS && new_state == IBQPS_SQD    &&\r
961             attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY               &&\r
962             attr->en_sqd_async_notify)\r
963                 sqd_event = (u32)(1 << 31);\r
964 \r
965         err = mthca_MODIFY_QP(dev, state_table[cur_state][new_state].trans,\r
966                               qp->qpn, 0, mailbox, sqd_event, &status);\r
967         if (err) {\r
968                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("mthca_MODIFY_QP returned error (qp-num = 0x%x) returned status %02x "\r
969                         "cur_state  = %d  new_state = %d attr_mask = %d req_param = %d opt_param = %d\n",\r
970                         ibqp->qp_num, status, cur_state, new_state, \r
971                         attr_mask, req_param, opt_param));        \r
972                 goto out_mailbox;\r
973         }\r
974         if (status) {\r
975                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("mthca_MODIFY_QP bad status(qp-num = 0x%x) returned status %02x "\r
976                         "cur_state  = %d  new_state = %d attr_mask = %d req_param = %d opt_param = %d\n",\r
977                         ibqp->qp_num, status, cur_state, new_state, \r
978                         attr_mask, req_param, opt_param));\r
979                 err = -EINVAL;\r
980                 goto out_mailbox;\r
981         }\r
982 \r
983         qp->state = new_state;\r
984         if (attr_mask & IB_QP_ACCESS_FLAGS)\r
985                 qp->atomic_rd_en = (u8)attr->qp_access_flags;\r
986         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)\r
987                 qp->resp_depth = attr->max_dest_rd_atomic;\r
988 \r
989         if (is_sqp(dev, qp))\r
990                 store_attrs(to_msqp(qp), attr, attr_mask);\r
991 \r
992         /*\r
993          * If we moved QP0 to RTR, bring the IB link up; if we moved\r
994          * QP0 to RESET or ERROR, bring the link back down.\r
995          */\r
996         if (is_qp0(dev, qp)) {\r
997                 if (cur_state != IBQPS_RTR &&\r
998                         new_state == IBQPS_RTR)\r
999                         init_port(dev, to_msqp(qp)->port);\r
1000 \r
1001                 if (cur_state != IBQPS_RESET &&\r
1002                         cur_state != IBQPS_ERR &&\r
1003                         (new_state == IBQPS_RESET ||\r
1004                         new_state == IBQPS_ERR))\r
1005                         mthca_CLOSE_IB(dev, to_msqp(qp)->port, &status);\r
1006         }\r
1007 \r
1008         /*\r
1009          * If we moved a kernel QP to RESET, clean up all old CQ\r
1010          * entries and reinitialize the QP.\r
1011          */\r
1012         if (new_state == IBQPS_RESET && !qp->ibqp.ucontext) {\r
1013                 mthca_cq_clean(dev, to_mcq(qp->ibqp.send_cq)->cqn, qp->qpn,\r
1014                                qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);\r
1015                 if (qp->ibqp.send_cq != qp->ibqp.recv_cq)\r
1016                         mthca_cq_clean(dev, to_mcq(qp->ibqp.recv_cq)->cqn, qp->qpn,\r
1017                                        qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);\r
1018 \r
1019                 mthca_wq_init(&qp->sq);\r
1020                 qp->sq.last = get_send_wqe(qp, qp->sq.max - 1);\r
1021                 mthca_wq_init(&qp->rq);\r
1022                 qp->rq.last = get_recv_wqe(qp, qp->rq.max - 1);\r
1023 \r
1024                 if (mthca_is_memfree(dev)) {\r
1025                         *qp->sq.db = 0;\r
1026                         *qp->rq.db = 0;\r
1027                 }\r
1028         }\r
1029 \r
1030 out_mailbox:\r
1031         mthca_free_mailbox(dev, mailbox);\r
1032 \r
1033 out:\r
1034         up( &qp->mutex );\r
1035         return err;\r
1036 }\r
1037 \r
1038 static int mthca_max_data_size(struct mthca_dev *dev, struct mthca_qp *qp, int desc_sz)\r
1039 {\r
1040 \r
1041         /*\r
1042          * Calculate the maximum size of WQE s/g segments, excluding\r
1043          * the next segment and other non-data segments.\r
1044          */\r
1045         int max_data_size = desc_sz - sizeof (struct mthca_next_seg);\r
1046 \r
1047         switch (qp->transport) {\r
1048         case MLX:\r
1049                 max_data_size -= 2 * sizeof (struct mthca_data_seg);\r
1050                 break;\r
1051 \r
1052         case UD:\r
1053                 if (mthca_is_memfree(dev))\r
1054                         max_data_size -= sizeof (struct mthca_arbel_ud_seg);\r
1055                 else\r
1056                         max_data_size -= sizeof (struct mthca_tavor_ud_seg);\r
1057                 break;\r
1058 \r
1059         default:\r
1060                 max_data_size -= sizeof (struct mthca_raddr_seg);\r
1061                 break;\r
1062         }\r
1063                 return max_data_size;\r
1064 }\r
1065 \r
1066 static inline int mthca_max_inline_data(int max_data_size)\r
1067 {\r
1068         return max_data_size - MTHCA_INLINE_HEADER_SIZE ;\r
1069 }\r
1070 \r
1071 static void mthca_adjust_qp_caps(struct mthca_dev *dev,\r
1072                                  struct mthca_qp *qp)\r
1073 {\r
1074         int max_data_size = mthca_max_data_size(dev, qp,\r
1075                 min(dev->limits.max_desc_sz, 1 << qp->sq.wqe_shift));\r
1076 \r
1077         qp->max_inline_data = mthca_max_inline_data( max_data_size);\r
1078 \r
1079         qp->sq.max_gs = min(dev->limits.max_sg,\r
1080                 (int)(max_data_size / sizeof (struct mthca_data_seg)));\r
1081         qp->rq.max_gs = min(dev->limits.max_sg,\r
1082                 (int)((min(dev->limits.max_desc_sz, 1 << qp->rq.wqe_shift) -\r
1083                 sizeof (struct mthca_next_seg)) / sizeof (struct mthca_data_seg)));     \r
1084 }\r
1085 \r
1086 /*\r
1087  * Allocate and register buffer for WQEs.  qp->rq.max, sq.max,\r
1088  * rq.max_gs and sq.max_gs must all be assigned.\r
1089  * mthca_alloc_wqe_buf will calculate rq.wqe_shift and\r
1090  * sq.wqe_shift (as well as send_wqe_offset, is_direct, and\r
1091  * queue)\r
1092  */\r
1093 static int mthca_alloc_wqe_buf(struct mthca_dev *dev,\r
1094                                struct mthca_pd *pd,\r
1095                                struct mthca_qp *qp)\r
1096 {\r
1097         int size;\r
1098         int err = -ENOMEM;\r
1099         \r
1100         HCA_ENTER(HCA_DBG_QP);\r
1101         size = sizeof (struct mthca_next_seg) +\r
1102                 qp->rq.max_gs * sizeof (struct mthca_data_seg);\r
1103 \r
1104         if (size > dev->limits.max_desc_sz)\r
1105                 return -EINVAL;\r
1106 \r
1107         for (qp->rq.wqe_shift = 6; 1 << qp->rq.wqe_shift < size;\r
1108              qp->rq.wqe_shift++)\r
1109                 ; /* nothing */\r
1110 \r
1111         size = qp->sq.max_gs * sizeof (struct mthca_data_seg);\r
1112         switch (qp->transport) {\r
1113                 case MLX:\r
1114                         size += 2 * sizeof (struct mthca_data_seg);\r
1115                         break;\r
1116 \r
1117                 case UD:\r
1118                         size += mthca_is_memfree(dev) ?\r
1119                                 sizeof (struct mthca_arbel_ud_seg) :\r
1120                                 sizeof (struct mthca_tavor_ud_seg);\r
1121                         break;\r
1122                 \r
1123                 case UC:\r
1124                         size += sizeof (struct mthca_raddr_seg);\r
1125                         break;\r
1126                 \r
1127                 case RC:\r
1128                         size += sizeof (struct mthca_raddr_seg);\r
1129                         /*\r
1130                          * An atomic op will require an atomic segment, a\r
1131                          * remote address segment and one scatter entry.\r
1132                          */\r
1133                         size = max(size,\r
1134                                  sizeof (struct mthca_atomic_seg) +\r
1135                                  sizeof (struct mthca_raddr_seg) +\r
1136                                  sizeof (struct mthca_data_seg));\r
1137                         break;\r
1138                         \r
1139                 default:\r
1140                         break;\r
1141         }\r
1142                 \r
1143         /* Make sure that we have enough space for a bind request */\r
1144         size = max(size, sizeof (struct mthca_bind_seg));\r
1145         \r
1146         size += sizeof (struct mthca_next_seg);\r
1147         \r
1148         if (size > dev->limits.max_desc_sz)\r
1149                 return -EINVAL;\r
1150 \r
1151         for (qp->sq.wqe_shift = 6; 1 << qp->sq.wqe_shift < size;\r
1152              qp->sq.wqe_shift++)\r
1153                 ; /* nothing */\r
1154 \r
1155         qp->send_wqe_offset = ALIGN(qp->rq.max << qp->rq.wqe_shift,\r
1156                                     1 << qp->sq.wqe_shift);\r
1157 \r
1158         /*\r
1159          * If this is a userspace QP, we don't actually have to\r
1160          * allocate anything.  All we need is to calculate the WQE\r
1161          * sizes and the send_wqe_offset, so we're done now.\r
1162          */\r
1163         if (pd->ibpd.ucontext)\r
1164                 return 0;\r
1165 \r
1166         size = (int)(LONG_PTR)NEXT_PAGE_ALIGN(qp->send_wqe_offset +\r
1167                           (qp->sq.max << qp->sq.wqe_shift));\r
1168 \r
1169         qp->wrid = kmalloc((qp->rq.max + qp->sq.max) * sizeof (u64),\r
1170                            GFP_KERNEL);\r
1171         if (!qp->wrid)\r
1172                 goto err_out;\r
1173 \r
1174         err = mthca_buf_alloc(dev, size, MTHCA_MAX_DIRECT_QP_SIZE,\r
1175                               &qp->queue, &qp->is_direct, pd, 0, &qp->mr);\r
1176         if (err)\r
1177                 goto err_out;\r
1178         \r
1179         HCA_EXIT(HCA_DBG_QP);\r
1180         return 0;\r
1181 \r
1182 err_out:\r
1183         kfree(qp->wrid);\r
1184         return err;\r
1185 }\r
1186 \r
1187 static void mthca_free_wqe_buf(struct mthca_dev *dev,\r
1188                                struct mthca_qp *qp)\r
1189 {\r
1190         mthca_buf_free(dev, (int)(LONG_PTR)NEXT_PAGE_ALIGN(qp->send_wqe_offset +\r
1191                                        (qp->sq.max << qp->sq.wqe_shift)),\r
1192                        &qp->queue, qp->is_direct, &qp->mr);\r
1193         kfree(qp->wrid);\r
1194 }\r
1195 \r
1196 static int mthca_map_memfree(struct mthca_dev *dev,\r
1197                              struct mthca_qp *qp)\r
1198 {\r
1199         int ret;\r
1200 \r
1201         if (mthca_is_memfree(dev)) {\r
1202                 ret = mthca_table_get(dev, dev->qp_table.qp_table, qp->qpn);\r
1203                 if (ret)\r
1204                         return ret;\r
1205 \r
1206                 ret = mthca_table_get(dev, dev->qp_table.eqp_table, qp->qpn);\r
1207                 if (ret)\r
1208                         goto err_qpc;\r
1209 \r
1210                 ret = mthca_table_get(dev, dev->qp_table.rdb_table,\r
1211                                       qp->qpn << dev->qp_table.rdb_shift);\r
1212                 if (ret)\r
1213                         goto err_eqpc;\r
1214 \r
1215         }\r
1216 \r
1217         return 0;\r
1218 \r
1219 err_eqpc:\r
1220         mthca_table_put(dev, dev->qp_table.eqp_table, qp->qpn);\r
1221 \r
1222 err_qpc:\r
1223         mthca_table_put(dev, dev->qp_table.qp_table, qp->qpn);\r
1224 \r
1225         return ret;\r
1226 }\r
1227 \r
1228 static void mthca_unmap_memfree(struct mthca_dev *dev,\r
1229                                 struct mthca_qp *qp)\r
1230 {\r
1231         mthca_table_put(dev, dev->qp_table.rdb_table,\r
1232                         qp->qpn << dev->qp_table.rdb_shift);\r
1233         mthca_table_put(dev, dev->qp_table.eqp_table, qp->qpn);\r
1234         mthca_table_put(dev, dev->qp_table.qp_table, qp->qpn);\r
1235 }\r
1236 \r
1237 static int mthca_alloc_memfree(struct mthca_dev *dev,\r
1238                                struct mthca_qp *qp)\r
1239 {\r
1240         int ret = 0;\r
1241 \r
1242         if (mthca_is_memfree(dev)) {\r
1243                 qp->rq.db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_RQ,\r
1244                                                  qp->qpn, &qp->rq.db);\r
1245                 if (qp->rq.db_index < 0)\r
1246                         return qp->rq.db_index;\r
1247 \r
1248                 qp->sq.db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_SQ,\r
1249                                                  qp->qpn, &qp->sq.db);\r
1250                 if (qp->sq.db_index < 0){\r
1251                         mthca_free_db(dev, MTHCA_DB_TYPE_RQ, qp->rq.db_index);\r
1252                         return qp->sq.db_index;\r
1253                 }\r
1254 \r
1255         }\r
1256 \r
1257         return ret;\r
1258 }\r
1259 \r
1260 static void mthca_free_memfree(struct mthca_dev *dev,\r
1261                                struct mthca_qp *qp)\r
1262 {\r
1263         if (mthca_is_memfree(dev)) {\r
1264                 mthca_free_db(dev, MTHCA_DB_TYPE_SQ, qp->sq.db_index);\r
1265                 mthca_free_db(dev, MTHCA_DB_TYPE_RQ, qp->rq.db_index);\r
1266         }\r
1267 }\r
1268 \r
1269 static int mthca_alloc_qp_common(struct mthca_dev *dev,\r
1270                                  struct mthca_pd *pd,\r
1271                                  struct mthca_cq *send_cq,\r
1272                                  struct mthca_cq *recv_cq,\r
1273                                  enum ib_sig_type send_policy,\r
1274                                  struct mthca_qp *qp)\r
1275 {\r
1276         int ret;\r
1277         int i;\r
1278 \r
1279         atomic_set(&qp->refcount, 1);\r
1280         init_waitqueue_head(&qp->wait);\r
1281         KeInitializeMutex(&qp->mutex, 0);\r
1282 \r
1283         qp->state        = IBQPS_RESET;\r
1284         qp->atomic_rd_en = 0;\r
1285         qp->resp_depth   = 0;\r
1286         qp->sq_policy    = send_policy;\r
1287         mthca_wq_init(&qp->sq);\r
1288         mthca_wq_init(&qp->rq);\r
1289 \r
1290         UNREFERENCED_PARAMETER(send_cq);\r
1291         UNREFERENCED_PARAMETER(recv_cq);\r
1292         \r
1293         ret = mthca_map_memfree(dev, qp);\r
1294         if (ret)\r
1295                 return ret;\r
1296 \r
1297         ret = mthca_alloc_wqe_buf(dev, pd, qp);\r
1298         if (ret) {\r
1299                 mthca_unmap_memfree(dev, qp);\r
1300                 return ret;\r
1301         }\r
1302 \r
1303         mthca_adjust_qp_caps(dev, qp);\r
1304 \r
1305         /*\r
1306          * If this is a userspace QP, we're done now.  The doorbells\r
1307          * will be allocated and buffers will be initialized in\r
1308          * userspace.\r
1309          */\r
1310         if (pd->ibpd.ucontext)\r
1311                 return 0;\r
1312 \r
1313         ret = mthca_alloc_memfree(dev, qp);\r
1314         if (ret) {\r
1315                 mthca_free_wqe_buf(dev, qp);\r
1316                 mthca_unmap_memfree(dev, qp);\r
1317                 return ret;\r
1318         }\r
1319 \r
1320         if (mthca_is_memfree(dev)) {\r
1321                 struct mthca_next_seg *next;\r
1322                 struct mthca_data_seg *scatter;\r
1323                 int size = (sizeof (struct mthca_next_seg) +\r
1324                             qp->rq.max_gs * sizeof (struct mthca_data_seg)) / 16;\r
1325 \r
1326                 for (i = 0; i < qp->rq.max; ++i) {\r
1327                         next = get_recv_wqe(qp, i);\r
1328                         next->nda_op = cl_hton32(((i + 1) & (qp->rq.max - 1)) <<\r
1329                                                    qp->rq.wqe_shift);\r
1330                         next->ee_nds = cl_hton32(size);\r
1331 \r
1332                         for (scatter = (void *) (next + 1);\r
1333                              (void *) scatter < (void *) ((u8*)next + (u32)(1 << qp->rq.wqe_shift));\r
1334                              ++scatter)\r
1335                                 scatter->lkey = cl_hton32(MTHCA_INVAL_LKEY);\r
1336                 }\r
1337 \r
1338                 for (i = 0; i < qp->sq.max; ++i) {\r
1339                         next = get_send_wqe(qp, i);\r
1340                         next->nda_op = cl_hton32((((i + 1) & (qp->sq.max - 1)) <<\r
1341                                                     qp->sq.wqe_shift) +\r
1342                                                    qp->send_wqe_offset);\r
1343                 }\r
1344         }\r
1345 \r
1346         qp->sq.last = get_send_wqe(qp, qp->sq.max - 1);\r
1347         qp->rq.last = get_recv_wqe(qp, qp->rq.max - 1);\r
1348 \r
1349         return 0;\r
1350 }\r
1351 \r
1352 static int mthca_set_qp_size(struct mthca_dev *dev, struct ib_qp_cap *cap,\r
1353         struct mthca_qp *qp)\r
1354 {\r
1355         int max_data_size = mthca_max_data_size(dev, qp, dev->limits.max_desc_sz);\r
1356 \r
1357         /* Sanity check QP size before proceeding */\r
1358         if (cap->max_send_wr     > (u32)dev->limits.max_wqes ||\r
1359             cap->max_recv_wr     > (u32)dev->limits.max_wqes ||\r
1360             cap->max_send_sge    > (u32)dev->limits.max_sg   ||\r
1361             cap->max_recv_sge    > (u32)dev->limits.max_sg   ||\r
1362             cap->max_inline_data > (u32)mthca_max_inline_data(max_data_size))\r
1363                 return -EINVAL;\r
1364 \r
1365         /*\r
1366          * For MLX transport we need 2 extra S/G entries:\r
1367          * one for the header and one for the checksum at the end\r
1368          */\r
1369         if (qp->transport == MLX && cap->max_recv_sge + 2 > (u32)dev->limits.max_sg)\r
1370                 return -EINVAL;\r
1371 \r
1372         /* Enable creating zero-sized QPs */\r
1373         if (!cap->max_recv_wr)\r
1374                 cap->max_recv_wr = 1;\r
1375         if (!cap->max_send_wr)\r
1376                 cap->max_send_wr = 1;\r
1377         \r
1378         if (mthca_is_memfree(dev)) {\r
1379                 qp->rq.max = cap->max_recv_wr ?\r
1380                         roundup_pow_of_two(cap->max_recv_wr) : 0;\r
1381                 qp->sq.max = cap->max_send_wr ?\r
1382                         roundup_pow_of_two(cap->max_send_wr) : 0;\r
1383         } else {\r
1384                 qp->rq.max = cap->max_recv_wr;\r
1385                 qp->sq.max = cap->max_send_wr;\r
1386         }\r
1387 \r
1388         qp->rq.max_gs = cap->max_recv_sge;\r
1389         qp->sq.max_gs = MAX(cap->max_send_sge,\r
1390                               ALIGN(cap->max_inline_data + MTHCA_INLINE_HEADER_SIZE,\r
1391                                     MTHCA_INLINE_CHUNK_SIZE) /\r
1392                               (int)sizeof (struct mthca_data_seg));\r
1393 \r
1394         return 0;\r
1395 }\r
1396 \r
1397 int mthca_alloc_qp(struct mthca_dev *dev,\r
1398                    struct mthca_pd *pd,\r
1399                    struct mthca_cq *send_cq,\r
1400                    struct mthca_cq *recv_cq,\r
1401                    enum ib_qp_type_t type,\r
1402                    enum ib_sig_type send_policy,\r
1403                    struct ib_qp_cap *cap,\r
1404                    struct mthca_qp *qp)\r
1405 {\r
1406         int err;\r
1407         SPIN_LOCK_PREP(lh);\r
1408 \r
1409         switch (type) {\r
1410         case IB_QPT_RELIABLE_CONN: qp->transport = RC; break;\r
1411         case IB_QPT_UNRELIABLE_CONN: qp->transport = UC; break;\r
1412         case IB_QPT_UNRELIABLE_DGRM: qp->transport = UD; break;\r
1413         default: return -EINVAL;\r
1414         }\r
1415 \r
1416         err = mthca_set_qp_size(dev, cap, qp);\r
1417         if (err)\r
1418                 return err;\r
1419 \r
1420         qp->qpn = mthca_alloc(&dev->qp_table.alloc);\r
1421         if (qp->qpn == -1)\r
1422                 return -ENOMEM;\r
1423 \r
1424         err = mthca_alloc_qp_common(dev, pd, send_cq, recv_cq,\r
1425                                     send_policy, qp);\r
1426         if (err) {\r
1427                 mthca_free(&dev->qp_table.alloc, qp->qpn);\r
1428                 return err;\r
1429         }\r
1430 \r
1431         spin_lock_irq(&dev->qp_table.lock, &lh);\r
1432         mthca_array_set(&dev->qp_table.qp,\r
1433                         qp->qpn & (dev->limits.num_qps - 1), qp);\r
1434         spin_unlock_irq(&lh);\r
1435 \r
1436         return 0;\r
1437 }\r
1438 \r
1439 int mthca_alloc_sqp(struct mthca_dev *dev,\r
1440                     struct mthca_pd *pd,\r
1441                     struct mthca_cq *send_cq,\r
1442                     struct mthca_cq *recv_cq,\r
1443                     enum ib_sig_type send_policy,\r
1444                     struct ib_qp_cap *cap,\r
1445                     int qpn,\r
1446                     int port,\r
1447                     struct mthca_sqp *sqp)\r
1448 {\r
1449         u32 mqpn = qpn * 2 + dev->qp_table.sqp_start + port - 1;\r
1450         int err;\r
1451         SPIN_LOCK_PREP(lhs);\r
1452         SPIN_LOCK_PREP(lhr);\r
1453         SPIN_LOCK_PREP(lht);\r
1454 \r
1455         err = mthca_set_qp_size(dev, cap, &sqp->qp);\r
1456         if (err)\r
1457                 return err;\r
1458 \r
1459         alloc_dma_zmem_map(dev, \r
1460                 sqp->qp.sq.max * MTHCA_UD_HEADER_SIZE, \r
1461                 PCI_DMA_BIDIRECTIONAL,\r
1462                 &sqp->sg);\r
1463         if (!sqp->sg.page)\r
1464                 return -ENOMEM;\r
1465 \r
1466         spin_lock_irq(&dev->qp_table.lock, &lht);\r
1467         if (mthca_array_get(&dev->qp_table.qp, mqpn))\r
1468                 err = -EBUSY;\r
1469         else\r
1470                 mthca_array_set(&dev->qp_table.qp, mqpn, sqp);\r
1471         spin_unlock_irq(&lht);\r
1472 \r
1473         if (err)\r
1474                 goto err_out;\r
1475 \r
1476         sqp->port = port;\r
1477         sqp->qp.qpn       = mqpn;\r
1478         sqp->qp.transport = MLX;\r
1479 \r
1480         err = mthca_alloc_qp_common(dev, pd, send_cq, recv_cq,\r
1481                                     send_policy, &sqp->qp);\r
1482         if (err)\r
1483                 goto err_out_free;\r
1484 \r
1485         atomic_inc(&pd->sqp_count);\r
1486 \r
1487         return 0;\r
1488 \r
1489  err_out_free:\r
1490         /*\r
1491          * Lock CQs here, so that CQ polling code can do QP lookup\r
1492          * without taking a lock.\r
1493          */\r
1494         spin_lock_irq(&send_cq->lock, &lhs);\r
1495         if (send_cq != recv_cq)\r
1496                 spin_lock(&recv_cq->lock, &lhr);\r
1497 \r
1498         spin_lock(&dev->qp_table.lock, &lht);\r
1499         mthca_array_clear(&dev->qp_table.qp, mqpn);\r
1500         spin_unlock(&lht);\r
1501 \r
1502         if (send_cq != recv_cq)\r
1503                 spin_unlock(&lhr);\r
1504         spin_unlock_irq(&lhs);\r
1505 \r
1506  err_out:\r
1507         free_dma_mem_map(dev, &sqp->sg, PCI_DMA_BIDIRECTIONAL);\r
1508 \r
1509         return err;\r
1510 }\r
1511 \r
1512 void mthca_free_qp(struct mthca_dev *dev,\r
1513                    struct mthca_qp *qp)\r
1514 {\r
1515         u8 status;\r
1516         struct mthca_cq *send_cq;\r
1517         struct mthca_cq *recv_cq;\r
1518         SPIN_LOCK_PREP(lhs);\r
1519         SPIN_LOCK_PREP(lhr);\r
1520         SPIN_LOCK_PREP(lht);\r
1521 \r
1522         send_cq = to_mcq(qp->ibqp.send_cq);\r
1523         recv_cq = to_mcq(qp->ibqp.recv_cq);\r
1524 \r
1525         /*\r
1526          * Lock CQs here, so that CQ polling code can do QP lookup\r
1527          * without taking a lock.\r
1528          */\r
1529         spin_lock_irq(&send_cq->lock, &lhs);\r
1530         if (send_cq != recv_cq)\r
1531                 spin_lock(&recv_cq->lock, &lhr);\r
1532 \r
1533         spin_lock(&dev->qp_table.lock, &lht);\r
1534         mthca_array_clear(&dev->qp_table.qp,\r
1535                           qp->qpn & (dev->limits.num_qps - 1));\r
1536         spin_unlock(&lht);\r
1537 \r
1538         if (send_cq != recv_cq)\r
1539                 spin_unlock(&lhr);\r
1540         spin_unlock_irq(&lhs);\r
1541 \r
1542         atomic_dec(&qp->refcount);\r
1543         wait_event(&qp->wait, !atomic_read(&qp->refcount));\r
1544 \r
1545         if (qp->state != IBQPS_RESET) {\r
1546                 mthca_MODIFY_QP(dev, MTHCA_TRANS_ANY2RST, qp->qpn, 0, NULL, 0, &status);\r
1547         }\r
1548 \r
1549         /*\r
1550          * If this is a userspace QP, the buffers, MR, CQs and so on\r
1551          * will be cleaned up in userspace, so all we have to do is\r
1552          * unref the mem-free tables and free the QPN in our table.\r
1553          */\r
1554         if (!qp->ibqp.ucontext) {\r
1555                 mthca_cq_clean(dev, to_mcq(qp->ibqp.send_cq)->cqn, qp->qpn,\r
1556                                qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);\r
1557                 if (qp->ibqp.send_cq != qp->ibqp.recv_cq)\r
1558                         mthca_cq_clean(dev, to_mcq(qp->ibqp.recv_cq)->cqn, qp->qpn,\r
1559                                        qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);\r
1560 \r
1561                 mthca_free_memfree(dev, qp);\r
1562                 mthca_free_wqe_buf(dev, qp);\r
1563         }\r
1564 \r
1565         mthca_unmap_memfree(dev, qp);\r
1566 \r
1567         if (is_sqp(dev, qp)) {\r
1568                 atomic_dec(&(to_mpd(qp->ibqp.pd)->sqp_count));\r
1569                 free_dma_mem_map(dev, &to_msqp(qp)->sg, PCI_DMA_BIDIRECTIONAL);\r
1570         } else\r
1571                 mthca_free(&dev->qp_table.alloc, qp->qpn);\r
1572 }\r
1573 \r
1574 static enum mthca_wr_opcode conv_ibal_wr_opcode(struct _ib_send_wr *wr)\r
1575 {\r
1576 \r
1577         enum mthca_wr_opcode opcode = -1; //= wr->wr_type;\r
1578 \r
1579         switch (wr->wr_type) {\r
1580                 case WR_SEND: \r
1581                         opcode = (wr->send_opt & IB_SEND_OPT_IMMEDIATE) ? MTHCA_OPCODE_SEND_IMM : MTHCA_OPCODE_SEND;\r
1582                         break;\r
1583                 case WR_RDMA_WRITE:     \r
1584                         opcode = (wr->send_opt & IB_SEND_OPT_IMMEDIATE) ? MTHCA_OPCODE_RDMA_WRITE_IMM : MTHCA_OPCODE_RDMA_WRITE;\r
1585                         break;\r
1586                 case WR_RDMA_READ:              opcode = MTHCA_OPCODE_RDMA_READ; break;\r
1587                 case WR_COMPARE_SWAP:           opcode = MTHCA_OPCODE_ATOMIC_CS; break;\r
1588                 case WR_FETCH_ADD:                      opcode = MTHCA_OPCODE_ATOMIC_FA; break;\r
1589                 default:                                                opcode = MTHCA_OPCODE_INVALID;break;\r
1590         }\r
1591         return opcode;\r
1592 }\r
1593 \r
1594 /* Create UD header for an MLX send and build a data segment for it */\r
1595 static int build_mlx_header(struct mthca_dev *dev, struct mthca_sqp *sqp,\r
1596                             int ind, struct _ib_send_wr *wr,\r
1597                             struct mthca_mlx_seg *mlx,\r
1598                             struct mthca_data_seg *data)\r
1599 {\r
1600         enum ib_wr_opcode opcode = conv_ibal_wr_opcode(wr);\r
1601         int header_size;\r
1602         int err;\r
1603         __be16 pkey;\r
1604         CPU_2_BE64_PREP;\r
1605 \r
1606         if (!wr->dgrm.ud.h_av) {\r
1607                 HCA_PRINT(TRACE_LEVEL_ERROR , HCA_DBG_AV, \r
1608                         ("absent AV in send wr %p\n", wr));\r
1609                 return -EINVAL;\r
1610         }\r
1611                 \r
1612         ib_ud_header_init(256, /* assume a MAD */\r
1613                 mthca_ah_grh_present(to_mah((struct ib_ah *)wr->dgrm.ud.h_av)),\r
1614                 &sqp->ud_header);\r
1615 \r
1616         err = mthca_read_ah(dev, to_mah((struct ib_ah *)wr->dgrm.ud.h_av), &sqp->ud_header);\r
1617         if (err){\r
1618                 HCA_PRINT(TRACE_LEVEL_ERROR , HCA_DBG_AV, ("read av error%p\n",\r
1619                         to_mah((struct ib_ah *)wr->dgrm.ud.h_av)->av));\r
1620                 return err;\r
1621         }\r
1622         mlx->flags &= ~cl_hton32(MTHCA_NEXT_SOLICIT | 1);\r
1623         mlx->flags |= cl_hton32((!sqp->qp.ibqp.qp_num ? MTHCA_MLX_VL15 : 0) |\r
1624                                   (sqp->ud_header.lrh.destination_lid ==\r
1625                                    IB_LID_PERMISSIVE ? MTHCA_MLX_SLR : 0) |\r
1626                                   (sqp->ud_header.lrh.service_level << 8));\r
1627         mlx->rlid = sqp->ud_header.lrh.destination_lid;\r
1628         mlx->vcrc = 0;\r
1629 \r
1630         switch (opcode) {\r
1631         case MTHCA_OPCODE_SEND:\r
1632                 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY;\r
1633                 sqp->ud_header.immediate_present = 0;\r
1634                 break;\r
1635         case MTHCA_OPCODE_SEND_IMM:\r
1636                 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;\r
1637                 sqp->ud_header.immediate_present = 1;\r
1638                 sqp->ud_header.immediate_data = wr->immediate_data;\r
1639                 break;\r
1640         default:\r
1641                 return -EINVAL;\r
1642         }\r
1643 \r
1644         sqp->ud_header.lrh.virtual_lane    = !sqp->qp.ibqp.qp_num ? 15 : 0;\r
1645         if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE)\r
1646                 sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE;\r
1647         sqp->ud_header.bth.solicited_event = (u8)!!(wr->send_opt & IB_SEND_OPT_SOLICITED);\r
1648         if (!sqp->qp.ibqp.qp_num)\r
1649                 ib_get_cached_pkey(&dev->ib_dev, (u8)sqp->port,\r
1650                                    sqp->pkey_index, &pkey);\r
1651         else\r
1652                 ib_get_cached_pkey(&dev->ib_dev, (u8)sqp->port,\r
1653                                    wr->dgrm.ud.pkey_index, &pkey);\r
1654         sqp->ud_header.bth.pkey = pkey;\r
1655         sqp->ud_header.bth.destination_qpn = wr->dgrm.ud.remote_qp;\r
1656         sqp->ud_header.bth.psn = cl_hton32((sqp->send_psn++) & ((1 << 24) - 1));\r
1657         sqp->ud_header.deth.qkey = wr->dgrm.ud.remote_qkey & 0x00000080 ?\r
1658                                                cl_hton32(sqp->qkey) : wr->dgrm.ud.remote_qkey;\r
1659         sqp->ud_header.deth.source_qpn = cl_hton32(sqp->qp.ibqp.qp_num);\r
1660 \r
1661         header_size = ib_ud_header_pack(&sqp->ud_header,\r
1662                                         (u8*)sqp->sg.page +\r
1663                                         ind * MTHCA_UD_HEADER_SIZE);\r
1664 \r
1665         data->byte_count = cl_hton32(header_size);\r
1666         data->lkey       = cl_hton32(to_mpd(sqp->qp.ibqp.pd)->ntmr.ibmr.lkey);\r
1667         data->addr       = CPU_2_BE64(sqp->sg.dma_address +\r
1668                                        ind * MTHCA_UD_HEADER_SIZE);\r
1669 \r
1670         return 0;\r
1671 }\r
1672 \r
1673 static inline int mthca_wq_overflow(struct mthca_wq *wq, int nreq,\r
1674                                     struct ib_cq *ib_cq)\r
1675 {\r
1676         unsigned cur;\r
1677         struct mthca_cq *cq;\r
1678         SPIN_LOCK_PREP(lh);\r
1679 \r
1680         cur = wq->head - wq->tail;\r
1681         if (likely((int)cur + nreq < wq->max))\r
1682                 return 0;\r
1683 \r
1684         cq = to_mcq(ib_cq);\r
1685         spin_lock_dpc(&cq->lock, &lh);\r
1686         cur = wq->head - wq->tail;\r
1687         spin_unlock_dpc(&lh);\r
1688 \r
1689         return (int)cur + nreq >= wq->max;\r
1690 }\r
1691 \r
1692 int mthca_tavor_post_send(struct ib_qp *ibqp, struct _ib_send_wr *wr,\r
1693                           struct _ib_send_wr **bad_wr)\r
1694 {\r
1695         struct mthca_dev *dev = to_mdev(ibqp->device);\r
1696         struct mthca_qp *qp = to_mqp(ibqp);\r
1697         u8 *wqe;\r
1698         u8 *prev_wqe;\r
1699         int err = 0;\r
1700         int nreq;\r
1701         int i;\r
1702         int size;\r
1703         int size0 = 0;\r
1704         u32 f0 = unlikely(wr->send_opt & IB_SEND_OPT_FENCE) ? MTHCA_SEND_DOORBELL_FENCE : 0;\r
1705         int ind;\r
1706         u8 op0 = 0;\r
1707         enum ib_wr_opcode opcode;\r
1708         SPIN_LOCK_PREP(lh);   \r
1709 \r
1710         spin_lock_irqsave(&qp->sq.lock, &lh);\r
1711     \r
1712         /* XXX check that state is OK to post send */\r
1713 \r
1714         ind = qp->sq.next_ind;\r
1715 \r
1716         for (nreq = 0; wr; ++nreq, wr = wr->p_next) {\r
1717                 if (mthca_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {\r
1718                         HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_QP,("SQ %06x full (%u head, %u tail,"\r
1719                                         " %d max, %d nreq)\n", qp->qpn,\r
1720                                         qp->sq.head, qp->sq.tail,\r
1721                                         qp->sq.max, nreq));\r
1722                         err = -ENOMEM;\r
1723                         if (bad_wr)\r
1724                                 *bad_wr = wr;\r
1725                         goto out;\r
1726                 }\r
1727 \r
1728                 wqe = get_send_wqe(qp, ind);\r
1729                 prev_wqe = qp->sq.last;\r
1730                 qp->sq.last = wqe;\r
1731                 opcode = conv_ibal_wr_opcode(wr);\r
1732 \r
1733                 ((struct mthca_next_seg *) wqe)->nda_op = 0;\r
1734                 ((struct mthca_next_seg *) wqe)->ee_nds = 0;\r
1735                 ((struct mthca_next_seg *) wqe)->flags =\r
1736                         ((wr->send_opt & IB_SEND_OPT_SIGNALED) ?\r
1737                          cl_hton32(MTHCA_NEXT_CQ_UPDATE) : 0) |\r
1738                         ((wr->send_opt & IB_SEND_OPT_SOLICITED) ?\r
1739                          cl_hton32(MTHCA_NEXT_SOLICIT) : 0)   |\r
1740                         cl_hton32(1);\r
1741                 if (opcode == MTHCA_OPCODE_SEND_IMM||\r
1742                     opcode == MTHCA_OPCODE_RDMA_WRITE_IMM)\r
1743                         ((struct mthca_next_seg *) wqe)->imm = wr->immediate_data;\r
1744 \r
1745                 wqe += sizeof (struct mthca_next_seg);\r
1746                 size = sizeof (struct mthca_next_seg) / 16;\r
1747 \r
1748                 switch (qp->transport) {\r
1749                 case RC:\r
1750                         switch (opcode) {\r
1751                         case MTHCA_OPCODE_ATOMIC_CS:\r
1752                         case MTHCA_OPCODE_ATOMIC_FA:\r
1753                                 ((struct mthca_raddr_seg *) wqe)->raddr =\r
1754                                         cl_hton64(wr->remote_ops.vaddr);\r
1755                                 ((struct mthca_raddr_seg *) wqe)->rkey =\r
1756                                         wr->remote_ops.rkey;\r
1757                                 ((struct mthca_raddr_seg *) wqe)->reserved = 0;\r
1758 \r
1759                                 wqe += sizeof (struct mthca_raddr_seg);\r
1760 \r
1761                                 if (opcode == MTHCA_OPCODE_ATOMIC_CS) {\r
1762                                         ((struct mthca_atomic_seg *) wqe)->swap_add =\r
1763                                                 (wr->remote_ops.atomic2);\r
1764                                         ((struct mthca_atomic_seg *) wqe)->compare =\r
1765                                                 (wr->remote_ops.atomic1);\r
1766                                 } else {\r
1767                                         ((struct mthca_atomic_seg *) wqe)->swap_add =\r
1768                                                 (wr->remote_ops.atomic1);\r
1769                                         ((struct mthca_atomic_seg *) wqe)->compare = 0;\r
1770                                 }\r
1771 \r
1772                                 wqe += sizeof (struct mthca_atomic_seg);\r
1773                                 size += (sizeof (struct mthca_raddr_seg) +\r
1774                                         sizeof (struct mthca_atomic_seg)) / 16 ;\r
1775                                 break;\r
1776 \r
1777                         case MTHCA_OPCODE_RDMA_READ:\r
1778                         case MTHCA_OPCODE_RDMA_WRITE:\r
1779                         case MTHCA_OPCODE_RDMA_WRITE_IMM:\r
1780                                 ((struct mthca_raddr_seg *) wqe)->raddr =\r
1781                                         cl_hton64(wr->remote_ops.vaddr);\r
1782                                 ((struct mthca_raddr_seg *) wqe)->rkey =\r
1783                                         wr->remote_ops.rkey;\r
1784                                 ((struct mthca_raddr_seg *) wqe)->reserved = 0;\r
1785                                 wqe += sizeof (struct mthca_raddr_seg);\r
1786                                 size += sizeof (struct mthca_raddr_seg) / 16;\r
1787                                 break;\r
1788 \r
1789                         default:\r
1790                                 /* No extra segments required for sends */\r
1791                                 break;\r
1792                         }\r
1793 \r
1794                         break;\r
1795 \r
1796                 case UC:\r
1797                         switch (opcode) {\r
1798                         case MTHCA_OPCODE_RDMA_WRITE:\r
1799                         case MTHCA_OPCODE_RDMA_WRITE_IMM:\r
1800                                 ((struct mthca_raddr_seg *) wqe)->raddr =\r
1801                                         cl_hton64(wr->remote_ops.vaddr);\r
1802                                 ((struct mthca_raddr_seg *) wqe)->rkey =\r
1803                                         wr->remote_ops.rkey;\r
1804                                 ((struct mthca_raddr_seg *) wqe)->reserved = 0;\r
1805                                 wqe += sizeof (struct mthca_raddr_seg);\r
1806                                 size += sizeof (struct mthca_raddr_seg) / 16;\r
1807                                 break;\r
1808 \r
1809                         default:\r
1810                                 /* No extra segments required for sends */\r
1811                                 break;\r
1812                         }\r
1813 \r
1814                         break;\r
1815 \r
1816                 case UD:\r
1817                         ((struct mthca_tavor_ud_seg *) wqe)->lkey =\r
1818                                 cl_hton32(to_mah((struct ib_ah *)wr->dgrm.ud.h_av)->key);\r
1819                         ((struct mthca_tavor_ud_seg *) wqe)->av_addr =\r
1820                                 cl_hton64(to_mah((struct ib_ah *)wr->dgrm.ud.h_av)->avdma);\r
1821                         ((struct mthca_tavor_ud_seg *) wqe)->dqpn = wr->dgrm.ud.remote_qp;\r
1822                         ((struct mthca_tavor_ud_seg *) wqe)->qkey = wr->dgrm.ud.remote_qkey;\r
1823 \r
1824                         wqe += sizeof (struct mthca_tavor_ud_seg);\r
1825                         size += sizeof (struct mthca_tavor_ud_seg) / 16;\r
1826                         break;\r
1827 \r
1828                 case MLX:\r
1829                         err = build_mlx_header(dev, to_msqp(qp), ind, wr,\r
1830                                                (void*)(wqe - sizeof (struct mthca_next_seg)),\r
1831                                                (void*)wqe);\r
1832                         if (err) {\r
1833                                 if (bad_wr)\r
1834                                         *bad_wr = wr;\r
1835                                 goto out;\r
1836                         }\r
1837                         wqe += sizeof (struct mthca_data_seg);\r
1838                         size += sizeof (struct mthca_data_seg) / 16;\r
1839                         break;\r
1840                 }\r
1841 \r
1842                 if ((int)(int)wr->num_ds > qp->sq.max_gs) {\r
1843                         HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_QP ,("SQ %06x too many gathers\n",qp->qpn));\r
1844                         err = -EINVAL;\r
1845                         if (bad_wr)\r
1846                                 *bad_wr = wr;\r
1847                         goto out;\r
1848                 }\r
1849                 if (wr->send_opt & IB_SEND_OPT_INLINE) {\r
1850                         if (wr->num_ds) {\r
1851                                 struct mthca_inline_seg *seg = (struct mthca_inline_seg *)wqe;\r
1852                                 uint32_t s = 0;\r
1853 \r
1854                                 wqe += sizeof *seg;\r
1855                                 for (i = 0; i < (int)wr->num_ds; ++i) {\r
1856                                         struct _ib_local_ds *sge = &wr->ds_array[i];\r
1857 \r
1858                                         s += sge->length;\r
1859 \r
1860                                         if (s > (uint32_t)qp->max_inline_data) {\r
1861                                                 err = -EINVAL;\r
1862                                                 if (bad_wr)\r
1863                                                         *bad_wr = wr;\r
1864                                                 goto out;\r
1865                                         }\r
1866 \r
1867                                         memcpy(wqe, (void *) (ULONG_PTR) sge->vaddr,\r
1868                                                sge->length);\r
1869                                         wqe += sge->length;\r
1870                                 }\r
1871 \r
1872                                 seg->byte_count = cl_hton32(MTHCA_INLINE_SEG | s);\r
1873                                 size += align(s + sizeof *seg, 16) / 16;\r
1874                         }\r
1875                 } else {\r
1876                         \r
1877                 for (i = 0; i < (int)wr->num_ds; ++i) {\r
1878                         ((struct mthca_data_seg *) wqe)->byte_count =\r
1879                                 cl_hton32(wr->ds_array[i].length);\r
1880                         ((struct mthca_data_seg *) wqe)->lkey =\r
1881                                 cl_hton32(wr->ds_array[i].lkey);\r
1882                         ((struct mthca_data_seg *) wqe)->addr =\r
1883                                 cl_hton64(wr->ds_array[i].vaddr);\r
1884                         wqe += sizeof (struct mthca_data_seg);\r
1885                         size += sizeof (struct mthca_data_seg) / 16;\r
1886                         HCA_PRINT(TRACE_LEVEL_VERBOSE ,HCA_DBG_QP ,("SQ %06x [%02x]  lkey 0x%08x vaddr 0x%I64x 0x%x\n",qp->qpn,i,\r
1887                                 (wr->ds_array[i].lkey),(wr->ds_array[i].vaddr),wr->ds_array[i].length));\r
1888                 }\r
1889         }\r
1890 \r
1891                 /* Add one more inline data segment for ICRC */\r
1892                 if (qp->transport == MLX) {\r
1893                         ((struct mthca_data_seg *) wqe)->byte_count =\r
1894                                 cl_hton32((unsigned long)((1 << 31) | 4));\r
1895                         ((u32 *) wqe)[1] = 0;\r
1896                         wqe += sizeof (struct mthca_data_seg);\r
1897                         size += sizeof (struct mthca_data_seg) / 16;\r
1898                 }\r
1899 \r
1900                 qp->wrid[ind + qp->rq.max] = wr->wr_id;\r
1901 \r
1902                 if (opcode == MTHCA_OPCODE_INVALID) {\r
1903                         HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_QP ,("SQ %06x opcode invalid\n",qp->qpn));\r
1904                         err = -EINVAL;\r
1905                         if (bad_wr)\r
1906                                 *bad_wr = wr;\r
1907                         goto out;\r
1908                 }\r
1909 \r
1910                 ((struct mthca_next_seg *) prev_wqe)->nda_op =\r
1911                         cl_hton32(((ind << qp->sq.wqe_shift) +  \r
1912                         qp->send_wqe_offset) |opcode);\r
1913                 wmb();\r
1914                 ((struct mthca_next_seg *) prev_wqe)->ee_nds =\r
1915                         cl_hton32((size0 ? 0 : MTHCA_NEXT_DBD) | size |\r
1916                                 ((wr->send_opt & IB_SEND_OPT_FENCE) ?\r
1917                                 MTHCA_NEXT_FENCE : 0));\r
1918 \r
1919                 if (!size0) {\r
1920                         size0 = size;\r
1921                         op0   = opcode;\r
1922                 }\r
1923 \r
1924                 dump_wqe( TRACE_LEVEL_VERBOSE, (u32*)qp->sq.last,qp);\r
1925 \r
1926                 ++ind;\r
1927                 if (unlikely(ind >= qp->sq.max))\r
1928                         ind -= qp->sq.max;\r
1929         }\r
1930 \r
1931 out:\r
1932         if (likely(nreq)) {\r
1933                 __be32 doorbell[2];\r
1934 \r
1935                 doorbell[0] = cl_hton32(((qp->sq.next_ind << qp->sq.wqe_shift) +\r
1936                                            qp->send_wqe_offset) | f0 | op0);\r
1937                 doorbell[1] = cl_hton32((qp->qpn << 8) | size0);\r
1938 \r
1939                 wmb();\r
1940 \r
1941                 mthca_write64(doorbell,\r
1942                               dev->kar + MTHCA_SEND_DOORBELL,\r
1943                               MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));\r
1944         }\r
1945 \r
1946         qp->sq.next_ind = ind;\r
1947         qp->sq.head    += nreq;\r
1948         \r
1949     spin_unlock_irqrestore(&lh);   \r
1950         return err;\r
1951 }\r
1952 \r
1953 int mthca_tavor_post_recv(struct ib_qp *ibqp, struct _ib_recv_wr *wr,\r
1954                              struct _ib_recv_wr **bad_wr)\r
1955 {\r
1956         struct mthca_dev *dev = to_mdev(ibqp->device);\r
1957         struct mthca_qp *qp = to_mqp(ibqp);\r
1958         __be32 doorbell[2];\r
1959         int err = 0;\r
1960         int nreq;\r
1961         int i;\r
1962         int size;\r
1963         int size0 = 0;\r
1964         int ind;\r
1965         u8 *wqe;\r
1966         u8 *prev_wqe;\r
1967         SPIN_LOCK_PREP(lh);\r
1968 \r
1969         spin_lock_irqsave(&qp->rq.lock, &lh);\r
1970 \r
1971         /* XXX check that state is OK to post receive */\r
1972 \r
1973         ind = qp->rq.next_ind;\r
1974 \r
1975         for (nreq = 0; wr; ++nreq, wr = wr->p_next) {\r
1976                 if (unlikely(nreq == MTHCA_TAVOR_MAX_WQES_PER_RECV_DB)) {\r
1977                         nreq = 0;\r
1978 \r
1979                         doorbell[0] = cl_hton32((qp->rq.next_ind << qp->rq.wqe_shift) | size0);\r
1980                         doorbell[1] = cl_hton32(qp->qpn << 8);\r
1981 \r
1982                         wmb();\r
1983 \r
1984                         mthca_write64(doorbell, dev->kar + MTHCA_RECV_DOORBELL,\r
1985                       MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));\r
1986 \r
1987                         qp->rq.head += MTHCA_TAVOR_MAX_WQES_PER_RECV_DB;\r
1988                         size0 = 0;\r
1989                 }\r
1990                 if (mthca_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {\r
1991                         HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_QP,("RQ %06x full (%u head, %u tail,"\r
1992                                         " %d max, %d nreq)\n", qp->qpn,\r
1993                                         qp->rq.head, qp->rq.tail,\r
1994                                         qp->rq.max, nreq));\r
1995                         err = -ENOMEM;\r
1996                         if (bad_wr)\r
1997                                 *bad_wr = wr;\r
1998                         goto out;\r
1999                 }\r
2000 \r
2001                 wqe = get_recv_wqe(qp, ind);\r
2002                 prev_wqe = qp->rq.last;\r
2003                 qp->rq.last = wqe;\r
2004 \r
2005                 ((struct mthca_next_seg *) wqe)->nda_op = 0;\r
2006                 ((struct mthca_next_seg *) wqe)->ee_nds =\r
2007                         cl_hton32(MTHCA_NEXT_DBD);\r
2008                 ((struct mthca_next_seg *) wqe)->flags = 0;\r
2009 \r
2010                 wqe += sizeof (struct mthca_next_seg);\r
2011                 size = sizeof (struct mthca_next_seg) / 16;\r
2012 \r
2013                 if (unlikely((int)wr->num_ds > qp->rq.max_gs)) {\r
2014                         HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_QP ,("RQ %06x too many gathers\n",qp->qpn));\r
2015                         err = -EINVAL;\r
2016                         if (bad_wr)\r
2017                                 *bad_wr = wr;\r
2018                         goto out;\r
2019                 }\r
2020 \r
2021                 for (i = 0; i < (int)wr->num_ds; ++i) {\r
2022                         ((struct mthca_data_seg *) wqe)->byte_count =\r
2023                                 cl_hton32(wr->ds_array[i].length);\r
2024                         ((struct mthca_data_seg *) wqe)->lkey =\r
2025                                 cl_hton32(wr->ds_array[i].lkey);\r
2026                         ((struct mthca_data_seg *) wqe)->addr =\r
2027                                 cl_hton64(wr->ds_array[i].vaddr);\r
2028                         wqe += sizeof (struct mthca_data_seg);\r
2029                         size += sizeof (struct mthca_data_seg) / 16;\r
2030 //                      HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_QP ,("RQ %06x [%02x]  lkey 0x%08x vaddr 0x%I64x 0x %x 0x%08x\n",i,qp->qpn,\r
2031 //                              (wr->ds_array[i].lkey),(wr->ds_array[i].vaddr),wr->ds_array[i].length, wr->wr_id));\r
2032                 }\r
2033 \r
2034                 qp->wrid[ind] = wr->wr_id;\r
2035 \r
2036                 ((struct mthca_next_seg *) prev_wqe)->nda_op =\r
2037                         cl_hton32((ind << qp->rq.wqe_shift) | 1);\r
2038                 wmb();\r
2039                 ((struct mthca_next_seg *) prev_wqe)->ee_nds =\r
2040                         cl_hton32(MTHCA_NEXT_DBD | size);\r
2041 \r
2042                 if (!size0)\r
2043                         size0 = size;\r
2044 \r
2045                 dump_wqe(TRACE_LEVEL_VERBOSE,  (u32*)wqe ,qp);\r
2046                 \r
2047                 ++ind;\r
2048                 if (unlikely(ind >= qp->rq.max))\r
2049                         ind -= qp->rq.max;\r
2050         }\r
2051 \r
2052 out:\r
2053         if (likely(nreq)) {\r
2054                 doorbell[0] = cl_hton32((qp->rq.next_ind << qp->rq.wqe_shift) | size0);\r
2055                 doorbell[1] = cl_hton32((qp->qpn << 8) | (nreq & 255));\r
2056 \r
2057                 wmb();\r
2058 \r
2059                 mthca_write64(doorbell, dev->kar + MTHCA_RECV_DOORBELL,\r
2060               MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));\r
2061         }\r
2062 \r
2063         qp->rq.next_ind = ind;\r
2064         qp->rq.head    += nreq;\r
2065 \r
2066         spin_unlock_irqrestore(&lh);\r
2067         return err;\r
2068 }\r
2069 \r
2070 int mthca_arbel_post_send(struct ib_qp *ibqp, struct _ib_send_wr *wr,\r
2071                           struct _ib_send_wr **bad_wr)\r
2072 {\r
2073         struct mthca_dev *dev = to_mdev(ibqp->device);\r
2074         struct mthca_qp *qp = to_mqp(ibqp);\r
2075         __be32 doorbell[2];\r
2076         u8 *wqe;\r
2077         u8 *prev_wqe;\r
2078         int err = 0;\r
2079         int nreq;\r
2080         int i;\r
2081         int size;\r
2082         int size0 = 0;\r
2083         u32 f0 = unlikely(wr->send_opt & IB_SEND_OPT_FENCE) ? MTHCA_SEND_DOORBELL_FENCE : 0;\r
2084         int ind;\r
2085         u8 op0 = 0;\r
2086         enum ib_wr_opcode opcode;\r
2087         SPIN_LOCK_PREP(lh);\r
2088 \r
2089         spin_lock_irqsave(&qp->sq.lock, &lh);\r
2090 \r
2091         /* XXX check that state is OK to post send */\r
2092 \r
2093         ind = qp->sq.head & (qp->sq.max - 1);\r
2094 \r
2095         for (nreq = 0; wr; ++nreq, wr = wr->p_next) {\r
2096                 if (unlikely(nreq == MTHCA_ARBEL_MAX_WQES_PER_SEND_DB)) {\r
2097                         nreq = 0;\r
2098                         doorbell[0] = cl_hton32((MTHCA_ARBEL_MAX_WQES_PER_SEND_DB << 24) |\r
2099                                 ((qp->sq.head & 0xffff) << 8) |f0 | op0);\r
2100                         doorbell[1] = cl_hton32((qp->qpn << 8) | size0);\r
2101                         qp->sq.head += MTHCA_ARBEL_MAX_WQES_PER_SEND_DB;\r
2102                         size0 = 0;\r
2103                         f0 = unlikely(wr->send_opt & IB_SEND_OPT_FENCE) ? MTHCA_SEND_DOORBELL_FENCE : 0;\r
2104 \r
2105                         /*\r
2106                          * Make sure that descriptors are written before\r
2107                          * doorbell record.\r
2108                          */\r
2109                         wmb();\r
2110                         *qp->sq.db = cl_hton32(qp->sq.head & 0xffff);\r
2111 \r
2112                         /*\r
2113                          * Make sure doorbell record is written before we\r
2114                          * write MMIO send doorbell.\r
2115                          */\r
2116                         wmb();\r
2117                         mthca_write64(doorbell, dev->kar + MTHCA_SEND_DOORBELL,\r
2118                                 MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));\r
2119                 }\r
2120 \r
2121                 if (mthca_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {\r
2122                         HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_QP,("SQ %06x full (%u head, %u tail,"\r
2123                                         " %d max, %d nreq)\n", qp->qpn,\r
2124                                         qp->sq.head, qp->sq.tail,\r
2125                                         qp->sq.max, nreq));\r
2126                         err = -ENOMEM;\r
2127                         if (bad_wr)\r
2128                                 *bad_wr = wr;\r
2129                         goto out;\r
2130                 }\r
2131 \r
2132                 wqe = get_send_wqe(qp, ind);\r
2133                 prev_wqe = qp->sq.last;\r
2134                 qp->sq.last = wqe;\r
2135                 opcode = conv_ibal_wr_opcode(wr);\r
2136 \r
2137                 ((struct mthca_next_seg *) wqe)->flags =\r
2138                         ((wr->send_opt & IB_SEND_OPT_SIGNALED) ?\r
2139                          cl_hton32(MTHCA_NEXT_CQ_UPDATE) : 0) |\r
2140                         ((wr->send_opt & IB_SEND_OPT_SOLICITED) ?\r
2141                          cl_hton32(MTHCA_NEXT_SOLICIT) : 0)   |\r
2142                         ((wr->send_opt & IB_SEND_OPT_TX_IP_CSUM) ?\r
2143                          cl_hton32(MTHCA_NEXT_IP_CSUM) : 0) |\r
2144                         ((wr->send_opt & IB_SEND_OPT_TX_TCP_UDP_CSUM) ?\r
2145                          cl_hton32(MTHCA_NEXT_TCP_UDP_CSUM) : 0) |\r
2146                          cl_hton32(1);\r
2147                 if (opcode == MTHCA_OPCODE_SEND_IMM||\r
2148                         opcode == MTHCA_OPCODE_RDMA_WRITE_IMM)\r
2149                         ((struct mthca_next_seg *) wqe)->imm = wr->immediate_data;\r
2150 \r
2151                 wqe += sizeof (struct mthca_next_seg);\r
2152                 size = sizeof (struct mthca_next_seg) / 16;\r
2153 \r
2154                 switch (qp->transport) {\r
2155                 case RC:\r
2156                         switch (opcode) {\r
2157                         case MTHCA_OPCODE_ATOMIC_CS:\r
2158                         case MTHCA_OPCODE_ATOMIC_FA:\r
2159                                 ((struct mthca_raddr_seg *) wqe)->raddr =\r
2160                                         cl_hton64(wr->remote_ops.vaddr);\r
2161                                 ((struct mthca_raddr_seg *) wqe)->rkey =\r
2162                                         wr->remote_ops.rkey;\r
2163                                 ((struct mthca_raddr_seg *) wqe)->reserved = 0;\r
2164 \r
2165                                 wqe += sizeof (struct mthca_raddr_seg);\r
2166 \r
2167                                 if (opcode == MTHCA_OPCODE_ATOMIC_CS) {\r
2168                                         ((struct mthca_atomic_seg *) wqe)->swap_add =\r
2169                                                 (wr->remote_ops.atomic2);\r
2170                                         ((struct mthca_atomic_seg *) wqe)->compare =\r
2171                                                 (wr->remote_ops.atomic1);\r
2172                                 } else {\r
2173                                         ((struct mthca_atomic_seg *) wqe)->swap_add =\r
2174                                                 (wr->remote_ops.atomic1);\r
2175                                         ((struct mthca_atomic_seg *) wqe)->compare = 0;\r
2176                                 }\r
2177 \r
2178                                 wqe += sizeof (struct mthca_atomic_seg);\r
2179                                 size += (sizeof (struct mthca_raddr_seg) +\r
2180                                         sizeof (struct mthca_atomic_seg)) / 16 ;\r
2181                                 break;\r
2182 \r
2183                         case MTHCA_OPCODE_RDMA_READ:\r
2184                         case MTHCA_OPCODE_RDMA_WRITE:\r
2185                         case MTHCA_OPCODE_RDMA_WRITE_IMM:\r
2186                                 ((struct mthca_raddr_seg *) wqe)->raddr =\r
2187                                         cl_hton64(wr->remote_ops.vaddr);\r
2188                                 ((struct mthca_raddr_seg *) wqe)->rkey =\r
2189                                         wr->remote_ops.rkey;\r
2190                                 ((struct mthca_raddr_seg *) wqe)->reserved = 0;\r
2191                                 wqe += sizeof (struct mthca_raddr_seg);\r
2192                                 size += sizeof (struct mthca_raddr_seg) / 16;\r
2193                                 break;\r
2194 \r
2195                         default:\r
2196                                 /* No extra segments required for sends */\r
2197                                 break;\r
2198                         }\r
2199 \r
2200                         break;\r
2201 \r
2202                 case UC:\r
2203                         switch (opcode) {\r
2204                         case MTHCA_OPCODE_RDMA_WRITE:\r
2205                         case MTHCA_OPCODE_RDMA_WRITE_IMM:\r
2206                                 ((struct mthca_raddr_seg *) wqe)->raddr =\r
2207                                         cl_hton64(wr->remote_ops.vaddr);\r
2208                                 ((struct mthca_raddr_seg *) wqe)->rkey =\r
2209                                         wr->remote_ops.rkey;\r
2210                                 ((struct mthca_raddr_seg *) wqe)->reserved = 0;\r
2211                                 wqe += sizeof (struct mthca_raddr_seg);\r
2212                                 size += sizeof (struct mthca_raddr_seg) / 16;\r
2213                                 break;\r
2214 \r
2215                         default:\r
2216                                 /* No extra segments required for sends */\r
2217                                 break;\r
2218                         }\r
2219 \r
2220                         break;\r
2221 \r
2222                 case UD:\r
2223                         memcpy(((struct mthca_arbel_ud_seg *) wqe)->av,\r
2224                                to_mah((struct ib_ah *)wr->dgrm.ud.h_av)->av, MTHCA_AV_SIZE);\r
2225                         ((struct mthca_arbel_ud_seg *) wqe)->dqpn = wr->dgrm.ud.remote_qp;\r
2226                         ((struct mthca_arbel_ud_seg *) wqe)->qkey = wr->dgrm.ud.remote_qkey;\r
2227 \r
2228                         wqe += sizeof (struct mthca_arbel_ud_seg);\r
2229                         size += sizeof (struct mthca_arbel_ud_seg) / 16;\r
2230                         break;\r
2231 \r
2232                 case MLX:\r
2233                         err = build_mlx_header(dev, to_msqp(qp), ind, wr,\r
2234                                                (void*)(wqe - sizeof (struct mthca_next_seg)),\r
2235                                                (void*)wqe);\r
2236                         if (err) {\r
2237                                 if (bad_wr)\r
2238                                         *bad_wr = wr;\r
2239                                 goto out;\r
2240                         }\r
2241                         wqe += sizeof (struct mthca_data_seg);\r
2242                         size += sizeof (struct mthca_data_seg) / 16;\r
2243                         break;\r
2244                 }\r
2245 \r
2246                 if ((int)wr->num_ds > qp->sq.max_gs) {\r
2247                         HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_QP ,("SQ %06x full too many gathers\n",qp->qpn));\r
2248                         err = -EINVAL;\r
2249                         if (bad_wr)\r
2250                                 *bad_wr = wr;\r
2251                         goto out;\r
2252                 }\r
2253         if (wr->send_opt & IB_SEND_OPT_INLINE) {\r
2254                         if (wr->num_ds) {\r
2255                                 struct mthca_inline_seg *seg = (struct mthca_inline_seg *)wqe;\r
2256                                 uint32_t s = 0;\r
2257 \r
2258                                 wqe += sizeof *seg;\r
2259                                 for (i = 0; i < (int)wr->num_ds; ++i) {\r
2260                                         struct _ib_local_ds *sge = &wr->ds_array[i];\r
2261 \r
2262                                         s += sge->length;\r
2263 \r
2264                                         if (s > (uint32_t)qp->max_inline_data) {\r
2265                                                 err = -EINVAL;\r
2266                                                 if (bad_wr)\r
2267                                                         *bad_wr = wr;\r
2268                                                 goto out;\r
2269                                         }\r
2270 \r
2271                                         memcpy(wqe, (void *) (uintptr_t) sge->vaddr,\r
2272                                                sge->length);\r
2273                                         wqe += sge->length;\r
2274                                 }\r
2275 \r
2276                                 seg->byte_count = cl_hton32(MTHCA_INLINE_SEG | s);\r
2277                                 size += align(s + sizeof *seg, 16) / 16;\r
2278                         }\r
2279                 } else {\r
2280                 for (i = 0; i < (int)wr->num_ds; ++i) {\r
2281                         ((struct mthca_data_seg *) wqe)->byte_count =\r
2282                                 cl_hton32(wr->ds_array[i].length);\r
2283                         ((struct mthca_data_seg *) wqe)->lkey =\r
2284                                 cl_hton32(wr->ds_array[i].lkey);\r
2285                         ((struct mthca_data_seg *) wqe)->addr =\r
2286                                 cl_hton64(wr->ds_array[i].vaddr);\r
2287                         wqe += sizeof (struct mthca_data_seg);\r
2288                         size += sizeof (struct mthca_data_seg) / 16;\r
2289                 }\r
2290         }\r
2291 \r
2292                 /* Add one more inline data segment for ICRC */\r
2293                 if (qp->transport == MLX) {\r
2294                         ((struct mthca_data_seg *) wqe)->byte_count =\r
2295                                 cl_hton32((unsigned long)((1 << 31) | 4));\r
2296                         ((u32 *) wqe)[1] = 0;\r
2297                         wqe += sizeof (struct mthca_data_seg);\r
2298                         size += sizeof (struct mthca_data_seg) / 16;\r
2299                 }\r
2300 \r
2301                 qp->wrid[ind + qp->rq.max] = wr->wr_id;\r
2302 \r
2303                 if (opcode == MTHCA_OPCODE_INVALID) {\r
2304                         HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_QP ,("SQ %06x opcode invalid\n",qp->qpn));\r
2305                         err = -EINVAL;\r
2306                         if (bad_wr)\r
2307                                 *bad_wr = wr;\r
2308                         goto out;\r
2309                 }\r
2310 \r
2311                 ((struct mthca_next_seg *) prev_wqe)->nda_op =\r
2312                         cl_hton32(((ind << qp->sq.wqe_shift) +\r
2313                         qp->send_wqe_offset) |opcode);\r
2314                 wmb();\r
2315                 ((struct mthca_next_seg *) prev_wqe)->ee_nds =\r
2316                         cl_hton32(MTHCA_NEXT_DBD | size |\r
2317                         ((wr->send_opt & IB_SEND_OPT_FENCE) ?\r
2318                         MTHCA_NEXT_FENCE : 0));\r
2319                 \r
2320                 if (!size0) {\r
2321                         size0 = size;\r
2322                         op0   = opcode;\r
2323                 }\r
2324 \r
2325                 ++ind;\r
2326                 if (unlikely(ind >= qp->sq.max))\r
2327                         ind -= qp->sq.max;\r
2328         }\r
2329 \r
2330 out:\r
2331         if (likely(nreq)) {\r
2332                 doorbell[0] = cl_hton32((nreq << 24) |\r
2333                         ((qp->sq.head & 0xffff) << 8) |f0 | op0);\r
2334                 doorbell[1] = cl_hton32((qp->qpn << 8) | size0);\r
2335                 qp->sq.head += nreq;\r
2336 \r
2337                 /*\r
2338                  * Make sure that descriptors are written before\r
2339                  * doorbell record.\r
2340                  */\r
2341                 wmb();\r
2342                 *qp->sq.db = cl_hton32(qp->sq.head & 0xffff);\r
2343 \r
2344                 /*\r
2345                  * Make sure doorbell record is written before we\r
2346                  * write MMIO send doorbell.\r
2347                  */\r
2348                 wmb();\r
2349                 mthca_write64(doorbell,\r
2350                               dev->kar + MTHCA_SEND_DOORBELL,\r
2351                               MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));\r
2352         }\r
2353 \r
2354         spin_unlock_irqrestore(&lh);\r
2355         return err;\r
2356 }\r
2357 \r
2358 int mthca_arbel_post_recv(struct ib_qp *ibqp, struct _ib_recv_wr *wr,\r
2359                              struct _ib_recv_wr **bad_wr)\r
2360 {\r
2361         struct mthca_qp *qp = to_mqp(ibqp);\r
2362         int err = 0;\r
2363         int nreq;\r
2364         int ind;\r
2365         int i;\r
2366         u8 *wqe;\r
2367         SPIN_LOCK_PREP(lh);\r
2368 \r
2369         spin_lock_irqsave(&qp->rq.lock, &lh);\r
2370 \r
2371         /* XXX check that state is OK to post receive */\r
2372 \r
2373         ind = qp->rq.head & (qp->rq.max - 1);\r
2374 \r
2375         for (nreq = 0; wr; ++nreq, wr = wr->p_next) {\r
2376                 if (mthca_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {\r
2377                         HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_QP,("RQ %06x full (%u head, %u tail,"\r
2378                                         " %d max, %d nreq)\n", qp->qpn,\r
2379                                         qp->rq.head, qp->rq.tail,\r
2380                                         qp->rq.max, nreq));\r
2381                         err = -ENOMEM;\r
2382                         if (bad_wr)\r
2383                                 *bad_wr = wr;\r
2384                         goto out;\r
2385                 }\r
2386 \r
2387                 wqe = get_recv_wqe(qp, ind);\r
2388 \r
2389                 ((struct mthca_next_seg *) wqe)->flags = 0;\r
2390 \r
2391                 wqe += sizeof (struct mthca_next_seg);\r
2392 \r
2393                 if (unlikely((int)wr->num_ds > qp->rq.max_gs)) {\r
2394                         HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_QP ,("RQ %06x full too many scatter\n",qp->qpn));\r
2395                         err = -EINVAL;\r
2396                         if (bad_wr)\r
2397                                 *bad_wr = wr;\r
2398                         goto out;\r
2399                 }\r
2400 \r
2401                 for (i = 0; i < (int)wr->num_ds; ++i) {\r
2402                         ((struct mthca_data_seg *) wqe)->byte_count =\r
2403                                 cl_hton32(wr->ds_array[i].length);\r
2404                         ((struct mthca_data_seg *) wqe)->lkey =\r
2405                                 cl_hton32(wr->ds_array[i].lkey);\r
2406                         ((struct mthca_data_seg *) wqe)->addr =\r
2407                                 cl_hton64(wr->ds_array[i].vaddr);\r
2408                         wqe += sizeof (struct mthca_data_seg);\r
2409                 }\r
2410 \r
2411                 if (i < qp->rq.max_gs) {\r
2412                         ((struct mthca_data_seg *) wqe)->byte_count = 0;\r
2413                         ((struct mthca_data_seg *) wqe)->lkey = cl_hton32(MTHCA_INVAL_LKEY);\r
2414                         ((struct mthca_data_seg *) wqe)->addr = 0;\r
2415                 }\r
2416 \r
2417                 qp->wrid[ind] = wr->wr_id;\r
2418 \r
2419                 ++ind;\r
2420                 if (unlikely(ind >= qp->rq.max))\r
2421                         ind -= qp->rq.max;\r
2422         }\r
2423 out:\r
2424         if (likely(nreq)) {\r
2425                 qp->rq.head += nreq;\r
2426 \r
2427                 /*\r
2428                  * Make sure that descriptors are written before\r
2429                  * doorbell record.\r
2430                  */\r
2431                 wmb();\r
2432                 *qp->rq.db = cl_hton32(qp->rq.head & 0xffff);\r
2433         }\r
2434 \r
2435         spin_unlock_irqrestore(&lh);\r
2436         return err;\r
2437 }\r
2438 \r
2439 void mthca_free_err_wqe(struct mthca_dev *dev, struct mthca_qp *qp, int is_send,\r
2440                        int index, int *dbd, __be32 *new_wqe)\r
2441 {\r
2442         struct mthca_next_seg *next;\r
2443 \r
2444         UNREFERENCED_PARAMETER(dev);\r
2445         \r
2446         /*\r
2447          * For SRQs, all WQEs generate a CQE, so we're always at the\r
2448          * end of the doorbell chain.\r
2449          */\r
2450         if (qp->ibqp.srq) {\r
2451                 *new_wqe = 0;\r
2452                 return;\r
2453         }\r
2454 \r
2455         if (is_send)\r
2456                 next = get_send_wqe(qp, index);\r
2457         else\r
2458                 next = get_recv_wqe(qp, index);\r
2459 \r
2460         *dbd = !!(next->ee_nds & cl_hton32(MTHCA_NEXT_DBD));\r
2461         if (next->ee_nds & cl_hton32(0x3f))\r
2462                 *new_wqe = (next->nda_op & cl_hton32((unsigned long)~0x3f)) |\r
2463                         (next->ee_nds & cl_hton32(0x3f));\r
2464         else\r
2465                 *new_wqe = 0;\r
2466 }\r
2467 \r
2468 int mthca_init_qp_table(struct mthca_dev *dev)\r
2469 {\r
2470         int err;\r
2471         u8 status;\r
2472         int i;\r
2473 \r
2474         spin_lock_init(&dev->qp_table.lock);\r
2475         fill_state_table();\r
2476 \r
2477         /*\r
2478          * We reserve 2 extra QPs per port for the special QPs.  The\r
2479          * special QP for port 1 has to be even, so round up.\r
2480          */\r
2481         dev->qp_table.sqp_start = (dev->limits.reserved_qps + 1) & ~1UL;\r
2482         err = mthca_alloc_init(&dev->qp_table.alloc,\r
2483                                dev->limits.num_qps,\r
2484                                (1 << 24) - 1,\r
2485                                dev->qp_table.sqp_start +\r
2486                                MTHCA_MAX_PORTS * 2);\r
2487         if (err)\r
2488                 return err;\r
2489 \r
2490         err = mthca_array_init(&dev->qp_table.qp,\r
2491                                dev->limits.num_qps);\r
2492         if (err) {\r
2493                 mthca_alloc_cleanup(&dev->qp_table.alloc);\r
2494                 return err;\r
2495         }\r
2496 \r
2497         for (i = 0; i < 2; ++i) {\r
2498                 err = mthca_CONF_SPECIAL_QP(dev, i ? IB_QPT_QP1 : IB_QPT_QP0,\r
2499                                             dev->qp_table.sqp_start + i * 2,\r
2500                                             &status);\r
2501                 if (err)\r
2502                         goto err_out;\r
2503                 if (status) {\r
2504                         HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("CONF_SPECIAL_QP returned "\r
2505                                    "status %02x, aborting.\n",\r
2506                                    status));\r
2507                         err = -EINVAL;\r
2508                         goto err_out;\r
2509                 }\r
2510         }\r
2511         return 0;\r
2512 \r
2513  err_out:\r
2514         mthca_CONF_SPECIAL_QP(dev, IB_QPT_QP1, 0, &status);\r
2515         mthca_CONF_SPECIAL_QP(dev, IB_QPT_QP0, 0, &status);\r
2516 \r
2517         mthca_array_cleanup(&dev->qp_table.qp, dev->limits.num_qps);\r
2518         mthca_alloc_cleanup(&dev->qp_table.alloc);\r
2519 \r
2520         return err;\r
2521 }\r
2522 \r
2523 void mthca_cleanup_qp_table(struct mthca_dev *dev)\r
2524 {\r
2525         u8 status;\r
2526 \r
2527         mthca_CONF_SPECIAL_QP(dev, IB_QPT_QP1, 0, &status);\r
2528         mthca_CONF_SPECIAL_QP(dev, IB_QPT_QP0, 0, &status);\r
2529 \r
2530         mthca_array_cleanup(&dev->qp_table.qp, dev->limits.num_qps);\r
2531         mthca_alloc_cleanup(&dev->qp_table.alloc);\r
2532 }\r
2533 \r
2534 \r
2535 \r