[MLX4] added debug prints in case of errors. [mlnx: 4664]
[mirror/winof/.git] / hw / mlx4 / kernel / bus / ib / cq.c
1 /*\r
2  * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.\r
3  *\r
4  * This software is available to you under a choice of one of two\r
5  * licenses.  You may choose to be licensed under the terms of the GNU\r
6  * General Public License (GPL) Version 2, available from the file\r
7  * COPYING in the main directory of this source tree, or the\r
8  * OpenIB.org BSD license below:\r
9  *\r
10  *     Redistribution and use in source and binary forms, with or\r
11  *     without modification, are permitted provided that the following\r
12  *     conditions are met:\r
13  *\r
14  *      - Redistributions of source code must retain the above\r
15  *        copyright notice, this list of conditions and the following\r
16  *        disclaimer.\r
17  *\r
18  *      - Redistributions in binary form must reproduce the above\r
19  *        copyright notice, this list of conditions and the following\r
20  *        disclaimer in the documentation and/or other materials\r
21  *        provided with the distribution.\r
22  *\r
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
30  * SOFTWARE.\r
31  */\r
32 #include <mlx4_debug.h>\r
33 #include "mlx4_ib.h"\r
34 #include "cq.h"\r
35 #include "qp.h"\r
36 #include "user.h"\r
37 \r
38 #if defined(EVENT_TRACING)\r
39 #ifdef offsetof\r
40 #undef offsetof\r
41 #endif\r
42 #include "cq.tmh"\r
43 #endif\r
44 \r
45 \r
46 static void mlx4_ib_cq_comp(struct mlx4_cq *cq)\r
47 {\r
48         struct ib_cq *ibcq = &to_mibcq(cq)->ibcq;\r
49         ibcq->comp_handler(ibcq->cq_context);\r
50 }\r
51 \r
52 static void mlx4_ib_cq_event(struct mlx4_cq *cq, enum mlx4_event type)\r
53 {\r
54         ib_event_rec_t event;\r
55         struct ib_cq *ibcq;\r
56 \r
57         if (type != MLX4_EVENT_TYPE_CQ_ERROR) {\r
58                 printk(KERN_WARNING "mlx4_ib: Unexpected event type %d "\r
59                        "on CQ %06x\n", type, cq->cqn);\r
60                 return;\r
61         }\r
62 \r
63         ibcq = &to_mibcq(cq)->ibcq;\r
64         event.type = IB_EVENT_CQ_ERR;\r
65         event.context = ibcq->cq_context;\r
66         event.vendor_specific = type;\r
67         ibcq->event_handler(&event);\r
68 }\r
69 \r
70 static void *get_cqe_from_buf(struct mlx4_ib_cq_buf *buf, int n)\r
71 {\r
72         int offset = n * sizeof (struct mlx4_cqe);\r
73 \r
74         if (buf->buf.nbufs == 1)\r
75                 return buf->buf.u.direct.buf + offset;\r
76         else\r
77                 return buf->buf.u.page_list[offset >> PAGE_SHIFT].buf +\r
78                         (offset & (PAGE_SIZE - 1));\r
79 }\r
80 \r
81 static void *get_cqe(struct mlx4_ib_cq *cq, int n)\r
82 {\r
83         return get_cqe_from_buf(&cq->buf, n);\r
84 }\r
85 \r
86 static void *get_sw_cqe(struct mlx4_ib_cq *cq, int n)\r
87 {\r
88         struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibcq.cqe);\r
89 \r
90         return (!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^\r
91                 !!(n & (cq->ibcq.cqe + 1))) ? NULL : cqe;\r
92 }\r
93 \r
94 static struct mlx4_cqe *next_cqe_sw(struct mlx4_ib_cq *cq)\r
95 {\r
96         return get_sw_cqe(cq, cq->mcq.cons_index);\r
97 }\r
98 \r
99 int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)\r
100 {\r
101         struct mlx4_ib_cq *mcq = to_mcq(cq);\r
102         struct mlx4_ib_dev *dev = to_mdev(cq->device);\r
103         struct mlx4_cq_context *context;\r
104         int err;\r
105 \r
106         if (mlx4_is_barred(dev->dev))\r
107                 return -EFAULT;\r
108 \r
109         context = kzalloc(sizeof *context, GFP_KERNEL);\r
110         if (!context)\r
111                 return -ENOMEM;\r
112 \r
113         context->cq_period = cpu_to_be16(cq_period);\r
114         context->cq_max_count = cpu_to_be16(cq_count);\r
115         err = mlx4_cq_modify(dev->dev, &mcq->mcq, context, 1);\r
116 \r
117         kfree(context);\r
118         return err;\r
119 }\r
120 \r
121 struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector,\r
122                                 struct ib_ucontext *context,\r
123                                 struct ib_udata *udata)\r
124 {\r
125         struct mlx4_ib_dev *dev = to_mdev(ibdev);\r
126         struct mlx4_ib_cq *cq;\r
127         struct mlx4_uar *uar;\r
128         int buf_size;\r
129         int err;\r
130 \r
131         if (mlx4_is_barred(ibdev->dma_device))\r
132                 return ERR_PTR(-EFAULT);\r
133 \r
134         if (entries < 1 || entries > dev->dev->caps.max_cqes)\r
135                 return ERR_PTR(-EINVAL);\r
136 \r
137         cq = kzalloc(sizeof *cq, GFP_KERNEL);\r
138         if (!cq) {\r
139                 printk(KERN_ERR "mlx4_ib_create_cq: -ENOMEM \n");\r
140                 return ERR_PTR(-ENOMEM);\r
141         }\r
142 \r
143         entries      = roundup_pow_of_two(entries + 1);\r
144         cq->ibcq.cqe = entries - 1;\r
145         buf_size     = entries * sizeof (struct mlx4_cqe);\r
146         spin_lock_init(&cq->lock);\r
147 \r
148         if (context && udata) {\r
149                 struct mlx4_ib_create_cq ucmd;\r
150 \r
151                 if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {\r
152                         err = -EFAULT;\r
153                         goto err_cq;\r
154                 }\r
155 \r
156                 cq->umem = ib_umem_get(context, ucmd.buf_addr, buf_size,\r
157                                        IB_ACCESS_LOCAL_WRITE, FALSE);\r
158                 if (IS_ERR(cq->umem)) {\r
159                         err = PTR_ERR(cq->umem);\r
160                         printk(KERN_ERR "mlx4_ib_create_cq: ib_umem_get failed with %d \n", err);\r
161                         goto err_cq;\r
162                 }\r
163 \r
164                 err = mlx4_mtt_init(dev->dev, ib_umem_page_count(cq->umem),\r
165                                     ilog2(cq->umem->page_size), &cq->buf.mtt);\r
166                 if (err) {\r
167                         printk(KERN_ERR "mlx4_ib_create_cq: mlx4_mtt_init failed with %d \n", err);\r
168                         goto err_buf;\r
169                 }\r
170 \r
171                 err = mlx4_ib_umem_write_mtt(dev, &cq->buf.mtt, cq->umem);\r
172                 if (err) {\r
173                         printk(KERN_ERR "mlx4_ib_create_cq: mlx4_ib_umem_write_mtt failed with %d \n", err);\r
174                         goto err_mtt;\r
175                 }\r
176 \r
177                 err = mlx4_ib_db_map_user(to_mucontext(context), ucmd.db_addr,\r
178                                           &cq->db);\r
179                 if (err) {\r
180                         printk(KERN_ERR "mlx4_ib_create_cq: mlx4_ib_db_map_user failed with %d \n", err);\r
181                         goto err_mtt;\r
182                 }\r
183 \r
184                 // add mapping to user's arm_sn variable\r
185                 // we have no way pass the completion event to provider library\r
186                 // so we'll increment user's arm_sn in kernel\r
187                 err = ib_umem_map( ucmd.arm_sn_addr, sizeof(int), \r
188                         IB_ACCESS_LOCAL_WRITE, &cq->mcq.mdl, &cq->mcq.p_u_arm_sn );\r
189                 if (err) {\r
190                         printk(KERN_ERR "mlx4_ib_create_cq: ib_umem_map failed with %d \n", err);\r
191                         goto err_dbmap;\r
192                 }\r
193 \r
194                 uar = &to_mucontext(context)->uar;\r
195         } else {\r
196                 err = mlx4_ib_db_alloc(dev, &cq->db, 1);\r
197                 if (err) {\r
198                         printk(KERN_ERR "mlx4_ib_create_cq: mlx4_ib_db_alloc failed with %d \n", err);\r
199                         goto err_cq;\r
200                 }\r
201 \r
202                 cq->mcq.set_ci_db  = cq->db.db;\r
203                 cq->mcq.arm_db     = cq->db.db + 1;\r
204                 *cq->mcq.set_ci_db = 0;\r
205                 *cq->mcq.arm_db    = 0;\r
206 \r
207                 if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, &cq->buf.buf)) {\r
208                         printk(KERN_ERR "mlx4_ib_create_cq: mlx4_buf_alloc failed for buf_size %d (entries %d) "\r
209                                    "on CQ %06x\n", buf_size, entries, cq->mcq.cqn);\r
210                         err = -ENOMEM;\r
211                         goto err_db;\r
212                 }\r
213 \r
214                 err = mlx4_mtt_init(dev->dev, cq->buf.buf.npages, cq->buf.buf.page_shift,\r
215                                     &cq->buf.mtt);\r
216                 if (err) {\r
217                         printk(KERN_ERR "mlx4_ib_create_cq: mlx4_mtt_init failed with %d \n", err);\r
218                         goto err_buf;\r
219                 }\r
220 \r
221                 err = mlx4_buf_write_mtt(dev->dev, &cq->buf.mtt, &cq->buf.buf);\r
222                 if (err) {\r
223                         printk(KERN_ERR "mlx4_ib_create_cq: mlx4_buf_write_mtt failed with %d \n", err);\r
224                         goto err_mtt;\r
225                 }\r
226 \r
227                 cq->mcq.p_u_arm_sn = NULL;\r
228                 uar = &dev->priv_uar;\r
229         }\r
230 \r
231         err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar,\r
232                 cq->db.dma.da, &cq->mcq, vector, 0);\r
233         if (err) {\r
234                 printk(KERN_ERR "mlx4_ib_create_cq: mlx4_cq_alloc failed with %d \n", err);\r
235                 goto err_dbmap;\r
236         }\r
237 \r
238         cq->mcq.comp  = mlx4_ib_cq_comp;\r
239         cq->mcq.event = mlx4_ib_cq_event;\r
240 \r
241         if (context) \r
242                 if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) {\r
243                         err = -EFAULT;\r
244                         goto err_dbmap;\r
245                 }\r
246 \r
247         return &cq->ibcq;\r
248 \r
249 err_dbmap:\r
250         ib_umem_unmap( cq->mcq.mdl, cq->mcq.p_u_arm_sn );\r
251         if (context)\r
252                 mlx4_ib_db_unmap_user(to_mucontext(context), &cq->db);\r
253 \r
254 err_mtt:\r
255         mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt);\r
256 \r
257 err_buf:\r
258         if (context)\r
259                 ib_umem_release(cq->umem);\r
260         else\r
261                 mlx4_buf_free(dev->dev, entries * sizeof (struct mlx4_cqe),\r
262                               &cq->buf.buf);\r
263 \r
264 err_db:\r
265         if (!context)\r
266                 mlx4_ib_db_free(dev, &cq->db);\r
267 \r
268 err_cq:\r
269         kfree(cq);\r
270 \r
271         return ERR_PTR(err);\r
272 }\r
273 \r
274 int mlx4_ib_destroy_cq(struct ib_cq *cq)\r
275 {\r
276         struct mlx4_ib_dev *dev = to_mdev(cq->device);\r
277         struct mlx4_ib_cq *mcq = to_mcq(cq);\r
278 \r
279         mlx4_cq_free(dev->dev, &mcq->mcq);\r
280         mlx4_mtt_cleanup(dev->dev, &mcq->buf.mtt);\r
281 \r
282         if (cq->p_uctx) {\r
283                 ib_umem_unmap( mcq->mcq.mdl, mcq->mcq.p_u_arm_sn );\r
284                 mlx4_ib_db_unmap_user(to_mucontext(cq->p_uctx), &mcq->db);\r
285                 ib_umem_release(mcq->umem);\r
286         } else {\r
287                 mlx4_buf_free(dev->dev, (cq->cqe + 1) * sizeof (struct mlx4_cqe),\r
288                               &mcq->buf.buf);\r
289                 mlx4_ib_db_free(dev, &mcq->db);\r
290         }\r
291 \r
292         kfree(mcq);\r
293 \r
294         return 0;\r
295 }\r
296 \r
297 static void dump_cqe(void *cqe)\r
298 {\r
299         __be32 *buf = cqe;\r
300 \r
301         MLX4_PRINT(TRACE_LEVEL_ERROR, MLX4_DBG_DRV, \r
302                 (KERN_DEBUG "CQE contents %08x %08x %08x %08x %08x %08x %08x %08x\n",\r
303                            be32_to_cpu(buf[0]), be32_to_cpu(buf[1]), be32_to_cpu(buf[2]),\r
304                            be32_to_cpu(buf[3]), be32_to_cpu(buf[4]), be32_to_cpu(buf[5]),\r
305                            be32_to_cpu(buf[6]), be32_to_cpu(buf[7])));\r
306 }\r
307 \r
308 static void mlx4_ib_handle_error_cqe(struct mlx4_err_cqe *cqe,\r
309                                      ib_wc_t *wc)\r
310 {\r
311         if (cqe->syndrome == MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR) {\r
312                 printk(KERN_DEBUG "local QP operation err "\r
313                        "(QPN %06x, WQE index %x, vendor syndrome %02x, "\r
314                        "opcode = %02x)\n",\r
315                        be32_to_cpu(cqe->my_qpn), be16_to_cpu(cqe->wqe_index),\r
316                        cqe->vendor_err_syndrome,\r
317                        cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);\r
318                 dump_cqe(cqe);\r
319         }\r
320 \r
321         switch (cqe->syndrome) {\r
322         case MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR:\r
323                 wc->status = IB_WCS_LOCAL_LEN_ERR;\r
324                 break;\r
325         case MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR:\r
326                 wc->status = IB_WCS_LOCAL_OP_ERR;\r
327                 break;\r
328         case MLX4_CQE_SYNDROME_LOCAL_PROT_ERR:\r
329                 wc->status = IB_WCS_LOCAL_PROTECTION_ERR;\r
330                 break;\r
331         case MLX4_CQE_SYNDROME_WR_FLUSH_ERR:\r
332                 wc->status = IB_WCS_WR_FLUSHED_ERR;\r
333                 break;\r
334         case MLX4_CQE_SYNDROME_MW_BIND_ERR:\r
335                 wc->status = IB_WCS_MEM_WINDOW_BIND_ERR;\r
336                 break;\r
337         case MLX4_CQE_SYNDROME_BAD_RESP_ERR:\r
338                 wc->status = IB_WCS_BAD_RESP_ERR;\r
339                 break;\r
340         case MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR:\r
341                 wc->status = IB_WCS_LOCAL_ACCESS_ERR;\r
342                 break;\r
343         case MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:\r
344                 wc->status = IB_WCS_REM_INVALID_REQ_ERR;\r
345                 break;\r
346         case MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR:\r
347                 wc->status = IB_WCS_REM_ACCESS_ERR;\r
348                 break;\r
349         case MLX4_CQE_SYNDROME_REMOTE_OP_ERR:\r
350                 wc->status = IB_WCS_REM_OP_ERR;\r
351                 break;\r
352         case MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:\r
353                 wc->status = IB_WCS_TIMEOUT_RETRY_ERR;\r
354                 break;\r
355         case MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR:\r
356                 wc->status = IB_WCS_RNR_RETRY_ERR;\r
357                 break;\r
358         case MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR:\r
359                 wc->status = IB_WCS_REM_ABORT_ERR;\r
360                 break;\r
361         default:\r
362                 wc->status = IB_WC_GENERAL_ERR;\r
363                 break;\r
364         }\r
365 \r
366         wc->vendor_specific = cqe->vendor_err_syndrome;\r
367 }\r
368 \r
369 static uint32_t mlx4_ib_ipoib_csum_ok(__be32 status, __be16 checksum) {\r
370         \r
371         #define CSUM_VALID_NUM 0xffff\r
372         uint32_t res = 0;\r
373 \r
374         // Verify that IP_OK bit is set and the packet is pure IPv4 packet\r
375         if ((status & cpu_to_be32(MLX4_CQE_IPOIB_STATUS_IPV4            |\r
376                                                         MLX4_CQE_IPOIB_STATUS_IPV4              |\r
377                                                         MLX4_CQE_IPOIB_STATUS_IPV4OPT   |\r
378                                                         MLX4_CQE_IPOIB_STATUS_IPV6              |\r
379                                                         MLX4_CQE_IPOIB_STATUS_IPOK))    ==\r
380                                 cpu_to_be32(MLX4_CQE_IPOIB_STATUS_IPV4          |\r
381                                                         MLX4_CQE_IPOIB_STATUS_IPOK))\r
382         {\r
383                 // IP checksum calculated by MLX4 matched the checksum in the receive packet's \r
384                 res |= MLX4_NdisPacketIpChecksumSucceeded;\r
385                 if (checksum == CSUM_VALID_NUM) {\r
386                                 // TCP or UDP checksum calculated by MLX4 matched the checksum in the receive packet's \r
387                                 res |= (MLX4_NdisPacketUdpChecksumSucceeded |\r
388                                                 MLX4_NdisPacketTcpChecksumSucceeded );\r
389                                 ASSERT( status & cpu_to_be32(MLX4_CQE_IPOIB_STATUS_TCP | MLX4_CQE_IPOIB_STATUS_UDP));\r
390                 }\r
391         }\r
392         return (( res << 8 ) & IB_RECV_OPT_CSUM_MASK );\r
393 }\r
394 \r
395 static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq,\r
396                             struct mlx4_ib_qp **cur_qp,\r
397                             ib_wc_t *wc)\r
398 {\r
399         struct mlx4_cqe *cqe;\r
400         struct mlx4_qp *mqp;\r
401         struct mlx4_ib_wq *wq;\r
402         struct mlx4_ib_srq *srq;\r
403         int is_send;\r
404         int is_error;\r
405         u16 wqe_ctr;\r
406 \r
407         cqe = next_cqe_sw(cq);\r
408         if (!cqe)\r
409                 return -EAGAIN;\r
410 \r
411         ++cq->mcq.cons_index;\r
412 \r
413         /*\r
414          * Make sure we read CQ entry contents after we've checked the\r
415          * ownership bit.\r
416          */\r
417         rmb();\r
418 \r
419         is_send  = cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK;\r
420         is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==\r
421                 MLX4_CQE_OPCODE_ERROR;\r
422 \r
423         if (!*cur_qp || (be32_to_cpu(cqe->my_qpn) & 0xffffff) != (u32)(*cur_qp)->mqp.qpn) {\r
424                 /*\r
425                  * We do not have to take the QP table lock here,\r
426                  * because CQs will be locked while QPs are removed\r
427                  * from the table.\r
428                  */\r
429 #if 1\r
430                 // radix_tree_insert in current implementation seems like\r
431                 // can cause radix_tree_lookup to miss an existing QP\r
432                 // so we call qp_lookup under the spinlock\r
433                 mqp = mlx4_qp_lookup_locked( to_mdev(cq->ibcq.device)->dev, be32_to_cpu(cqe->my_qpn));\r
434 #else\r
435                 mqp = __mlx4_qp_lookup( to_mdev(cq->ibcq.device)->dev, be32_to_cpu(cqe->my_qpn));\r
436 #endif\r
437 \r
438                 if (unlikely(!mqp)) {\r
439                         printk(KERN_WARNING "CQ %06x with entry for unknown QPN %06x\n",\r
440                                 cq->mcq.cqn, be32_to_cpu(cqe->my_qpn) & 0xffffff);\r
441                         return -EINVAL;\r
442                 }\r
443 \r
444                 *cur_qp = to_mibqp(mqp);\r
445         }\r
446 \r
447         if (is_send) {\r
448                 wq = &(*cur_qp)->sq;\r
449                 wqe_ctr = be16_to_cpu(cqe->wqe_index);\r
450                 wq->tail += (u16) (wqe_ctr - (u16) wq->tail);\r
451                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];\r
452                 ++wq->tail;\r
453         } else if ((*cur_qp)->ibqp.srq) {\r
454                 srq = to_msrq((*cur_qp)->ibqp.srq);\r
455                 wqe_ctr = be16_to_cpu(cqe->wqe_index);\r
456                 wc->wr_id = srq->wrid[wqe_ctr];\r
457                 mlx4_ib_free_srq_wqe(srq, wqe_ctr);\r
458         } else {\r
459                 wq        = &(*cur_qp)->rq;\r
460                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];\r
461                 ++wq->tail;\r
462         }\r
463 \r
464         if (is_send) {\r
465                 wc->recv.ud.recv_opt = 0;\r
466                 switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {\r
467                 case MLX4_OPCODE_RDMA_WRITE_IMM:\r
468                         wc->recv.ud.recv_opt |= IB_RECV_OPT_IMMEDIATE;\r
469                 case MLX4_OPCODE_RDMA_WRITE:\r
470                         wc->wc_type    = IB_WC_RDMA_WRITE;\r
471                         break;\r
472                 case MLX4_OPCODE_SEND_IMM:\r
473                         wc->recv.ud.recv_opt |= IB_RECV_OPT_IMMEDIATE;\r
474                 case MLX4_OPCODE_SEND:\r
475                         wc->wc_type    = IB_WC_SEND;\r
476                         break;\r
477                 case MLX4_OPCODE_RDMA_READ:\r
478                         wc->wc_type    = IB_WC_RDMA_READ;\r
479                         wc->length  = be32_to_cpu(cqe->byte_cnt);\r
480                         break;\r
481                 case MLX4_OPCODE_ATOMIC_CS:\r
482                         wc->wc_type    = IB_WC_COMPARE_SWAP;\r
483                         wc->length  = 8;\r
484                         break;\r
485                 case MLX4_OPCODE_ATOMIC_FA:\r
486                         wc->wc_type    = IB_WC_FETCH_ADD;\r
487                         wc->length  = 8;\r
488                         break;\r
489                 case MLX4_OPCODE_BIND_MW:\r
490                         wc->wc_type    = IB_WC_MW_BIND;\r
491                         break;\r
492                 case MLX4_OPCODE_LSO:\r
493                         wc->wc_type    = IB_WC_LSO;\r
494                         break;\r
495                 default:\r
496                         wc->wc_type       = IB_WC_SEND;\r
497                         break;\r
498                 }\r
499         } else {\r
500                 wc->length = be32_to_cpu(cqe->byte_cnt);\r
501 \r
502                 switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {\r
503                 case MLX4_RECV_OPCODE_RDMA_WRITE_IMM:\r
504                         wc->wc_type   = IB_WC_RECV_RDMA_WRITE;\r
505                         wc->recv.ud.recv_opt = IB_RECV_OPT_IMMEDIATE;\r
506                         wc->recv.ud.immediate_data = cqe->immed_rss_invalid;\r
507                         break;\r
508                 case MLX4_RECV_OPCODE_SEND:\r
509                         wc->wc_type   = IB_WC_RECV;\r
510                         wc->recv.ud.recv_opt = 0;\r
511                         break;\r
512                 case MLX4_RECV_OPCODE_SEND_IMM:\r
513                         wc->wc_type   = IB_WC_RECV;\r
514                         wc->recv.ud.recv_opt = IB_RECV_OPT_IMMEDIATE;\r
515                         wc->recv.ud.immediate_data = cqe->immed_rss_invalid;\r
516                         break;\r
517                 default:\r
518                         wc->recv.ud.recv_opt = 0;\r
519                         wc->wc_type = IB_WC_RECV;\r
520                         break;\r
521                 }\r
522 \r
523                 wc->recv.ud.remote_lid  = cqe->rlid;\r
524                 wc->recv.ud.remote_sl           = cqe->sl >> 4;\r
525                 wc->recv.ud.remote_qp   = cqe->g_mlpath_rqpn & 0xffffff00;\r
526                 wc->recv.ud.path_bits           = (u8)(cqe->g_mlpath_rqpn & 0x7f);\r
527                 wc->recv.ud.recv_opt            |= cqe->g_mlpath_rqpn & 0x080 ? IB_RECV_OPT_GRH_VALID : 0;\r
528                 wc->recv.ud.pkey_index  = (u16)(be32_to_cpu(cqe->immed_rss_invalid)  & 0x7f);\r
529                 wc->recv.ud.recv_opt |= mlx4_ib_ipoib_csum_ok(cqe->ipoib_status,cqe->checksum);\r
530         }\r
531         if (!is_send && cqe->rlid == 0){\r
532                 MLX4_PRINT(TRACE_LEVEL_INFORMATION,MLX4_DBG_CQ,("found rlid == 0 \n "));\r
533                 wc->recv.ud.recv_opt         |= IB_RECV_OPT_FORWARD;\r
534         }\r
535 \r
536         if (unlikely(is_error))\r
537                 mlx4_ib_handle_error_cqe((struct mlx4_err_cqe *) cqe, wc);\r
538         else\r
539                 wc->status = IB_WCS_SUCCESS;\r
540 \r
541         return 0;\r
542 }\r
543 \r
544 int mlx4_ib_poll_cq(\r
545         IN              struct ib_cq *ibcq, \r
546         IN      OUT                     ib_wc_t** const                         pp_free_wclist,\r
547                 OUT                     ib_wc_t** const                         pp_done_wclist )\r
548 {\r
549         struct mlx4_ib_cq *cq = to_mcq(ibcq);\r
550         struct mlx4_ib_qp *cur_qp = NULL;\r
551         unsigned long flags;\r
552         int err = 0;\r
553         int npolled = 0;\r
554         ib_wc_t         *wc_p, **next_pp;\r
555 \r
556         spin_lock_irqsave(&cq->lock, &flags);\r
557 \r
558         // loop through CQ\r
559         next_pp = pp_done_wclist;\r
560         wc_p = *pp_free_wclist;\r
561         while( wc_p ) {\r
562                 // poll one CQE\r
563                 err = mlx4_ib_poll_one(cq, &cur_qp, wc_p);\r
564                 if (err)\r
565                         break;\r
566 \r
567                 // prepare for the next loop\r
568                 *next_pp = wc_p;\r
569                 next_pp = &wc_p->p_next;\r
570                 wc_p = wc_p->p_next;\r
571                 ++npolled;\r
572         }\r
573 \r
574         // prepare the results\r
575         *pp_free_wclist = wc_p;         /* Set the head of the free list. */\r
576         *next_pp = NULL;                                                /* Clear the tail of the done list. */\r
577 \r
578         // update consumer index\r
579         if (npolled)\r
580                 mlx4_cq_set_ci(&cq->mcq);\r
581 \r
582         spin_unlock_irqrestore(&cq->lock, flags);\r
583         return (err == 0 || err == -EAGAIN)? npolled : err;\r
584 }\r
585 \r
586 int mlx4_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)\r
587 {\r
588         if (!mlx4_is_barred(ibcq->device->dma_device))\r
589                 mlx4_cq_arm(&to_mcq(ibcq)->mcq,\r
590                     (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?\r
591                     MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT,\r
592                     to_mdev(ibcq->device)->uar_map,\r
593                     MLX4_GET_DOORBELL_LOCK(&to_mdev(ibcq->device)->uar_lock));\r
594 \r
595         return 0;\r
596 }\r
597 \r
598 void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)\r
599 {\r
600         u32 prod_index;\r
601         int nfreed = 0;\r
602         struct mlx4_cqe *cqe, *dest;\r
603         u8 owner_bit;\r
604 \r
605         /*\r
606          * First we need to find the current producer index, so we\r
607          * know where to start cleaning from.  It doesn't matter if HW\r
608          * adds new entries after this loop -- the QP we're worried\r
609          * about is already in RESET, so the new entries won't come\r
610          * from our QP and therefore don't need to be checked.\r
611          */\r
612         for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); ++prod_index)\r
613                 if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe)\r
614                         break;\r
615 \r
616         /*\r
617          * Now sweep backwards through the CQ, removing CQ entries\r
618          * that match our QP by copying older entries on top of them.\r
619          */\r
620         while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) {\r
621                 cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);\r
622                 if ((be32_to_cpu(cqe->my_qpn) & 0xffffff) == qpn) {\r
623                         if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK))\r
624                                 mlx4_ib_free_srq_wqe(srq, be16_to_cpu(cqe->wqe_index));\r
625                         ++nfreed;\r
626                 } else if (nfreed) {\r
627                         dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe);\r
628                         owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK;\r
629                         memcpy(dest, cqe, sizeof *cqe);\r
630                         dest->owner_sr_opcode = owner_bit |\r
631                                 (dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);\r
632                 }\r
633         }\r
634 \r
635         if (nfreed) {\r
636                 cq->mcq.cons_index += nfreed;\r
637                 /*\r
638                  * Make sure update of buffer contents is done before\r
639                  * updating consumer index.\r
640                  */\r
641                 wmb();\r
642                 mlx4_cq_set_ci(&cq->mcq);\r
643         }\r
644 }\r
645 \r
646 void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)\r
647 {\r
648         spin_lock_irq(&cq->lock);\r
649         __mlx4_ib_cq_clean(cq, qpn, srq);\r
650         spin_unlock_irq(&cq->lock);\r
651 }\r