77f56d3fd6c6dd288da8c36fda8cb4862422d4db
[mirror/winof/.git] / hw / mlx4 / kernel / bus / ib / cq.c
1 /*\r
2  * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.\r
3  *\r
4  * This software is available to you under a choice of one of two\r
5  * licenses.  You may choose to be licensed under the terms of the GNU\r
6  * General Public License (GPL) Version 2, available from the file\r
7  * COPYING in the main directory of this source tree, or the\r
8  * OpenIB.org BSD license below:\r
9  *\r
10  *     Redistribution and use in source and binary forms, with or\r
11  *     without modification, are permitted provided that the following\r
12  *     conditions are met:\r
13  *\r
14  *      - Redistributions of source code must retain the above\r
15  *        copyright notice, this list of conditions and the following\r
16  *        disclaimer.\r
17  *\r
18  *      - Redistributions in binary form must reproduce the above\r
19  *        copyright notice, this list of conditions and the following\r
20  *        disclaimer in the documentation and/or other materials\r
21  *        provided with the distribution.\r
22  *\r
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
30  * SOFTWARE.\r
31  */\r
32 #include <mlx4_debug.h>\r
33 #include "mlx4_ib.h"\r
34 #include "cq.h"\r
35 #include "qp.h"\r
36 #include "user.h"\r
37 \r
38 #if defined(EVENT_TRACING)\r
39 #ifdef offsetof\r
40 #undef offsetof\r
41 #endif\r
42 #include "cq.tmh"\r
43 #endif\r
44 \r
45 \r
46 static void mlx4_ib_cq_comp(struct mlx4_cq *cq)\r
47 {\r
48         struct ib_cq *ibcq = &to_mibcq(cq)->ibcq;\r
49         ibcq->comp_handler(ibcq->cq_context);\r
50 }\r
51 \r
52 static void mlx4_ib_cq_event(struct mlx4_cq *cq, enum mlx4_event type)\r
53 {\r
54         ib_event_rec_t event;\r
55         struct ib_cq *ibcq;\r
56 \r
57         if (type != MLX4_EVENT_TYPE_CQ_ERROR) {\r
58                 printk(KERN_WARNING "mlx4_ib: Unexpected event type %d "\r
59                        "on CQ %06x\n", type, cq->cqn);\r
60                 return;\r
61         }\r
62 \r
63         ibcq = &to_mibcq(cq)->ibcq;\r
64         event.type = IB_EVENT_CQ_ERR;\r
65         event.context = ibcq->cq_context;\r
66         event.vendor_specific = type;\r
67         ibcq->event_handler(&event);\r
68 }\r
69 \r
70 static void *get_cqe_from_buf(struct mlx4_ib_cq_buf *buf, int n)\r
71 {\r
72         int offset = n * sizeof (struct mlx4_cqe);\r
73 \r
74         if (buf->buf.nbufs == 1)\r
75                 return buf->buf.u.direct.buf + offset;\r
76         else\r
77                 return buf->buf.u.page_list[offset >> PAGE_SHIFT].buf +\r
78                         (offset & (PAGE_SIZE - 1));\r
79 }\r
80 \r
81 static void *get_cqe(struct mlx4_ib_cq *cq, int n)\r
82 {\r
83         return get_cqe_from_buf(&cq->buf, n);\r
84 }\r
85 \r
86 static void *get_sw_cqe(struct mlx4_ib_cq *cq, int n)\r
87 {\r
88         struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibcq.cqe);\r
89 \r
90         return (!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^\r
91                 !!(n & (cq->ibcq.cqe + 1))) ? NULL : cqe;\r
92 }\r
93 \r
94 static struct mlx4_cqe *next_cqe_sw(struct mlx4_ib_cq *cq)\r
95 {\r
96         return get_sw_cqe(cq, cq->mcq.cons_index);\r
97 }\r
98 \r
99 int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)\r
100 {\r
101         struct mlx4_ib_cq *mcq = to_mcq(cq);\r
102         struct mlx4_ib_dev *dev = to_mdev(cq->device);\r
103         struct mlx4_cq_context *context;\r
104         int err;\r
105 \r
106         if (mlx4_is_barred(dev->dev))\r
107                 return -EFAULT;\r
108 \r
109         context = kzalloc(sizeof *context, GFP_KERNEL);\r
110         if (!context)\r
111                 return -ENOMEM;\r
112 \r
113         context->cq_period = cpu_to_be16(cq_period);\r
114         context->cq_max_count = cpu_to_be16(cq_count);\r
115         err = mlx4_cq_modify(dev->dev, &mcq->mcq, context, 1);\r
116 \r
117         kfree(context);\r
118         return err;\r
119 }\r
120 \r
121 struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector,\r
122                                 struct ib_ucontext *context,\r
123                                 struct ib_udata *udata)\r
124 {\r
125         struct mlx4_ib_dev *dev = to_mdev(ibdev);\r
126         struct mlx4_ib_cq *cq;\r
127         struct mlx4_uar *uar;\r
128         int buf_size;\r
129         int err;\r
130 \r
131         if (mlx4_is_barred(ibdev->dma_device))\r
132                 return ERR_PTR(-EFAULT);\r
133 \r
134         if (entries < 1 || entries > dev->dev->caps.max_cqes)\r
135                 return ERR_PTR(-EINVAL);\r
136 \r
137         cq = kzalloc(sizeof *cq, GFP_KERNEL);\r
138         if (!cq)\r
139                 return ERR_PTR(-ENOMEM);\r
140 \r
141         entries      = roundup_pow_of_two(entries + 1);\r
142         cq->ibcq.cqe = entries - 1;\r
143         buf_size     = entries * sizeof (struct mlx4_cqe);\r
144         spin_lock_init(&cq->lock);\r
145 \r
146         if (context && udata) {\r
147                 struct mlx4_ib_create_cq ucmd;\r
148 \r
149                 if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {\r
150                         err = -EFAULT;\r
151                         goto err_cq;\r
152                 }\r
153 \r
154                 cq->umem = ib_umem_get(context, ucmd.buf_addr, buf_size,\r
155                                        IB_ACCESS_LOCAL_WRITE, FALSE);\r
156                 if (IS_ERR(cq->umem)) {\r
157                         err = PTR_ERR(cq->umem);\r
158                         goto err_cq;\r
159                 }\r
160 \r
161                 err = mlx4_mtt_init(dev->dev, ib_umem_page_count(cq->umem),\r
162                                     ilog2(cq->umem->page_size), &cq->buf.mtt);\r
163                 if (err)\r
164                         goto err_buf;\r
165 \r
166                 err = mlx4_ib_umem_write_mtt(dev, &cq->buf.mtt, cq->umem);\r
167                 if (err)\r
168                         goto err_mtt;\r
169 \r
170                 err = mlx4_ib_db_map_user(to_mucontext(context), ucmd.db_addr,\r
171                                           &cq->db);\r
172                 if (err)\r
173                         goto err_mtt;\r
174 \r
175                 // add mapping to user's arm_sn variable\r
176                 // we have no way pass the completion event to provider library\r
177                 // so we'll increment user's arm_sn in kernel\r
178                 err = ib_umem_map( ucmd.arm_sn_addr, sizeof(int), \r
179                         IB_ACCESS_LOCAL_WRITE, &cq->mcq.mdl, &cq->mcq.p_u_arm_sn );\r
180                 if (err)\r
181                         goto err_dbmap;\r
182 \r
183                 uar = &to_mucontext(context)->uar;\r
184         } else {\r
185                 err = mlx4_ib_db_alloc(dev, &cq->db, 1);\r
186                 if (err)\r
187                         goto err_cq;\r
188 \r
189                 cq->mcq.set_ci_db  = cq->db.db;\r
190                 cq->mcq.arm_db     = cq->db.db + 1;\r
191                 *cq->mcq.set_ci_db = 0;\r
192                 *cq->mcq.arm_db    = 0;\r
193 \r
194                 if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, &cq->buf.buf)) {\r
195                         err = -ENOMEM;\r
196                         goto err_db;\r
197                 }\r
198 \r
199                 err = mlx4_mtt_init(dev->dev, cq->buf.buf.npages, cq->buf.buf.page_shift,\r
200                                     &cq->buf.mtt);\r
201                 if (err)\r
202                         goto err_buf;\r
203 \r
204                 err = mlx4_buf_write_mtt(dev->dev, &cq->buf.mtt, &cq->buf.buf);\r
205                 if (err)\r
206                         goto err_mtt;\r
207 \r
208                 cq->mcq.p_u_arm_sn = NULL;\r
209                 uar = &dev->priv_uar;\r
210         }\r
211 \r
212         err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar,\r
213                 cq->db.dma.da, &cq->mcq, vector, 0);\r
214         if (err)\r
215                 goto err_dbmap;\r
216 \r
217         cq->mcq.comp  = mlx4_ib_cq_comp;\r
218         cq->mcq.event = mlx4_ib_cq_event;\r
219 \r
220         if (context) \r
221                 if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) {\r
222                         err = -EFAULT;\r
223                         goto err_dbmap;\r
224                 }\r
225 \r
226         return &cq->ibcq;\r
227 \r
228 err_dbmap:\r
229         ib_umem_unmap( cq->mcq.mdl, cq->mcq.p_u_arm_sn );\r
230         if (context)\r
231                 mlx4_ib_db_unmap_user(to_mucontext(context), &cq->db);\r
232 \r
233 err_mtt:\r
234         mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt);\r
235 \r
236 err_buf:\r
237         if (context)\r
238                 ib_umem_release(cq->umem);\r
239         else\r
240                 mlx4_buf_free(dev->dev, entries * sizeof (struct mlx4_cqe),\r
241                               &cq->buf.buf);\r
242 \r
243 err_db:\r
244         if (!context)\r
245                 mlx4_ib_db_free(dev, &cq->db);\r
246 \r
247 err_cq:\r
248         kfree(cq);\r
249 \r
250         return ERR_PTR(err);\r
251 }\r
252 \r
253 int mlx4_ib_destroy_cq(struct ib_cq *cq)\r
254 {\r
255         struct mlx4_ib_dev *dev = to_mdev(cq->device);\r
256         struct mlx4_ib_cq *mcq = to_mcq(cq);\r
257 \r
258         mlx4_cq_free(dev->dev, &mcq->mcq);\r
259         mlx4_mtt_cleanup(dev->dev, &mcq->buf.mtt);\r
260 \r
261         if (cq->p_uctx) {\r
262                 ib_umem_unmap( mcq->mcq.mdl, mcq->mcq.p_u_arm_sn );\r
263                 mlx4_ib_db_unmap_user(to_mucontext(cq->p_uctx), &mcq->db);\r
264                 ib_umem_release(mcq->umem);\r
265         } else {\r
266                 mlx4_buf_free(dev->dev, (cq->cqe + 1) * sizeof (struct mlx4_cqe),\r
267                               &mcq->buf.buf);\r
268                 mlx4_ib_db_free(dev, &mcq->db);\r
269         }\r
270 \r
271         kfree(mcq);\r
272 \r
273         return 0;\r
274 }\r
275 \r
276 static void dump_cqe(void *cqe)\r
277 {\r
278         __be32 *buf = cqe;\r
279 \r
280         MLX4_PRINT(TRACE_LEVEL_ERROR, MLX4_DBG_DRV, \r
281                 (KERN_DEBUG "CQE contents %08x %08x %08x %08x %08x %08x %08x %08x\n",\r
282                            be32_to_cpu(buf[0]), be32_to_cpu(buf[1]), be32_to_cpu(buf[2]),\r
283                            be32_to_cpu(buf[3]), be32_to_cpu(buf[4]), be32_to_cpu(buf[5]),\r
284                            be32_to_cpu(buf[6]), be32_to_cpu(buf[7])));\r
285 }\r
286 \r
287 static void mlx4_ib_handle_error_cqe(struct mlx4_err_cqe *cqe,\r
288                                      ib_wc_t *wc)\r
289 {\r
290         if (cqe->syndrome == MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR) {\r
291                 printk(KERN_DEBUG "local QP operation err "\r
292                        "(QPN %06x, WQE index %x, vendor syndrome %02x, "\r
293                        "opcode = %02x)\n",\r
294                        be32_to_cpu(cqe->my_qpn), be16_to_cpu(cqe->wqe_index),\r
295                        cqe->vendor_err_syndrome,\r
296                        cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);\r
297                 dump_cqe(cqe);\r
298         }\r
299 \r
300         switch (cqe->syndrome) {\r
301         case MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR:\r
302                 wc->status = IB_WCS_LOCAL_LEN_ERR;\r
303                 break;\r
304         case MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR:\r
305                 wc->status = IB_WCS_LOCAL_OP_ERR;\r
306                 break;\r
307         case MLX4_CQE_SYNDROME_LOCAL_PROT_ERR:\r
308                 wc->status = IB_WCS_LOCAL_PROTECTION_ERR;\r
309                 break;\r
310         case MLX4_CQE_SYNDROME_WR_FLUSH_ERR:\r
311                 wc->status = IB_WCS_WR_FLUSHED_ERR;\r
312                 break;\r
313         case MLX4_CQE_SYNDROME_MW_BIND_ERR:\r
314                 wc->status = IB_WCS_MEM_WINDOW_BIND_ERR;\r
315                 break;\r
316         case MLX4_CQE_SYNDROME_BAD_RESP_ERR:\r
317                 wc->status = IB_WCS_BAD_RESP_ERR;\r
318                 break;\r
319         case MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR:\r
320                 wc->status = IB_WCS_LOCAL_ACCESS_ERR;\r
321                 break;\r
322         case MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:\r
323                 wc->status = IB_WCS_REM_INVALID_REQ_ERR;\r
324                 break;\r
325         case MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR:\r
326                 wc->status = IB_WCS_REM_ACCESS_ERR;\r
327                 break;\r
328         case MLX4_CQE_SYNDROME_REMOTE_OP_ERR:\r
329                 wc->status = IB_WCS_REM_OP_ERR;\r
330                 break;\r
331         case MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:\r
332                 wc->status = IB_WCS_TIMEOUT_RETRY_ERR;\r
333                 break;\r
334         case MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR:\r
335                 wc->status = IB_WCS_RNR_RETRY_ERR;\r
336                 break;\r
337         case MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR:\r
338                 wc->status = IB_WCS_REM_ABORT_ERR;\r
339                 break;\r
340         default:\r
341                 wc->status = IB_WC_GENERAL_ERR;\r
342                 break;\r
343         }\r
344 \r
345         wc->vendor_specific = cqe->vendor_err_syndrome;\r
346 }\r
347 \r
348 static uint32_t mlx4_ib_ipoib_csum_ok(__be32 status, __be16 checksum) {\r
349         \r
350         #define CSUM_VALID_NUM 0xffff\r
351         uint32_t res = 0;\r
352 \r
353         // Verify that IP_OK bit is set and the packet is pure IPv4 packet\r
354         if ((status & cpu_to_be32(MLX4_CQE_IPOIB_STATUS_IPV4            |\r
355                                                         MLX4_CQE_IPOIB_STATUS_IPV4              |\r
356                                                         MLX4_CQE_IPOIB_STATUS_IPV4OPT   |\r
357                                                         MLX4_CQE_IPOIB_STATUS_IPV6              |\r
358                                                         MLX4_CQE_IPOIB_STATUS_IPOK))    ==\r
359                                 cpu_to_be32(MLX4_CQE_IPOIB_STATUS_IPV4          |\r
360                                                         MLX4_CQE_IPOIB_STATUS_IPOK))\r
361         {\r
362                 // IP checksum calculated by MLX4 matched the checksum in the receive packet's \r
363                 res |= MLX4_NdisPacketIpChecksumSucceeded;\r
364                 if (checksum == CSUM_VALID_NUM) {\r
365                                 // TCP or UDP checksum calculated by MLX4 matched the checksum in the receive packet's \r
366                                 res |= (MLX4_NdisPacketUdpChecksumSucceeded |\r
367                                                 MLX4_NdisPacketTcpChecksumSucceeded );\r
368                                 ASSERT( status & cpu_to_be32(MLX4_CQE_IPOIB_STATUS_TCP | MLX4_CQE_IPOIB_STATUS_UDP));\r
369                 }\r
370         }\r
371         return (( res << 8 ) & IB_RECV_OPT_CSUM_MASK );\r
372 }\r
373 \r
374 static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq,\r
375                             struct mlx4_ib_qp **cur_qp,\r
376                             ib_wc_t *wc)\r
377 {\r
378         struct mlx4_cqe *cqe;\r
379         struct mlx4_qp *mqp;\r
380         struct mlx4_ib_wq *wq;\r
381         struct mlx4_ib_srq *srq;\r
382         int is_send;\r
383         int is_error;\r
384         u16 wqe_ctr;\r
385 \r
386         cqe = next_cqe_sw(cq);\r
387         if (!cqe)\r
388                 return -EAGAIN;\r
389 \r
390         ++cq->mcq.cons_index;\r
391 \r
392         /*\r
393          * Make sure we read CQ entry contents after we've checked the\r
394          * ownership bit.\r
395          */\r
396         rmb();\r
397 \r
398         is_send  = cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK;\r
399         is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==\r
400                 MLX4_CQE_OPCODE_ERROR;\r
401 \r
402         if (!*cur_qp || (be32_to_cpu(cqe->my_qpn) & 0xffffff) != (u32)(*cur_qp)->mqp.qpn) {\r
403                 /*\r
404                  * We do not have to take the QP table lock here,\r
405                  * because CQs will be locked while QPs are removed\r
406                  * from the table.\r
407                  */\r
408 #if 1\r
409                 // radix_tree_insert in current implementation seems like\r
410                 // can cause radix_tree_lookup to miss an existing QP\r
411                 // so we call qp_lookup under the spinlock\r
412                 mqp = mlx4_qp_lookup_locked( to_mdev(cq->ibcq.device)->dev, be32_to_cpu(cqe->my_qpn));\r
413 #else\r
414                 mqp = __mlx4_qp_lookup( to_mdev(cq->ibcq.device)->dev, be32_to_cpu(cqe->my_qpn));\r
415 #endif\r
416 \r
417                 if (unlikely(!mqp)) {\r
418                         printk(KERN_WARNING "CQ %06x with entry for unknown QPN %06x\n",\r
419                                 cq->mcq.cqn, be32_to_cpu(cqe->my_qpn) & 0xffffff);\r
420                         return -EINVAL;\r
421                 }\r
422 \r
423                 *cur_qp = to_mibqp(mqp);\r
424         }\r
425 \r
426         if (is_send) {\r
427                 wq = &(*cur_qp)->sq;\r
428                 wqe_ctr = be16_to_cpu(cqe->wqe_index);\r
429                 wq->tail += (u16) (wqe_ctr - (u16) wq->tail);\r
430                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];\r
431                 ++wq->tail;\r
432         } else if ((*cur_qp)->ibqp.srq) {\r
433                 srq = to_msrq((*cur_qp)->ibqp.srq);\r
434                 wqe_ctr = be16_to_cpu(cqe->wqe_index);\r
435                 wc->wr_id = srq->wrid[wqe_ctr];\r
436                 mlx4_ib_free_srq_wqe(srq, wqe_ctr);\r
437         } else {\r
438                 wq        = &(*cur_qp)->rq;\r
439                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];\r
440                 ++wq->tail;\r
441         }\r
442 \r
443         if (is_send) {\r
444                 wc->recv.ud.recv_opt = 0;\r
445                 switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {\r
446                 case MLX4_OPCODE_RDMA_WRITE_IMM:\r
447                         wc->recv.ud.recv_opt |= IB_RECV_OPT_IMMEDIATE;\r
448                 case MLX4_OPCODE_RDMA_WRITE:\r
449                         wc->wc_type    = IB_WC_RDMA_WRITE;\r
450                         break;\r
451                 case MLX4_OPCODE_SEND_IMM:\r
452                         wc->recv.ud.recv_opt |= IB_RECV_OPT_IMMEDIATE;\r
453                 case MLX4_OPCODE_SEND:\r
454                         wc->wc_type    = IB_WC_SEND;\r
455                         break;\r
456                 case MLX4_OPCODE_RDMA_READ:\r
457                         wc->wc_type    = IB_WC_RDMA_READ;\r
458                         wc->length  = be32_to_cpu(cqe->byte_cnt);\r
459                         break;\r
460                 case MLX4_OPCODE_ATOMIC_CS:\r
461                         wc->wc_type    = IB_WC_COMPARE_SWAP;\r
462                         wc->length  = 8;\r
463                         break;\r
464                 case MLX4_OPCODE_ATOMIC_FA:\r
465                         wc->wc_type    = IB_WC_FETCH_ADD;\r
466                         wc->length  = 8;\r
467                         break;\r
468                 case MLX4_OPCODE_BIND_MW:\r
469                         wc->wc_type    = IB_WC_MW_BIND;\r
470                         break;\r
471                 case MLX4_OPCODE_LSO:\r
472                         wc->wc_type    = IB_WC_LSO;\r
473                         break;\r
474                 default:\r
475                         wc->wc_type       = IB_WC_SEND;\r
476                         break;\r
477                 }\r
478         } else {\r
479                 wc->length = be32_to_cpu(cqe->byte_cnt);\r
480 \r
481                 switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {\r
482                 case MLX4_RECV_OPCODE_RDMA_WRITE_IMM:\r
483                         wc->wc_type   = IB_WC_RECV_RDMA_WRITE;\r
484                         wc->recv.ud.recv_opt = IB_RECV_OPT_IMMEDIATE;\r
485                         wc->recv.ud.immediate_data = cqe->immed_rss_invalid;\r
486                         break;\r
487                 case MLX4_RECV_OPCODE_SEND:\r
488                         wc->wc_type   = IB_WC_RECV;\r
489                         wc->recv.ud.recv_opt = 0;\r
490                         break;\r
491                 case MLX4_RECV_OPCODE_SEND_IMM:\r
492                         wc->wc_type   = IB_WC_RECV;\r
493                         wc->recv.ud.recv_opt = IB_RECV_OPT_IMMEDIATE;\r
494                         wc->recv.ud.immediate_data = cqe->immed_rss_invalid;\r
495                         break;\r
496                 default:\r
497                         wc->recv.ud.recv_opt = 0;\r
498                         wc->wc_type = IB_WC_RECV;\r
499                         break;\r
500                 }\r
501 \r
502                 wc->recv.ud.remote_lid  = cqe->rlid;\r
503                 wc->recv.ud.remote_sl           = cqe->sl >> 4;\r
504                 wc->recv.ud.remote_qp   = cqe->g_mlpath_rqpn & 0xffffff00;\r
505                 wc->recv.ud.path_bits           = (u8)(cqe->g_mlpath_rqpn & 0x7f);\r
506                 wc->recv.ud.recv_opt            |= cqe->g_mlpath_rqpn & 0x080 ? IB_RECV_OPT_GRH_VALID : 0;\r
507                 wc->recv.ud.pkey_index  = (u16)(be32_to_cpu(cqe->immed_rss_invalid)  & 0x7f);\r
508                 wc->recv.ud.recv_opt |= mlx4_ib_ipoib_csum_ok(cqe->ipoib_status,cqe->checksum);\r
509         }\r
510         if (!is_send && cqe->rlid == 0){\r
511                 MLX4_PRINT(TRACE_LEVEL_INFORMATION,MLX4_DBG_CQ,("found rlid == 0 \n "));\r
512                 wc->recv.ud.recv_opt         |= IB_RECV_OPT_FORWARD;\r
513         }\r
514 \r
515         if (unlikely(is_error))\r
516                 mlx4_ib_handle_error_cqe((struct mlx4_err_cqe *) cqe, wc);\r
517         else\r
518                 wc->status = IB_WCS_SUCCESS;\r
519 \r
520         return 0;\r
521 }\r
522 \r
523 int mlx4_ib_poll_cq(\r
524         IN              struct ib_cq *ibcq, \r
525         IN      OUT                     ib_wc_t** const                         pp_free_wclist,\r
526                 OUT                     ib_wc_t** const                         pp_done_wclist )\r
527 {\r
528         struct mlx4_ib_cq *cq = to_mcq(ibcq);\r
529         struct mlx4_ib_qp *cur_qp = NULL;\r
530         unsigned long flags;\r
531         int err = 0;\r
532         int npolled = 0;\r
533         ib_wc_t         *wc_p, **next_pp;\r
534 \r
535         spin_lock_irqsave(&cq->lock, &flags);\r
536 \r
537         // loop through CQ\r
538         next_pp = pp_done_wclist;\r
539         wc_p = *pp_free_wclist;\r
540         while( wc_p ) {\r
541                 // poll one CQE\r
542                 err = mlx4_ib_poll_one(cq, &cur_qp, wc_p);\r
543                 if (err)\r
544                         break;\r
545 \r
546                 // prepare for the next loop\r
547                 *next_pp = wc_p;\r
548                 next_pp = &wc_p->p_next;\r
549                 wc_p = wc_p->p_next;\r
550                 ++npolled;\r
551         }\r
552 \r
553         // prepare the results\r
554         *pp_free_wclist = wc_p;         /* Set the head of the free list. */\r
555         *next_pp = NULL;                                                /* Clear the tail of the done list. */\r
556 \r
557         // update consumer index\r
558         if (npolled)\r
559                 mlx4_cq_set_ci(&cq->mcq);\r
560 \r
561         spin_unlock_irqrestore(&cq->lock, flags);\r
562         return (err == 0 || err == -EAGAIN)? npolled : err;\r
563 }\r
564 \r
565 int mlx4_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)\r
566 {\r
567         if (!mlx4_is_barred(ibcq->device->dma_device))\r
568                 mlx4_cq_arm(&to_mcq(ibcq)->mcq,\r
569                     (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?\r
570                     MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT,\r
571                     to_mdev(ibcq->device)->uar_map,\r
572                     MLX4_GET_DOORBELL_LOCK(&to_mdev(ibcq->device)->uar_lock));\r
573 \r
574         return 0;\r
575 }\r
576 \r
577 void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)\r
578 {\r
579         u32 prod_index;\r
580         int nfreed = 0;\r
581         struct mlx4_cqe *cqe, *dest;\r
582         u8 owner_bit;\r
583 \r
584         /*\r
585          * First we need to find the current producer index, so we\r
586          * know where to start cleaning from.  It doesn't matter if HW\r
587          * adds new entries after this loop -- the QP we're worried\r
588          * about is already in RESET, so the new entries won't come\r
589          * from our QP and therefore don't need to be checked.\r
590          */\r
591         for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); ++prod_index)\r
592                 if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe)\r
593                         break;\r
594 \r
595         /*\r
596          * Now sweep backwards through the CQ, removing CQ entries\r
597          * that match our QP by copying older entries on top of them.\r
598          */\r
599         while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) {\r
600                 cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);\r
601                 if ((be32_to_cpu(cqe->my_qpn) & 0xffffff) == qpn) {\r
602                         if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK))\r
603                                 mlx4_ib_free_srq_wqe(srq, be16_to_cpu(cqe->wqe_index));\r
604                         ++nfreed;\r
605                 } else if (nfreed) {\r
606                         dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe);\r
607                         owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK;\r
608                         memcpy(dest, cqe, sizeof *cqe);\r
609                         dest->owner_sr_opcode = owner_bit |\r
610                                 (dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);\r
611                 }\r
612         }\r
613 \r
614         if (nfreed) {\r
615                 cq->mcq.cons_index += nfreed;\r
616                 /*\r
617                  * Make sure update of buffer contents is done before\r
618                  * updating consumer index.\r
619                  */\r
620                 wmb();\r
621                 mlx4_cq_set_ci(&cq->mcq);\r
622         }\r
623 }\r
624 \r
625 void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)\r
626 {\r
627         spin_lock_irq(&cq->lock);\r
628         __mlx4_ib_cq_clean(cq, qpn, srq);\r
629         spin_unlock_irq(&cq->lock);\r
630 }\r