hw/mthca/kernel/mthca_qp.c

   1 /*\r
   2  * Copyright (c) 2004 Topspin Communications.  All rights reserved.\r
   3  * Copyright (c) 2005 Cisco Systems. All rights reserved.\r
   4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.\r
   5  * Copyright (c) 2004 Voltaire, Inc. All rights reserved. \r
   6  *\r
   7  * This software is available to you under a choice of one of two\r
   8  * licenses.  You may choose to be licensed under the terms of the GNU\r
   9  * General Public License (GPL) Version 2, available from the file\r
  10  * COPYING in the main directory of this source tree, or the\r
  11  * OpenIB.org BSD license below:\r
  12  *\r
  13  *     Redistribution and use in source and binary forms, with or\r
  14  *     without modification, are permitted provided that the following\r
  15  *     conditions are met:\r
  16  *\r
  17  *      - Redistributions of source code must retain the above\r
  18  *        copyright notice, this list of conditions and the following\r
  19  *        disclaimer.\r
  20  *\r
  21  *      - Redistributions in binary form must reproduce the above\r
  22  *        copyright notice, this list of conditions and the following\r
  23  *        disclaimer in the documentation and/or other materials\r
  24  *        provided with the distribution.\r
  25  *\r
  26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
  27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
  28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\r
  29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
  30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
  31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
  32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
  33  * SOFTWARE.\r
  34  *\r
  35  * $Id$\r
  36  */\r
  37 \r
  38 #include <ib_verbs.h>\r
  39 #include <ib_cache.h>\r
  40 #include <ib_pack.h>\r
  41 \r
  42 #include "mthca_dev.h"\r
  43 #if defined(EVENT_TRACING)\r
  44 #ifdef offsetof\r
  45 #undef offsetof\r
  46 #endif\r
  47 #include "mthca_qp.tmh"\r
  48 #endif\r
  49 #include "mthca_cmd.h"\r
  50 #include "mthca_memfree.h"\r
  51 #include "mthca_wqe.h"\r
  52 \r
  53 \r
  54 enum {\r
  55         MTHCA_MAX_DIRECT_QP_SIZE = 4 * PAGE_SIZE,\r
  56         MTHCA_ACK_REQ_FREQ       = 10,\r
  57         MTHCA_FLIGHT_LIMIT       = 9,\r
  58         MTHCA_UD_HEADER_SIZE     = 72, /* largest UD header possible */\r
  59         MTHCA_INLINE_HEADER_SIZE = 4,  /* data segment overhead for inline */\r
  60         MTHCA_INLINE_CHUNK_SIZE  = 16  /* inline data segment chunk */\r
  61 };\r
  62 \r
  63 enum {\r
  64         MTHCA_QP_STATE_RST  = 0,\r
  65         MTHCA_QP_STATE_INIT = 1,\r
  66         MTHCA_QP_STATE_RTR  = 2,\r
  67         MTHCA_QP_STATE_RTS  = 3,\r
  68         MTHCA_QP_STATE_SQE  = 4,\r
  69         MTHCA_QP_STATE_SQD  = 5,\r
  70         MTHCA_QP_STATE_ERR  = 6,\r
  71         MTHCA_QP_STATE_DRAINING = 7\r
  72 };\r
  73 \r
  74 enum {\r
  75         MTHCA_QP_ST_RC  = 0x0,\r
  76         MTHCA_QP_ST_UC  = 0x1,\r
  77         MTHCA_QP_ST_RD  = 0x2,\r
  78         MTHCA_QP_ST_UD  = 0x3,\r
  79         MTHCA_QP_ST_MLX = 0x7\r
  80 };\r
  81 \r
  82 enum {\r
  83         MTHCA_QP_PM_MIGRATED = 0x3,\r
  84         MTHCA_QP_PM_ARMED    = 0x0,\r
  85         MTHCA_QP_PM_REARM    = 0x1\r
  86 };\r
  87 \r
  88 enum {\r
  89         /* qp_context flags */\r
  90         MTHCA_QP_BIT_DE  = 1 <<  8,\r
  91         /* params1 */\r
  92         MTHCA_QP_BIT_SRE = 1 << 15,\r
  93         MTHCA_QP_BIT_SWE = 1 << 14,\r
  94         MTHCA_QP_BIT_SAE = 1 << 13,\r
  95         MTHCA_QP_BIT_SIC = 1 <<  4,\r
  96         MTHCA_QP_BIT_SSC = 1 <<  3,\r
  97         /* params2 */\r
  98         MTHCA_QP_BIT_RRE = 1 << 15,\r
  99         MTHCA_QP_BIT_RWE = 1 << 14,\r
 100         MTHCA_QP_BIT_RAE = 1 << 13,\r
 101         MTHCA_QP_BIT_RIC = 1 <<  4,\r
 102         MTHCA_QP_BIT_RSC = 1 <<  3\r
 103 };\r
 104 \r
 105 #pragma pack(push,1)\r
 106 struct mthca_qp_path {\r
 107         __be32 port_pkey;\r
 108         u8     rnr_retry;\r
 109         u8     g_mylmc;\r
 110         __be16 rlid;\r
 111         u8     ackto;\r
 112         u8     mgid_index;\r
 113         u8     static_rate;\r
 114         u8     hop_limit;\r
 115         __be32 sl_tclass_flowlabel;\r
 116         u8     rgid[16];\r
 117 } ;\r
 118 \r
 119 struct mthca_qp_context {\r
 120         __be32 flags;\r
 121         __be32 tavor_sched_queue; /* Reserved on Arbel */\r
 122         u8     mtu_msgmax;\r
 123         u8     rq_size_stride;  /* Reserved on Tavor */\r
 124         u8     sq_size_stride;  /* Reserved on Tavor */\r
 125         u8     rlkey_arbel_sched_queue; /* Reserved on Tavor */\r
 126         __be32 usr_page;\r
 127         __be32 local_qpn;\r
 128         __be32 remote_qpn;\r
 129         u32    reserved1[2];\r
 130         struct mthca_qp_path pri_path;\r
 131         struct mthca_qp_path alt_path;\r
 132         __be32 rdd;\r
 133         __be32 pd;\r
 134         __be32 wqe_base;\r
 135         __be32 wqe_lkey;\r
 136         __be32 params1;\r
 137         __be32 reserved2;\r
 138         __be32 next_send_psn;\r
 139         __be32 cqn_snd;\r
 140         __be32 snd_wqe_base_l;  /* Next send WQE on Tavor */\r
 141         __be32 snd_db_index;    /* (debugging only entries) */\r
 142         __be32 last_acked_psn;\r
 143         __be32 ssn;\r
 144         __be32 params2;\r
 145         __be32 rnr_nextrecvpsn;\r
 146         __be32 ra_buff_indx;\r
 147         __be32 cqn_rcv;\r
 148         __be32 rcv_wqe_base_l;  /* Next recv WQE on Tavor */\r
 149         __be32 rcv_db_index;    /* (debugging only entries) */\r
 150         __be32 qkey;\r
 151         __be32 srqn;\r
 152         __be32 rmsn;\r
 153         __be16 rq_wqe_counter;  /* reserved on Tavor */\r
 154         __be16 sq_wqe_counter;  /* reserved on Tavor */\r
 155         u32    reserved3[18];\r
 156 } ;\r
 157 \r
 158 struct mthca_qp_param {\r
 159         __be32 opt_param_mask;\r
 160         u32    reserved1;\r
 161         struct mthca_qp_context context;\r
 162         u32    reserved2[62];\r
 163 } ;\r
 164 #pragma pack(pop)\r
 165 \r
 166 enum {\r
 167         MTHCA_QP_OPTPAR_ALT_ADDR_PATH     = 1 << 0,\r
 168         MTHCA_QP_OPTPAR_RRE               = 1 << 1,\r
 169         MTHCA_QP_OPTPAR_RAE               = 1 << 2,\r
 170         MTHCA_QP_OPTPAR_RWE               = 1 << 3,\r
 171         MTHCA_QP_OPTPAR_PKEY_INDEX        = 1 << 4,\r
 172         MTHCA_QP_OPTPAR_Q_KEY             = 1 << 5,\r
 173         MTHCA_QP_OPTPAR_RNR_TIMEOUT       = 1 << 6,\r
 174         MTHCA_QP_OPTPAR_PRIMARY_ADDR_PATH = 1 << 7,\r
 175         MTHCA_QP_OPTPAR_SRA_MAX           = 1 << 8,\r
 176         MTHCA_QP_OPTPAR_RRA_MAX           = 1 << 9,\r
 177         MTHCA_QP_OPTPAR_PM_STATE          = 1 << 10,\r
 178         MTHCA_QP_OPTPAR_PORT_NUM          = 1 << 11,\r
 179         MTHCA_QP_OPTPAR_RETRY_COUNT       = 1 << 12,\r
 180         MTHCA_QP_OPTPAR_ALT_RNR_RETRY     = 1 << 13,\r
 181         MTHCA_QP_OPTPAR_ACK_TIMEOUT       = 1 << 14,\r
 182         MTHCA_QP_OPTPAR_RNR_RETRY         = 1 << 15,\r
 183         MTHCA_QP_OPTPAR_SCHED_QUEUE       = 1 << 16\r
 184 };\r
 185 \r
 186 static const u8 mthca_opcode[] = {\r
 187         MTHCA_OPCODE_RDMA_WRITE,\r
 188         MTHCA_OPCODE_RDMA_WRITE_IMM,\r
 189         MTHCA_OPCODE_SEND,\r
 190         MTHCA_OPCODE_SEND_IMM,\r
 191         MTHCA_OPCODE_RDMA_READ,\r
 192         MTHCA_OPCODE_ATOMIC_CS,\r
 193         MTHCA_OPCODE_ATOMIC_FA\r
 194 };\r
 195 \r
 196 \r
 197 enum { RC, UC, UD, RD, RDEE, MLX, NUM_TRANS };\r
 198 \r
 199 static struct _state_table {\r
 200         int trans;\r
 201         u32 req_param[NUM_TRANS];\r
 202         u32 opt_param[NUM_TRANS];\r
 203 } state_table[IBQPS_ERR + 1][IBQPS_ERR + 1]= {0};\r
 204 \r
 205 static void fill_state_table()\r
 206 {\r
 207         struct _state_table *t;\r
 208         RtlZeroMemory( state_table, sizeof(state_table) );\r
 209 \r
 210         /* IBQPS_RESET */       \r
 211         t = &state_table[IBQPS_RESET][0];\r
 212         t[IBQPS_RESET].trans                                    = MTHCA_TRANS_ANY2RST;\r
 213         t[IBQPS_ERR].trans                                              = MTHCA_TRANS_ANY2ERR;\r
 214 \r
 215         t[IBQPS_INIT].trans                                             = MTHCA_TRANS_RST2INIT;\r
 216         t[IBQPS_INIT].req_param[UD]     = IB_QP_PKEY_INDEX |IB_QP_PORT |IB_QP_QKEY;\r
 217         t[IBQPS_INIT].req_param[UC]     = IB_QP_PKEY_INDEX |IB_QP_PORT |IB_QP_ACCESS_FLAGS;\r
 218         t[IBQPS_INIT].req_param[RC]     = IB_QP_PKEY_INDEX |IB_QP_PORT |IB_QP_ACCESS_FLAGS;\r
 219         t[IBQPS_INIT].req_param[MLX]    = IB_QP_PKEY_INDEX |IB_QP_QKEY;\r
 220         t[IBQPS_INIT].opt_param[MLX]    = IB_QP_PORT;\r
 221 \r
 222         /* IBQPS_INIT */        \r
 223         t = &state_table[IBQPS_INIT][0];\r
 224         t[IBQPS_RESET].trans                                    = MTHCA_TRANS_ANY2RST;\r
 225         t[IBQPS_ERR].trans                                              = MTHCA_TRANS_ANY2ERR;\r
 226 \r
 227         t[IBQPS_INIT].trans                                             = MTHCA_TRANS_INIT2INIT;\r
 228         t[IBQPS_INIT].opt_param[UD]     = IB_QP_PKEY_INDEX |IB_QP_PORT |IB_QP_QKEY;\r
 229         t[IBQPS_INIT].opt_param[UC]     = IB_QP_PKEY_INDEX |IB_QP_PORT |IB_QP_ACCESS_FLAGS;\r
 230         t[IBQPS_INIT].opt_param[RC]     = IB_QP_PKEY_INDEX |IB_QP_PORT |IB_QP_ACCESS_FLAGS;\r
 231         t[IBQPS_INIT].opt_param[MLX]    = IB_QP_PKEY_INDEX |IB_QP_QKEY;\r
 232 \r
 233         t[IBQPS_RTR].trans                                              = MTHCA_TRANS_INIT2RTR;\r
 234         t[IBQPS_RTR].req_param[UC]      = \r
 235                 IB_QP_AV |IB_QP_PATH_MTU |IB_QP_DEST_QPN |IB_QP_RQ_PSN;\r
 236         t[IBQPS_RTR].req_param[RC]      = \r
 237                 IB_QP_AV |IB_QP_PATH_MTU |IB_QP_DEST_QPN |IB_QP_RQ_PSN |IB_QP_MAX_DEST_RD_ATOMIC |IB_QP_MIN_RNR_TIMER;\r
 238         t[IBQPS_RTR].opt_param[UD]      = IB_QP_PKEY_INDEX |IB_QP_QKEY;\r
 239         t[IBQPS_RTR].opt_param[UC]      = IB_QP_PKEY_INDEX |IB_QP_ALT_PATH |IB_QP_ACCESS_FLAGS;\r
 240         t[IBQPS_RTR].opt_param[RC]      = IB_QP_PKEY_INDEX |IB_QP_ALT_PATH |IB_QP_ACCESS_FLAGS;\r
 241         t[IBQPS_RTR].opt_param[MLX]     = IB_QP_PKEY_INDEX |IB_QP_QKEY;\r
 242 \r
 243 /* IBQPS_RTR */ \r
 244         t = &state_table[IBQPS_RTR][0];\r
 245         t[IBQPS_RESET].trans                                    = MTHCA_TRANS_ANY2RST;\r
 246         t[IBQPS_ERR].trans                                              = MTHCA_TRANS_ANY2ERR;\r
 247 \r
 248         t[IBQPS_RTS].trans                                              = MTHCA_TRANS_RTR2RTS;\r
 249         t[IBQPS_RTS].req_param[UD]      = IB_QP_SQ_PSN;\r
 250         t[IBQPS_RTS].req_param[UC]      = IB_QP_SQ_PSN;\r
 251         t[IBQPS_RTS].req_param[RC]      = \r
 252                 IB_QP_TIMEOUT |IB_QP_RETRY_CNT |IB_QP_RNR_RETRY |IB_QP_SQ_PSN |IB_QP_MAX_QP_RD_ATOMIC;\r
 253         t[IBQPS_RTS].req_param[MLX]     = IB_QP_SQ_PSN;\r
 254         t[IBQPS_RTS].opt_param[UD]      = IB_QP_CUR_STATE |IB_QP_QKEY;\r
 255         t[IBQPS_RTS].opt_param[UC]      = \r
 256                 IB_QP_CUR_STATE |IB_QP_ALT_PATH |IB_QP_ACCESS_FLAGS |IB_QP_PATH_MIG_STATE;\r
 257         t[IBQPS_RTS].opt_param[RC]      =       IB_QP_CUR_STATE |IB_QP_ALT_PATH |\r
 258                 IB_QP_ACCESS_FLAGS |IB_QP_MIN_RNR_TIMER |IB_QP_PATH_MIG_STATE;\r
 259         t[IBQPS_RTS].opt_param[MLX]     = IB_QP_CUR_STATE |IB_QP_QKEY;\r
 260 \r
 261         /* IBQPS_RTS */ \r
 262         t = &state_table[IBQPS_RTS][0];\r
 263         t[IBQPS_RESET].trans                                    = MTHCA_TRANS_ANY2RST;\r
 264         t[IBQPS_ERR].trans                                              = MTHCA_TRANS_ANY2ERR;\r
 265 \r
 266         t[IBQPS_RTS].trans                                              = MTHCA_TRANS_RTS2RTS;\r
 267         t[IBQPS_RTS].opt_param[UD]      = IB_QP_CUR_STATE |IB_QP_QKEY;\r
 268         t[IBQPS_RTS].opt_param[UC]      = IB_QP_ACCESS_FLAGS |IB_QP_ALT_PATH |IB_QP_PATH_MIG_STATE;\r
 269         t[IBQPS_RTS].opt_param[RC]      =       IB_QP_ACCESS_FLAGS |\r
 270                 IB_QP_ALT_PATH |IB_QP_PATH_MIG_STATE |IB_QP_MIN_RNR_TIMER;\r
 271         t[IBQPS_RTS].opt_param[MLX]     = IB_QP_CUR_STATE |IB_QP_QKEY;\r
 272 \r
 273         t[IBQPS_SQD].trans                                              = MTHCA_TRANS_RTS2SQD;\r
 274         t[IBQPS_SQD].opt_param[UD]      = IB_QP_EN_SQD_ASYNC_NOTIFY;\r
 275         t[IBQPS_SQD].opt_param[UC]      = IB_QP_EN_SQD_ASYNC_NOTIFY;\r
 276         t[IBQPS_SQD].opt_param[RC]      =       IB_QP_EN_SQD_ASYNC_NOTIFY;\r
 277         t[IBQPS_SQD].opt_param[MLX]     = IB_QP_EN_SQD_ASYNC_NOTIFY;\r
 278 \r
 279         /* IBQPS_SQD */ \r
 280         t = &state_table[IBQPS_SQD][0];\r
 281         t[IBQPS_RESET].trans                                    = MTHCA_TRANS_ANY2RST;\r
 282         t[IBQPS_ERR].trans                                              = MTHCA_TRANS_ANY2ERR;\r
 283 \r
 284         t[IBQPS_RTS].trans                                              = MTHCA_TRANS_SQD2RTS;\r
 285         t[IBQPS_RTS].opt_param[UD]      = IB_QP_CUR_STATE |IB_QP_QKEY;\r
 286         t[IBQPS_RTS].opt_param[UC]      = IB_QP_CUR_STATE |\r
 287                 IB_QP_ALT_PATH |IB_QP_ACCESS_FLAGS |IB_QP_PATH_MIG_STATE;\r
 288         t[IBQPS_RTS].opt_param[RC]      =       IB_QP_CUR_STATE |IB_QP_ALT_PATH |\r
 289                 IB_QP_ACCESS_FLAGS |IB_QP_MIN_RNR_TIMER |IB_QP_PATH_MIG_STATE;\r
 290         t[IBQPS_RTS].opt_param[MLX]     = IB_QP_CUR_STATE |IB_QP_QKEY;\r
 291 \r
 292         t[IBQPS_SQD].trans                                              = MTHCA_TRANS_SQD2SQD;\r
 293         t[IBQPS_SQD].opt_param[UD]      = IB_QP_PKEY_INDEX |IB_QP_QKEY;\r
 294         t[IBQPS_SQD].opt_param[UC]      = IB_QP_AV |    IB_QP_CUR_STATE |\r
 295                 IB_QP_ALT_PATH |IB_QP_ACCESS_FLAGS |IB_QP_PKEY_INDEX |IB_QP_PATH_MIG_STATE;\r
 296         t[IBQPS_SQD].opt_param[RC]      =       IB_QP_AV |IB_QP_TIMEOUT |IB_QP_RETRY_CNT |IB_QP_RNR_RETRY |\r
 297                 IB_QP_MAX_QP_RD_ATOMIC |IB_QP_MAX_DEST_RD_ATOMIC |IB_QP_CUR_STATE |IB_QP_ALT_PATH |\r
 298                 IB_QP_ACCESS_FLAGS |IB_QP_PKEY_INDEX |IB_QP_MIN_RNR_TIMER |IB_QP_PATH_MIG_STATE;\r
 299         t[IBQPS_SQD].opt_param[MLX]     = IB_QP_PKEY_INDEX |IB_QP_QKEY;\r
 300 \r
 301         /* IBQPS_SQE */ \r
 302         t = &state_table[IBQPS_SQE][0];\r
 303         t[IBQPS_RESET].trans                                    = MTHCA_TRANS_ANY2RST;\r
 304         t[IBQPS_ERR].trans                                              = MTHCA_TRANS_ANY2ERR;\r
 305 \r
 306         t[IBQPS_RTS].trans                                              = MTHCA_TRANS_SQERR2RTS;\r
 307         t[IBQPS_RTS].opt_param[UD]      = IB_QP_CUR_STATE |IB_QP_QKEY;\r
 308         t[IBQPS_RTS].opt_param[UC]      = IB_QP_CUR_STATE | IB_QP_ACCESS_FLAGS;\r
 309 //      t[IBQPS_RTS].opt_param[RC]      =       IB_QP_CUR_STATE |IB_QP_MIN_RNR_TIMER;\r
 310         t[IBQPS_RTS].opt_param[MLX]     = IB_QP_CUR_STATE |IB_QP_QKEY;\r
 311 \r
 312         /* IBQPS_ERR */ \r
 313         t = &state_table[IBQPS_ERR][0];\r
 314         t[IBQPS_RESET].trans                                    = MTHCA_TRANS_ANY2RST;\r
 315         t[IBQPS_ERR].trans                                              = MTHCA_TRANS_ANY2ERR;\r
 316 \r
 317 };\r
 318 \r
 319 \r
 320 static int is_sqp(struct mthca_dev *dev, struct mthca_qp *qp)\r
 321 {\r
 322         return qp->qpn >= (u32)dev->qp_table.sqp_start &&\r
 323                 qp->qpn <= (u32)dev->qp_table.sqp_start + 3;\r
 324 }\r
 325 \r
 326 static int is_qp0(struct mthca_dev *dev, struct mthca_qp *qp)\r
 327 {\r
 328         return qp->qpn >= (u32)dev->qp_table.sqp_start &&\r
 329                 qp->qpn <= (u32)(dev->qp_table.sqp_start + 1);\r
 330 }\r
 331 \r
 332 \r
 333 static void dump_wqe(u32 print_lvl, u32 *wqe_ptr , struct mthca_qp *qp_ptr)\r
 334 {\r
 335         __be32 *wqe = wqe_ptr;\r
 336 \r
 337         UNUSED_PARAM_WOWPP(qp_ptr);\r
 338         UNUSED_PARAM_WOWPP(print_lvl);\r
 339 \r
 340         (void) wqe;     /* avoid warning if mthca_dbg compiled away... */\r
 341         HCA_PRINT(print_lvl,HCA_DBG_QP,("WQE contents  QPN 0x%06x \n",qp_ptr->qpn));\r
 342         HCA_PRINT(print_lvl,HCA_DBG_QP,("WQE contents [%02x] %08x %08x %08x %08x \n",0\r
 343                 , cl_ntoh32(wqe[0]), cl_ntoh32(wqe[1]), cl_ntoh32(wqe[2]), cl_ntoh32(wqe[3])));\r
 344         HCA_PRINT(print_lvl,HCA_DBG_QP,("WQE contents [%02x] %08x %08x %08x %08x \n",4\r
 345                 , cl_ntoh32(wqe[4]), cl_ntoh32(wqe[5]), cl_ntoh32(wqe[6]), cl_ntoh32(wqe[7])));\r
 346         HCA_PRINT(print_lvl,HCA_DBG_QP,("WQE contents [%02x] %08x %08x %08x %08x \n",8\r
 347                 , cl_ntoh32(wqe[8]), cl_ntoh32(wqe[9]), cl_ntoh32(wqe[10]), cl_ntoh32(wqe[11])));\r
 348         HCA_PRINT(print_lvl,HCA_DBG_QP,("WQE contents [%02x] %08x %08x %08x %08x \n",12\r
 349                 , cl_ntoh32(wqe[12]), cl_ntoh32(wqe[13]), cl_ntoh32(wqe[14]), cl_ntoh32(wqe[15])));\r
 350 \r
 351 }\r
 352 \r
 353 \r
 354 static void *get_recv_wqe(struct mthca_qp *qp, int n)\r
 355 {\r
 356         if (qp->is_direct)\r
 357                 return (u8*)qp->queue.direct.page + (n << qp->rq.wqe_shift);\r
 358         else\r
 359                 return (u8*)qp->queue.page_list[(n << qp->rq.wqe_shift) >> PAGE_SHIFT].page +\r
 360                         ((n << qp->rq.wqe_shift) & (PAGE_SIZE - 1));\r
 361 }\r
 362 \r
 363 static void *get_send_wqe(struct mthca_qp *qp, int n)\r
 364 {\r
 365         if (qp->is_direct)\r
 366                 return (u8*)qp->queue.direct.page + qp->send_wqe_offset +\r
 367                         (n << qp->sq.wqe_shift);\r
 368         else\r
 369                 return (u8*)qp->queue.page_list[(qp->send_wqe_offset +\r
 370                                             (n << qp->sq.wqe_shift)) >>\r
 371                                            PAGE_SHIFT].page +\r
 372                         ((qp->send_wqe_offset + (n << qp->sq.wqe_shift)) &\r
 373                          (PAGE_SIZE - 1));\r
 374 }\r
 375 \r
 376 static void mthca_wq_init(struct mthca_wq *wq)\r
 377 {       \r
 378         spin_lock_init(&wq->lock);      \r
 379         wq->next_ind  = 0;      \r
 380         wq->last_comp = wq->max - 1;    \r
 381         wq->head      = 0;      \r
 382         wq->tail      = 0;      \r
 383 }\r
 384 \r
 385 void mthca_qp_event(struct mthca_dev *dev, u32 qpn,\r
 386                     enum ib_event_type event_type, u8 vendor_code)\r
 387 {\r
 388         struct mthca_qp *qp;\r
 389         ib_event_rec_t event;\r
 390         SPIN_LOCK_PREP(lh);\r
 391 \r
 392         spin_lock(&dev->qp_table.lock, &lh);\r
 393         qp = mthca_array_get(&dev->qp_table.qp, qpn & (dev->limits.num_qps - 1));\r
 394         if (qp)\r
 395                 atomic_inc(&qp->refcount);\r
 396         spin_unlock(&lh);\r
 397 \r
 398         if (!qp) {\r
 399                 HCA_PRINT(TRACE_LEVEL_WARNING,HCA_DBG_QP,("QP %06x Async event for bogus \n", qpn));\r
 400                 return;\r
 401         }\r
 402 \r
 403         event.type = event_type;\r
 404         event.context = qp->ibqp.qp_context;\r
 405         event.vendor_specific = vendor_code;\r
 406         HCA_PRINT(TRACE_LEVEL_WARNING,HCA_DBG_QP,("QP %06x Async event  event_type 0x%x vendor_code 0x%x\n",\r
 407                 qpn,event_type,vendor_code));\r
 408         qp->ibqp.event_handler(&event);\r
 409 \r
 410         if (atomic_dec_and_test(&qp->refcount))\r
 411                 wake_up(&qp->wait);\r
 412 }\r
 413 \r
 414 static int to_mthca_state(enum ib_qp_state ib_state)\r
 415 {\r
 416         switch (ib_state) {\r
 417         case IBQPS_RESET: return MTHCA_QP_STATE_RST;\r
 418         case IBQPS_INIT:  return MTHCA_QP_STATE_INIT;\r
 419         case IBQPS_RTR:   return MTHCA_QP_STATE_RTR;\r
 420         case IBQPS_RTS:   return MTHCA_QP_STATE_RTS;\r
 421         case IBQPS_SQD:   return MTHCA_QP_STATE_SQD;\r
 422         case IBQPS_SQE:   return MTHCA_QP_STATE_SQE;\r
 423         case IBQPS_ERR:   return MTHCA_QP_STATE_ERR;\r
 424         default:                return -1;\r
 425         }\r
 426 }\r
 427 \r
 428 static int to_mthca_st(int transport)\r
 429 {\r
 430         switch (transport) {\r
 431         case RC:  return MTHCA_QP_ST_RC;\r
 432         case UC:  return MTHCA_QP_ST_UC;\r
 433         case UD:  return MTHCA_QP_ST_UD;\r
 434         case RD:  return MTHCA_QP_ST_RD;\r
 435         case MLX: return MTHCA_QP_ST_MLX;\r
 436         default:  return -1;\r
 437         }\r
 438 }\r
 439 \r
 440 static inline enum ib_qp_state to_ib_qp_state(int mthca_state)\r
 441 {\r
 442         switch (mthca_state) {\r
 443         case MTHCA_QP_STATE_RST: return IBQPS_RESET;\r
 444         case MTHCA_QP_STATE_INIT:  return IBQPS_INIT;\r
 445         case MTHCA_QP_STATE_RTR:   return IBQPS_RTR;\r
 446         case MTHCA_QP_STATE_RTS:   return IBQPS_RTS;\r
 447         case MTHCA_QP_STATE_SQD:   return IBQPS_SQD;\r
 448         case MTHCA_QP_STATE_DRAINING:   return IBQPS_SQD;\r
 449         case MTHCA_QP_STATE_SQE:   return IBQPS_SQE;\r
 450         case MTHCA_QP_STATE_ERR:   return IBQPS_ERR;\r
 451         default:                return -1;\r
 452         }\r
 453 }\r
 454 \r
 455 static inline enum ib_mig_state to_ib_mig_state(int mthca_mig_state)\r
 456 {\r
 457         switch (mthca_mig_state) {\r
 458         case 0:  return IB_MIG_ARMED;\r
 459         case 1:  return IB_MIG_REARM;\r
 460         case 3:  return IB_MIG_MIGRATED;\r
 461         default: return -1;\r
 462         }\r
 463 }\r
 464 \r
 465 static int to_ib_qp_access_flags(int mthca_flags)\r
 466 {\r
 467         int ib_flags = 0;\r
 468 \r
 469         if (mthca_flags & MTHCA_QP_BIT_RRE)\r
 470                 ib_flags |= MTHCA_ACCESS_REMOTE_READ;\r
 471         if (mthca_flags & MTHCA_QP_BIT_RWE)\r
 472                 ib_flags |= MTHCA_ACCESS_REMOTE_WRITE;\r
 473         if (mthca_flags & MTHCA_QP_BIT_RAE)\r
 474                 ib_flags |= MTHCA_ACCESS_REMOTE_ATOMIC;\r
 475 \r
 476         return ib_flags;\r
 477 }\r
 478 \r
 479 static void to_ib_ah_attr(struct mthca_dev *dev, struct ib_ah_attr *ib_ah_attr,\r
 480                                 struct mthca_qp_path *path)\r
 481 {\r
 482         memset(ib_ah_attr, 0, sizeof *ib_ah_attr);\r
 483         ib_ah_attr->port_num      = (u8)((cl_ntoh32(path->port_pkey) >> 24) & 0x3);\r
 484 \r
 485         if (ib_ah_attr->port_num == 0 || ib_ah_attr->port_num > dev->limits.num_ports)\r
 486                 return;\r
 487 \r
 488         ib_ah_attr->dlid          = cl_ntoh16(path->rlid);\r
 489         ib_ah_attr->sl            = (u8)(cl_ntoh32(path->sl_tclass_flowlabel) >> 28);\r
 490         ib_ah_attr->src_path_bits = path->g_mylmc & 0x7f;\r
 491         //TODO: work around: set always full speed      - really, it's much more complicate\r
 492         ib_ah_attr->static_rate   = 0;\r
 493         ib_ah_attr->ah_flags      = (path->g_mylmc & (1 << 7)) ? IB_AH_GRH : 0;\r
 494         if (ib_ah_attr->ah_flags) {\r
 495                 ib_ah_attr->grh.sgid_index = (u8)(path->mgid_index & (dev->limits.gid_table_len - 1));\r
 496                 ib_ah_attr->grh.hop_limit  = path->hop_limit;\r
 497                 ib_ah_attr->grh.traffic_class =\r
 498                         (u8)((cl_ntoh32(path->sl_tclass_flowlabel) >> 20) & 0xff);\r
 499                 ib_ah_attr->grh.flow_label =\r
 500                         cl_ntoh32(path->sl_tclass_flowlabel) & 0xfffff;\r
 501                 memcpy(ib_ah_attr->grh.dgid.raw,\r
 502                         path->rgid, sizeof ib_ah_attr->grh.dgid.raw);\r
 503         }\r
 504 }\r
 505 \r
 506 int mthca_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,\r
 507                    struct ib_qp_init_attr *qp_init_attr)\r
 508 {\r
 509         struct mthca_dev *dev = to_mdev(ibqp->device);\r
 510         struct mthca_qp *qp = to_mqp(ibqp);\r
 511         int err = 0;\r
 512         struct mthca_mailbox *mailbox = NULL;\r
 513         struct mthca_qp_param *qp_param;\r
 514         struct mthca_qp_context *context;\r
 515         int mthca_state;\r
 516         u8 status;\r
 517 \r
 518         UNUSED_PARAM(qp_attr_mask);\r
 519         \r
 520         down( &qp->mutex );\r
 521 \r
 522         if (qp->state == IBQPS_RESET) {\r
 523                 qp_attr->qp_state = IBQPS_RESET;\r
 524                 goto done;\r
 525         }\r
 526 \r
 527         mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);\r
 528         if (IS_ERR(mailbox)) {\r
 529                 err = PTR_ERR(mailbox);\r
 530                 goto out;\r
 531         }\r
 532 \r
 533         err = mthca_QUERY_QP(dev, qp->qpn, 0, mailbox, &status);\r
 534         if (err)\r
 535                 goto out_mailbox;\r
 536         if (status) {\r
 537                 HCA_PRINT(TRACE_LEVEL_WARNING ,HCA_DBG_QP,\r
 538                         ("QUERY_QP returned status %02x\n", status));\r
 539                 err = -EINVAL;\r
 540                 goto out_mailbox;\r
 541         }\r
 542 \r
 543         qp_param    = mailbox->buf;\r
 544         context     = &qp_param->context;\r
 545         mthca_state = cl_ntoh32(context->flags) >> 28;\r
 546 \r
 547         qp->state                    = to_ib_qp_state(mthca_state);\r
 548         qp_attr->qp_state            = qp->state;\r
 549         qp_attr->path_mtu            = context->mtu_msgmax >> 5;\r
 550         qp_attr->path_mig_state      =\r
 551                 to_ib_mig_state((cl_ntoh32(context->flags) >> 11) & 0x3);\r
 552         qp_attr->qkey                = cl_ntoh32(context->qkey);\r
 553         qp_attr->rq_psn              = cl_ntoh32(context->rnr_nextrecvpsn) & 0xffffff;\r
 554         qp_attr->sq_psn              = cl_ntoh32(context->next_send_psn) & 0xffffff;\r
 555         qp_attr->dest_qp_num         = cl_ntoh32(context->remote_qpn) & 0xffffff;\r
 556         qp_attr->qp_access_flags     =\r
 557                 to_ib_qp_access_flags(cl_ntoh32(context->params2));\r
 558 \r
 559         if (qp->transport == RC || qp->transport == UC) {\r
 560                 to_ib_ah_attr(dev, &qp_attr->ah_attr, &context->pri_path);\r
 561                 to_ib_ah_attr(dev, &qp_attr->alt_ah_attr, &context->alt_path);\r
 562                 qp_attr->alt_pkey_index =\r
 563                         (u16)(cl_ntoh32(context->alt_path.port_pkey) & 0x7f);\r
 564                 qp_attr->alt_port_num   = qp_attr->alt_ah_attr.port_num;\r
 565         }\r
 566 \r
 567         qp_attr->pkey_index = (u16)(cl_ntoh32(context->pri_path.port_pkey) & 0x7f);\r
 568         qp_attr->port_num   =\r
 569                 (u8)((cl_ntoh32(context->pri_path.port_pkey) >> 24) & 0x3);\r
 570 \r
 571         /* qp_attr->en_sqd_async_notify is only applicable in modify qp */\r
 572         qp_attr->sq_draining = (u8)(mthca_state == MTHCA_QP_STATE_DRAINING);\r
 573 \r
 574         qp_attr->max_rd_atomic = (u8)(1 << ((cl_ntoh32(context->params1) >> 21) & 0x7));\r
 575 \r
 576         qp_attr->max_dest_rd_atomic =\r
 577                 (u8)(1 << ((cl_ntoh32(context->params2) >> 21) & 0x7));\r
 578         qp_attr->min_rnr_timer      =\r
 579                 (u8)((cl_ntoh32(context->rnr_nextrecvpsn) >> 24) & 0x1f);\r
 580         qp_attr->timeout            = context->pri_path.ackto >> 3;\r
 581         qp_attr->retry_cnt          = (u8)((cl_ntoh32(context->params1) >> 16) & 0x7);\r
 582         qp_attr->rnr_retry          = context->pri_path.rnr_retry >> 5;\r
 583         qp_attr->alt_timeout        = context->alt_path.ackto >> 3;\r
 584 \r
 585 done:\r
 586         qp_attr->cur_qp_state        = qp_attr->qp_state;\r
 587         qp_attr->cap.max_send_wr     = qp->sq.max;\r
 588         qp_attr->cap.max_recv_wr     = qp->rq.max;\r
 589         qp_attr->cap.max_send_sge    = qp->sq.max_gs;\r
 590         qp_attr->cap.max_recv_sge    = qp->rq.max_gs;\r
 591         qp_attr->cap.max_inline_data = qp->max_inline_data;\r
 592 \r
 593         qp_init_attr->cap            = qp_attr->cap;\r
 594 \r
 595 out_mailbox:\r
 596         mthca_free_mailbox(dev, mailbox);\r
 597 \r
 598 out:\r
 599         up(&qp->mutex);\r
 600         return err;\r
 601 }\r
 602 \r
 603 static void store_attrs(struct mthca_sqp *sqp, struct ib_qp_attr *attr,\r
 604                         int attr_mask)\r
 605 {\r
 606         if (attr_mask & IB_QP_PKEY_INDEX)\r
 607                 sqp->pkey_index = attr->pkey_index;\r
 608         if (attr_mask & IB_QP_QKEY)\r
 609                 sqp->qkey = attr->qkey;\r
 610         if (attr_mask & IB_QP_SQ_PSN)\r
 611                 sqp->send_psn = attr->sq_psn;\r
 612 }\r
 613 \r
 614 static void init_port(struct mthca_dev *dev, int port)\r
 615 {\r
 616         int err;\r
 617         u8 status;\r
 618         struct mthca_init_ib_param param;\r
 619 \r
 620         RtlZeroMemory(&param, sizeof param);\r
 621 \r
 622         param.port_width    = dev->limits.port_width_cap;\r
 623         param.vl_cap    = dev->limits.vl_cap;\r
 624         param.mtu_cap   = dev->limits.mtu_cap;\r
 625         param.gid_cap   = (u16)dev->limits.gid_table_len;\r
 626         param.pkey_cap  = (u16)dev->limits.pkey_table_len;\r
 627 \r
 628         err = mthca_INIT_IB(dev, &param, port, &status);\r
 629         if (err)\r
 630                 HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_QP  ,("INIT_IB failed, return code %d.\n", err));\r
 631         if (status)\r
 632                 HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_QP  ,("INIT_IB returned status %02x.\n", status));\r
 633 }\r
 634 \r
 635 \r
 636 static __be32 get_hw_access_flags(struct mthca_qp *qp, struct ib_qp_attr *attr,\r
 637                                   int attr_mask)\r
 638 {\r
 639         u8 dest_rd_atomic;\r
 640         u32 access_flags;\r
 641         u32 hw_access_flags = 0;\r
 642 \r
 643         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)\r
 644                 dest_rd_atomic = attr->max_dest_rd_atomic;\r
 645         else\r
 646                 dest_rd_atomic = qp->resp_depth;\r
 647 \r
 648         if (attr_mask & IB_QP_ACCESS_FLAGS)\r
 649                 access_flags = attr->qp_access_flags;\r
 650         else\r
 651                 access_flags = qp->atomic_rd_en;\r
 652 \r
 653         if (!dest_rd_atomic)\r
 654                 access_flags &= MTHCA_ACCESS_REMOTE_WRITE;\r
 655 \r
 656         if (access_flags & MTHCA_ACCESS_REMOTE_READ)\r
 657                 hw_access_flags |= MTHCA_QP_BIT_RRE;\r
 658         if (access_flags & MTHCA_ACCESS_REMOTE_ATOMIC)\r
 659                 hw_access_flags |= MTHCA_QP_BIT_RAE;\r
 660         if (access_flags & MTHCA_ACCESS_REMOTE_WRITE)\r
 661                 hw_access_flags |= MTHCA_QP_BIT_RWE;\r
 662 \r
 663         return cl_hton32(hw_access_flags);\r
 664 }\r
 665 \r
 666 int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask)\r
 667 {\r
 668         struct mthca_dev *dev = to_mdev(ibqp->device);\r
 669         struct mthca_qp *qp = to_mqp(ibqp);\r
 670         enum ib_qp_state cur_state, new_state;\r
 671         struct mthca_mailbox *mailbox;\r
 672         struct mthca_qp_param *qp_param;\r
 673         struct mthca_qp_context *qp_context;\r
 674         u32 req_param, opt_param;\r
 675         u32 sqd_event = 0;\r
 676         u8 status;\r
 677         int err = -EINVAL;\r
 678         SPIN_LOCK_PREP(lhs);\r
 679         SPIN_LOCK_PREP(lhr);\r
 680 \r
 681         down( &qp->mutex );\r
 682 \r
 683         if (attr_mask & IB_QP_CUR_STATE) {\r
 684                 if (attr->cur_qp_state != IBQPS_RTR &&\r
 685                         attr->cur_qp_state != IBQPS_RTS &&\r
 686                         attr->cur_qp_state != IBQPS_SQD &&\r
 687                         attr->cur_qp_state != IBQPS_SQE)\r
 688                         goto out;\r
 689                 else\r
 690                         cur_state = attr->cur_qp_state;\r
 691         } else {\r
 692                 spin_lock_irq(&qp->sq.lock, &lhs);\r
 693                 spin_lock(&qp->rq.lock, &lhr);\r
 694                 cur_state = qp->state;\r
 695                 spin_unlock(&lhr);\r
 696                 spin_unlock_irq(&lhs);\r
 697         }\r
 698 \r
 699         if (attr_mask & IB_QP_STATE) {\r
 700                 if (attr->qp_state < 0 || attr->qp_state > IBQPS_ERR)\r
 701                         goto out;\r
 702                 new_state = attr->qp_state;\r
 703         } else\r
 704                 new_state = cur_state;\r
 705 \r
 706         if (state_table[cur_state][new_state].trans == MTHCA_TRANS_INVALID) {\r
 707                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("Illegal QP transition "\r
 708                           "%d->%d\n", cur_state, new_state));\r
 709                 goto out;\r
 710         }\r
 711 \r
 712         req_param = state_table[cur_state][new_state].req_param[qp->transport];\r
 713         opt_param = state_table[cur_state][new_state].opt_param[qp->transport];\r
 714 \r
 715         if ((req_param & attr_mask) != req_param) {\r
 716                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("QP transition "\r
 717                           "%d->%d missing req attr 0x%08x\n",\r
 718                           cur_state, new_state,\r
 719                           req_param & ~attr_mask));\r
 720                 //NB: IBAL doesn't use all the fields, so we can miss some mandatory flags\r
 721                 goto out;\r
 722         }\r
 723 \r
 724         if (attr_mask & ~(req_param | opt_param | IB_QP_STATE)) {\r
 725                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("QP transition (transport %d) "\r
 726                           "%d->%d has extra attr 0x%08x\n",\r
 727                           qp->transport,\r
 728                           cur_state, new_state,\r
 729                           attr_mask & ~(req_param | opt_param |\r
 730                                                  IB_QP_STATE)));\r
 731                 //NB: The old code sometimes uses optional flags that are not so in this code\r
 732                 goto out;\r
 733         }\r
 734 \r
 735         if ((attr_mask & IB_QP_PKEY_INDEX) && \r
 736                 attr->pkey_index >= dev->limits.pkey_table_len) {\r
 737                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("PKey index (%u) too large. max is %d\n",\r
 738                           attr->pkey_index,dev->limits.pkey_table_len-1)); \r
 739                 goto out;\r
 740         }\r
 741 \r
 742         if ((attr_mask & IB_QP_PORT) &&\r
 743                 (attr->port_num == 0 || attr->port_num > dev->limits.num_ports)) {\r
 744                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("Port number (%u) is invalid\n", attr->port_num));\r
 745                 goto out;\r
 746         }\r
 747 \r
 748         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&\r
 749                 attr->max_rd_atomic > dev->limits.max_qp_init_rdma) {\r
 750                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("Max rdma_atomic as initiator %u too large (max is %d)\n",\r
 751                           attr->max_rd_atomic, dev->limits.max_qp_init_rdma));\r
 752                 goto out;\r
 753         }\r
 754 \r
 755         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&\r
 756             attr->max_dest_rd_atomic > 1 << dev->qp_table.rdb_shift) {\r
 757                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("Max rdma_atomic as responder %u too large (max %d)\n",\r
 758                           attr->max_dest_rd_atomic, 1 << dev->qp_table.rdb_shift));\r
 759                 goto out;\r
 760         }\r
 761 \r
 762         mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);\r
 763         if (IS_ERR(mailbox)) {\r
 764                 err = PTR_ERR(mailbox);\r
 765                 goto out;\r
 766         }\r
 767         qp_param = mailbox->buf;\r
 768         qp_context = &qp_param->context;\r
 769         RtlZeroMemory(qp_param, sizeof *qp_param);\r
 770 \r
 771         qp_context->flags      = cl_hton32((to_mthca_state(new_state) << 28) |\r
 772                                              (to_mthca_st(qp->transport) << 16));\r
 773         qp_context->flags     |= cl_hton32(MTHCA_QP_BIT_DE);\r
 774         if (!(attr_mask & IB_QP_PATH_MIG_STATE))\r
 775                 qp_context->flags |= cl_hton32(MTHCA_QP_PM_MIGRATED << 11);\r
 776         else {\r
 777                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_PM_STATE);\r
 778                 switch (attr->path_mig_state) {\r
 779                 case IB_APM_MIGRATED:\r
 780                         qp_context->flags |= cl_hton32(MTHCA_QP_PM_MIGRATED << 11);\r
 781                         break;\r
 782                 case IB_APM_REARM:\r
 783                         qp_context->flags |= cl_hton32(MTHCA_QP_PM_REARM << 11);\r
 784                         break;\r
 785                 case IB_APM_ARMED:\r
 786                         qp_context->flags |= cl_hton32(MTHCA_QP_PM_ARMED << 11);\r
 787                         break;\r
 788                 }\r
 789         }\r
 790 \r
 791         /* leave tavor_sched_queue as 0 */\r
 792 \r
 793         if (qp->transport == MLX || qp->transport == UD)\r
 794                 qp_context->mtu_msgmax = (IB_MTU_LEN_2048 << 5) | 11;\r
 795         else if (attr_mask & IB_QP_PATH_MTU) {\r
 796                 if (attr->path_mtu < IB_MTU_LEN_256 || attr->path_mtu > IB_MTU_LEN_2048) {\r
 797                         HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,\r
 798                                 ("path MTU (%u) is invalid\n", attr->path_mtu));\r
 799                         goto out_mailbox;\r
 800                 }\r
 801                 qp_context->mtu_msgmax = (u8)((attr->path_mtu << 5) | 31);\r
 802         }\r
 803 \r
 804         if (mthca_is_memfree(dev)) {\r
 805                 if (qp->rq.max)\r
 806                         qp_context->rq_size_stride = (u8)(long_log2(qp->rq.max) << 3);\r
 807                 qp_context->rq_size_stride |= qp->rq.wqe_shift - 4;\r
 808 \r
 809                 if (qp->sq.max)\r
 810                         qp_context->sq_size_stride = (u8)(long_log2(qp->sq.max) << 3);\r
 811                 qp_context->sq_size_stride |= qp->sq.wqe_shift - 4;\r
 812         }\r
 813 \r
 814         /* leave arbel_sched_queue as 0 */\r
 815 \r
 816         if (qp->ibqp.ucontext)\r
 817                 qp_context->usr_page =\r
 818                         cl_hton32(to_mucontext(qp->ibqp.ucontext)->uar.index);\r
 819         else\r
 820                 qp_context->usr_page = cl_hton32(dev->driver_uar.index);\r
 821         qp_context->local_qpn  = cl_hton32(qp->qpn);\r
 822         if (attr_mask & IB_QP_DEST_QPN) {\r
 823                 qp_context->remote_qpn = cl_hton32(attr->dest_qp_num);\r
 824         }\r
 825 \r
 826         if (qp->transport == MLX)\r
 827                 qp_context->pri_path.port_pkey |=\r
 828                         cl_hton32(to_msqp(qp)->port << 24);\r
 829         else {\r
 830                 if (attr_mask & IB_QP_PORT) {\r
 831                         qp_context->pri_path.port_pkey |=\r
 832                                 cl_hton32(attr->port_num << 24);\r
 833                         qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_PORT_NUM);\r
 834                 }\r
 835         }\r
 836 \r
 837         if (attr_mask & IB_QP_PKEY_INDEX) {\r
 838                 qp_context->pri_path.port_pkey |=\r
 839                         cl_hton32(attr->pkey_index);\r
 840                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_PKEY_INDEX);\r
 841         }\r
 842 \r
 843         if (attr_mask & IB_QP_RNR_RETRY) {\r
 844                 qp_context->pri_path.rnr_retry = attr->rnr_retry << 5;\r
 845                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_RNR_RETRY);\r
 846         }\r
 847 \r
 848         if (attr_mask & IB_QP_AV) {\r
 849                 qp_context->pri_path.g_mylmc     = attr->ah_attr.src_path_bits & 0x7f;\r
 850                 qp_context->pri_path.rlid        = cl_hton16(attr->ah_attr.dlid);\r
 851                 //TODO: work around: set always full speed  - really, it's much more complicate\r
 852                 qp_context->pri_path.static_rate = 0;\r
 853                 if (attr->ah_attr.ah_flags & IB_AH_GRH) {\r
 854                         qp_context->pri_path.g_mylmc |= 1 << 7;\r
 855                         qp_context->pri_path.mgid_index = attr->ah_attr.grh.sgid_index;\r
 856                         qp_context->pri_path.hop_limit = attr->ah_attr.grh.hop_limit;\r
 857                         qp_context->pri_path.sl_tclass_flowlabel =\r
 858                                 cl_hton32((attr->ah_attr.sl << 28)                |\r
 859                                             (attr->ah_attr.grh.traffic_class << 20) |\r
 860                                             (attr->ah_attr.grh.flow_label));\r
 861                         memcpy(qp_context->pri_path.rgid,\r
 862                                attr->ah_attr.grh.dgid.raw, 16);\r
 863                 } else {\r
 864                         qp_context->pri_path.sl_tclass_flowlabel =\r
 865                                 cl_hton32(attr->ah_attr.sl << 28);\r
 866                 }\r
 867                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_PRIMARY_ADDR_PATH);\r
 868         }\r
 869 \r
 870         if (attr_mask & IB_QP_TIMEOUT) {\r
 871                 qp_context->pri_path.ackto = attr->timeout << 3;\r
 872                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_ACK_TIMEOUT);\r
 873         }\r
 874 \r
 875         /* XXX alt_path */\r
 876 \r
 877         /* leave rdd as 0 */\r
 878         qp_context->pd         = cl_hton32(to_mpd(ibqp->pd)->pd_num);\r
 879         /* leave wqe_base as 0 (we always create an MR based at 0 for WQs) */\r
 880         qp_context->wqe_lkey   = cl_hton32(qp->mr.ibmr.lkey);\r
 881         qp_context->params1    = cl_hton32((unsigned long)(\r
 882                 (MTHCA_ACK_REQ_FREQ << 28) |\r
 883                 (MTHCA_FLIGHT_LIMIT << 24) |\r
 884                 MTHCA_QP_BIT_SWE));\r
 885         if (qp->sq_policy == IB_SIGNAL_ALL_WR)\r
 886                 qp_context->params1 |= cl_hton32(MTHCA_QP_BIT_SSC);\r
 887         if (attr_mask & IB_QP_RETRY_CNT) {\r
 888                 qp_context->params1 |= cl_hton32(attr->retry_cnt << 16);\r
 889                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_RETRY_COUNT);\r
 890         }\r
 891 \r
 892         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {\r
 893                 if (attr->max_rd_atomic) {\r
 894                         qp_context->params1 |=\r
 895                                 cl_hton32(MTHCA_QP_BIT_SRE |\r
 896                                             MTHCA_QP_BIT_SAE);\r
 897                         qp_context->params1 |=\r
 898                                 cl_hton32(fls(attr->max_rd_atomic - 1) << 21);\r
 899                 }\r
 900                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_SRA_MAX);\r
 901         }\r
 902 \r
 903         if (attr_mask & IB_QP_SQ_PSN)\r
 904                 qp_context->next_send_psn = cl_hton32(attr->sq_psn);\r
 905         qp_context->cqn_snd = cl_hton32(to_mcq(ibqp->send_cq)->cqn);\r
 906 \r
 907         if (mthca_is_memfree(dev)) {\r
 908                 qp_context->snd_wqe_base_l = cl_hton32(qp->send_wqe_offset);\r
 909                 qp_context->snd_db_index   = cl_hton32(qp->sq.db_index);\r
 910         }\r
 911 \r
 912         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {\r
 913 \r
 914                 if (attr->max_dest_rd_atomic)\r
 915                         qp_context->params2 |=\r
 916                                 cl_hton32(fls(attr->max_dest_rd_atomic - 1) << 21);\r
 917 \r
 918                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_RRA_MAX);\r
 919 \r
 920         }\r
 921 \r
 922         if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {\r
 923                 qp_context->params2      |= get_hw_access_flags(qp, attr, attr_mask);\r
 924                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_RWE |\r
 925                                                         MTHCA_QP_OPTPAR_RRE |\r
 926                                                         MTHCA_QP_OPTPAR_RAE);\r
 927         }\r
 928 \r
 929         qp_context->params2 |= cl_hton32(MTHCA_QP_BIT_RSC);\r
 930 \r
 931         if (ibqp->srq)\r
 932                 qp_context->params2 |= cl_hton32(MTHCA_QP_BIT_RIC);\r
 933 \r
 934         if (attr_mask & IB_QP_MIN_RNR_TIMER) {\r
 935                 qp_context->rnr_nextrecvpsn |= cl_hton32(attr->min_rnr_timer << 24);\r
 936                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_RNR_TIMEOUT);\r
 937         }\r
 938         if (attr_mask & IB_QP_RQ_PSN)\r
 939                 qp_context->rnr_nextrecvpsn |= cl_hton32(attr->rq_psn);\r
 940 \r
 941         qp_context->ra_buff_indx =\r
 942                 cl_hton32(dev->qp_table.rdb_base +\r
 943                             ((qp->qpn & (dev->limits.num_qps - 1)) * MTHCA_RDB_ENTRY_SIZE <<\r
 944                              dev->qp_table.rdb_shift));\r
 945 \r
 946         qp_context->cqn_rcv = cl_hton32(to_mcq(ibqp->recv_cq)->cqn);\r
 947 \r
 948         if (mthca_is_memfree(dev))\r
 949                 qp_context->rcv_db_index   = cl_hton32(qp->rq.db_index);\r
 950 \r
 951         if (attr_mask & IB_QP_QKEY) {\r
 952                 qp_context->qkey = cl_hton32(attr->qkey);\r
 953                 qp_param->opt_param_mask |= cl_hton32(MTHCA_QP_OPTPAR_Q_KEY);\r
 954         }\r
 955 \r
 956         if (ibqp->srq)\r
 957                 qp_context->srqn = cl_hton32(1 << 24 |\r
 958                                                to_msrq(ibqp->srq)->srqn);\r
 959 \r
 960         if (cur_state == IBQPS_RTS && new_state == IBQPS_SQD    &&\r
 961             attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY               &&\r
 962             attr->en_sqd_async_notify)\r
 963                 sqd_event = (u32)(1 << 31);\r
 964 \r
 965         err = mthca_MODIFY_QP(dev, state_table[cur_state][new_state].trans,\r
 966                               qp->qpn, 0, mailbox, sqd_event, &status);\r
 967         if (err) {\r
 968                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("mthca_MODIFY_QP returned error (qp-num = 0x%x) returned status %02x "\r
 969                         "cur_state  = %d  new_state = %d attr_mask = %d req_param = %d opt_param = %d\n",\r
 970                         ibqp->qp_num, status, cur_state, new_state, \r
 971                         attr_mask, req_param, opt_param));        \r
 972                 goto out_mailbox;\r
 973         }\r
 974         if (status) {\r
 975                 HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("mthca_MODIFY_QP bad status(qp-num = 0x%x) returned status %02x "\r
 976                         "cur_state  = %d  new_state = %d attr_mask = %d req_param = %d opt_param = %d\n",\r
 977                         ibqp->qp_num, status, cur_state, new_state, \r
 978                         attr_mask, req_param, opt_param));\r
 979                 err = -EINVAL;\r
 980                 goto out_mailbox;\r
 981         }\r
 982 \r
 983         qp->state = new_state;\r
 984         if (attr_mask & IB_QP_ACCESS_FLAGS)\r
 985                 qp->atomic_rd_en = (u8)attr->qp_access_flags;\r
 986         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)\r
 987                 qp->resp_depth = attr->max_dest_rd_atomic;\r
 988 \r
 989         if (is_sqp(dev, qp))\r
 990                 store_attrs(to_msqp(qp), attr, attr_mask);\r
 991 \r
 992         /*\r
 993          * If we moved QP0 to RTR, bring the IB link up; if we moved\r
 994          * QP0 to RESET or ERROR, bring the link back down.\r
 995          */\r
 996         if (is_qp0(dev, qp)) {\r
 997                 if (cur_state != IBQPS_RTR &&\r
 998                         new_state == IBQPS_RTR)\r
 999                         init_port(dev, to_msqp(qp)->port);\r
1000 \r
1001                 if (cur_state != IBQPS_RESET &&\r
1002                         cur_state != IBQPS_ERR &&\r
1003                         (new_state == IBQPS_RESET ||\r
1004                         new_state == IBQPS_ERR))\r
1005                         mthca_CLOSE_IB(dev, to_msqp(qp)->port, &status);\r
1006         }\r
1007 \r
1008         /*\r
1009          * If we moved a kernel QP to RESET, clean up all old CQ\r
1010          * entries and reinitialize the QP.\r
1011          */\r
1012         if (new_state == IBQPS_RESET && !qp->ibqp.ucontext) {\r
1013                 mthca_cq_clean(dev, to_mcq(qp->ibqp.send_cq)->cqn, qp->qpn,\r
1014                                qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);\r
1015                 if (qp->ibqp.send_cq != qp->ibqp.recv_cq)\r
1016                         mthca_cq_clean(dev, to_mcq(qp->ibqp.recv_cq)->cqn, qp->qpn,\r
1017                                        qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);\r
1018 \r
1019                 mthca_wq_init(&qp->sq);\r
1020                 qp->sq.last = get_send_wqe(qp, qp->sq.max - 1);\r
1021                 mthca_wq_init(&qp->rq);\r
1022                 qp->rq.last = get_recv_wqe(qp, qp->rq.max - 1);\r
1023 \r
1024                 if (mthca_is_memfree(dev)) {\r
1025                         *qp->sq.db = 0;\r
1026                         *qp->rq.db = 0;\r
1027                 }\r
1028         }\r
1029 \r
1030 out_mailbox:\r
1031         mthca_free_mailbox(dev, mailbox);\r
1032 \r
1033 out:\r
1034         up( &qp->mutex );\r
1035         return err;\r
1036 }\r
1037 \r
1038 static int mthca_max_data_size(struct mthca_dev *dev, struct mthca_qp *qp, int desc_sz)\r
1039 {\r
1040 \r
1041         /*\r
1042          * Calculate the maximum size of WQE s/g segments, excluding\r
1043          * the next segment and other non-data segments.\r
1044          */\r
1045         int max_data_size = desc_sz - sizeof (struct mthca_next_seg);\r
1046 \r
1047         switch (qp->transport) {\r
1048         case MLX:\r
1049                 max_data_size -= 2 * sizeof (struct mthca_data_seg);\r
1050                 break;\r
1051 \r
1052         case UD:\r
1053                 if (mthca_is_memfree(dev))\r
1054                         max_data_size -= sizeof (struct mthca_arbel_ud_seg);\r
1055                 else\r
1056                         max_data_size -= sizeof (struct mthca_tavor_ud_seg);\r
1057                 break;\r
1058 \r
1059         default:\r
1060                 max_data_size -= sizeof (struct mthca_raddr_seg);\r
1061                 break;\r
1062         }\r
1063                 return max_data_size;\r
1064 }\r
1065 \r
1066 static inline int mthca_max_inline_data(int max_data_size)\r
1067 {\r
1068         return max_data_size - MTHCA_INLINE_HEADER_SIZE ;\r
1069 }\r
1070 \r
1071 static void mthca_adjust_qp_caps(struct mthca_dev *dev,\r
1072                                  struct mthca_qp *qp)\r
1073 {\r
1074         int max_data_size = mthca_max_data_size(dev, qp,\r
1075                 min(dev->limits.max_desc_sz, 1 << qp->sq.wqe_shift));\r
1076 \r
1077         qp->max_inline_data = mthca_max_inline_data( max_data_size);\r
1078 \r
1079         qp->sq.max_gs = min(dev->limits.max_sg,\r
1080                 (int)(max_data_size / sizeof (struct mthca_data_seg)));\r
1081         qp->rq.max_gs = min(dev->limits.max_sg,\r
1082                 (int)((min(dev->limits.max_desc_sz, 1 << qp->rq.wqe_shift) -\r
1083                 sizeof (struct mthca_next_seg)) / sizeof (struct mthca_data_seg)));     \r
1084 }\r
1085 \r
1086 /*\r
1087  * Allocate and register buffer for WQEs.  qp->rq.max, sq.max,\r
1088  * rq.max_gs and sq.max_gs must all be assigned.\r
1089  * mthca_alloc_wqe_buf will calculate rq.wqe_shift and\r
1090  * sq.wqe_shift (as well as send_wqe_offset, is_direct, and\r
1091  * queue)\r
1092  */\r
1093 static int mthca_alloc_wqe_buf(struct mthca_dev *dev,\r
1094                                struct mthca_pd *pd,\r
1095                                struct mthca_qp *qp)\r
1096 {\r
1097         int size;\r
1098         int err = -ENOMEM;\r
1099         \r
1100         HCA_ENTER(HCA_DBG_QP);\r
1101         size = sizeof (struct mthca_next_seg) +\r
1102                 qp->rq.max_gs * sizeof (struct mthca_data_seg);\r
1103 \r
1104         if (size > dev->limits.max_desc_sz)\r
1105                 return -EINVAL;\r
1106 \r
1107         for (qp->rq.wqe_shift = 6; 1 << qp->rq.wqe_shift < size;\r
1108              qp->rq.wqe_shift++)\r
1109                 ; /* nothing */\r
1110 \r
1111         size = qp->sq.max_gs * sizeof (struct mthca_data_seg);\r
1112         switch (qp->transport) {\r
1113                 case MLX:\r
1114                         size += 2 * sizeof (struct mthca_data_seg);\r
1115                         break;\r
1116 \r
1117                 case UD:\r
1118                         size += mthca_is_memfree(dev) ?\r
1119                                 sizeof (struct mthca_arbel_ud_seg) :\r
1120                                 sizeof (struct mthca_tavor_ud_seg);\r
1121                         break;\r
1122                 \r
1123                 case UC:\r
1124                         size += sizeof (struct mthca_raddr_seg);\r
1125                         break;\r
1126                 \r
1127                 case RC:\r
1128                         size += sizeof (struct mthca_raddr_seg);\r
1129                         /*\r
1130                          * An atomic op will require an atomic segment, a\r
1131                          * remote address segment and one scatter entry.\r
1132                          */\r
1133                         size = max(size,\r
1134                                  sizeof (struct mthca_atomic_seg) +\r
1135                                  sizeof (struct mthca_raddr_seg) +\r
1136                                  sizeof (struct mthca_data_seg));\r
1137                         break;\r
1138                         \r
1139                 default:\r
1140                         break;\r
1141         }\r
1142                 \r
1143         /* Make sure that we have enough space for a bind request */\r
1144         size = max(size, sizeof (struct mthca_bind_seg));\r
1145         \r
1146         size += sizeof (struct mthca_next_seg);\r
1147         \r
1148         if (size > dev->limits.max_desc_sz)\r
1149                 return -EINVAL;\r
1150 \r
1151         for (qp->sq.wqe_shift = 6; 1 << qp->sq.wqe_shift < size;\r
1152              qp->sq.wqe_shift++)\r
1153                 ; /* nothing */\r
1154 \r
1155         qp->send_wqe_offset = ALIGN(qp->rq.max << qp->rq.wqe_shift,\r
1156                                     1 << qp->sq.wqe_shift);\r
1157 \r
1158         /*\r
1159          * If this is a userspace QP, we don't actually have to\r
1160          * allocate anything.  All we need is to calculate the WQE\r
1161          * sizes and the send_wqe_offset, so we're done now.\r
1162          */\r
1163         if (pd->ibpd.ucontext)\r
1164                 return 0;\r
1165 \r
1166         size = (int)(LONG_PTR)NEXT_PAGE_ALIGN(qp->send_wqe_offset +\r
1167                           (qp->sq.max << qp->sq.wqe_shift));\r
1168 \r
1169         qp->wrid = kmalloc((qp->rq.max + qp->sq.max) * sizeof (u64),\r
1170                            GFP_KERNEL);\r
1171         if (!qp->wrid)\r
1172                 goto err_out;\r
1173 \r
1174         err = mthca_buf_alloc(dev, size, MTHCA_MAX_DIRECT_QP_SIZE,\r
1175                               &qp->queue, &qp->is_direct, pd, 0, &qp->mr);\r
1176         if (err)\r
1177                 goto err_out;\r
1178         \r
1179         HCA_EXIT(HCA_DBG_QP);\r
1180         return 0;\r
1181 \r
1182 err_out:\r
1183         kfree(qp->wrid);\r
1184         return err;\r
1185 }\r
1186 \r
1187 static void mthca_free_wqe_buf(struct mthca_dev *dev,\r
1188                                struct mthca_qp *qp)\r
1189 {\r
1190         mthca_buf_free(dev, (int)(LONG_PTR)NEXT_PAGE_ALIGN(qp->send_wqe_offset +\r
1191                                        (qp->sq.max << qp->sq.wqe_shift)),\r
1192                        &qp->queue, qp->is_direct, &qp->mr);\r
1193         kfree(qp->wrid);\r
1194 }\r
1195 \r
1196 static int mthca_map_memfree(struct mthca_dev *dev,\r
1197                              struct mthca_qp *qp)\r
1198 {\r
1199         int ret;\r
1200 \r
1201         if (mthca_is_memfree(dev)) {\r
1202                 ret = mthca_table_get(dev, dev->qp_table.qp_table, qp->qpn);\r
1203                 if (ret)\r
1204                         return ret;\r
1205 \r
1206                 ret = mthca_table_get(dev, dev->qp_table.eqp_table, qp->qpn);\r
1207                 if (ret)\r
1208                         goto err_qpc;\r
1209 \r
1210                 ret = mthca_table_get(dev, dev->qp_table.rdb_table,\r
1211                                       qp->qpn << dev->qp_table.rdb_shift);\r
1212                 if (ret)\r
1213                         goto err_eqpc;\r
1214 \r
1215         }\r
1216 \r
1217         return 0;\r
1218 \r
1219 err_eqpc:\r
1220         mthca_table_put(dev, dev->qp_table.eqp_table, qp->qpn);\r
1221 \r
1222 err_qpc:\r
1223         mthca_table_put(dev, dev->qp_table.qp_table, qp->qpn);\r
1224 \r
1225         return ret;\r
1226 }\r
1227 \r
1228 static void mthca_unmap_memfree(struct mthca_dev *dev,\r
1229                                 struct mthca_qp *qp)\r
1230 {\r
1231         mthca_table_put(dev, dev->qp_table.rdb_table,\r
1232                         qp->qpn << dev->qp_table.rdb_shift);\r
1233         mthca_table_put(dev, dev->qp_table.eqp_table, qp->qpn);\r
1234         mthca_table_put(dev, dev->qp_table.qp_table, qp->qpn);\r
1235 }\r
1236 \r
1237 static int mthca_alloc_memfree(struct mthca_dev *dev,\r
1238                                struct mthca_qp *qp)\r
1239 {\r
1240         int ret = 0;\r
1241 \r
1242         if (mthca_is_memfree(dev)) {\r
1243                 qp->rq.db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_RQ,\r
1244                                                  qp->qpn, &qp->rq.db);\r
1245                 if (qp->rq.db_index < 0)\r
1246                         return qp->rq.db_index;\r
1247 \r
1248                 qp->sq.db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_SQ,\r
1249                                                  qp->qpn, &qp->sq.db);\r
1250                 if (qp->sq.db_index < 0){\r
1251                         mthca_free_db(dev, MTHCA_DB_TYPE_RQ, qp->rq.db_index);\r
1252                         return qp->sq.db_index;\r
1253                 }\r
1254 \r
1255         }\r
1256 \r
1257         return ret;\r
1258 }\r
1259 \r
1260 static void mthca_free_memfree(struct mthca_dev *dev,\r
1261                                struct mthca_qp *qp)\r
1262 {\r
1263         if (mthca_is_memfree(dev)) {\r
1264                 mthca_free_db(dev, MTHCA_DB_TYPE_SQ, qp->sq.db_index);\r
1265                 mthca_free_db(dev, MTHCA_DB_TYPE_RQ, qp->rq.db_index);\r
1266         }\r
1267 }\r
1268 \r
1269 static int mthca_alloc_qp_common(struct mthca_dev *dev,\r
1270                                  struct mthca_pd *pd,\r
1271                                  struct mthca_cq *send_cq,\r
1272                                  struct mthca_cq *recv_cq,\r
1273                                  enum ib_sig_type send_policy,\r
1274                                  struct mthca_qp *qp)\r
1275 {\r
1276         int ret;\r
1277         int i;\r
1278 \r
1279         atomic_set(&qp->refcount, 1);\r
1280         init_waitqueue_head(&qp->wait);\r
1281         KeInitializeMutex(&qp->mutex, 0);\r
1282 \r
1283         qp->state        = IBQPS_RESET;\r
1284         qp->atomic_rd_en = 0;\r
1285         qp->resp_depth   = 0;\r
1286         qp->sq_policy    = send_policy;\r
1287         mthca_wq_init(&qp->sq);\r
1288         mthca_wq_init(&qp->rq);\r
1289 \r
1290         UNREFERENCED_PARAMETER(send_cq);\r
1291         UNREFERENCED_PARAMETER(recv_cq);\r
1292         \r
1293         ret = mthca_map_memfree(dev, qp);\r
1294         if (ret)\r
1295                 return ret;\r
1296 \r
1297         ret = mthca_alloc_wqe_buf(dev, pd, qp);\r
1298         if (ret) {\r
1299                 mthca_unmap_memfree(dev, qp);\r
1300                 return ret;\r
1301         }\r
1302 \r
1303         mthca_adjust_qp_caps(dev, qp);\r
1304 \r
1305         /*\r
1306          * If this is a userspace QP, we're done now.  The doorbells\r
1307          * will be allocated and buffers will be initialized in\r
1308          * userspace.\r
1309          */\r
1310         if (pd->ibpd.ucontext)\r
1311                 return 0;\r
1312 \r
1313         ret = mthca_alloc_memfree(dev, qp);\r
1314         if (ret) {\r
1315                 mthca_free_wqe_buf(dev, qp);\r
1316                 mthca_unmap_memfree(dev, qp);\r
1317                 return ret;\r
1318         }\r
1319 \r
1320         if (mthca_is_memfree(dev)) {\r
1321                 struct mthca_next_seg *next;\r
1322                 struct mthca_data_seg *scatter;\r
1323                 int size = (sizeof (struct mthca_next_seg) +\r
1324                             qp->rq.max_gs * sizeof (struct mthca_data_seg)) / 16;\r
1325 \r
1326                 for (i = 0; i < qp->rq.max; ++i) {\r
1327                         next = get_recv_wqe(qp, i);\r
1328                         next->nda_op = cl_hton32(((i + 1) & (qp->rq.max - 1)) <<\r
1329                                                    qp->rq.wqe_shift);\r
1330                         next->ee_nds = cl_hton32(size);\r
1331 \r
1332                         for (scatter = (void *) (next + 1);\r
1333                              (void *) scatter < (void *) ((u8*)next + (u32)(1 << qp->rq.wqe_shift));\r
1334                              ++scatter)\r
1335                                 scatter->lkey = cl_hton32(MTHCA_INVAL_LKEY);\r
1336                 }\r
1337 \r
1338                 for (i = 0; i < qp->sq.max; ++i) {\r
1339                         next = get_send_wqe(qp, i);\r
1340                         next->nda_op = cl_hton32((((i + 1) & (qp->sq.max - 1)) <<\r
1341                                                     qp->sq.wqe_shift) +\r
1342                                                    qp->send_wqe_offset);\r
1343                 }\r
1344         }\r
1345 \r
1346         qp->sq.last = get_send_wqe(qp, qp->sq.max - 1);\r
1347         qp->rq.last = get_recv_wqe(qp, qp->rq.max - 1);\r
1348 \r
1349         return 0;\r
1350 }\r
1351 \r
1352 static int mthca_set_qp_size(struct mthca_dev *dev, struct ib_qp_cap *cap,\r
1353         struct mthca_qp *qp)\r
1354 {\r
1355         int max_data_size = mthca_max_data_size(dev, qp, dev->limits.max_desc_sz);\r
1356 \r
1357         /* Sanity check QP size before proceeding */\r
1358         if (cap->max_send_wr     > (u32)dev->limits.max_wqes ||\r
1359             cap->max_recv_wr     > (u32)dev->limits.max_wqes ||\r
1360             cap->max_send_sge    > (u32)dev->limits.max_sg   ||\r
1361             cap->max_recv_sge    > (u32)dev->limits.max_sg   ||\r
1362             cap->max_inline_data > (u32)mthca_max_inline_data(max_data_size))\r
1363                 return -EINVAL;\r
1364 \r
1365         /*\r
1366          * For MLX transport we need 2 extra S/G entries:\r
1367          * one for the header and one for the checksum at the end\r
1368          */\r
1369         if (qp->transport == MLX && cap->max_recv_sge + 2 > (u32)dev->limits.max_sg)\r
1370                 return -EINVAL;\r
1371 \r
1372         /* Enable creating zero-sized QPs */\r
1373         if (!cap->max_recv_wr)\r
1374                 cap->max_recv_wr = 1;\r
1375         if (!cap->max_send_wr)\r
1376                 cap->max_send_wr = 1;\r
1377         \r
1378         if (mthca_is_memfree(dev)) {\r
1379                 qp->rq.max = cap->max_recv_wr ?\r
1380                         roundup_pow_of_two(cap->max_recv_wr) : 0;\r
1381                 qp->sq.max = cap->max_send_wr ?\r
1382                         roundup_pow_of_two(cap->max_send_wr) : 0;\r
1383         } else {\r
1384                 qp->rq.max = cap->max_recv_wr;\r
1385                 qp->sq.max = cap->max_send_wr;\r
1386         }\r
1387 \r
1388         qp->rq.max_gs = cap->max_recv_sge;\r
1389         qp->sq.max_gs = MAX(cap->max_send_sge,\r
1390                               ALIGN(cap->max_inline_data + MTHCA_INLINE_HEADER_SIZE,\r
1391                                     MTHCA_INLINE_CHUNK_SIZE) /\r
1392                               (int)sizeof (struct mthca_data_seg));\r
1393 \r
1394         return 0;\r
1395 }\r
1396 \r
1397 int mthca_alloc_qp(struct mthca_dev *dev,\r
1398                    struct mthca_pd *pd,\r
1399                    struct mthca_cq *send_cq,\r
1400                    struct mthca_cq *recv_cq,\r
1401                    enum ib_qp_type_t type,\r
1402                    enum ib_sig_type send_policy,\r
1403                    struct ib_qp_cap *cap,\r
1404                    struct mthca_qp *qp)\r
1405 {\r
1406         int err;\r
1407         SPIN_LOCK_PREP(lh);\r
1408 \r
1409         switch (type) {\r
1410         case IB_QPT_RELIABLE_CONN: qp->transport = RC; break;\r
1411         case IB_QPT_UNRELIABLE_CONN: qp->transport = UC; break;\r
1412         case IB_QPT_UNRELIABLE_DGRM: qp->transport = UD; break;\r
1413         default: return -EINVAL;\r
1414         }\r
1415 \r
1416         err = mthca_set_qp_size(dev, cap, qp);\r
1417         if (err)\r
1418                 return err;\r
1419 \r
1420         qp->qpn = mthca_alloc(&dev->qp_table.alloc);\r
1421         if (qp->qpn == -1)\r
1422                 return -ENOMEM;\r
1423 \r
1424         err = mthca_alloc_qp_common(dev, pd, send_cq, recv_cq,\r
1425                                     send_policy, qp);\r
1426         if (err) {\r
1427                 mthca_free(&dev->qp_table.alloc, qp->qpn);\r
1428                 return err;\r
1429         }\r
1430 \r
1431         spin_lock_irq(&dev->qp_table.lock, &lh);\r
1432         mthca_array_set(&dev->qp_table.qp,\r
1433                         qp->qpn & (dev->limits.num_qps - 1), qp);\r
1434         spin_unlock_irq(&lh);\r
1435 \r
1436         return 0;\r
1437 }\r
1438 \r
1439 int mthca_alloc_sqp(struct mthca_dev *dev,\r
1440                     struct mthca_pd *pd,\r
1441                     struct mthca_cq *send_cq,\r
1442                     struct mthca_cq *recv_cq,\r
1443                     enum ib_sig_type send_policy,\r
1444                     struct ib_qp_cap *cap,\r
1445                     int qpn,\r
1446                     int port,\r
1447                     struct mthca_sqp *sqp)\r
1448 {\r
1449         u32 mqpn = qpn * 2 + dev->qp_table.sqp_start + port - 1;\r
1450         int err;\r
1451         SPIN_LOCK_PREP(lhs);\r
1452         SPIN_LOCK_PREP(lhr);\r
1453         SPIN_LOCK_PREP(lht);\r
1454 \r
1455         err = mthca_set_qp_size(dev, cap, &sqp->qp);\r
1456         if (err)\r
1457                 return err;\r
1458 \r
1459         alloc_dma_zmem_map(dev, \r
1460                 sqp->qp.sq.max * MTHCA_UD_HEADER_SIZE, \r
1461                 PCI_DMA_BIDIRECTIONAL,\r
1462                 &sqp->sg);\r
1463         if (!sqp->sg.page)\r
1464                 return -ENOMEM;\r
1465 \r
1466         spin_lock_irq(&dev->qp_table.lock, &lht);\r
1467         if (mthca_array_get(&dev->qp_table.qp, mqpn))\r
1468                 err = -EBUSY;\r
1469         else\r
1470                 mthca_array_set(&dev->qp_table.qp, mqpn, sqp);\r
1471         spin_unlock_irq(&lht);\r
1472 \r
1473         if (err)\r
1474                 goto err_out;\r
1475 \r
1476         sqp->port = port;\r
1477         sqp->qp.qpn       = mqpn;\r
1478         sqp->qp.transport = MLX;\r
1479 \r
1480         err = mthca_alloc_qp_common(dev, pd, send_cq, recv_cq,\r
1481                                     send_policy, &sqp->qp);\r
1482         if (err)\r
1483                 goto err_out_free;\r
1484 \r
1485         atomic_inc(&pd->sqp_count);\r
1486 \r
1487         return 0;\r
1488 \r
1489  err_out_free:\r
1490         /*\r
1491          * Lock CQs here, so that CQ polling code can do QP lookup\r
1492          * without taking a lock.\r
1493          */\r
1494         spin_lock_irq(&send_cq->lock, &lhs);\r
1495         if (send_cq != recv_cq)\r
1496                 spin_lock(&recv_cq->lock, &lhr);\r
1497 \r
1498         spin_lock(&dev->qp_table.lock, &lht);\r
1499         mthca_array_clear(&dev->qp_table.qp, mqpn);\r
1500         spin_unlock(&lht);\r
1501 \r
1502         if (send_cq != recv_cq)\r
1503                 spin_unlock(&lhr);\r
1504         spin_unlock_irq(&lhs);\r
1505 \r
1506  err_out:\r
1507         free_dma_mem_map(dev, &sqp->sg, PCI_DMA_BIDIRECTIONAL);\r
1508 \r
1509         return err;\r
1510 }\r
1511 \r
1512 void mthca_free_qp(struct mthca_dev *dev,\r
1513                    struct mthca_qp *qp)\r
1514 {\r
1515         u8 status;\r
1516         struct mthca_cq *send_cq;\r
1517         struct mthca_cq *recv_cq;\r
1518         SPIN_LOCK_PREP(lhs);\r
1519         SPIN_LOCK_PREP(lhr);\r
1520         SPIN_LOCK_PREP(lht);\r
1521 \r
1522         send_cq = to_mcq(qp->ibqp.send_cq);\r
1523         recv_cq = to_mcq(qp->ibqp.recv_cq);\r
1524 \r
1525         /*\r
1526          * Lock CQs here, so that CQ polling code can do QP lookup\r
1527          * without taking a lock.\r
1528          */\r
1529         spin_lock_irq(&send_cq->lock, &lhs);\r
1530         if (send_cq != recv_cq)\r
1531                 spin_lock(&recv_cq->lock, &lhr);\r
1532 \r
1533         spin_lock(&dev->qp_table.lock, &lht);\r
1534         mthca_array_clear(&dev->qp_table.qp,\r
1535                           qp->qpn & (dev->limits.num_qps - 1));\r
1536         spin_unlock(&lht);\r
1537 \r
1538         if (send_cq != recv_cq)\r
1539                 spin_unlock(&lhr);\r
1540         spin_unlock_irq(&lhs);\r
1541 \r
1542         atomic_dec(&qp->refcount);\r
1543         wait_event(&qp->wait, !atomic_read(&qp->refcount));\r
1544 \r
1545         if (qp->state != IBQPS_RESET) {\r
1546                 mthca_MODIFY_QP(dev, MTHCA_TRANS_ANY2RST, qp->qpn, 0, NULL, 0, &status);\r
1547         }\r
1548 \r
1549         /*\r
1550          * If this is a userspace QP, the buffers, MR, CQs and so on\r
1551          * will be cleaned up in userspace, so all we have to do is\r
1552          * unref the mem-free tables and free the QPN in our table.\r
1553          */\r
1554         if (!qp->ibqp.ucontext) {\r
1555                 mthca_cq_clean(dev, to_mcq(qp->ibqp.send_cq)->cqn, qp->qpn,\r
1556                                qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);\r
1557                 if (qp->ibqp.send_cq != qp->ibqp.recv_cq)\r
1558                         mthca_cq_clean(dev, to_mcq(qp->ibqp.recv_cq)->cqn, qp->qpn,\r
1559                                        qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);\r
1560 \r
1561                 mthca_free_memfree(dev, qp);\r
1562                 mthca_free_wqe_buf(dev, qp);\r
1563         }\r
1564 \r
1565         mthca_unmap_memfree(dev, qp);\r
1566 \r
1567         if (is_sqp(dev, qp)) {\r
1568                 atomic_dec(&(to_mpd(qp->ibqp.pd)->sqp_count));\r
1569                 free_dma_mem_map(dev, &to_msqp(qp)->sg, PCI_DMA_BIDIRECTIONAL);\r
1570         } else\r
1571                 mthca_free(&dev->qp_table.alloc, qp->qpn);\r
1572 }\r
1573 \r
1574 static enum mthca_wr_opcode conv_ibal_wr_opcode(struct _ib_send_wr *wr)\r
1575 {\r
1576 \r
1577         enum mthca_wr_opcode opcode = -1; //= wr->wr_type;\r
1578 \r
1579         switch (wr->wr_type) {\r
1580                 case WR_SEND: \r
1581                         opcode = (wr->send_opt & IB_SEND_OPT_IMMEDIATE) ? MTHCA_OPCODE_SEND_IMM : MTHCA_OPCODE_SEND;\r
1582                         break;\r
1583                 case WR_RDMA_WRITE:     \r
1584                         opcode = (wr->send_opt & IB_SEND_OPT_IMMEDIATE) ? MTHCA_OPCODE_RDMA_WRITE_IMM : MTHCA_OPCODE_RDMA_WRITE;\r
1585                         break;\r
1586                 case WR_RDMA_READ:              opcode = MTHCA_OPCODE_RDMA_READ; break;\r
1587                 case WR_COMPARE_SWAP:           opcode = MTHCA_OPCODE_ATOMIC_CS; break;\r
1588                 case WR_FETCH_ADD:                      opcode = MTHCA_OPCODE_ATOMIC_FA; break;\r
1589                 default:                                                opcode = MTHCA_OPCODE_INVALID;break;\r
1590         }\r
1591         return opcode;\r
1592 }\r
1593 \r
1594 /* Create UD header for an MLX send and build a data segment for it */\r
1595 static int build_mlx_header(struct mthca_dev *dev, struct mthca_sqp *sqp,\r
1596                             int ind, struct _ib_send_wr *wr,\r
1597                             struct mthca_mlx_seg *mlx,\r
1598                             struct mthca_data_seg *data)\r
1599 {\r
1600         enum ib_wr_opcode opcode = conv_ibal_wr_opcode(wr);\r
1601         int header_size;\r
1602         int err;\r
1603         __be16 pkey;\r
1604         CPU_2_BE64_PREP;\r
1605 \r
1606         if (!wr->dgrm.ud.h_av) {\r
1607                 HCA_PRINT(TRACE_LEVEL_ERROR , HCA_DBG_AV, \r
1608                         ("absent AV in send wr %p\n", wr));\r
1609                 return -EINVAL;\r
1610         }\r
1611                 \r
1612         ib_ud_header_init(256, /* assume a MAD */\r
1613                 mthca_ah_grh_present(to_mah((struct ib_ah *)wr->dgrm.ud.h_av)),\r
1614                 &sqp->ud_header);\r
1615 \r
1616         err = mthca_read_ah(dev, to_mah((struct ib_ah *)wr->dgrm.ud.h_av), &sqp->ud_header);\r
1617         if (err){\r
1618                 HCA_PRINT(TRACE_LEVEL_ERROR , HCA_DBG_AV, ("read av error%p\n",\r
1619                         to_mah((struct ib_ah *)wr->dgrm.ud.h_av)->av));\r
1620                 return err;\r
1621         }\r
1622         mlx->flags &= ~cl_hton32(MTHCA_NEXT_SOLICIT | 1);\r
1623         mlx->flags |= cl_hton32((!sqp->qp.ibqp.qp_num ? MTHCA_MLX_VL15 : 0) |\r
1624                                   (sqp->ud_header.lrh.destination_lid ==\r
1625                                    IB_LID_PERMISSIVE ? MTHCA_MLX_SLR : 0) |\r
1626                                   (sqp->ud_header.lrh.service_level << 8));\r
1627         mlx->rlid = sqp->ud_header.lrh.destination_lid;\r
1628         mlx->vcrc = 0;\r
1629 \r
1630         switch (opcode) {\r
1631         case MTHCA_OPCODE_SEND:\r
1632                 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY;\r
1633                 sqp->ud_header.immediate_present = 0;\r
1634                 break;\r
1635         case MTHCA_OPCODE_SEND_IMM:\r
1636                 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;\r
1637                 sqp->ud_header.immediate_present = 1;\r
1638                 sqp->ud_header.immediate_data = wr->immediate_data;\r
1639                 break;\r
1640         default:\r
1641                 return -EINVAL;\r
1642         }\r
1643 \r
1644         sqp->ud_header.lrh.virtual_lane    = !sqp->qp.ibqp.qp_num ? 15 : 0;\r
1645         if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE)\r
1646                 sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE;\r
1647         sqp->ud_header.bth.solicited_event = (u8)!!(wr->send_opt & IB_SEND_OPT_SOLICITED);\r
1648         if (!sqp->qp.ibqp.qp_num)\r
1649                 ib_get_cached_pkey(&dev->ib_dev, (u8)sqp->port,\r
1650                                    sqp->pkey_index, &pkey);\r
1651         else\r
1652                 ib_get_cached_pkey(&dev->ib_dev, (u8)sqp->port,\r
1653                                    wr->dgrm.ud.pkey_index, &pkey);\r
1654         sqp->ud_header.bth.pkey = pkey;\r
1655         sqp->ud_header.bth.destination_qpn = wr->dgrm.ud.remote_qp;\r
1656         sqp->ud_header.bth.psn = cl_hton32((sqp->send_psn++) & ((1 << 24) - 1));\r
1657         sqp->ud_header.deth.qkey = wr->dgrm.ud.remote_qkey & 0x00000080 ?\r
1658                                                cl_hton32(sqp->qkey) : wr->dgrm.ud.remote_qkey;\r
1659         sqp->ud_header.deth.source_qpn = cl_hton32(sqp->qp.ibqp.qp_num);\r
1660 \r
1661         header_size = ib_ud_header_pack(&sqp->ud_header,\r
1662                                         (u8*)sqp->sg.page +\r
1663                                         ind * MTHCA_UD_HEADER_SIZE);\r
1664 \r
1665         data->byte_count = cl_hton32(header_size);\r
1666         data->lkey       = cl_hton32(to_mpd(sqp->qp.ibqp.pd)->ntmr.ibmr.lkey);\r
1667         data->addr       = CPU_2_BE64(sqp->sg.dma_address +\r
1668                                        ind * MTHCA_UD_HEADER_SIZE);\r
1669 \r
1670         return 0;\r
1671 }\r
1672 \r
1673 static inline int mthca_wq_overflow(struct mthca_wq *wq, int nreq,\r
1674                                     struct ib_cq *ib_cq)\r
1675 {\r
1676         unsigned cur;\r
1677         struct mthca_cq *cq;\r
1678         SPIN_LOCK_PREP(lh);\r
1679 \r
1680         cur = wq->head - wq->tail;\r
1681         if (likely((int)cur + nreq < wq->max))\r
1682                 return 0;\r
1683 \r
1684         cq = to_mcq(ib_cq);\r
1685         spin_lock_dpc(&cq->lock, &lh);\r
1686         cur = wq->head - wq->tail;\r
1687         spin_unlock_dpc(&lh);\r
1688 \r
1689         return (int)cur + nreq >= wq->max;\r
1690 }\r
1691 \r
1692 int mthca_tavor_post_send(struct ib_qp *ibqp, struct _ib_send_wr *wr,\r
1693                           struct _ib_send_wr **bad_wr)\r
1694 {\r
1695         struct mthca_dev *dev = to_mdev(ibqp->device);\r
1696         struct mthca_qp *qp = to_mqp(ibqp);\r
1697         u8 *wqe;\r
1698         u8 *prev_wqe;\r
1699         int err = 0;\r
1700         int nreq;\r
1701         int i;\r
1702         int size;\r
1703         int size0 = 0;\r
1704         u32 f0 = unlikely(wr->send_opt & IB_SEND_OPT_FENCE) ? MTHCA_SEND_DOORBELL_FENCE : 0;\r
1705         int ind;\r
1706         u8 op0 = 0;\r
1707         enum ib_wr_opcode opcode;\r
1708         SPIN_LOCK_PREP(lh);   \r
1709 \r
1710         spin_lock_irqsave(&qp->sq.lock, &lh);\r
1711     \r
1712         /* XXX check that state is OK to post send */\r
1713 \r
1714         ind = qp->sq.next_ind;\r
1715 \r
1716         for (nreq = 0; wr; ++nreq, wr = wr->p_next) {\r
1717                 if (mthca_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {\r
1718                         HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_QP,("SQ %06x full (%u head, %u tail,"\r
1719                                         " %d max, %d nreq)\n", qp->qpn,\r
1720                                         qp->sq.head, qp->sq.tail,\r
1721                                         qp->sq.max, nreq));\r
1722                         err = -ENOMEM;\r
1723                         if (bad_wr)\r
1724                                 *bad_wr = wr;\r
1725                         goto out;\r
1726                 }\r
1727 \r
1728                 wqe = get_send_wqe(qp, ind);\r
1729                 prev_wqe = qp->sq.last;\r
1730                 qp->sq.last = wqe;\r
1731                 opcode = conv_ibal_wr_opcode(wr);\r
1732 \r
1733                 ((struct mthca_next_seg *) wqe)->nda_op = 0;\r
1734                 ((struct mthca_next_seg *) wqe)->ee_nds = 0;\r
1735                 ((struct mthca_next_seg *) wqe)->flags =\r
1736                         ((wr->send_opt & IB_SEND_OPT_SIGNALED) ?\r
1737                          cl_hton32(MTHCA_NEXT_CQ_UPDATE) : 0) |\r
1738                         ((wr->send_opt & IB_SEND_OPT_SOLICITED) ?\r
1739                          cl_hton32(MTHCA_NEXT_SOLICIT) : 0)   |\r
1740                         cl_hton32(1);\r
1741                 if (opcode == MTHCA_OPCODE_SEND_IMM||\r
1742                     opcode == MTHCA_OPCODE_RDMA_WRITE_IMM)\r
1743                         ((struct mthca_next_seg *) wqe)->imm = wr->immediate_data;\r
1744 \r
1745                 wqe += sizeof (struct mthca_next_seg);\r
1746                 size = sizeof (struct mthca_next_seg) / 16;\r
1747 \r
1748                 switch (qp->transport) {\r
1749                 case RC:\r
1750                         switch (opcode) {\r
1751                         case MTHCA_OPCODE_ATOMIC_CS:\r
1752                         case MTHCA_OPCODE_ATOMIC_FA:\r
1753                                 ((struct mthca_raddr_seg *) wqe)->raddr =\r
1754                                         cl_hton64(wr->remote_ops.vaddr);\r
1755                                 ((struct mthca_raddr_seg *) wqe)->rkey =\r
1756                                         wr->remote_ops.rkey;\r
1757                                 ((struct mthca_raddr_seg *) wqe)->reserved = 0;\r
1758 \r
1759                                 wqe += sizeof (struct mthca_raddr_seg);\r
1760 \r
1761                                 if (opcode == MTHCA_OPCODE_ATOMIC_CS) {\r
1762                                         ((struct mthca_atomic_seg *) wqe)->swap_add =\r
1763                                                 (wr->remote_ops.atomic2);\r
1764                                         ((struct mthca_atomic_seg *) wqe)->compare =\r
1765                                                 (wr->remote_ops.atomic1);\r
1766                                 } else {\r
1767                                         ((struct mthca_atomic_seg *) wqe)->swap_add =\r
1768                                                 (wr->remote_ops.atomic1);\r
1769                                         ((struct mthca_atomic_seg *) wqe)->compare = 0;\r
1770                                 }\r
1771 \r
1772                                 wqe += sizeof (struct mthca_atomic_seg);\r
1773                                 size += (sizeof (struct mthca_raddr_seg) +\r
1774                                         sizeof (struct mthca_atomic_seg)) / 16 ;\r
1775                                 break;\r
1776 \r
1777                         case MTHCA_OPCODE_RDMA_READ:\r
1778                         case MTHCA_OPCODE_RDMA_WRITE:\r
1779                         case MTHCA_OPCODE_RDMA_WRITE_IMM:\r
1780                                 ((struct mthca_raddr_seg *) wqe)->raddr =\r
1781                                         cl_hton64(wr->remote_ops.vaddr);\r
1782                                 ((struct mthca_raddr_seg *) wqe)->rkey =\r
1783                                         wr->remote_ops.rkey;\r
1784                                 ((struct mthca_raddr_seg *) wqe)->reserved = 0;\r
1785                                 wqe += sizeof (struct mthca_raddr_seg);\r
1786                                 size += sizeof (struct mthca_raddr_seg) / 16;\r
1787                                 break;\r
1788 \r
1789                         default:\r
1790                                 /* No extra segments required for sends */\r
1791                                 break;\r
1792                         }\r
1793 \r
1794                         break;\r
1795 \r
1796                 case UC:\r
1797                         switch (opcode) {\r
1798                         case MTHCA_OPCODE_RDMA_WRITE:\r
1799                         case MTHCA_OPCODE_RDMA_WRITE_IMM:\r
1800                                 ((struct mthca_raddr_seg *) wqe)->raddr =\r
1801                                         cl_hton64(wr->remote_ops.vaddr);\r
1802                                 ((struct mthca_raddr_seg *) wqe)->rkey =\r
1803                                         wr->remote_ops.rkey;\r
1804                                 ((struct mthca_raddr_seg *) wqe)->reserved = 0;\r
1805                                 wqe += sizeof (struct mthca_raddr_seg);\r
1806                                 size += sizeof (struct mthca_raddr_seg) / 16;\r
1807                                 break;\r
1808 \r
1809                         default:\r
1810                                 /* No extra segments required for sends */\r
1811                                 break;\r
1812                         }\r
1813 \r
1814                         break;\r
1815 \r
1816                 case UD:\r
1817                         ((struct mthca_tavor_ud_seg *) wqe)->lkey =\r
1818                                 cl_hton32(to_mah((struct ib_ah *)wr->dgrm.ud.h_av)->key);\r
1819                         ((struct mthca_tavor_ud_seg *) wqe)->av_addr =\r
1820                                 cl_hton64(to_mah((struct ib_ah *)wr->dgrm.ud.h_av)->avdma);\r
1821                         ((struct mthca_tavor_ud_seg *) wqe)->dqpn = wr->dgrm.ud.remote_qp;\r
1822                         ((struct mthca_tavor_ud_seg *) wqe)->qkey = wr->dgrm.ud.remote_qkey;\r
1823 \r
1824                         wqe += sizeof (struct mthca_tavor_ud_seg);\r
1825                         size += sizeof (struct mthca_tavor_ud_seg) / 16;\r
1826                         break;\r
1827 \r
1828                 case MLX:\r
1829                         err = build_mlx_header(dev, to_msqp(qp), ind, wr,\r
1830                                                (void*)(wqe - sizeof (struct mthca_next_seg)),\r
1831                                                (void*)wqe);\r
1832                         if (err) {\r
1833                                 if (bad_wr)\r
1834                                         *bad_wr = wr;\r
1835                                 goto out;\r
1836                         }\r
1837                         wqe += sizeof (struct mthca_data_seg);\r
1838                         size += sizeof (struct mthca_data_seg) / 16;\r
1839                         break;\r
1840                 }\r
1841 \r
1842                 if ((int)(int)wr->num_ds > qp->sq.max_gs) {\r
1843                         HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_QP ,("SQ %06x too many gathers\n",qp->qpn));\r
1844                         err = -EINVAL;\r
1845                         if (bad_wr)\r
1846                                 *bad_wr = wr;\r
1847                         goto out;\r
1848                 }\r
1849                 if (wr->send_opt & IB_SEND_OPT_INLINE) {\r
1850                         if (wr->num_ds) {\r
1851                                 struct mthca_inline_seg *seg = (struct mthca_inline_seg *)wqe;\r
1852                                 uint32_t s = 0;\r
1853 \r
1854                                 wqe += sizeof *seg;\r
1855                                 for (i = 0; i < (int)wr->num_ds; ++i) {\r
1856                                         struct _ib_local_ds *sge = &wr->ds_array[i];\r
1857 \r
1858                                         s += sge->length;\r
1859 \r
1860                                         if (s > (uint32_t)qp->max_inline_data) {\r
1861                                                 err = -EINVAL;\r
1862                                                 if (bad_wr)\r
1863                                                         *bad_wr = wr;\r
1864                                                 goto out;\r
1865                                         }\r
1866 \r
1867                                         memcpy(wqe, (void *) (ULONG_PTR) sge->vaddr,\r
1868                                                sge->length);\r
1869                                         wqe += sge->length;\r
1870                                 }\r
1871 \r
1872                                 seg->byte_count = cl_hton32(MTHCA_INLINE_SEG | s);\r
1873                                 size += align(s + sizeof *seg, 16) / 16;\r
1874                         }\r
1875                 } else {\r
1876                         \r
1877                 for (i = 0; i < (int)wr->num_ds; ++i) {\r
1878                         ((struct mthca_data_seg *) wqe)->byte_count =\r
1879                                 cl_hton32(wr->ds_array[i].length);\r
1880                         ((struct mthca_data_seg *) wqe)->lkey =\r
1881                                 cl_hton32(wr->ds_array[i].lkey);\r
1882                         ((struct mthca_data_seg *) wqe)->addr =\r
1883                                 cl_hton64(wr->ds_array[i].vaddr);\r
1884                         wqe += sizeof (struct mthca_data_seg);\r
1885                         size += sizeof (struct mthca_data_seg) / 16;\r
1886                         HCA_PRINT(TRACE_LEVEL_VERBOSE ,HCA_DBG_QP ,("SQ %06x [%02x]  lkey 0x%08x vaddr 0x%I64x 0x%x\n",qp->qpn,i,\r
1887                                 (wr->ds_array[i].lkey),(wr->ds_array[i].vaddr),wr->ds_array[i].length));\r
1888                 }\r
1889         }\r
1890 \r
1891                 /* Add one more inline data segment for ICRC */\r
1892                 if (qp->transport == MLX) {\r
1893                         ((struct mthca_data_seg *) wqe)->byte_count =\r
1894                                 cl_hton32((unsigned long)((1 << 31) | 4));\r
1895                         ((u32 *) wqe)[1] = 0;\r
1896                         wqe += sizeof (struct mthca_data_seg);\r
1897                         size += sizeof (struct mthca_data_seg) / 16;\r
1898                 }\r
1899 \r
1900                 qp->wrid[ind + qp->rq.max] = wr->wr_id;\r
1901 \r
1902                 if (opcode == MTHCA_OPCODE_INVALID) {\r
1903                         HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_QP ,("SQ %06x opcode invalid\n",qp->qpn));\r
1904                         err = -EINVAL;\r
1905                         if (bad_wr)\r
1906                                 *bad_wr = wr;\r
1907                         goto out;\r
1908                 }\r
1909 \r
1910                 ((struct mthca_next_seg *) prev_wqe)->nda_op =\r
1911                         cl_hton32(((ind << qp->sq.wqe_shift) +  \r
1912                         qp->send_wqe_offset) |opcode);\r
1913                 wmb();\r
1914                 ((struct mthca_next_seg *) prev_wqe)->ee_nds =\r
1915                         cl_hton32((size0 ? 0 : MTHCA_NEXT_DBD) | size |\r
1916                                 ((wr->send_opt & IB_SEND_OPT_FENCE) ?\r
1917                                 MTHCA_NEXT_FENCE : 0));\r
1918 \r
1919                 if (!size0) {\r
1920                         size0 = size;\r
1921                         op0   = opcode;\r
1922                 }\r
1923 \r
1924                 dump_wqe( TRACE_LEVEL_VERBOSE, (u32*)qp->sq.last,qp);\r
1925 \r
1926                 ++ind;\r
1927                 if (unlikely(ind >= qp->sq.max))\r
1928                         ind -= qp->sq.max;\r
1929         }\r
1930 \r
1931 out:\r
1932         if (likely(nreq)) {\r
1933                 __be32 doorbell[2];\r
1934 \r
1935                 doorbell[0] = cl_hton32(((qp->sq.next_ind << qp->sq.wqe_shift) +\r
1936                                            qp->send_wqe_offset) | f0 | op0);\r
1937                 doorbell[1] = cl_hton32((qp->qpn << 8) | size0);\r
1938 \r
1939                 wmb();\r
1940 \r
1941                 mthca_write64(doorbell,\r
1942                               dev->kar + MTHCA_SEND_DOORBELL,\r
1943                               MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));\r
1944         }\r
1945 \r
1946         qp->sq.next_ind = ind;\r
1947         qp->sq.head    += nreq;\r
1948         \r
1949     spin_unlock_irqrestore(&lh);   \r
1950         return err;\r
1951 }\r
1952 \r
1953 int mthca_tavor_post_recv(struct ib_qp *ibqp, struct _ib_recv_wr *wr,\r
1954                              struct _ib_recv_wr **bad_wr)\r
1955 {\r
1956         struct mthca_dev *dev = to_mdev(ibqp->device);\r
1957         struct mthca_qp *qp = to_mqp(ibqp);\r
1958         __be32 doorbell[2];\r
1959         int err = 0;\r
1960         int nreq;\r
1961         int i;\r
1962         int size;\r
1963         int size0 = 0;\r
1964         int ind;\r
1965         u8 *wqe;\r
1966         u8 *prev_wqe;\r
1967         SPIN_LOCK_PREP(lh);\r
1968 \r
1969         spin_lock_irqsave(&qp->rq.lock, &lh);\r
1970 \r
1971         /* XXX check that state is OK to post receive */\r
1972 \r
1973         ind = qp->rq.next_ind;\r
1974 \r
1975         for (nreq = 0; wr; ++nreq, wr = wr->p_next) {\r
1976                 if (unlikely(nreq == MTHCA_TAVOR_MAX_WQES_PER_RECV_DB)) {\r
1977                         nreq = 0;\r
1978 \r
1979                         doorbell[0] = cl_hton32((qp->rq.next_ind << qp->rq.wqe_shift) | size0);\r
1980                         doorbell[1] = cl_hton32(qp->qpn << 8);\r
1981 \r
1982                         wmb();\r
1983 \r
1984                         mthca_write64(doorbell, dev->kar + MTHCA_RECV_DOORBELL,\r
1985                       MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));\r
1986 \r
1987                         qp->rq.head += MTHCA_TAVOR_MAX_WQES_PER_RECV_DB;\r
1988                         size0 = 0;\r
1989                 }\r
1990                 if (mthca_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {\r
1991                         HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_QP,("RQ %06x full (%u head, %u tail,"\r
1992                                         " %d max, %d nreq)\n", qp->qpn,\r
1993                                         qp->rq.head, qp->rq.tail,\r
1994                                         qp->rq.max, nreq));\r
1995                         err = -ENOMEM;\r
1996                         if (bad_wr)\r
1997                                 *bad_wr = wr;\r
1998                         goto out;\r
1999                 }\r
2000 \r
2001                 wqe = get_recv_wqe(qp, ind);\r
2002                 prev_wqe = qp->rq.last;\r
2003                 qp->rq.last = wqe;\r
2004 \r
2005                 ((struct mthca_next_seg *) wqe)->nda_op = 0;\r
2006                 ((struct mthca_next_seg *) wqe)->ee_nds =\r
2007                         cl_hton32(MTHCA_NEXT_DBD);\r
2008                 ((struct mthca_next_seg *) wqe)->flags = 0;\r
2009 \r
2010                 wqe += sizeof (struct mthca_next_seg);\r
2011                 size = sizeof (struct mthca_next_seg) / 16;\r
2012 \r
2013                 if (unlikely((int)wr->num_ds > qp->rq.max_gs)) {\r
2014                         HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_QP ,("RQ %06x too many gathers\n",qp->qpn));\r
2015                         err = -EINVAL;\r
2016                         if (bad_wr)\r
2017                                 *bad_wr = wr;\r
2018                         goto out;\r
2019                 }\r
2020 \r
2021                 for (i = 0; i < (int)wr->num_ds; ++i) {\r
2022                         ((struct mthca_data_seg *) wqe)->byte_count =\r
2023                                 cl_hton32(wr->ds_array[i].length);\r
2024                         ((struct mthca_data_seg *) wqe)->lkey =\r
2025                                 cl_hton32(wr->ds_array[i].lkey);\r
2026                         ((struct mthca_data_seg *) wqe)->addr =\r
2027                                 cl_hton64(wr->ds_array[i].vaddr);\r
2028                         wqe += sizeof (struct mthca_data_seg);\r
2029                         size += sizeof (struct mthca_data_seg) / 16;\r
2030 //                      HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_QP ,("RQ %06x [%02x]  lkey 0x%08x vaddr 0x%I64x 0x %x 0x%08x\n",i,qp->qpn,\r
2031 //                              (wr->ds_array[i].lkey),(wr->ds_array[i].vaddr),wr->ds_array[i].length, wr->wr_id));\r
2032                 }\r
2033 \r
2034                 qp->wrid[ind] = wr->wr_id;\r
2035 \r
2036                 ((struct mthca_next_seg *) prev_wqe)->nda_op =\r
2037                         cl_hton32((ind << qp->rq.wqe_shift) | 1);\r
2038                 wmb();\r
2039                 ((struct mthca_next_seg *) prev_wqe)->ee_nds =\r
2040                         cl_hton32(MTHCA_NEXT_DBD | size);\r
2041 \r
2042                 if (!size0)\r
2043                         size0 = size;\r
2044 \r
2045                 dump_wqe(TRACE_LEVEL_VERBOSE,  (u32*)wqe ,qp);\r
2046                 \r
2047                 ++ind;\r
2048                 if (unlikely(ind >= qp->rq.max))\r
2049                         ind -= qp->rq.max;\r
2050         }\r
2051 \r
2052 out:\r
2053         if (likely(nreq)) {\r
2054                 doorbell[0] = cl_hton32((qp->rq.next_ind << qp->rq.wqe_shift) | size0);\r
2055                 doorbell[1] = cl_hton32((qp->qpn << 8) | (nreq & 255));\r
2056 \r
2057                 wmb();\r
2058 \r
2059                 mthca_write64(doorbell, dev->kar + MTHCA_RECV_DOORBELL,\r
2060               MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));\r
2061         }\r
2062 \r
2063         qp->rq.next_ind = ind;\r
2064         qp->rq.head    += nreq;\r
2065 \r
2066         spin_unlock_irqrestore(&lh);\r
2067         return err;\r
2068 }\r
2069 \r
2070 int mthca_arbel_post_send(struct ib_qp *ibqp, struct _ib_send_wr *wr,\r
2071                           struct _ib_send_wr **bad_wr)\r
2072 {\r
2073         struct mthca_dev *dev = to_mdev(ibqp->device);\r
2074         struct mthca_qp *qp = to_mqp(ibqp);\r
2075         __be32 doorbell[2];\r
2076         u8 *wqe;\r
2077         u8 *prev_wqe;\r
2078         int err = 0;\r
2079         int nreq;\r
2080         int i;\r
2081         int size;\r
2082         int size0 = 0;\r
2083         u32 f0 = unlikely(wr->send_opt & IB_SEND_OPT_FENCE) ? MTHCA_SEND_DOORBELL_FENCE : 0;\r
2084         int ind;\r
2085         u8 op0 = 0;\r
2086         enum ib_wr_opcode opcode;\r
2087         SPIN_LOCK_PREP(lh);\r
2088 \r
2089         spin_lock_irqsave(&qp->sq.lock, &lh);\r
2090 \r
2091         /* XXX check that state is OK to post send */\r
2092 \r
2093         ind = qp->sq.head & (qp->sq.max - 1);\r
2094 \r
2095         for (nreq = 0; wr; ++nreq, wr = wr->p_next) {\r
2096                 if (unlikely(nreq == MTHCA_ARBEL_MAX_WQES_PER_SEND_DB)) {\r
2097                         nreq = 0;\r
2098                         doorbell[0] = cl_hton32((MTHCA_ARBEL_MAX_WQES_PER_SEND_DB << 24) |\r
2099                                 ((qp->sq.head & 0xffff) << 8) |f0 | op0);\r
2100                         doorbell[1] = cl_hton32((qp->qpn << 8) | size0);\r
2101                         qp->sq.head += MTHCA_ARBEL_MAX_WQES_PER_SEND_DB;\r
2102                         size0 = 0;\r
2103                         f0 = unlikely(wr->send_opt & IB_SEND_OPT_FENCE) ? MTHCA_SEND_DOORBELL_FENCE : 0;\r
2104 \r
2105                         /*\r
2106                          * Make sure that descriptors are written before\r
2107                          * doorbell record.\r
2108                          */\r
2109                         wmb();\r
2110                         *qp->sq.db = cl_hton32(qp->sq.head & 0xffff);\r
2111 \r
2112                         /*\r
2113                          * Make sure doorbell record is written before we\r
2114                          * write MMIO send doorbell.\r
2115                          */\r
2116                         wmb();\r
2117                         mthca_write64(doorbell, dev->kar + MTHCA_SEND_DOORBELL,\r
2118                                 MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));\r
2119                 }\r
2120 \r
2121                 if (mthca_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {\r
2122                         HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_QP,("SQ %06x full (%u head, %u tail,"\r
2123                                         " %d max, %d nreq)\n", qp->qpn,\r
2124                                         qp->sq.head, qp->sq.tail,\r
2125                                         qp->sq.max, nreq));\r
2126                         err = -ENOMEM;\r
2127                         if (bad_wr)\r
2128                                 *bad_wr = wr;\r
2129                         goto out;\r
2130                 }\r
2131 \r
2132                 wqe = get_send_wqe(qp, ind);\r
2133                 prev_wqe = qp->sq.last;\r
2134                 qp->sq.last = wqe;\r
2135                 opcode = conv_ibal_wr_opcode(wr);\r
2136 \r
2137                 ((struct mthca_next_seg *) wqe)->flags =\r
2138                         ((wr->send_opt & IB_SEND_OPT_SIGNALED) ?\r
2139                          cl_hton32(MTHCA_NEXT_CQ_UPDATE) : 0) |\r
2140                         ((wr->send_opt & IB_SEND_OPT_SOLICITED) ?\r
2141                          cl_hton32(MTHCA_NEXT_SOLICIT) : 0)   |\r
2142                         ((wr->send_opt & IB_SEND_OPT_TX_IP_CSUM) ?\r
2143                          cl_hton32(MTHCA_NEXT_IP_CSUM) : 0) |\r
2144                         ((wr->send_opt & IB_SEND_OPT_TX_TCP_UDP_CSUM) ?\r
2145                          cl_hton32(MTHCA_NEXT_TCP_UDP_CSUM) : 0) |\r
2146                          cl_hton32(1);\r
2147                 if (opcode == MTHCA_OPCODE_SEND_IMM||\r
2148                         opcode == MTHCA_OPCODE_RDMA_WRITE_IMM)\r
2149                         ((struct mthca_next_seg *) wqe)->imm = wr->immediate_data;\r
2150 \r
2151                 wqe += sizeof (struct mthca_next_seg);\r
2152                 size = sizeof (struct mthca_next_seg) / 16;\r
2153 \r
2154                 switch (qp->transport) {\r
2155                 case RC:\r
2156                         switch (opcode) {\r
2157                         case MTHCA_OPCODE_ATOMIC_CS:\r
2158                         case MTHCA_OPCODE_ATOMIC_FA:\r
2159                                 ((struct mthca_raddr_seg *) wqe)->raddr =\r
2160                                         cl_hton64(wr->remote_ops.vaddr);\r
2161                                 ((struct mthca_raddr_seg *) wqe)->rkey =\r
2162                                         wr->remote_ops.rkey;\r
2163                                 ((struct mthca_raddr_seg *) wqe)->reserved = 0;\r
2164 \r
2165                                 wqe += sizeof (struct mthca_raddr_seg);\r
2166 \r
2167                                 if (opcode == MTHCA_OPCODE_ATOMIC_CS) {\r
2168                                         ((struct mthca_atomic_seg *) wqe)->swap_add =\r
2169                                                 (wr->remote_ops.atomic2);\r
2170                                         ((struct mthca_atomic_seg *) wqe)->compare =\r
2171                                                 (wr->remote_ops.atomic1);\r
2172                                 } else {\r
2173                                         ((struct mthca_atomic_seg *) wqe)->swap_add =\r
2174                                                 (wr->remote_ops.atomic1);\r
2175                                         ((struct mthca_atomic_seg *) wqe)->compare = 0;\r
2176                                 }\r
2177 \r
2178                                 wqe += sizeof (struct mthca_atomic_seg);\r
2179                                 size += (sizeof (struct mthca_raddr_seg) +\r
2180                                         sizeof (struct mthca_atomic_seg)) / 16 ;\r
2181                                 break;\r
2182 \r
2183                         case MTHCA_OPCODE_RDMA_READ:\r
2184                         case MTHCA_OPCODE_RDMA_WRITE:\r
2185                         case MTHCA_OPCODE_RDMA_WRITE_IMM:\r
2186                                 ((struct mthca_raddr_seg *) wqe)->raddr =\r
2187                                         cl_hton64(wr->remote_ops.vaddr);\r
2188                                 ((struct mthca_raddr_seg *) wqe)->rkey =\r
2189                                         wr->remote_ops.rkey;\r
2190                                 ((struct mthca_raddr_seg *) wqe)->reserved = 0;\r
2191                                 wqe += sizeof (struct mthca_raddr_seg);\r
2192                                 size += sizeof (struct mthca_raddr_seg) / 16;\r
2193                                 break;\r
2194 \r
2195                         default:\r
2196                                 /* No extra segments required for sends */\r
2197                                 break;\r
2198                         }\r
2199 \r
2200                         break;\r
2201 \r
2202                 case UC:\r
2203                         switch (opcode) {\r
2204                         case MTHCA_OPCODE_RDMA_WRITE:\r
2205                         case MTHCA_OPCODE_RDMA_WRITE_IMM:\r
2206                                 ((struct mthca_raddr_seg *) wqe)->raddr =\r
2207                                         cl_hton64(wr->remote_ops.vaddr);\r
2208                                 ((struct mthca_raddr_seg *) wqe)->rkey =\r
2209                                         wr->remote_ops.rkey;\r
2210                                 ((struct mthca_raddr_seg *) wqe)->reserved = 0;\r
2211                                 wqe += sizeof (struct mthca_raddr_seg);\r
2212                                 size += sizeof (struct mthca_raddr_seg) / 16;\r
2213                                 break;\r
2214 \r
2215                         default:\r
2216                                 /* No extra segments required for sends */\r
2217                                 break;\r
2218                         }\r
2219 \r
2220                         break;\r
2221 \r
2222                 case UD:\r
2223                         memcpy(((struct mthca_arbel_ud_seg *) wqe)->av,\r
2224                                to_mah((struct ib_ah *)wr->dgrm.ud.h_av)->av, MTHCA_AV_SIZE);\r
2225                         ((struct mthca_arbel_ud_seg *) wqe)->dqpn = wr->dgrm.ud.remote_qp;\r
2226                         ((struct mthca_arbel_ud_seg *) wqe)->qkey = wr->dgrm.ud.remote_qkey;\r
2227 \r
2228                         wqe += sizeof (struct mthca_arbel_ud_seg);\r
2229                         size += sizeof (struct mthca_arbel_ud_seg) / 16;\r
2230                         break;\r
2231 \r
2232                 case MLX:\r
2233                         err = build_mlx_header(dev, to_msqp(qp), ind, wr,\r
2234                                                (void*)(wqe - sizeof (struct mthca_next_seg)),\r
2235                                                (void*)wqe);\r
2236                         if (err) {\r
2237                                 if (bad_wr)\r
2238                                         *bad_wr = wr;\r
2239                                 goto out;\r
2240                         }\r
2241                         wqe += sizeof (struct mthca_data_seg);\r
2242                         size += sizeof (struct mthca_data_seg) / 16;\r
2243                         break;\r
2244                 }\r
2245 \r
2246                 if ((int)wr->num_ds > qp->sq.max_gs) {\r
2247                         HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_QP ,("SQ %06x full too many gathers\n",qp->qpn));\r
2248                         err = -EINVAL;\r
2249                         if (bad_wr)\r
2250                                 *bad_wr = wr;\r
2251                         goto out;\r
2252                 }\r
2253         if (wr->send_opt & IB_SEND_OPT_INLINE) {\r
2254                         if (wr->num_ds) {\r
2255                                 struct mthca_inline_seg *seg = (struct mthca_inline_seg *)wqe;\r
2256                                 uint32_t s = 0;\r
2257 \r
2258                                 wqe += sizeof *seg;\r
2259                                 for (i = 0; i < (int)wr->num_ds; ++i) {\r
2260                                         struct _ib_local_ds *sge = &wr->ds_array[i];\r
2261 \r
2262                                         s += sge->length;\r
2263 \r
2264                                         if (s > (uint32_t)qp->max_inline_data) {\r
2265                                                 err = -EINVAL;\r
2266                                                 if (bad_wr)\r
2267                                                         *bad_wr = wr;\r
2268                                                 goto out;\r
2269                                         }\r
2270 \r
2271                                         memcpy(wqe, (void *) (uintptr_t) sge->vaddr,\r
2272                                                sge->length);\r
2273                                         wqe += sge->length;\r
2274                                 }\r
2275 \r
2276                                 seg->byte_count = cl_hton32(MTHCA_INLINE_SEG | s);\r
2277                                 size += align(s + sizeof *seg, 16) / 16;\r
2278                         }\r
2279                 } else {\r
2280                 for (i = 0; i < (int)wr->num_ds; ++i) {\r
2281                         ((struct mthca_data_seg *) wqe)->byte_count =\r
2282                                 cl_hton32(wr->ds_array[i].length);\r
2283                         ((struct mthca_data_seg *) wqe)->lkey =\r
2284                                 cl_hton32(wr->ds_array[i].lkey);\r
2285                         ((struct mthca_data_seg *) wqe)->addr =\r
2286                                 cl_hton64(wr->ds_array[i].vaddr);\r
2287                         wqe += sizeof (struct mthca_data_seg);\r
2288                         size += sizeof (struct mthca_data_seg) / 16;\r
2289                 }\r
2290         }\r
2291 \r
2292                 /* Add one more inline data segment for ICRC */\r
2293                 if (qp->transport == MLX) {\r
2294                         ((struct mthca_data_seg *) wqe)->byte_count =\r
2295                                 cl_hton32((unsigned long)((1 << 31) | 4));\r
2296                         ((u32 *) wqe)[1] = 0;\r
2297                         wqe += sizeof (struct mthca_data_seg);\r
2298                         size += sizeof (struct mthca_data_seg) / 16;\r
2299                 }\r
2300 \r
2301                 qp->wrid[ind + qp->rq.max] = wr->wr_id;\r
2302 \r
2303                 if (opcode == MTHCA_OPCODE_INVALID) {\r
2304                         HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_QP ,("SQ %06x opcode invalid\n",qp->qpn));\r
2305                         err = -EINVAL;\r
2306                         if (bad_wr)\r
2307                                 *bad_wr = wr;\r
2308                         goto out;\r
2309                 }\r
2310 \r
2311                 ((struct mthca_next_seg *) prev_wqe)->nda_op =\r
2312                         cl_hton32(((ind << qp->sq.wqe_shift) +\r
2313                         qp->send_wqe_offset) |opcode);\r
2314                 wmb();\r
2315                 ((struct mthca_next_seg *) prev_wqe)->ee_nds =\r
2316                         cl_hton32(MTHCA_NEXT_DBD | size |\r
2317                         ((wr->send_opt & IB_SEND_OPT_FENCE) ?\r
2318                         MTHCA_NEXT_FENCE : 0));\r
2319                 \r
2320                 if (!size0) {\r
2321                         size0 = size;\r
2322                         op0   = opcode;\r
2323                 }\r
2324 \r
2325                 ++ind;\r
2326                 if (unlikely(ind >= qp->sq.max))\r
2327                         ind -= qp->sq.max;\r
2328         }\r
2329 \r
2330 out:\r
2331         if (likely(nreq)) {\r
2332                 doorbell[0] = cl_hton32((nreq << 24) |\r
2333                         ((qp->sq.head & 0xffff) << 8) |f0 | op0);\r
2334                 doorbell[1] = cl_hton32((qp->qpn << 8) | size0);\r
2335                 qp->sq.head += nreq;\r
2336 \r
2337                 /*\r
2338                  * Make sure that descriptors are written before\r
2339                  * doorbell record.\r
2340                  */\r
2341                 wmb();\r
2342                 *qp->sq.db = cl_hton32(qp->sq.head & 0xffff);\r
2343 \r
2344                 /*\r
2345                  * Make sure doorbell record is written before we\r
2346                  * write MMIO send doorbell.\r
2347                  */\r
2348                 wmb();\r
2349                 mthca_write64(doorbell,\r
2350                               dev->kar + MTHCA_SEND_DOORBELL,\r
2351                               MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));\r
2352         }\r
2353 \r
2354         spin_unlock_irqrestore(&lh);\r
2355         return err;\r
2356 }\r
2357 \r
2358 int mthca_arbel_post_recv(struct ib_qp *ibqp, struct _ib_recv_wr *wr,\r
2359                              struct _ib_recv_wr **bad_wr)\r
2360 {\r
2361         struct mthca_qp *qp = to_mqp(ibqp);\r
2362         int err = 0;\r
2363         int nreq;\r
2364         int ind;\r
2365         int i;\r
2366         u8 *wqe;\r
2367         SPIN_LOCK_PREP(lh);\r
2368 \r
2369         spin_lock_irqsave(&qp->rq.lock, &lh);\r
2370 \r
2371         /* XXX check that state is OK to post receive */\r
2372 \r
2373         ind = qp->rq.head & (qp->rq.max - 1);\r
2374 \r
2375         for (nreq = 0; wr; ++nreq, wr = wr->p_next) {\r
2376                 if (mthca_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {\r
2377                         HCA_PRINT(TRACE_LEVEL_ERROR,HCA_DBG_QP,("RQ %06x full (%u head, %u tail,"\r
2378                                         " %d max, %d nreq)\n", qp->qpn,\r
2379                                         qp->rq.head, qp->rq.tail,\r
2380                                         qp->rq.max, nreq));\r
2381                         err = -ENOMEM;\r
2382                         if (bad_wr)\r
2383                                 *bad_wr = wr;\r
2384                         goto out;\r
2385                 }\r
2386 \r
2387                 wqe = get_recv_wqe(qp, ind);\r
2388 \r
2389                 ((struct mthca_next_seg *) wqe)->flags = 0;\r
2390 \r
2391                 wqe += sizeof (struct mthca_next_seg);\r
2392 \r
2393                 if (unlikely((int)wr->num_ds > qp->rq.max_gs)) {\r
2394                         HCA_PRINT(TRACE_LEVEL_ERROR  ,HCA_DBG_QP ,("RQ %06x full too many scatter\n",qp->qpn));\r
2395                         err = -EINVAL;\r
2396                         if (bad_wr)\r
2397                                 *bad_wr = wr;\r
2398                         goto out;\r
2399                 }\r
2400 \r
2401                 for (i = 0; i < (int)wr->num_ds; ++i) {\r
2402                         ((struct mthca_data_seg *) wqe)->byte_count =\r
2403                                 cl_hton32(wr->ds_array[i].length);\r
2404                         ((struct mthca_data_seg *) wqe)->lkey =\r
2405                                 cl_hton32(wr->ds_array[i].lkey);\r
2406                         ((struct mthca_data_seg *) wqe)->addr =\r
2407                                 cl_hton64(wr->ds_array[i].vaddr);\r
2408                         wqe += sizeof (struct mthca_data_seg);\r
2409                 }\r
2410 \r
2411                 if (i < qp->rq.max_gs) {\r
2412                         ((struct mthca_data_seg *) wqe)->byte_count = 0;\r
2413                         ((struct mthca_data_seg *) wqe)->lkey = cl_hton32(MTHCA_INVAL_LKEY);\r
2414                         ((struct mthca_data_seg *) wqe)->addr = 0;\r
2415                 }\r
2416 \r
2417                 qp->wrid[ind] = wr->wr_id;\r
2418 \r
2419                 ++ind;\r
2420                 if (unlikely(ind >= qp->rq.max))\r
2421                         ind -= qp->rq.max;\r
2422         }\r
2423 out:\r
2424         if (likely(nreq)) {\r
2425                 qp->rq.head += nreq;\r
2426 \r
2427                 /*\r
2428                  * Make sure that descriptors are written before\r
2429                  * doorbell record.\r
2430                  */\r
2431                 wmb();\r
2432                 *qp->rq.db = cl_hton32(qp->rq.head & 0xffff);\r
2433         }\r
2434 \r
2435         spin_unlock_irqrestore(&lh);\r
2436         return err;\r
2437 }\r
2438 \r
2439 void mthca_free_err_wqe(struct mthca_dev *dev, struct mthca_qp *qp, int is_send,\r
2440                        int index, int *dbd, __be32 *new_wqe)\r
2441 {\r
2442         struct mthca_next_seg *next;\r
2443 \r
2444         UNREFERENCED_PARAMETER(dev);\r
2445         \r
2446         /*\r
2447          * For SRQs, all WQEs generate a CQE, so we're always at the\r
2448          * end of the doorbell chain.\r
2449          */\r
2450         if (qp->ibqp.srq) {\r
2451                 *new_wqe = 0;\r
2452                 return;\r
2453         }\r
2454 \r
2455         if (is_send)\r
2456                 next = get_send_wqe(qp, index);\r
2457         else\r
2458                 next = get_recv_wqe(qp, index);\r
2459 \r
2460         *dbd = !!(next->ee_nds & cl_hton32(MTHCA_NEXT_DBD));\r
2461         if (next->ee_nds & cl_hton32(0x3f))\r
2462                 *new_wqe = (next->nda_op & cl_hton32((unsigned long)~0x3f)) |\r
2463                         (next->ee_nds & cl_hton32(0x3f));\r
2464         else\r
2465                 *new_wqe = 0;\r
2466 }\r
2467 \r
2468 int mthca_init_qp_table(struct mthca_dev *dev)\r
2469 {\r
2470         int err;\r
2471         u8 status;\r
2472         int i;\r
2473 \r
2474         spin_lock_init(&dev->qp_table.lock);\r
2475         fill_state_table();\r
2476 \r
2477         /*\r
2478          * We reserve 2 extra QPs per port for the special QPs.  The\r
2479          * special QP for port 1 has to be even, so round up.\r
2480          */\r
2481         dev->qp_table.sqp_start = (dev->limits.reserved_qps + 1) & ~1UL;\r
2482         err = mthca_alloc_init(&dev->qp_table.alloc,\r
2483                                dev->limits.num_qps,\r
2484                                (1 << 24) - 1,\r
2485                                dev->qp_table.sqp_start +\r
2486                                MTHCA_MAX_PORTS * 2);\r
2487         if (err)\r
2488                 return err;\r
2489 \r
2490         err = mthca_array_init(&dev->qp_table.qp,\r
2491                                dev->limits.num_qps);\r
2492         if (err) {\r
2493                 mthca_alloc_cleanup(&dev->qp_table.alloc);\r
2494                 return err;\r
2495         }\r
2496 \r
2497         for (i = 0; i < 2; ++i) {\r
2498                 err = mthca_CONF_SPECIAL_QP(dev, i ? IB_QPT_QP1 : IB_QPT_QP0,\r
2499                                             dev->qp_table.sqp_start + i * 2,\r
2500                                             &status);\r
2501                 if (err)\r
2502                         goto err_out;\r
2503                 if (status) {\r
2504                         HCA_PRINT(TRACE_LEVEL_ERROR ,HCA_DBG_QP,("CONF_SPECIAL_QP returned "\r
2505                                    "status %02x, aborting.\n",\r
2506                                    status));\r
2507                         err = -EINVAL;\r
2508                         goto err_out;\r
2509                 }\r
2510         }\r
2511         return 0;\r
2512 \r
2513  err_out:\r
2514         mthca_CONF_SPECIAL_QP(dev, IB_QPT_QP1, 0, &status);\r
2515         mthca_CONF_SPECIAL_QP(dev, IB_QPT_QP0, 0, &status);\r
2516 \r
2517         mthca_array_cleanup(&dev->qp_table.qp, dev->limits.num_qps);\r
2518         mthca_alloc_cleanup(&dev->qp_table.alloc);\r
2519 \r
2520         return err;\r
2521 }\r
2522 \r
2523 void mthca_cleanup_qp_table(struct mthca_dev *dev)\r
2524 {\r
2525         u8 status;\r
2526 \r
2527         mthca_CONF_SPECIAL_QP(dev, IB_QPT_QP1, 0, &status);\r
2528         mthca_CONF_SPECIAL_QP(dev, IB_QPT_QP0, 0, &status);\r
2529 \r
2530         mthca_array_cleanup(&dev->qp_table.qp, dev->limits.num_qps);\r
2531         mthca_alloc_cleanup(&dev->qp_table.alloc);\r
2532 }\r
2533 \r
2534 \r
2535 \r