srpt/src/ib_srpt.c

   1 /*
   2  * Copyright (c) 2006 - 2009 Mellanox Technology Inc.  All rights reserved.
   3  * Copyright (C) 2008 Vladislav Bolkhovitin <vst@vlnb.net>
   4  * Copyright (C) 2008 - 2009 Bart Van Assche <bart.vanassche@gmail.com>
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  *
  34  */
  35
  36 #include <linux/module.h>
  37 #include <linux/init.h>
  38 #include <linux/slab.h>
  39 #include <linux/err.h>
  40 #include <linux/ctype.h>
  41 #include <linux/string.h>
  42 #include <linux/kthread.h>
  43 #include <asm/atomic.h>
  44 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
  45 #include <linux/proc_fs.h>
  46 #include <linux/seq_file.h>
  47 #endif
  48 #include "ib_srpt.h"
  49 #include "scst_debug.h"
  50
  51 /* Name of this kernel module. */
  52 #define DRV_NAME                "ib_srpt"
  53 /* Prefix for printk() kernel messages. */
  54 #define LOG_PFX                 DRV_NAME ": "
  55 #define DRV_VERSION             "1.0.1"
  56 #define DRV_RELDATE             "July 10, 2008"
  57 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
  58 /* Flags to be used in SCST debug tracing statements. */
  59 #define DEFAULT_SRPT_TRACE_FLAGS (TRACE_OUT_OF_MEM | TRACE_MINOR \
  60                                   | TRACE_MGMT | TRACE_SPECIAL)
  61 /* Name of the entry that will be created under /proc/scsi_tgt/ib_srpt. */
  62 #define SRPT_PROC_TRACE_LEVEL_NAME      "trace_level"
  63 #endif
  64
  65 #define MELLANOX_SRPT_ID_STRING "SCST SRP target"
  66
  67 MODULE_AUTHOR("Vu Pham");
  68 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
  69                    "v" DRV_VERSION " (" DRV_RELDATE ")");
  70 MODULE_LICENSE("Dual BSD/GPL");
  71
  72 struct srpt_thread {
  73         /* Protects thread_ioctx_list. */
  74         spinlock_t thread_lock;
  75         /* I/O contexts to be processed by the kernel thread. */
  76         struct list_head thread_ioctx_list;
  77         /* SRPT kernel thread. */
  78         struct task_struct *thread;
  79 };
  80
  81 /*
  82  * Global Variables
  83  */
  84
  85 static u64 srpt_service_guid;
  86 /* List of srpt_device structures. */
  87 static atomic_t srpt_device_count;
  88 static int use_port_guid_in_session_name;
  89 static int thread;
  90 static struct srpt_thread srpt_thread;
  91 static DECLARE_WAIT_QUEUE_HEAD(ioctx_list_waitQ);
  92 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
  93 static unsigned long trace_flag = DEFAULT_SRPT_TRACE_FLAGS;
  94 module_param(trace_flag, long, 0644);
  95 MODULE_PARM_DESC(trace_flag,
  96                  "Trace flags for the ib_srpt kernel module.");
  97 #endif
  98
  99 module_param(thread, int, 0444);
 100 MODULE_PARM_DESC(thread,
 101                  "Executing ioctx in thread context. Default 0, i.e. soft IRQ, "
 102                  "where possible");
 103
 104 module_param(use_port_guid_in_session_name, bool, 0444);
 105 MODULE_PARM_DESC(use_port_guid_in_session_name,
 106                  "Use target port ID in the SCST session name such that"
 107                  " redundant paths between multiport systems can be masked.");
 108
 109 static void srpt_add_one(struct ib_device *device);
 110 static void srpt_remove_one(struct ib_device *device);
 111 static void srpt_unregister_mad_agent(struct srpt_device *sdev);
 112 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt);
 113
 114 static struct ib_client srpt_client = {
 115         .name = DRV_NAME,
 116         .add = srpt_add_one,
 117         .remove = srpt_remove_one
 118 };
 119
 120 /**
 121  * Atomically test and set the channel state.
 122  * @ch: RDMA channel.
 123  * @old: channel state to compare with.
 124  * @new: state to change the channel state to if the current state matches the
 125  *       argument 'old'.
 126  *
 127  * Returns true if the channel state matched old upon entry of this function,
 128  * and false otherwise.
 129  */
 130 static bool srpt_test_and_set_channel_state(struct srpt_rdma_ch *ch,
 131                                             enum rdma_ch_state old,
 132                                             enum rdma_ch_state new)
 133 {
 134         unsigned long flags;
 135         enum rdma_ch_state cur;
 136
 137         spin_lock_irqsave(&ch->spinlock, flags);
 138         cur = ch->state;
 139         if (cur == old)
 140                 ch->state = new;
 141         spin_unlock_irqrestore(&ch->spinlock, flags);
 142
 143         return cur == old;
 144 }
 145
 146 /*
 147  * Callback function called by the InfiniBand core when an asynchronous IB
 148  * event occurs. This callback may occur in interrupt context. See also
 149  * section 11.5.2, Set Asynchronous Event Handler in the InfiniBand
 150  * Architecture Specification.
 151  */
 152 static void srpt_event_handler(struct ib_event_handler *handler,
 153                                struct ib_event *event)
 154 {
 155         struct srpt_device *sdev;
 156         struct srpt_port *sport;
 157
 158         sdev = ib_get_client_data(event->device, &srpt_client);
 159         if (!sdev || sdev->device != event->device)
 160                 return;
 161
 162         TRACE_DBG("ASYNC event= %d on device= %s",
 163                   event->event, sdev->device->name);
 164
 165         switch (event->event) {
 166         case IB_EVENT_PORT_ERR:
 167                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
 168                         sport = &sdev->port[event->element.port_num - 1];
 169                         sport->lid = 0;
 170                         sport->sm_lid = 0;
 171                 }
 172                 break;
 173         case IB_EVENT_PORT_ACTIVE:
 174         case IB_EVENT_LID_CHANGE:
 175         case IB_EVENT_PKEY_CHANGE:
 176         case IB_EVENT_SM_CHANGE:
 177         case IB_EVENT_CLIENT_REREGISTER:
 178                 /*
 179                  * Refresh port data asynchronously. Note: it is safe to call
 180                  * schedule_work() even if &sport->work is already on the
 181                  * global workqueue because schedule_work() tests for the
 182                  * work_pending() condition before adding &sport->work to the
 183                  * global work queue.
 184                  */
 185                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
 186                         sport = &sdev->port[event->element.port_num - 1];
 187                         if (!sport->lid && !sport->sm_lid)
 188                                 schedule_work(&sport->work);
 189                 }
 190                 break;
 191         default:
 192                 break;
 193         }
 194
 195 }
 196
 197 /*
 198  * Callback function called by the InfiniBand core for SRQ (shared receive
 199  * queue) events.
 200  */
 201 static void srpt_srq_event(struct ib_event *event, void *ctx)
 202 {
 203         TRACE_DBG("SRQ event %d", event->event);
 204 }
 205
 206 /*
 207  * Callback function called by the InfiniBand core for QP (queue pair) events.
 208  */
 209 static void srpt_qp_event(struct ib_event *event, void *ctx)
 210 {
 211         struct srpt_rdma_ch *ch = ctx;
 212
 213         TRACE_DBG("QP event %d on cm_id=%p sess_name=%s state=%d",
 214                   event->event, ch->cm_id, ch->sess_name, ch->state);
 215
 216         switch (event->event) {
 217         case IB_EVENT_COMM_EST:
 218 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20) || defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
 219                 ib_cm_notify(ch->cm_id, event->event);
 220 #else
 221                 /* Vanilla 2.6.19 kernel (or before) without OFED. */
 222                 PRINT_ERROR("%s", "how to perform ib_cm_notify() on a"
 223                             " vanilla 2.6.18 kernel ???");
 224 #endif
 225                 break;
 226         case IB_EVENT_QP_LAST_WQE_REACHED:
 227                 if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_LIVE,
 228                                         RDMA_CHANNEL_DISCONNECTING)) {
 229                         PRINT_INFO("disconnected session %s.", ch->sess_name);
 230                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
 231                 }
 232                 break;
 233         default:
 234                 break;
 235         }
 236 }
 237
 238 /*
 239  * Helper function for filling in an InfiniBand IOUnitInfo structure. Copies
 240  * the lowest four bits of value in element slot of the array of four bit
 241  * elements called c_list (controller list). The index slot is one-based.
 242  *
 243  * @pre 1 <= slot && 0 <= value && value < 16
 244  */
 245 static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
 246 {
 247         u16 id;
 248         u8 tmp;
 249
 250         id = (slot - 1) / 2;
 251         if (slot & 0x1) {
 252                 tmp = c_list[id] & 0xf;
 253                 c_list[id] = (value << 4) | tmp;
 254         } else {
 255                 tmp = c_list[id] & 0xf0;
 256                 c_list[id] = (value & 0xf) | tmp;
 257         }
 258 }
 259
 260 /*
 261  * Write InfiniBand ClassPortInfo to mad. See also section 16.3.3.1
 262  * ClassPortInfo in the InfiniBand Architecture Specification.
 263  */
 264 static void srpt_get_class_port_info(struct ib_dm_mad *mad)
 265 {
 266         struct ib_class_port_info *cif;
 267
 268         cif = (struct ib_class_port_info *)mad->data;
 269         memset(cif, 0, sizeof *cif);
 270         cif->base_version = 1;
 271         cif->class_version = 1;
 272         cif->resp_time_value = 20;
 273
 274         mad->mad_hdr.status = 0;
 275 }
 276
 277 /*
 278  * Write IOUnitInfo to mad. See also section 16.3.3.3 IOUnitInfo in the
 279  * InfiniBand Architecture Specification. See also section B.7,
 280  * table B.6 in the T10 SRP r16a document.
 281  */
 282 static void srpt_get_iou(struct ib_dm_mad *mad)
 283 {
 284         struct ib_dm_iou_info *ioui;
 285         u8 slot;
 286         int i;
 287
 288         ioui = (struct ib_dm_iou_info *)mad->data;
 289         ioui->change_id = 1;
 290         ioui->max_controllers = 16;
 291
 292         /* set present for slot 1 and empty for the rest */
 293         srpt_set_ioc(ioui->controller_list, 1, 1);
 294         for (i = 1, slot = 2; i < 16; i++, slot++)
 295                 srpt_set_ioc(ioui->controller_list, slot, 0);
 296
 297         mad->mad_hdr.status = 0;
 298 }
 299
 300 /*
 301  * Write IOControllerprofile to mad for I/O controller (sdev, slot). See also
 302  * section 16.3.3.4 IOControllerProfile in the InfiniBand Architecture
 303  * Specification. See also section B.7, table B.7 in the T10 SRP r16a
 304  * document.
 305  */
 306 static void srpt_get_ioc(struct srpt_device *sdev, u32 slot,
 307                          struct ib_dm_mad *mad)
 308 {
 309         struct ib_dm_ioc_profile *iocp;
 310
 311         iocp = (struct ib_dm_ioc_profile *)mad->data;
 312
 313         if (!slot || slot > 16) {
 314                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
 315                 return;
 316         }
 317
 318         if (slot > 2) {
 319                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
 320                 return;
 321         }
 322
 323         memset(iocp, 0, sizeof *iocp);
 324         strcpy(iocp->id_string, MELLANOX_SRPT_ID_STRING);
 325         iocp->guid = cpu_to_be64(srpt_service_guid);
 326         iocp->vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
 327         iocp->device_id = cpu_to_be32(sdev->dev_attr.vendor_part_id);
 328         iocp->device_version = cpu_to_be16(sdev->dev_attr.hw_ver);
 329         iocp->subsys_vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
 330         iocp->subsys_device_id = 0x0;
 331         iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
 332         iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
 333         iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
 334         iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
 335         iocp->send_queue_depth = cpu_to_be16(SRPT_SRQ_SIZE);
 336         iocp->rdma_read_depth = 4;
 337         iocp->send_size = cpu_to_be32(MAX_MESSAGE_SIZE);
 338         iocp->rdma_size = cpu_to_be32(MAX_RDMA_SIZE);
 339         iocp->num_svc_entries = 1;
 340         iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
 341                 SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
 342
 343         mad->mad_hdr.status = 0;
 344 }
 345
 346 /*
 347  * Device management: write ServiceEntries to mad for the given slot. See also
 348  * section 16.3.3.5 ServiceEntries in the InfiniBand Architecture
 349  * Specification. See also section B.7, table B.8 in the T10 SRP r16a document.
 350  */
 351 static void srpt_get_svc_entries(u64 ioc_guid,
 352                                  u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
 353 {
 354         struct ib_dm_svc_entries *svc_entries;
 355
 356         WARN_ON(!ioc_guid);
 357
 358         if (!slot || slot > 16) {
 359                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
 360                 return;
 361         }
 362
 363         if (slot > 2 || lo > hi || hi > 1) {
 364                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
 365                 return;
 366         }
 367
 368         svc_entries = (struct ib_dm_svc_entries *)mad->data;
 369         memset(svc_entries, 0, sizeof *svc_entries);
 370         svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid);
 371         snprintf(svc_entries->service_entries[0].name,
 372                  sizeof(svc_entries->service_entries[0].name),
 373                  "%s%016llx",
 374                  SRP_SERVICE_NAME_PREFIX,
 375                  (unsigned long long)ioc_guid);
 376
 377         mad->mad_hdr.status = 0;
 378 }
 379
 380 /*
 381  * Actual processing of a received MAD *rq_mad received through source port *sp
 382  * (MAD = InfiniBand management datagram). The response to be sent back is
 383  * written to *rsp_mad.
 384  */
 385 static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
 386                                  struct ib_dm_mad *rsp_mad)
 387 {
 388         u16 attr_id;
 389         u32 slot;
 390         u8 hi, lo;
 391
 392         attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
 393         switch (attr_id) {
 394         case DM_ATTR_CLASS_PORT_INFO:
 395                 srpt_get_class_port_info(rsp_mad);
 396                 break;
 397         case DM_ATTR_IOU_INFO:
 398                 srpt_get_iou(rsp_mad);
 399                 break;
 400         case DM_ATTR_IOC_PROFILE:
 401                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
 402                 srpt_get_ioc(sp->sdev, slot, rsp_mad);
 403                 break;
 404         case DM_ATTR_SVC_ENTRIES:
 405                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
 406                 hi = (u8) ((slot >> 8) & 0xff);
 407                 lo = (u8) (slot & 0xff);
 408                 slot = (u16) ((slot >> 16) & 0xffff);
 409                 srpt_get_svc_entries(srpt_service_guid,
 410                                      slot, hi, lo, rsp_mad);
 411                 break;
 412         default:
 413                 rsp_mad->mad_hdr.status =
 414                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
 415                 break;
 416         }
 417 }
 418
 419 /*
 420  * Callback function that is called by the InfiniBand core after transmission of
 421  * a MAD. (MAD = management datagram; AH = address handle.)
 422  */
 423 static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
 424                                   struct ib_mad_send_wc *mad_wc)
 425 {
 426         ib_destroy_ah(mad_wc->send_buf->ah);
 427         ib_free_send_mad(mad_wc->send_buf);
 428 }
 429
 430 /*
 431  * Callback function that is called by the InfiniBand core after reception of
 432  * a MAD (management datagram).
 433  */
 434 static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
 435                                   struct ib_mad_recv_wc *mad_wc)
 436 {
 437         struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
 438         struct ib_ah *ah;
 439         struct ib_mad_send_buf *rsp;
 440         struct ib_dm_mad *dm_mad;
 441
 442         if (!mad_wc || !mad_wc->recv_buf.mad)
 443                 return;
 444
 445         ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
 446                                   mad_wc->recv_buf.grh, mad_agent->port_num);
 447         if (IS_ERR(ah))
 448                 goto err;
 449
 450         BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR);
 451
 452         rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
 453                                  mad_wc->wc->pkey_index, 0,
 454                                  IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA,
 455                                  GFP_KERNEL);
 456         if (IS_ERR(rsp))
 457                 goto err_rsp;
 458
 459         rsp->ah = ah;
 460
 461         dm_mad = rsp->mad;
 462         memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof *dm_mad);
 463         dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
 464         dm_mad->mad_hdr.status = 0;
 465
 466         switch (mad_wc->recv_buf.mad->mad_hdr.method) {
 467         case IB_MGMT_METHOD_GET:
 468                 srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
 469                 break;
 470         case IB_MGMT_METHOD_SET:
 471                 dm_mad->mad_hdr.status =
 472                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
 473                 break;
 474         default:
 475                 dm_mad->mad_hdr.status =
 476                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
 477                 break;
 478         }
 479
 480         if (!ib_post_send_mad(rsp, NULL)) {
 481                 ib_free_recv_mad(mad_wc);
 482                 /* will destroy_ah & free_send_mad in send completion */
 483                 return;
 484         }
 485
 486         ib_free_send_mad(rsp);
 487
 488 err_rsp:
 489         ib_destroy_ah(ah);
 490 err:
 491         ib_free_recv_mad(mad_wc);
 492 }
 493
 494 /*
 495  * Enable InfiniBand management datagram processing, update the cached sm_lid,
 496  * lid and gid values, and register a callback function for processing MADs
 497  * on the specified port. It is safe to call this function more than once for
 498  * the same port.
 499  */
 500 static int srpt_refresh_port(struct srpt_port *sport)
 501 {
 502         struct ib_mad_reg_req reg_req;
 503         struct ib_port_modify port_modify;
 504         struct ib_port_attr port_attr;
 505         int ret;
 506
 507         memset(&port_modify, 0, sizeof port_modify);
 508         port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
 509         port_modify.clr_port_cap_mask = 0;
 510
 511         ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
 512         if (ret)
 513                 goto err_mod_port;
 514
 515         ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
 516         if (ret)
 517                 goto err_query_port;
 518
 519         sport->sm_lid = port_attr.sm_lid;
 520         sport->lid = port_attr.lid;
 521
 522         ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid);
 523         if (ret)
 524                 goto err_query_port;
 525
 526         if (!sport->mad_agent) {
 527                 memset(&reg_req, 0, sizeof reg_req);
 528                 reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
 529                 reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
 530                 set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
 531                 set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
 532
 533                 sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
 534                                                          sport->port,
 535                                                          IB_QPT_GSI,
 536                                                          &reg_req, 0,
 537                                                          srpt_mad_send_handler,
 538                                                          srpt_mad_recv_handler,
 539                                                          sport);
 540                 if (IS_ERR(sport->mad_agent)) {
 541                         ret = PTR_ERR(sport->mad_agent);
 542                         sport->mad_agent = NULL;
 543                         goto err_query_port;
 544                 }
 545         }
 546
 547         return 0;
 548
 549 err_query_port:
 550
 551         port_modify.set_port_cap_mask = 0;
 552         port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
 553         ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
 554
 555 err_mod_port:
 556
 557         return ret;
 558 }
 559
 560 /*
 561  * Unregister the callback function for processing MADs and disable MAD
 562  * processing for all ports of the specified device. It is safe to call this
 563  * function more than once for the same device.
 564  */
 565 static void srpt_unregister_mad_agent(struct srpt_device *sdev)
 566 {
 567         struct ib_port_modify port_modify = {
 568                 .clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP,
 569         };
 570         struct srpt_port *sport;
 571         int i;
 572
 573         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
 574                 sport = &sdev->port[i - 1];
 575                 WARN_ON(sport->port != i);
 576                 if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
 577                         PRINT_ERROR("%s", "disabling MAD processing failed.");
 578                 if (sport->mad_agent) {
 579                         ib_unregister_mad_agent(sport->mad_agent);
 580                         sport->mad_agent = NULL;
 581                 }
 582         }
 583 }
 584
 585 /*
 586  * Allocate and initialize an SRPT I/O context structure.
 587  */
 588 static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev)
 589 {
 590         struct srpt_ioctx *ioctx;
 591
 592         ioctx = kmalloc(sizeof *ioctx, GFP_KERNEL);
 593         if (!ioctx)
 594                 goto out;
 595
 596         ioctx->buf = kzalloc(MAX_MESSAGE_SIZE, GFP_KERNEL);
 597         if (!ioctx->buf)
 598                 goto out_free_ioctx;
 599
 600         ioctx->dma = dma_map_single(sdev->device->dma_device, ioctx->buf,
 601                                     MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
 602 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
 603         if (dma_mapping_error(sdev->device->dma_device, ioctx->dma))
 604 #else
 605         if (dma_mapping_error(ioctx->dma))
 606 #endif
 607                 goto out_free_buf;
 608
 609         return ioctx;
 610
 611 out_free_buf:
 612         kfree(ioctx->buf);
 613 out_free_ioctx:
 614         kfree(ioctx);
 615 out:
 616         return NULL;
 617 }
 618
 619 /*
 620  * Deallocate an SRPT I/O context structure.
 621  */
 622 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
 623 {
 624         if (!ioctx)
 625                 return;
 626
 627         dma_unmap_single(sdev->device->dma_device, ioctx->dma,
 628                          MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
 629         kfree(ioctx->buf);
 630         kfree(ioctx);
 631 }
 632
 633 /*
 634  * Associate a ring of SRPT I/O context structures with the specified device.
 635  */
 636 static int srpt_alloc_ioctx_ring(struct srpt_device *sdev)
 637 {
 638         int i;
 639
 640         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
 641                 sdev->ioctx_ring[i] = srpt_alloc_ioctx(sdev);
 642
 643                 if (!sdev->ioctx_ring[i])
 644                         goto err;
 645
 646                 sdev->ioctx_ring[i]->index = i;
 647         }
 648
 649         return 0;
 650
 651 err:
 652         while (--i > 0) {
 653                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
 654                 sdev->ioctx_ring[i] = NULL;
 655         }
 656         return -ENOMEM;
 657 }
 658
 659 /* Free the ring of SRPT I/O context structures. */
 660 static void srpt_free_ioctx_ring(struct srpt_device *sdev)
 661 {
 662         int i;
 663
 664         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
 665                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
 666                 sdev->ioctx_ring[i] = NULL;
 667         }
 668 }
 669
 670 /*
 671  * Post a receive request on the work queue of InfiniBand device 'sdev'.
 672  */
 673 static int srpt_post_recv(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
 674 {
 675         struct ib_sge list;
 676         struct ib_recv_wr wr, *bad_wr;
 677
 678         wr.wr_id = ioctx->index | SRPT_OP_RECV;
 679
 680         list.addr = ioctx->dma;
 681         list.length = MAX_MESSAGE_SIZE;
 682         list.lkey = sdev->mr->lkey;
 683
 684         wr.next = NULL;
 685         wr.sg_list = &list;
 686         wr.num_sge = 1;
 687
 688         return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);
 689 }
 690
 691 /*
 692  * Post an IB send request.
 693  * @ch: RDMA channel to post the send request on.
 694  * @ioctx: I/O context of the send request.
 695  * @len: length of the request to be sent in bytes.
 696  *
 697  * Returns zero upon success and a non-zero value upon failure.
 698  */
 699 static int srpt_post_send(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
 700                           int len)
 701 {
 702         struct ib_sge list;
 703         struct ib_send_wr wr, *bad_wr;
 704         struct srpt_device *sdev = ch->sport->sdev;
 705
 706         dma_sync_single_for_device(sdev->device->dma_device, ioctx->dma,
 707                                    MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
 708
 709         list.addr = ioctx->dma;
 710         list.length = len;
 711         list.lkey = sdev->mr->lkey;
 712
 713         wr.next = NULL;
 714         wr.wr_id = ioctx->index;
 715         wr.sg_list = &list;
 716         wr.num_sge = 1;
 717         wr.opcode = IB_WR_SEND;
 718         wr.send_flags = IB_SEND_SIGNALED;
 719
 720         return ib_post_send(ch->qp, &wr, &bad_wr);
 721 }
 722
 723 static int srpt_get_desc_tbl(struct srpt_ioctx *ioctx, struct srp_cmd *srp_cmd,
 724                              int *ind)
 725 {
 726         struct srp_indirect_buf *idb;
 727         struct srp_direct_buf *db;
 728
 729         *ind = 0;
 730         if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
 731             ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
 732                 ioctx->n_rbuf = 1;
 733                 ioctx->rbufs = &ioctx->single_rbuf;
 734
 735                 db = (void *)srp_cmd->add_data;
 736                 memcpy(ioctx->rbufs, db, sizeof *db);
 737                 ioctx->data_len = be32_to_cpu(db->len);
 738         } else {
 739                 idb = (void *)srp_cmd->add_data;
 740
 741                 ioctx->n_rbuf = be32_to_cpu(idb->table_desc.len) / sizeof *db;
 742
 743                 if (ioctx->n_rbuf >
 744                     (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
 745                         *ind = 1;
 746                         ioctx->n_rbuf = 0;
 747                         goto out;
 748                 }
 749
 750                 if (ioctx->n_rbuf == 1)
 751                         ioctx->rbufs = &ioctx->single_rbuf;
 752                 else
 753                         ioctx->rbufs =
 754                                 kmalloc(ioctx->n_rbuf * sizeof *db, GFP_ATOMIC);
 755                 if (!ioctx->rbufs) {
 756                         ioctx->n_rbuf = 0;
 757                         return -ENOMEM;
 758                 }
 759
 760                 db = idb->desc_list;
 761                 memcpy(ioctx->rbufs, db, ioctx->n_rbuf * sizeof *db);
 762                 ioctx->data_len = be32_to_cpu(idb->len);
 763         }
 764 out:
 765         return 0;
 766 }
 767
 768 /*
 769  * Modify the attributes of queue pair 'qp': allow local write, remote read,
 770  * and remote write. Also transition 'qp' to state IB_QPS_INIT.
 771  */
 772 static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
 773 {
 774         struct ib_qp_attr *attr;
 775         int ret;
 776
 777         attr = kzalloc(sizeof *attr, GFP_KERNEL);
 778         if (!attr)
 779                 return -ENOMEM;
 780
 781         attr->qp_state = IB_QPS_INIT;
 782         attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ |
 783             IB_ACCESS_REMOTE_WRITE;
 784         attr->port_num = ch->sport->port;
 785         attr->pkey_index = 0;
 786
 787         ret = ib_modify_qp(qp, attr,
 788                            IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
 789                            IB_QP_PKEY_INDEX);
 790
 791         kfree(attr);
 792         return ret;
 793 }
 794
 795 /**
 796  * Change the state of a channel to 'ready to receive' (RTR).
 797  * @ch: channel of the queue pair.
 798  * @qp: queue pair to change the state of.
 799  *
 800  * Returns zero upon success and a negative value upon failure.
 801  *
 802  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
 803  * If this structure ever becomes larger, it might be necessary to allocate
 804  * it dynamically instead of on the stack.
 805  */
 806 static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
 807 {
 808         struct ib_qp_attr qp_attr;
 809         int attr_mask;
 810         int ret;
 811
 812         qp_attr.qp_state = IB_QPS_RTR;
 813         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
 814         if (ret)
 815                 goto out;
 816
 817         qp_attr.max_dest_rd_atomic = 4;
 818
 819         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
 820
 821 out:
 822         return ret;
 823 }
 824
 825 /**
 826  * Change the state of a channel to 'ready to send' (RTS).
 827  * @ch: channel of the queue pair.
 828  * @qp: queue pair to change the state of.
 829  *
 830  * Returns zero upon success and a negative value upon failure.
 831  *
 832  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
 833  * If this structure ever becomes larger, it might be necessary to allocate
 834  * it dynamically instead of on the stack.
 835  */
 836 static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
 837 {
 838         struct ib_qp_attr qp_attr;
 839         int attr_mask;
 840         int ret;
 841
 842         qp_attr.qp_state = IB_QPS_RTS;
 843         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
 844         if (ret)
 845                 goto out;
 846
 847         qp_attr.max_rd_atomic = 4;
 848
 849         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
 850
 851 out:
 852         return ret;
 853 }
 854
 855 static void srpt_reset_ioctx(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
 856 {
 857         int i;
 858
 859         if (ioctx->n_rdma_ius > 0 && ioctx->rdma_ius) {
 860                 struct rdma_iu *riu = ioctx->rdma_ius;
 861
 862                 for (i = 0; i < ioctx->n_rdma_ius; ++i, ++riu)
 863                         kfree(riu->sge);
 864                 kfree(ioctx->rdma_ius);
 865         }
 866
 867         if (ioctx->n_rbuf > 1)
 868                 kfree(ioctx->rbufs);
 869
 870         if (srpt_post_recv(ch->sport->sdev, ioctx))
 871                 PRINT_ERROR("%s", "SRQ post_recv failed - this is serious.");
 872                 /* we should queue it back to free_ioctx queue */
 873         else
 874                 atomic_inc(&ch->req_lim_delta);
 875 }
 876
 877 static void srpt_abort_scst_cmd(struct srpt_device *sdev,
 878                                 struct scst_cmd *scmnd,
 879                                 bool tell_initiator)
 880 {
 881         struct srpt_ioctx *ioctx;
 882         scst_data_direction dir;
 883         enum srpt_command_state orig_ioctx_state;
 884
 885         ioctx = scst_cmd_get_tgt_priv(scmnd);
 886         BUG_ON(!ioctx);
 887         dir = scst_cmd_get_data_direction(scmnd);
 888         if (dir != SCST_DATA_NONE) {
 889                 dma_unmap_sg(sdev->device->dma_device,
 890                              scst_cmd_get_sg(scmnd),
 891                              scst_cmd_get_sg_cnt(scmnd),
 892                              scst_to_tgt_dma_dir(dir));
 893
 894 #ifdef CONFIG_SCST_EXTRACHECKS
 895                 switch (scmnd->state) {
 896                 case SCST_CMD_STATE_DATA_WAIT:
 897                         WARN_ON(ioctx->state != SRPT_STATE_NEED_DATA);
 898                         break;
 899                 case SCST_CMD_STATE_XMIT_WAIT:
 900                         WARN_ON(ioctx->state != SRPT_STATE_PROCESSED);
 901                         break;
 902                 default:
 903                         WARN_ON(ioctx->state == SRPT_STATE_NEED_DATA ||
 904                                 ioctx->state == SRPT_STATE_PROCESSED);
 905                 }
 906 #endif
 907         }
 908
 909         orig_ioctx_state = ioctx->state;
 910         ioctx->state = SRPT_STATE_ABORTED;
 911
 912         if (orig_ioctx_state == SRPT_STATE_NEED_DATA) {
 913                 WARN_ON(scst_cmd_get_data_direction(ioctx->scmnd)
 914                         == SCST_DATA_READ);
 915                 scst_rx_data(scmnd,
 916                              tell_initiator ? SCST_RX_STATUS_ERROR
 917                              : SCST_RX_STATUS_ERROR_FATAL,
 918                              SCST_CONTEXT_THREAD);
 919                 goto out;
 920         } else if (ioctx->state == SRPT_STATE_PROCESSED)
 921                 ;
 922         else
 923                 WARN_ON("unexpected cmd state");
 924
 925         scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_FAILED);
 926         WARN_ON(scmnd->state != SCST_CMD_STATE_XMIT_WAIT);
 927         scst_tgt_cmd_done(scmnd, scst_estimate_context());
 928 out:
 929         return;
 930 }
 931
 932 static void srpt_handle_err_comp(struct srpt_rdma_ch *ch, struct ib_wc *wc)
 933 {
 934         struct srpt_ioctx *ioctx;
 935         struct srpt_device *sdev = ch->sport->sdev;
 936
 937         if (wc->wr_id & SRPT_OP_RECV) {
 938                 ioctx = sdev->ioctx_ring[wc->wr_id & ~SRPT_OP_RECV];
 939                 PRINT_ERROR("%s", "This is serious - SRQ is in bad state.");
 940         } else {
 941                 ioctx = sdev->ioctx_ring[wc->wr_id];
 942
 943                 if (ioctx->scmnd)
 944                         srpt_abort_scst_cmd(sdev, ioctx->scmnd, true);
 945                 else
 946                         srpt_reset_ioctx(ch, ioctx);
 947         }
 948 }
 949
 950 static void srpt_handle_send_comp(struct srpt_rdma_ch *ch,
 951                                   struct srpt_ioctx *ioctx,
 952                                   enum scst_exec_context context)
 953 {
 954         if (ioctx->scmnd) {
 955                 scst_data_direction dir =
 956                         scst_cmd_get_data_direction(ioctx->scmnd);
 957
 958                 if (dir != SCST_DATA_NONE)
 959                         dma_unmap_sg(ch->sport->sdev->device->dma_device,
 960                                      scst_cmd_get_sg(ioctx->scmnd),
 961                                      scst_cmd_get_sg_cnt(ioctx->scmnd),
 962                                      scst_to_tgt_dma_dir(dir));
 963
 964                 WARN_ON(ioctx->scmnd->state != SCST_CMD_STATE_XMIT_WAIT);
 965                 scst_tgt_cmd_done(ioctx->scmnd, context);
 966         } else
 967                 srpt_reset_ioctx(ch, ioctx);
 968 }
 969
 970 /** Process an RDMA completion notification. */
 971 static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch,
 972                                   struct srpt_ioctx *ioctx)
 973 {
 974         if (!ioctx->scmnd) {
 975                 WARN_ON("ERROR: ioctx->scmnd == NULL");
 976                 srpt_reset_ioctx(ch, ioctx);
 977                 return;
 978         }
 979
 980         /*
 981          * If an RDMA completion notification has been received for a write
 982          * command, tell SCST that processing can continue by calling
 983          * scst_rx_data().
 984          */
 985         if (ioctx->state == SRPT_STATE_NEED_DATA) {
 986                 WARN_ON(scst_cmd_get_data_direction(ioctx->scmnd)
 987                         == SCST_DATA_READ);
 988                 ioctx->state = SRPT_STATE_DATA_IN;
 989                 scst_rx_data(ioctx->scmnd, SCST_RX_STATUS_SUCCESS,
 990                              scst_estimate_context());
 991         }
 992 }
 993
 994 /**
 995  * Build an SRP_RSP response.
 996  * @ch: RDMA channel through which the request has been received.
 997  * @ioctx: I/O context in which the SRP_RSP response will be built.
 998  * @s_key: sense key that will be stored in the response.
 999  * @s_code: value that will be stored in the asc_ascq field of the sense data.
1000  * @tag: tag of the request for which this response is being generated.
1001  *
1002  * Returns the size in bytes of the SRP_RSP response.
1003  *
1004  * An SRP_RSP response contains a SCSI status or service response. See also
1005  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
1006  * response. See also SPC-2 for more information about sense data.
1007  */
1008 static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
1009                               struct srpt_ioctx *ioctx, u8 s_key, u8 s_code,
1010                               u64 tag)
1011 {
1012         struct srp_rsp *srp_rsp;
1013         struct sense_data *sense;
1014         int limit_delta;
1015         int sense_data_len = 0;
1016
1017         srp_rsp = ioctx->buf;
1018         memset(srp_rsp, 0, sizeof *srp_rsp);
1019
1020         limit_delta = atomic_read(&ch->req_lim_delta);
1021         atomic_sub(limit_delta, &ch->req_lim_delta);
1022
1023         srp_rsp->opcode = SRP_RSP;
1024         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1025         srp_rsp->tag = tag;
1026
1027         if (s_key != NO_SENSE) {
1028                 sense_data_len = sizeof *sense + (sizeof *sense % 4);
1029                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1030                 srp_rsp->status = SAM_STAT_CHECK_CONDITION;
1031                 srp_rsp->sense_data_len = cpu_to_be32(sense_data_len);
1032
1033                 sense = (struct sense_data *)(srp_rsp + 1);
1034                 sense->err_code = 0x70;
1035                 sense->key = s_key;
1036                 sense->asc_ascq = s_code;
1037         }
1038
1039         return sizeof(*srp_rsp) + sense_data_len;
1040 }
1041
1042 /**
1043  * Build a task management response, which is a specific SRP_RSP response.
1044  * @ch: RDMA channel through which the request has been received.
1045  * @ioctx: I/O context in which the SRP_RSP response will be built.
1046  * @rsp_code: RSP_CODE that will be stored in the response.
1047  * @tag: tag of the request for which this response is being generated.
1048  *
1049  * Returns the size in bytes of the SRP_RSP response.
1050  *
1051  * An SRP_RSP response contains a SCSI status or service response. See also
1052  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
1053  * response.
1054  */
1055 static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
1056                                   struct srpt_ioctx *ioctx, u8 rsp_code,
1057                                   u64 tag)
1058 {
1059         struct srp_rsp *srp_rsp;
1060         int limit_delta;
1061         int resp_data_len = 0;
1062
1063         dma_sync_single_for_cpu(ch->sport->sdev->device->dma_device, ioctx->dma,
1064                                 MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
1065
1066         srp_rsp = ioctx->buf;
1067         memset(srp_rsp, 0, sizeof *srp_rsp);
1068
1069         limit_delta = atomic_read(&ch->req_lim_delta);
1070         atomic_sub(limit_delta, &ch->req_lim_delta);
1071
1072         srp_rsp->opcode = SRP_RSP;
1073         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1074         srp_rsp->tag = tag;
1075
1076         if (rsp_code != SRP_TSK_MGMT_SUCCESS) {
1077                 resp_data_len = 4;
1078                 srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
1079                 srp_rsp->resp_data_len = cpu_to_be32(resp_data_len);
1080                 srp_rsp->data[3] = rsp_code;
1081         }
1082
1083         return sizeof(*srp_rsp) + resp_data_len;
1084 }
1085
1086 /*
1087  * Process SRP_CMD.
1088  */
1089 static int srpt_handle_cmd(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
1090 {
1091         struct scst_cmd *scmnd;
1092         struct srp_cmd *srp_cmd;
1093         struct srp_rsp *srp_rsp;
1094         scst_data_direction dir;
1095         int indirect_desc = 0;
1096         int ret;
1097         unsigned long flags;
1098
1099         srp_cmd = ioctx->buf;
1100         srp_rsp = ioctx->buf;
1101
1102         dir = SCST_DATA_NONE;
1103         if (srp_cmd->buf_fmt) {
1104                 ret = srpt_get_desc_tbl(ioctx, srp_cmd, &indirect_desc);
1105                 if (ret) {
1106                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1107                                            NO_ADD_SENSE, srp_cmd->tag);
1108                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1109                         goto err;
1110                 }
1111
1112                 if (indirect_desc) {
1113                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1114                                            NO_ADD_SENSE, srp_cmd->tag);
1115                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1116                         goto err;
1117                 }
1118
1119                 /*
1120                  * The lower four bits of the buffer format field contain the
1121                  * DATA-IN buffer descriptor format, and the highest four bits
1122                  * contain the DATA-OUT buffer descriptor format.
1123                  */
1124                 if (srp_cmd->buf_fmt & 0xf)
1125                         /* DATA-IN: transfer data from target to initiator. */
1126                         dir = SCST_DATA_READ;
1127                 else if (srp_cmd->buf_fmt >> 4)
1128                         /* DATA-OUT: transfer data from initiator to target. */
1129                         dir = SCST_DATA_WRITE;
1130         }
1131
1132         scmnd = scst_rx_cmd(ch->scst_sess, (u8 *) &srp_cmd->lun,
1133                             sizeof srp_cmd->lun, srp_cmd->cdb, 16,
1134                             thread ? SCST_NON_ATOMIC : SCST_ATOMIC);
1135         if (!scmnd) {
1136                 srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1137                                    NO_ADD_SENSE, srp_cmd->tag);
1138                 srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1139                 goto err;
1140         }
1141
1142         ioctx->scmnd = scmnd;
1143
1144         switch (srp_cmd->task_attr) {
1145         case SRP_CMD_HEAD_OF_Q:
1146                 scmnd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1147                 break;
1148         case SRP_CMD_ORDERED_Q:
1149                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1150                 break;
1151         case SRP_CMD_SIMPLE_Q:
1152                 scmnd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1153                 break;
1154         case SRP_CMD_ACA:
1155                 scmnd->queue_type = SCST_CMD_QUEUE_ACA;
1156                 break;
1157         default:
1158                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1159                 break;
1160         }
1161
1162         scst_cmd_set_tag(scmnd, srp_cmd->tag);
1163         scst_cmd_set_tgt_priv(scmnd, ioctx);
1164         scst_cmd_set_expected(scmnd, dir, ioctx->data_len);
1165
1166         spin_lock_irqsave(&ch->spinlock, flags);
1167         list_add_tail(&ioctx->scmnd_list, &ch->active_scmnd_list);
1168         ch->active_scmnd_cnt++;
1169         spin_unlock_irqrestore(&ch->spinlock, flags);
1170
1171         scst_cmd_init_done(scmnd, scst_estimate_context());
1172
1173         return 0;
1174
1175 err:
1176         WARN_ON(srp_rsp->opcode != SRP_RSP);
1177
1178         return -1;
1179 }
1180
1181 /*
1182  * Process an SRP_TSK_MGMT request.
1183  *
1184  * Returns 0 upon success and -1 upon failure.
1185  *
1186  * Each task management function is performed by calling one of the
1187  * scst_rx_mgmt_fn*() functions. These functions will either report failure
1188  * or process the task management function asynchronously. The function
1189  * srpt_tsk_mgmt_done() will be called by the SCST core upon completion of the
1190  * task management function. When srpt_handle_tsk_mgmt() reports failure
1191  * (i.e. returns -1) a response will have been built in ioctx->buf. This
1192  * information unit has to be sent back by the caller.
1193  *
1194  * For more information about SRP_TSK_MGMT information units, see also section
1195  * 6.7 in the T10 SRP r16a document.
1196  */
1197 static int srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
1198                                 struct srpt_ioctx *ioctx)
1199 {
1200         struct srp_tsk_mgmt *srp_tsk;
1201         struct srpt_mgmt_ioctx *mgmt_ioctx;
1202         int ret;
1203
1204         srp_tsk = ioctx->buf;
1205
1206         TRACE_DBG("recv_tsk_mgmt= %d for task_tag= %lld"
1207                   " using tag= %lld cm_id= %p sess= %p",
1208                   srp_tsk->tsk_mgmt_func,
1209                   (unsigned long long) srp_tsk->task_tag,
1210                   (unsigned long long) srp_tsk->tag,
1211                   ch->cm_id, ch->scst_sess);
1212
1213         mgmt_ioctx = kmalloc(sizeof *mgmt_ioctx, GFP_ATOMIC);
1214         if (!mgmt_ioctx) {
1215                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1216                                        srp_tsk->tag);
1217                 goto err;
1218         }
1219
1220         mgmt_ioctx->ioctx = ioctx;
1221         mgmt_ioctx->ch = ch;
1222         mgmt_ioctx->tag = srp_tsk->tag;
1223
1224         switch (srp_tsk->tsk_mgmt_func) {
1225         case SRP_TSK_ABORT_TASK:
1226                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK");
1227                 ret = scst_rx_mgmt_fn_tag(ch->scst_sess,
1228                                           SCST_ABORT_TASK,
1229                                           srp_tsk->task_tag,
1230                                           thread ?
1231                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1232                                           mgmt_ioctx);
1233                 break;
1234         case SRP_TSK_ABORT_TASK_SET:
1235                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK_SET");
1236                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1237                                           SCST_ABORT_TASK_SET,
1238                                           (u8 *) &srp_tsk->lun,
1239                                           sizeof srp_tsk->lun,
1240                                           thread ?
1241                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1242                                           mgmt_ioctx);
1243                 break;
1244         case SRP_TSK_CLEAR_TASK_SET:
1245                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_TASK_SET");
1246                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1247                                           SCST_CLEAR_TASK_SET,
1248                                           (u8 *) &srp_tsk->lun,
1249                                           sizeof srp_tsk->lun,
1250                                           thread ?
1251                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1252                                           mgmt_ioctx);
1253                 break;
1254         case SRP_TSK_LUN_RESET:
1255                 TRACE_DBG("%s", "Processing SRP_TSK_LUN_RESET");
1256                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1257                                           SCST_LUN_RESET,
1258                                           (u8 *) &srp_tsk->lun,
1259                                           sizeof srp_tsk->lun,
1260                                           thread ?
1261                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1262                                           mgmt_ioctx);
1263                 break;
1264         case SRP_TSK_CLEAR_ACA:
1265                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_ACA");
1266                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1267                                           SCST_CLEAR_ACA,
1268                                           (u8 *) &srp_tsk->lun,
1269                                           sizeof srp_tsk->lun,
1270                                           thread ?
1271                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1272                                           mgmt_ioctx);
1273                 break;
1274         default:
1275                 TRACE_DBG("%s", "Unsupported task management function.");
1276                 srpt_build_tskmgmt_rsp(ch, ioctx,
1277                                        SRP_TSK_MGMT_FUNC_NOT_SUPP,
1278                                        srp_tsk->tag);
1279                 goto err;
1280         }
1281
1282         if (ret) {
1283                 TRACE_DBG("%s", "Processing task management function failed.");
1284                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1285                                        srp_tsk->tag);
1286                 goto err;
1287         }
1288
1289         WARN_ON(srp_tsk->opcode == SRP_RSP);
1290
1291         return 0;
1292
1293 err:
1294         WARN_ON(srp_tsk->opcode != SRP_RSP);
1295
1296         kfree(mgmt_ioctx);
1297         return -1;
1298 }
1299
1300 /**
1301  * Process a receive completion event.
1302  * @ch: RDMA channel for which the completion event has been received.
1303  * @ioctx: SRPT I/O context for which the completion event has been received.
1304  */
1305 static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
1306                                struct srpt_ioctx *ioctx)
1307 {
1308         struct srp_cmd *srp_cmd;
1309         struct srp_rsp *srp_rsp;
1310         unsigned long flags;
1311         int len;
1312
1313         spin_lock_irqsave(&ch->spinlock, flags);
1314         if (ch->state != RDMA_CHANNEL_LIVE) {
1315                 if (ch->state == RDMA_CHANNEL_CONNECTING) {
1316                         list_add_tail(&ioctx->wait_list, &ch->cmd_wait_list);
1317                         spin_unlock_irqrestore(&ch->spinlock, flags);
1318                         return;
1319                 } else {
1320                         spin_unlock_irqrestore(&ch->spinlock, flags);
1321                         srpt_reset_ioctx(ch, ioctx);
1322                         return;
1323                 }
1324         }
1325         spin_unlock_irqrestore(&ch->spinlock, flags);
1326
1327         dma_sync_single_for_cpu(ch->sport->sdev->device->dma_device, ioctx->dma,
1328                                 MAX_MESSAGE_SIZE, DMA_FROM_DEVICE);
1329
1330         ioctx->data_len = 0;
1331         ioctx->n_rbuf = 0;
1332         ioctx->rbufs = NULL;
1333         ioctx->n_rdma = 0;
1334         ioctx->n_rdma_ius = 0;
1335         ioctx->rdma_ius = NULL;
1336         ioctx->scmnd = NULL;
1337         ioctx->state = SRPT_STATE_NEW;
1338
1339         srp_cmd = ioctx->buf;
1340         srp_rsp = ioctx->buf;
1341
1342         switch (srp_cmd->opcode) {
1343         case SRP_CMD:
1344                 if (srpt_handle_cmd(ch, ioctx) < 0)
1345                         goto err;
1346                 break;
1347
1348         case SRP_TSK_MGMT:
1349                 if (srpt_handle_tsk_mgmt(ch, ioctx) < 0)
1350                         goto err;
1351                 break;
1352
1353         case SRP_I_LOGOUT:
1354         case SRP_AER_REQ:
1355         default:
1356                 srpt_build_cmd_rsp(ch, ioctx, ILLEGAL_REQUEST, INVALID_CDB,
1357                                    srp_cmd->tag);
1358                 goto err;
1359         }
1360
1361         dma_sync_single_for_device(ch->sport->sdev->device->dma_device,
1362                                    ioctx->dma, MAX_MESSAGE_SIZE,
1363                                    DMA_FROM_DEVICE);
1364
1365         return;
1366
1367 err:
1368         WARN_ON(srp_rsp->opcode != SRP_RSP);
1369         len = (sizeof *srp_rsp) + be32_to_cpu(srp_rsp->sense_data_len);
1370
1371         if (ch->state != RDMA_CHANNEL_LIVE) {
1372                 /* Give up if another thread modified the channel state. */
1373                 PRINT_ERROR("%s: channel is in state %d", __func__, ch->state);
1374                 srpt_reset_ioctx(ch, ioctx);
1375         } else if (srpt_post_send(ch, ioctx, len)) {
1376                 PRINT_ERROR("%s: sending SRP_RSP response failed", __func__);
1377                 srpt_reset_ioctx(ch, ioctx);
1378         }
1379 }
1380
1381 /*
1382  * Returns true if the ioctx list is non-empty or if the ib_srpt kernel thread
1383  * should stop.
1384  * @pre thread != 0
1385  */
1386 static inline int srpt_test_ioctx_list(void)
1387 {
1388         int res = (!list_empty(&srpt_thread.thread_ioctx_list) ||
1389                    unlikely(kthread_should_stop()));
1390         return res;
1391 }
1392
1393 /*
1394  * Add 'ioctx' to the tail of the ioctx list and wake up the kernel thread.
1395  *
1396  * @pre thread != 0
1397  */
1398 static inline void srpt_schedule_thread(struct srpt_ioctx *ioctx)
1399 {
1400         unsigned long flags;
1401
1402         spin_lock_irqsave(&srpt_thread.thread_lock, flags);
1403         list_add_tail(&ioctx->comp_list, &srpt_thread.thread_ioctx_list);
1404         spin_unlock_irqrestore(&srpt_thread.thread_lock, flags);
1405         wake_up(&ioctx_list_waitQ);
1406 }
1407
1408 /**
1409  * InfiniBand completion queue callback function.
1410  * @cq: completion queue.
1411  * @ctx: completion queue context, which was passed as the fourth argument of
1412  *       the function ib_create_cq().
1413  */
1414 static void srpt_completion(struct ib_cq *cq, void *ctx)
1415 {
1416         struct srpt_rdma_ch *ch = ctx;
1417         struct srpt_device *sdev = ch->sport->sdev;
1418         struct ib_wc wc;
1419         struct srpt_ioctx *ioctx;
1420
1421         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1422         while (ib_poll_cq(ch->cq, 1, &wc) > 0) {
1423                 if (wc.status) {
1424                         PRINT_ERROR("failed %s status= %d",
1425                                wc.wr_id & SRPT_OP_RECV ? "receive" : "send",
1426                                wc.status);
1427                         srpt_handle_err_comp(ch, &wc);
1428                         break;
1429                 }
1430
1431                 if (wc.wr_id & SRPT_OP_RECV) {
1432                         ioctx = sdev->ioctx_ring[wc.wr_id & ~SRPT_OP_RECV];
1433                         if (thread) {
1434                                 ioctx->ch = ch;
1435                                 ioctx->op = IB_WC_RECV;
1436                                 srpt_schedule_thread(ioctx);
1437                         } else
1438                                 srpt_handle_new_iu(ch, ioctx);
1439                         continue;
1440                 } else
1441                         ioctx = sdev->ioctx_ring[wc.wr_id];
1442
1443                 if (thread) {
1444                         ioctx->ch = ch;
1445                         ioctx->op = wc.opcode;
1446                         srpt_schedule_thread(ioctx);
1447                 } else {
1448                         switch (wc.opcode) {
1449                         case IB_WC_SEND:
1450                                 srpt_handle_send_comp(ch, ioctx,
1451                                         scst_estimate_context());
1452                                 break;
1453                         case IB_WC_RDMA_WRITE:
1454                         case IB_WC_RDMA_READ:
1455                                 srpt_handle_rdma_comp(ch, ioctx);
1456                                 break;
1457                         default:
1458                                 break;
1459                         }
1460                 }
1461         }
1462 }
1463
1464 /*
1465  * Create a completion queue on the specified device.
1466  */
1467 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
1468 {
1469         struct ib_qp_init_attr *qp_init;
1470         struct srpt_device *sdev = ch->sport->sdev;
1471         int cqe;
1472         int ret;
1473
1474         qp_init = kzalloc(sizeof *qp_init, GFP_KERNEL);
1475         if (!qp_init)
1476                 return -ENOMEM;
1477
1478         /* Create a completion queue (CQ). */
1479
1480         cqe = SRPT_RQ_SIZE + SRPT_SQ_SIZE - 1;
1481 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(RHEL_RELEASE_CODE)
1482         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe);
1483 #else
1484         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe, 0);
1485 #endif
1486         if (IS_ERR(ch->cq)) {
1487                 ret = PTR_ERR(ch->cq);
1488                 PRINT_ERROR("failed to create_cq cqe= %d ret= %d", cqe, ret);
1489                 goto out;
1490         }
1491
1492         /* Request completion notification. */
1493
1494         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1495
1496         /* Create a queue pair (QP). */
1497
1498         qp_init->qp_context = (void *)ch;
1499         qp_init->event_handler = srpt_qp_event;
1500         qp_init->send_cq = ch->cq;
1501         qp_init->recv_cq = ch->cq;
1502         qp_init->srq = sdev->srq;
1503         qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
1504         qp_init->qp_type = IB_QPT_RC;
1505         qp_init->cap.max_send_wr = SRPT_SQ_SIZE;
1506         qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE;
1507
1508         ch->qp = ib_create_qp(sdev->pd, qp_init);
1509         if (IS_ERR(ch->qp)) {
1510                 ret = PTR_ERR(ch->qp);
1511                 ib_destroy_cq(ch->cq);
1512                 PRINT_ERROR("failed to create_qp ret= %d", ret);
1513                 goto out;
1514         }
1515
1516         TRACE_DBG("%s: max_cqe= %d max_sge= %d cm_id= %p",
1517                __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
1518                ch->cm_id);
1519
1520         /* Modify the attributes and the state of queue pair ch->qp. */
1521
1522         ret = srpt_init_ch_qp(ch, ch->qp);
1523         if (ret) {
1524                 ib_destroy_qp(ch->qp);
1525                 ib_destroy_cq(ch->cq);
1526                 goto out;
1527         }
1528
1529         atomic_set(&ch->req_lim_delta, SRPT_RQ_SIZE);
1530 out:
1531         kfree(qp_init);
1532         return ret;
1533 }
1534
1535 /**
1536  * Look up the RDMA channel that corresponds to the specified cm_id.
1537  *
1538  * Return NULL if no matching RDMA channel has been found.
1539  */
1540 static struct srpt_rdma_ch *srpt_find_channel(struct ib_cm_id *cm_id, bool del)
1541 {
1542         struct srpt_device *sdev = cm_id->context;
1543         struct srpt_rdma_ch *ch;
1544
1545         spin_lock_irq(&sdev->spinlock);
1546         list_for_each_entry(ch, &sdev->rch_list, list) {
1547                 if (ch->cm_id == cm_id) {
1548                         if (del)
1549                                 list_del(&ch->list);
1550                         spin_unlock_irq(&sdev->spinlock);
1551                         return ch;
1552                 }
1553         }
1554
1555         spin_unlock_irq(&sdev->spinlock);
1556
1557         return NULL;
1558 }
1559
1560 /**
1561  * Release all resources associated with the specified RDMA channel.
1562  *
1563  * Note: the caller must have removed the channel from the channel list
1564  * before calling this function.
1565  */
1566 static void srpt_release_channel(struct srpt_rdma_ch *ch, int destroy_cmid)
1567 {
1568         TRACE_ENTRY();
1569
1570         WARN_ON(srpt_find_channel(ch->cm_id, false) == ch);
1571
1572         if (ch->cm_id && destroy_cmid) {
1573                 TRACE_DBG("%s: destroy cm_id= %p", __func__, ch->cm_id);
1574                 ib_destroy_cm_id(ch->cm_id);
1575                 ch->cm_id = NULL;
1576         }
1577
1578         ib_destroy_qp(ch->qp);
1579         ib_destroy_cq(ch->cq);
1580
1581         if (ch->scst_sess) {
1582                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1583
1584                 if (ch->active_scmnd_cnt)
1585                         PRINT_INFO("Releasing session %s which still has %d"
1586                                    " active commands",
1587                                    ch->sess_name, ch->active_scmnd_cnt);
1588                 else
1589                         PRINT_INFO("Releasing session %s", ch->sess_name);
1590
1591                 spin_lock_irq(&ch->spinlock);
1592                 list_for_each_entry_safe(ioctx, ioctx_tmp,
1593                                          &ch->active_scmnd_list, scmnd_list) {
1594                         spin_unlock_irq(&ch->spinlock);
1595
1596                         if (ioctx->scmnd)
1597                                 srpt_abort_scst_cmd(ch->sport->sdev,
1598                                                     ioctx->scmnd, true);
1599
1600                         spin_lock_irq(&ch->spinlock);
1601                 }
1602                 WARN_ON(!list_empty(&ch->active_scmnd_list));
1603                 WARN_ON(ch->active_scmnd_cnt != 0);
1604                 spin_unlock_irq(&ch->spinlock);
1605
1606                 scst_unregister_session(ch->scst_sess, 0, NULL);
1607                 ch->scst_sess = NULL;
1608         }
1609
1610         kfree(ch);
1611
1612         TRACE_EXIT();
1613 }
1614
1615 static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
1616                             struct ib_cm_req_event_param *param,
1617                             void *private_data)
1618 {
1619         struct srpt_device *sdev = cm_id->context;
1620         struct srp_login_req *req;
1621         struct srp_login_rsp *rsp;
1622         struct srp_login_rej *rej;
1623         struct ib_cm_rep_param *rep_param;
1624         struct srpt_rdma_ch *ch, *tmp_ch;
1625         u32 it_iu_len;
1626         int ret = 0;
1627
1628 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
1629         WARN_ON(!sdev || !private_data);
1630         if (!sdev || !private_data)
1631                 return -EINVAL;
1632 #else
1633         if (WARN_ON(!sdev || !private_data))
1634                 return -EINVAL;
1635 #endif
1636
1637         req = (struct srp_login_req *)private_data;
1638
1639         it_iu_len = be32_to_cpu(req->req_it_iu_len);
1640
1641         PRINT_INFO("Received SRP_LOGIN_REQ with"
1642             " i_port_id 0x%llx:0x%llx, t_port_id 0x%llx:0x%llx and length %d"
1643             " on port %d (guid=0x%llx:0x%llx)",
1644             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[0]),
1645             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[8]),
1646             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[0]),
1647             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[8]),
1648             it_iu_len,
1649             param->port,
1650             (unsigned long long)be64_to_cpu(*(u64 *)
1651                                 &sdev->port[param->port - 1].gid.raw[0]),
1652             (unsigned long long)be64_to_cpu(*(u64 *)
1653                                 &sdev->port[param->port - 1].gid.raw[8]));
1654
1655         rsp = kzalloc(sizeof *rsp, GFP_KERNEL);
1656         rej = kzalloc(sizeof *rej, GFP_KERNEL);
1657         rep_param = kzalloc(sizeof *rep_param, GFP_KERNEL);
1658
1659         if (!rsp || !rej || !rep_param) {
1660                 ret = -ENOMEM;
1661                 goto out;
1662         }
1663
1664         if (it_iu_len > MAX_MESSAGE_SIZE || it_iu_len < 64) {
1665                 rej->reason =
1666                     cpu_to_be32(SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
1667                 ret = -EINVAL;
1668                 PRINT_ERROR("rejected SRP_LOGIN_REQ because its"
1669                             " length (%d bytes) is invalid", it_iu_len);
1670                 goto reject;
1671         }
1672
1673         if ((req->req_flags & 0x3) == SRP_MULTICHAN_SINGLE) {
1674                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
1675
1676                 spin_lock_irq(&sdev->spinlock);
1677
1678                 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
1679                         if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
1680                             && !memcmp(ch->t_port_id, req->target_port_id, 16)
1681                             && param->port == ch->sport->port
1682                             && param->listen_id == ch->sport->sdev->cm_id
1683                             && ch->cm_id) {
1684                                 enum rdma_ch_state prev_state;
1685
1686                                 /* found an existing channel */
1687                                 TRACE_DBG("Found existing channel name= %s"
1688                                           " cm_id= %p state= %d",
1689                                           ch->sess_name, ch->cm_id, ch->state);
1690
1691                                 prev_state = ch->state;
1692                                 if (ch->state == RDMA_CHANNEL_LIVE)
1693                                         ch->state = RDMA_CHANNEL_DISCONNECTING;
1694                                 else if (ch->state == RDMA_CHANNEL_CONNECTING)
1695                                         list_del(&ch->list);
1696
1697                                 spin_unlock_irq(&sdev->spinlock);
1698
1699                                 rsp->rsp_flags =
1700                                         SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
1701
1702                                 if (prev_state == RDMA_CHANNEL_LIVE) {
1703                                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
1704                                         PRINT_INFO("disconnected"
1705                                           " session %s because a new"
1706                                           " SRP_LOGIN_REQ has been received.",
1707                                           ch->sess_name);
1708                                 } else if (prev_state ==
1709                                          RDMA_CHANNEL_CONNECTING) {
1710                                         PRINT_ERROR("%s", "rejected"
1711                                           " SRP_LOGIN_REQ because another login"
1712                                           " request is being processed.");
1713                                         ib_send_cm_rej(ch->cm_id,
1714                                                        IB_CM_REJ_NO_RESOURCES,
1715                                                        NULL, 0, NULL, 0);
1716                                         srpt_release_channel(ch, 1);
1717                                 }
1718
1719                                 spin_lock_irq(&sdev->spinlock);
1720                         }
1721                 }
1722
1723                 spin_unlock_irq(&sdev->spinlock);
1724
1725         } else
1726                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
1727
1728         if (((u64) (*(u64 *) req->target_port_id) !=
1729              cpu_to_be64(srpt_service_guid)) ||
1730             ((u64) (*(u64 *) (req->target_port_id + 8)) !=
1731              cpu_to_be64(srpt_service_guid))) {
1732                 rej->reason =
1733                     cpu_to_be32(SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
1734                 ret = -ENOMEM;
1735                 PRINT_ERROR("%s", "rejected SRP_LOGIN_REQ because it"
1736                        " has an invalid target port identifier.");
1737                 goto reject;
1738         }
1739
1740         ch = kzalloc(sizeof *ch, GFP_KERNEL);
1741         if (!ch) {
1742                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1743                 PRINT_ERROR("%s",
1744                             "rejected SRP_LOGIN_REQ because out of memory.");
1745                 ret = -ENOMEM;
1746                 goto reject;
1747         }
1748
1749         spin_lock_init(&ch->spinlock);
1750         memcpy(ch->i_port_id, req->initiator_port_id, 16);
1751         memcpy(ch->t_port_id, req->target_port_id, 16);
1752         ch->sport = &sdev->port[param->port - 1];
1753         ch->cm_id = cm_id;
1754         ch->state = RDMA_CHANNEL_CONNECTING;
1755         INIT_LIST_HEAD(&ch->cmd_wait_list);
1756         INIT_LIST_HEAD(&ch->active_scmnd_list);
1757
1758         ret = srpt_create_ch_ib(ch);
1759         if (ret) {
1760                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1761                 PRINT_ERROR("%s", "rejected SRP_LOGIN_REQ because creating"
1762                             " a new RDMA channel failed.");
1763                 goto free_ch;
1764         }
1765
1766         ret = srpt_ch_qp_rtr(ch, ch->qp);
1767         if (ret) {
1768                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1769                 PRINT_ERROR("rejected SRP_LOGIN_REQ because enabling"
1770                        " RTR failed (error code = %d)", ret);
1771                 goto destroy_ib;
1772         }
1773
1774         if (use_port_guid_in_session_name) {
1775                 /*
1776                  * If the kernel module parameter use_port_guid_in_session_name
1777                  * has been specified, use a combination of the target port
1778                  * GUID and the initiator port ID as the session name. This
1779                  * was the original behavior of the SRP target implementation
1780                  * (i.e. before the SRPT was included in OFED 1.3).
1781                  */
1782                 snprintf(ch->sess_name, sizeof(ch->sess_name),
1783                          "0x%016llx%016llx",
1784                          (unsigned long long)be64_to_cpu(*(u64 *)
1785                                 &sdev->port[param->port - 1].gid.raw[8]),
1786                          (unsigned long long)be64_to_cpu(*(u64 *)
1787                                 (ch->i_port_id + 8)));
1788         } else {
1789                 /*
1790                  * Default behavior: use the initator port identifier as the
1791                  * session name.
1792                  */
1793                 snprintf(ch->sess_name, sizeof(ch->sess_name),
1794                          "0x%016llx%016llx",
1795                          (unsigned long long)be64_to_cpu(*(u64 *)ch->i_port_id),
1796                          (unsigned long long)be64_to_cpu(*(u64 *)
1797                                  (ch->i_port_id + 8)));
1798         }
1799
1800         TRACE_DBG("registering session %s", ch->sess_name);
1801
1802         BUG_ON(!sdev->scst_tgt);
1803         ch->scst_sess = scst_register_session(sdev->scst_tgt, 0, ch->sess_name,
1804                                               NULL, NULL);
1805         if (!ch->scst_sess) {
1806                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1807                 TRACE_DBG("%s", "Failed to create scst sess");
1808                 goto destroy_ib;
1809         }
1810
1811         TRACE_DBG("Establish connection sess=%p name=%s cm_id=%p",
1812                   ch->scst_sess, ch->sess_name, ch->cm_id);
1813
1814         scst_sess_set_tgt_priv(ch->scst_sess, ch);
1815
1816         /* create srp_login_response */
1817         rsp->opcode = SRP_LOGIN_RSP;
1818         rsp->tag = req->tag;
1819         rsp->max_it_iu_len = req->req_it_iu_len;
1820         rsp->max_ti_iu_len = req->req_it_iu_len;
1821         rsp->buf_fmt =
1822             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1823         rsp->req_lim_delta = cpu_to_be32(SRPT_RQ_SIZE);
1824         atomic_set(&ch->req_lim_delta, 0);
1825
1826         /* create cm reply */
1827         rep_param->qp_num = ch->qp->qp_num;
1828         rep_param->private_data = (void *)rsp;
1829         rep_param->private_data_len = sizeof *rsp;
1830         rep_param->rnr_retry_count = 7;
1831         rep_param->flow_control = 1;
1832         rep_param->failover_accepted = 0;
1833         rep_param->srq = 1;
1834         rep_param->responder_resources = 4;
1835         rep_param->initiator_depth = 4;
1836
1837         ret = ib_send_cm_rep(cm_id, rep_param);
1838         if (ret) {
1839                 PRINT_ERROR("sending SRP_LOGIN_REQ response failed"
1840                             " (error code = %d)", ret);
1841                 goto release_channel;
1842         }
1843
1844         spin_lock_irq(&sdev->spinlock);
1845         list_add_tail(&ch->list, &sdev->rch_list);
1846         spin_unlock_irq(&sdev->spinlock);
1847
1848         goto out;
1849
1850 release_channel:
1851         scst_unregister_session(ch->scst_sess, 0, NULL);
1852         ch->scst_sess = NULL;
1853
1854 destroy_ib:
1855         ib_destroy_qp(ch->qp);
1856         ib_destroy_cq(ch->cq);
1857
1858 free_ch:
1859         kfree(ch);
1860
1861 reject:
1862         rej->opcode = SRP_LOGIN_REJ;
1863         rej->tag = req->tag;
1864         rej->buf_fmt =
1865             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1866
1867         ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
1868                              (void *)rej, sizeof *rej);
1869
1870 out:
1871         kfree(rep_param);
1872         kfree(rsp);
1873         kfree(rej);
1874
1875         return ret;
1876 }
1877
1878 /**
1879  * Release the channel with the specified cm_id.
1880  *
1881  * Returns one to indicate that the caller of srpt_cm_handler() should destroy
1882  * the cm_id.
1883  */
1884 static void srpt_find_and_release_channel(struct ib_cm_id *cm_id)
1885 {
1886         struct srpt_rdma_ch *ch;
1887
1888         ch = srpt_find_channel(cm_id, true);
1889         if (ch)
1890                 srpt_release_channel(ch, 0);
1891 }
1892
1893 static void srpt_cm_rej_recv(struct ib_cm_id *cm_id)
1894 {
1895         PRINT_INFO("%s", "Received InfiniBand REJ packet.");
1896         srpt_find_and_release_channel(cm_id);
1897 }
1898
1899 /**
1900  * Process an IB_CM_RTU_RECEIVED or IB_CM_USER_ESTABLISHED event.
1901  *
1902  * An IB_CM_RTU_RECEIVED message indicates that the connection is established
1903  * and that the recipient may begin transmitting (RTU = ready to use).
1904  */
1905 static int srpt_cm_rtu_recv(struct ib_cm_id *cm_id)
1906 {
1907         struct srpt_rdma_ch *ch;
1908         int ret;
1909
1910         ch = srpt_find_channel(cm_id, false);
1911         if (!ch)
1912                 return -EINVAL;
1913
1914         if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_CONNECTING,
1915                                             RDMA_CHANNEL_LIVE)) {
1916                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1917
1918                 ret = srpt_ch_qp_rts(ch, ch->qp);
1919
1920                 list_for_each_entry_safe(ioctx, ioctx_tmp, &ch->cmd_wait_list,
1921                                          wait_list) {
1922                         list_del(&ioctx->wait_list);
1923                         srpt_handle_new_iu(ch, ioctx);
1924                 }
1925                 if (ret && srpt_test_and_set_channel_state(ch,
1926                                         RDMA_CHANNEL_LIVE,
1927                                         RDMA_CHANNEL_DISCONNECTING)) {
1928                         TRACE_DBG("cm_id=%p sess_name=%s state=%d",
1929                                   cm_id, ch->sess_name, ch->state);
1930                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
1931                 }
1932         } else if (ch->state == RDMA_CHANNEL_DISCONNECTING) {
1933                 TRACE_DBG("cm_id=%p sess_name=%s state=%d",
1934                           cm_id, ch->sess_name, ch->state);
1935                 ib_send_cm_dreq(ch->cm_id, NULL, 0);
1936                 ret = -EAGAIN;
1937         } else
1938                 ret = 0;
1939
1940         return ret;
1941 }
1942
1943 static void srpt_cm_timewait_exit(struct ib_cm_id *cm_id)
1944 {
1945         PRINT_INFO("%s", "Received InfiniBand TimeWait exit.");
1946         srpt_find_and_release_channel(cm_id);
1947 }
1948
1949 static void srpt_cm_rep_error(struct ib_cm_id *cm_id)
1950 {
1951         PRINT_INFO("%s", "Received InfiniBand REP error.");
1952         srpt_find_and_release_channel(cm_id);
1953 }
1954
1955 static int srpt_cm_dreq_recv(struct ib_cm_id *cm_id)
1956 {
1957         struct srpt_rdma_ch *ch;
1958
1959         ch = srpt_find_channel(cm_id, false);
1960         if (!ch)
1961                 return -EINVAL;
1962
1963         TRACE_DBG("%s: cm_id= %p ch->state= %d",
1964                  __func__, cm_id, ch->state);
1965
1966         switch (ch->state) {
1967         case RDMA_CHANNEL_LIVE:
1968         case RDMA_CHANNEL_CONNECTING:
1969                 ib_send_cm_drep(ch->cm_id, NULL, 0);
1970                 PRINT_INFO("Received DREQ and sent DREP for session %s.",
1971                            ch->sess_name);
1972                 break;
1973         case RDMA_CHANNEL_DISCONNECTING:
1974         default:
1975                 break;
1976         }
1977
1978         return 0;
1979 }
1980
1981 static void srpt_cm_drep_recv(struct ib_cm_id *cm_id)
1982 {
1983         PRINT_INFO("%s", "Received InfiniBand DREP message.");
1984         srpt_find_and_release_channel(cm_id);
1985 }
1986
1987 /**
1988  * IB connection manager callback function.
1989  *
1990  * A non-zero return value will make the caller destroy the CM ID.
1991  *
1992  * Note: srpt_add_one passes a struct srpt_device* as the third argument to
1993  * the ib_create_cm_id() call.
1994  */
1995 static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
1996 {
1997         int ret = 0;
1998
1999         switch (event->event) {
2000         case IB_CM_REQ_RECEIVED:
2001                 ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd,
2002                                        event->private_data);
2003                 break;
2004         case IB_CM_REJ_RECEIVED:
2005                 srpt_cm_rej_recv(cm_id);
2006                 ret = -EINVAL;
2007                 break;
2008         case IB_CM_RTU_RECEIVED:
2009         case IB_CM_USER_ESTABLISHED:
2010                 ret = srpt_cm_rtu_recv(cm_id);
2011                 break;
2012         case IB_CM_DREQ_RECEIVED:
2013                 ret = srpt_cm_dreq_recv(cm_id);
2014                 break;
2015         case IB_CM_DREP_RECEIVED:
2016                 srpt_cm_drep_recv(cm_id);
2017                 ret = -EINVAL;
2018                 break;
2019         case IB_CM_TIMEWAIT_EXIT:
2020                 srpt_cm_timewait_exit(cm_id);
2021                 ret = -EINVAL;
2022                 break;
2023         case IB_CM_REP_ERROR:
2024                 srpt_cm_rep_error(cm_id);
2025                 ret = -EINVAL;
2026                 break;
2027         default:
2028                 break;
2029         }
2030
2031         return ret;
2032 }
2033
2034 static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
2035                                  struct srpt_ioctx *ioctx,
2036                                  struct scst_cmd *scmnd)
2037 {
2038         struct scatterlist *scat;
2039         scst_data_direction dir;
2040         struct rdma_iu *riu;
2041         struct srp_direct_buf *db;
2042         dma_addr_t dma_addr;
2043         struct ib_sge *sge;
2044         u64 raddr;
2045         u32 rsize;
2046         u32 tsize;
2047         u32 dma_len;
2048         int count, nrdma;
2049         int i, j, k;
2050
2051         scat = scst_cmd_get_sg(scmnd);
2052         dir = scst_cmd_get_data_direction(scmnd);
2053         count = dma_map_sg(ch->sport->sdev->device->dma_device, scat,
2054                            scst_cmd_get_sg_cnt(scmnd),
2055                            scst_to_tgt_dma_dir(dir));
2056         if (unlikely(!count))
2057                 return -EBUSY;
2058
2059         if (ioctx->rdma_ius && ioctx->n_rdma_ius)
2060                 nrdma = ioctx->n_rdma_ius;
2061         else {
2062                 nrdma = count / SRPT_DEF_SG_PER_WQE + ioctx->n_rbuf;
2063
2064                 ioctx->rdma_ius = kzalloc(nrdma * sizeof *riu,
2065                                           scst_cmd_atomic(scmnd)
2066                                           ? GFP_ATOMIC : GFP_KERNEL);
2067                 if (!ioctx->rdma_ius) {
2068                         dma_unmap_sg(ch->sport->sdev->device->dma_device,
2069                                      scat, scst_cmd_get_sg_cnt(scmnd),
2070                                      scst_to_tgt_dma_dir(dir));
2071                         return -ENOMEM;
2072                 }
2073
2074                 ioctx->n_rdma_ius = nrdma;
2075         }
2076
2077         db = ioctx->rbufs;
2078         tsize = (dir == SCST_DATA_READ) ?
2079                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2080         dma_len = sg_dma_len(&scat[0]);
2081         riu = ioctx->rdma_ius;
2082
2083         /*
2084          * For each remote desc - calculate the #ib_sge.
2085          * If #ib_sge < SRPT_DEF_SG_PER_WQE per rdma operation then
2086          *      each remote desc rdma_iu is required a rdma wr;
2087          * else
2088          *      we need to allocate extra rdma_iu to carry extra #ib_sge in
2089          *      another rdma wr
2090          */
2091         for (i = 0, j = 0;
2092              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2093                 rsize = be32_to_cpu(db->len);
2094                 raddr = be64_to_cpu(db->va);
2095                 riu->raddr = raddr;
2096                 riu->rkey = be32_to_cpu(db->key);
2097                 riu->sge_cnt = 0;
2098
2099                 /* calculate how many sge required for this remote_buf */
2100                 while (rsize > 0 && tsize > 0) {
2101
2102                         if (rsize >= dma_len) {
2103                                 tsize -= dma_len;
2104                                 rsize -= dma_len;
2105                                 raddr += dma_len;
2106
2107                                 if (tsize > 0) {
2108                                         ++j;
2109                                         if (j < count)
2110                                                 dma_len = sg_dma_len(&scat[j]);
2111                                 }
2112                         } else {
2113                                 tsize -= rsize;
2114                                 dma_len -= rsize;
2115                                 rsize = 0;
2116                         }
2117
2118                         ++riu->sge_cnt;
2119
2120                         if (rsize > 0 && riu->sge_cnt == SRPT_DEF_SG_PER_WQE) {
2121                                 riu->sge =
2122                                     kmalloc(riu->sge_cnt * sizeof *riu->sge,
2123                                             scst_cmd_atomic(scmnd)
2124                                             ? GFP_ATOMIC : GFP_KERNEL);
2125                                 if (!riu->sge)
2126                                         goto free_mem;
2127
2128                                 ++ioctx->n_rdma;
2129                                 ++riu;
2130                                 riu->sge_cnt = 0;
2131                                 riu->raddr = raddr;
2132                                 riu->rkey = be32_to_cpu(db->key);
2133                         }
2134                 }
2135
2136                 riu->sge = kmalloc(riu->sge_cnt * sizeof *riu->sge,
2137                                    scst_cmd_atomic(scmnd)
2138                                    ? GFP_ATOMIC : GFP_KERNEL);
2139
2140                 if (!riu->sge)
2141                         goto free_mem;
2142
2143                 ++ioctx->n_rdma;
2144         }
2145
2146         db = ioctx->rbufs;
2147         scat = scst_cmd_get_sg(scmnd);
2148         tsize = (dir == SCST_DATA_READ) ?
2149                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2150         riu = ioctx->rdma_ius;
2151         dma_len = sg_dma_len(&scat[0]);
2152         dma_addr = sg_dma_address(&scat[0]);
2153
2154         /* this second loop is really mapped sg_addres to rdma_iu->ib_sge */
2155         for (i = 0, j = 0;
2156              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2157                 rsize = be32_to_cpu(db->len);
2158                 sge = riu->sge;
2159                 k = 0;
2160
2161                 while (rsize > 0 && tsize > 0) {
2162                         sge->addr = dma_addr;
2163                         sge->lkey = ch->sport->sdev->mr->lkey;
2164
2165                         if (rsize >= dma_len) {
2166                                 sge->length =
2167                                         (tsize < dma_len) ? tsize : dma_len;
2168                                 tsize -= dma_len;
2169                                 rsize -= dma_len;
2170
2171                                 if (tsize > 0) {
2172                                         ++j;
2173                                         if (j < count) {
2174                                                 dma_len = sg_dma_len(&scat[j]);
2175                                                 dma_addr =
2176                                                     sg_dma_address(&scat[j]);
2177                                         }
2178                                 }
2179                         } else {
2180                                 sge->length = (tsize < rsize) ? tsize : rsize;
2181                                 tsize -= rsize;
2182                                 dma_len -= rsize;
2183                                 dma_addr += rsize;
2184                                 rsize = 0;
2185                         }
2186
2187                         ++k;
2188                         if (k == riu->sge_cnt && rsize > 0) {
2189                                 ++riu;
2190                                 sge = riu->sge;
2191                                 k = 0;
2192                         } else if (rsize > 0)
2193                                 ++sge;
2194                 }
2195         }
2196
2197         return 0;
2198
2199 free_mem:
2200         while (ioctx->n_rdma)
2201                 kfree(ioctx->rdma_ius[ioctx->n_rdma--].sge);
2202
2203         kfree(ioctx->rdma_ius);
2204
2205         dma_unmap_sg(ch->sport->sdev->device->dma_device,
2206                      scat, scst_cmd_get_sg_cnt(scmnd),
2207                      scst_to_tgt_dma_dir(dir));
2208
2209         return -ENOMEM;
2210 }
2211
2212 static int srpt_perform_rdmas(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2213                               scst_data_direction dir)
2214 {
2215         struct ib_send_wr wr;
2216         struct ib_send_wr *bad_wr;
2217         struct rdma_iu *riu;
2218         int i;
2219         int ret = 0;
2220
2221         riu = ioctx->rdma_ius;
2222         memset(&wr, 0, sizeof wr);
2223
2224         for (i = 0; i < ioctx->n_rdma; ++i, ++riu) {
2225                 wr.opcode = (dir == SCST_DATA_READ) ?
2226                     IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;
2227                 wr.next = NULL;
2228                 wr.wr_id = ioctx->index;
2229                 wr.wr.rdma.remote_addr = riu->raddr;
2230                 wr.wr.rdma.rkey = riu->rkey;
2231                 wr.num_sge = riu->sge_cnt;
2232                 wr.sg_list = riu->sge;
2233
2234                 /* only get completion event for the last rdma wr */
2235                 if (i == (ioctx->n_rdma - 1) && dir == SCST_DATA_WRITE)
2236                         wr.send_flags = IB_SEND_SIGNALED;
2237
2238                 ret = ib_post_send(ch->qp, &wr, &bad_wr);
2239                 if (ret)
2240                         break;
2241         }
2242
2243         return ret;
2244 }
2245
2246 /*
2247  * Start data transfer between initiator and target. Must not block.
2248  */
2249 static int srpt_xfer_data(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2250                           struct scst_cmd *scmnd)
2251 {
2252         int ret;
2253
2254         ret = srpt_map_sg_to_ib_sge(ch, ioctx, scmnd);
2255         if (ret) {
2256                 PRINT_ERROR("%s[%d] ret=%d", __func__, __LINE__, ret);
2257                 ret = SCST_TGT_RES_QUEUE_FULL;
2258                 goto out;
2259         }
2260
2261         ret = srpt_perform_rdmas(ch, ioctx, scst_cmd_get_data_direction(scmnd));
2262         if (ret) {
2263                 PRINT_ERROR("%s[%d] ret=%d", __func__, __LINE__, ret);
2264                 if (ret == -EAGAIN || ret == -ENOMEM)
2265                         ret = SCST_TGT_RES_QUEUE_FULL;
2266                 else
2267                         ret = SCST_TGT_RES_FATAL_ERROR;
2268                 goto out;
2269         }
2270
2271         ret = SCST_TGT_RES_SUCCESS;
2272
2273 out:
2274         return ret;
2275 }
2276
2277 /*
2278  * Called by the SCST core to inform ib_srpt that data reception from the
2279  * initiator should start (SCST_DATA_WRITE). Must not block.
2280  */
2281 static int srpt_rdy_to_xfer(struct scst_cmd *scmnd)
2282 {
2283         struct srpt_rdma_ch *ch;
2284         struct srpt_ioctx *ioctx;
2285
2286         ioctx = scst_cmd_get_tgt_priv(scmnd);
2287         BUG_ON(!ioctx);
2288
2289         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2290         BUG_ON(!ch);
2291
2292         if (ch->state == RDMA_CHANNEL_DISCONNECTING)
2293                 return SCST_TGT_RES_FATAL_ERROR;
2294         else if (ch->state == RDMA_CHANNEL_CONNECTING)
2295                 return SCST_TGT_RES_QUEUE_FULL;
2296
2297         ioctx->state = SRPT_STATE_NEED_DATA;
2298
2299         return srpt_xfer_data(ch, ioctx, scmnd);
2300 }
2301
2302 /*
2303  * Called by the SCST core. Transmits the response buffer and status held in
2304  * 'scmnd'. Must not block.
2305  */
2306 static int srpt_xmit_response(struct scst_cmd *scmnd)
2307 {
2308         struct srpt_rdma_ch *ch;
2309         struct srpt_ioctx *ioctx;
2310         struct srp_rsp *srp_rsp;
2311         u64 tag;
2312         int ret = SCST_TGT_RES_SUCCESS;
2313         int dir;
2314         int status;
2315
2316         ioctx = scst_cmd_get_tgt_priv(scmnd);
2317         BUG_ON(!ioctx);
2318
2319         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2320         BUG_ON(!ch);
2321
2322         tag = scst_cmd_get_tag(scmnd);
2323
2324         ioctx->state = SRPT_STATE_PROCESSED;
2325
2326         if (ch->state != RDMA_CHANNEL_LIVE) {
2327                 PRINT_ERROR("%s: tag= %lld channel in bad state %d",
2328                        __func__, (unsigned long long)tag, ch->state);
2329
2330                 if (ch->state == RDMA_CHANNEL_DISCONNECTING)
2331                         ret = SCST_TGT_RES_FATAL_ERROR;
2332                 else if (ch->state == RDMA_CHANNEL_CONNECTING)
2333                         ret = SCST_TGT_RES_QUEUE_FULL;
2334
2335                 if (unlikely(scst_cmd_aborted(scmnd)))
2336                         goto out_aborted;
2337
2338                 goto out;
2339         }
2340
2341         dma_sync_single_for_cpu(ch->sport->sdev->device->dma_device, ioctx->dma,
2342                                 MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
2343
2344         srp_rsp = ioctx->buf;
2345
2346         if (unlikely(scst_cmd_aborted(scmnd))) {
2347                 PRINT_ERROR("%s: tag= %lld already get aborted",
2348                        __func__, (unsigned long long)tag);
2349                 goto out_aborted;
2350         }
2351
2352         dir = scst_cmd_get_data_direction(scmnd);
2353         status = scst_cmd_get_status(scmnd) & 0xff;
2354
2355         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE, NO_ADD_SENSE, tag);
2356
2357         if (SCST_SENSE_VALID(scst_cmd_get_sense_buffer(scmnd))) {
2358                 srp_rsp->sense_data_len = scst_cmd_get_sense_buffer_len(scmnd);
2359                 if (srp_rsp->sense_data_len >
2360                     (MAX_MESSAGE_SIZE - sizeof *srp_rsp))
2361                         srp_rsp->sense_data_len =
2362                             MAX_MESSAGE_SIZE - sizeof *srp_rsp;
2363
2364                 memcpy((u8 *) (srp_rsp + 1), scst_cmd_get_sense_buffer(scmnd),
2365                        srp_rsp->sense_data_len);
2366
2367                 srp_rsp->sense_data_len = cpu_to_be32(srp_rsp->sense_data_len);
2368                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
2369
2370                 if (!status)
2371                         status = SAM_STAT_CHECK_CONDITION;
2372         }
2373
2374         srp_rsp->status = status;
2375
2376         /* For read commands, transfer the data to the initiator. */
2377         if (dir == SCST_DATA_READ && scst_cmd_get_resp_data_len(scmnd)) {
2378                 ret = srpt_xfer_data(ch, ioctx, scmnd);
2379                 if (ret != SCST_TGT_RES_SUCCESS) {
2380                         PRINT_ERROR("%s: tag= %lld xfer_data failed",
2381                                     __func__, (unsigned long long)tag);
2382                         goto out;
2383                 }
2384         }
2385
2386         if (srpt_post_send(ch, ioctx,
2387                            sizeof *srp_rsp +
2388                            be32_to_cpu(srp_rsp->sense_data_len))) {
2389                 PRINT_ERROR("%s: ch->state= %d tag= %lld",
2390                             __func__, ch->state,
2391                             (unsigned long long)tag);
2392                 ret = SCST_TGT_RES_FATAL_ERROR;
2393         }
2394
2395 out:
2396         return ret;
2397
2398 out_aborted:
2399         ret = SCST_TGT_RES_SUCCESS;
2400         scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_ABORTED);
2401         ioctx->state = SRPT_STATE_ABORTED;
2402         WARN_ON(scmnd->state != SCST_CMD_STATE_XMIT_WAIT);
2403         scst_tgt_cmd_done(scmnd, SCST_CONTEXT_SAME);
2404         goto out;
2405 }
2406
2407 /*
2408  * Called by the SCST core to inform ib_srpt that a received task management
2409  * function has been completed. Must not block.
2410  */
2411 static void srpt_tsk_mgmt_done(struct scst_mgmt_cmd *mcmnd)
2412 {
2413         struct srpt_rdma_ch *ch;
2414         struct srpt_mgmt_ioctx *mgmt_ioctx;
2415         struct srpt_ioctx *ioctx;
2416         int rsp_len;
2417
2418         mgmt_ioctx = scst_mgmt_cmd_get_tgt_priv(mcmnd);
2419         BUG_ON(!mgmt_ioctx);
2420
2421         ch = mgmt_ioctx->ch;
2422         BUG_ON(!ch);
2423
2424         ioctx = mgmt_ioctx->ioctx;
2425         BUG_ON(!ioctx);
2426
2427         TRACE_DBG("%s: tsk_mgmt_done for tag= %lld status=%d",
2428                   __func__, (unsigned long long)mgmt_ioctx->tag,
2429                   scst_mgmt_cmd_get_status(mcmnd));
2430
2431         rsp_len = srpt_build_tskmgmt_rsp(ch, ioctx,
2432                                          (scst_mgmt_cmd_get_status(mcmnd) ==
2433                                           SCST_MGMT_STATUS_SUCCESS) ?
2434                                          SRP_TSK_MGMT_SUCCESS :
2435                                          SRP_TSK_MGMT_FAILED,
2436                                          mgmt_ioctx->tag);
2437         srpt_post_send(ch, ioctx, rsp_len);
2438
2439         scst_mgmt_cmd_set_tgt_priv(mcmnd, NULL);
2440
2441         kfree(mgmt_ioctx);
2442 }
2443
2444 /*
2445  * Called by the SCST core to inform ib_srpt that the command 'scmnd' is about
2446  * to be freed. May be called in IRQ context.
2447  */
2448 static void srpt_on_free_cmd(struct scst_cmd *scmnd)
2449 {
2450         struct srpt_rdma_ch *ch;
2451         struct srpt_ioctx *ioctx;
2452
2453         ioctx = scst_cmd_get_tgt_priv(scmnd);
2454         BUG_ON(!ioctx);
2455
2456         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2457         BUG_ON(!ch);
2458
2459         spin_lock_irq(&ch->spinlock);
2460         list_del(&ioctx->scmnd_list);
2461         ch->active_scmnd_cnt--;
2462         spin_unlock_irq(&ch->spinlock);
2463
2464         srpt_reset_ioctx(ch, ioctx);
2465         scst_cmd_set_tgt_priv(scmnd, NULL);
2466 }
2467
2468 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2469 /* A vanilla 2.6.19 or older kernel without backported OFED kernel headers. */
2470 static void srpt_refresh_port_work(void *ctx)
2471 #else
2472 static void srpt_refresh_port_work(struct work_struct *work)
2473 #endif
2474 {
2475 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2476         struct srpt_port *sport = (struct srpt_port *)ctx;
2477 #else
2478         struct srpt_port *sport = container_of(work, struct srpt_port, work);
2479 #endif
2480
2481         srpt_refresh_port(sport);
2482 }
2483
2484 /*
2485  * Called by the SCST core to detect target adapters. Returns the number of
2486  * detected target adapters.
2487  */
2488 static int srpt_detect(struct scst_tgt_template *tp)
2489 {
2490         int device_count;
2491
2492         TRACE_ENTRY();
2493
2494         device_count = atomic_read(&srpt_device_count);
2495
2496         TRACE_EXIT_RES(device_count);
2497
2498         return device_count;
2499 }
2500
2501 /*
2502  * Callback function called by the SCST core from scst_unregister() to free up
2503  * the resources associated with device scst_tgt.
2504  */
2505 static int srpt_release(struct scst_tgt *scst_tgt)
2506 {
2507         struct srpt_device *sdev = scst_tgt_get_tgt_priv(scst_tgt);
2508         struct srpt_rdma_ch *ch, *tmp_ch;
2509
2510         TRACE_ENTRY();
2511
2512         BUG_ON(!scst_tgt);
2513 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
2514         WARN_ON(!sdev);
2515         if (!sdev)
2516                 return -ENODEV;
2517 #else
2518         if (WARN_ON(!sdev))
2519                 return -ENODEV;
2520 #endif
2521
2522         srpt_unregister_procfs_entry(scst_tgt->tgtt);
2523
2524         spin_lock_irq(&sdev->spinlock);
2525         list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
2526                 list_del(&ch->list);
2527                 spin_unlock_irq(&sdev->spinlock);
2528                 srpt_release_channel(ch, 1);
2529                 spin_lock_irq(&sdev->spinlock);
2530         }
2531         spin_unlock_irq(&sdev->spinlock);
2532
2533         srpt_unregister_mad_agent(sdev);
2534
2535         scst_tgt_set_tgt_priv(scst_tgt, NULL);
2536
2537         TRACE_EXIT();
2538
2539         return 0;
2540 }
2541
2542 /*
2543  * Entry point for ib_srpt's kernel thread. This kernel thread is only created
2544  * when the module parameter 'thread' is not zero (the default is zero).
2545  * This thread processes the ioctx list srpt_thread.thread_ioctx_list.
2546  *
2547  * @pre thread != 0
2548  */
2549 static int srpt_ioctx_thread(void *arg)
2550 {
2551         struct srpt_ioctx *ioctx;
2552
2553         /* Hibernation / freezing of the SRPT kernel thread is not supported. */
2554         current->flags |= PF_NOFREEZE;
2555
2556         spin_lock_irq(&srpt_thread.thread_lock);
2557         while (!kthread_should_stop()) {
2558                 wait_queue_t wait;
2559                 init_waitqueue_entry(&wait, current);
2560
2561                 if (!srpt_test_ioctx_list()) {
2562                         add_wait_queue_exclusive(&ioctx_list_waitQ, &wait);
2563
2564                         for (;;) {
2565                                 set_current_state(TASK_INTERRUPTIBLE);
2566                                 if (srpt_test_ioctx_list())
2567                                         break;
2568                                 spin_unlock_irq(&srpt_thread.thread_lock);
2569                                 schedule();
2570                                 spin_lock_irq(&srpt_thread.thread_lock);
2571                         }
2572                         set_current_state(TASK_RUNNING);
2573                         remove_wait_queue(&ioctx_list_waitQ, &wait);
2574                 }
2575
2576                 while (!list_empty(&srpt_thread.thread_ioctx_list)) {
2577                         ioctx = list_entry(srpt_thread.thread_ioctx_list.next,
2578                                            struct srpt_ioctx, comp_list);
2579
2580                         list_del(&ioctx->comp_list);
2581
2582                         spin_unlock_irq(&srpt_thread.thread_lock);
2583                         switch (ioctx->op) {
2584                         case IB_WC_SEND:
2585                                 srpt_handle_send_comp(ioctx->ch, ioctx,
2586                                         SCST_CONTEXT_DIRECT);
2587                                 break;
2588                         case IB_WC_RDMA_WRITE:
2589                         case IB_WC_RDMA_READ:
2590                                 srpt_handle_rdma_comp(ioctx->ch, ioctx);
2591                                 break;
2592                         case IB_WC_RECV:
2593                                 srpt_handle_new_iu(ioctx->ch, ioctx);
2594                                 break;
2595                         default:
2596                                 break;
2597                         }
2598                         spin_lock_irq(&srpt_thread.thread_lock);
2599                 }
2600         }
2601         spin_unlock_irq(&srpt_thread.thread_lock);
2602
2603         return 0;
2604 }
2605
2606 /* SCST target template for the SRP target implementation. */
2607 static struct scst_tgt_template srpt_template = {
2608         .name = DRV_NAME,
2609         .sg_tablesize = SRPT_DEF_SG_TABLESIZE,
2610         .xmit_response_atomic = 1,
2611         .rdy_to_xfer_atomic = 1,
2612         .no_proc_entry = 0,
2613         .detect = srpt_detect,
2614         .release = srpt_release,
2615         .xmit_response = srpt_xmit_response,
2616         .rdy_to_xfer = srpt_rdy_to_xfer,
2617         .on_free_cmd = srpt_on_free_cmd,
2618         .task_mgmt_fn_done = srpt_tsk_mgmt_done
2619 };
2620
2621 /*
2622  * The callback function srpt_release_class_dev() is called whenever a
2623  * device is removed from the /sys/class/infiniband_srpt device class.
2624  */
2625 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2626 static void srpt_release_class_dev(struct class_device *class_dev)
2627 #else
2628 static void srpt_release_class_dev(struct device *dev)
2629 #endif
2630 {
2631 }
2632
2633 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2634 static int srpt_trace_level_show(struct seq_file *seq, void *v)
2635 {
2636         return scst_proc_log_entry_read(seq, trace_flag, NULL);
2637 }
2638
2639 static ssize_t srpt_proc_trace_level_write(struct file *file,
2640         const char __user *buf, size_t length, loff_t *off)
2641 {
2642         return scst_proc_log_entry_write(file, buf, length, &trace_flag,
2643                 DEFAULT_SRPT_TRACE_FLAGS, NULL);
2644 }
2645
2646 static struct scst_proc_data srpt_log_proc_data = {
2647         SCST_DEF_RW_SEQ_OP(srpt_proc_trace_level_write)
2648         .show = srpt_trace_level_show,
2649 };
2650 #endif
2651
2652 static struct class_attribute srpt_class_attrs[] = {
2653         __ATTR_NULL,
2654 };
2655
2656 static struct class srpt_class = {
2657         .name = "infiniband_srpt",
2658 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2659         .release = srpt_release_class_dev,
2660 #else
2661         .dev_release = srpt_release_class_dev,
2662 #endif
2663         .class_attrs = srpt_class_attrs,
2664 };
2665
2666 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2667 static ssize_t show_login_info(struct class_device *class_dev, char *buf)
2668 #else
2669 static ssize_t show_login_info(struct device *dev,
2670                                struct device_attribute *attr, char *buf)
2671 #endif
2672 {
2673         struct srpt_device *sdev =
2674 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2675                 container_of(class_dev, struct srpt_device, class_dev);
2676 #else
2677                 container_of(dev, struct srpt_device, dev);
2678 #endif
2679         struct srpt_port *sport;
2680         int i;
2681         int len = 0;
2682
2683         for (i = 0; i < sdev->device->phys_port_cnt; i++) {
2684                 sport = &sdev->port[i];
2685
2686                 len += sprintf(buf + len,
2687                                "tid_ext=%016llx,ioc_guid=%016llx,pkey=ffff,"
2688                                "dgid=%04x%04x%04x%04x%04x%04x%04x%04x,"
2689                                "service_id=%016llx\n",
2690                                (unsigned long long) srpt_service_guid,
2691                                (unsigned long long) srpt_service_guid,
2692                                be16_to_cpu(((__be16 *) sport->gid.raw)[0]),
2693                                be16_to_cpu(((__be16 *) sport->gid.raw)[1]),
2694                                be16_to_cpu(((__be16 *) sport->gid.raw)[2]),
2695                                be16_to_cpu(((__be16 *) sport->gid.raw)[3]),
2696                                be16_to_cpu(((__be16 *) sport->gid.raw)[4]),
2697                                be16_to_cpu(((__be16 *) sport->gid.raw)[5]),
2698                                be16_to_cpu(((__be16 *) sport->gid.raw)[6]),
2699                                be16_to_cpu(((__be16 *) sport->gid.raw)[7]),
2700                                (unsigned long long) srpt_service_guid);
2701         }
2702
2703         return len;
2704 }
2705
2706 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2707 static CLASS_DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2708 #else
2709 static DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2710 #endif
2711
2712 /*
2713  * Callback function called by the InfiniBand core when either an InfiniBand
2714  * device has been added or during the ib_register_client() call for each
2715  * registered InfiniBand device.
2716  */
2717 static void srpt_add_one(struct ib_device *device)
2718 {
2719         struct srpt_device *sdev;
2720         struct srpt_port *sport;
2721         struct ib_srq_init_attr srq_attr;
2722         int i;
2723
2724         TRACE_ENTRY();
2725
2726         sdev = kzalloc(sizeof *sdev, GFP_KERNEL);
2727         if (!sdev)
2728                 return;
2729
2730         sdev->device = device;
2731
2732 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2733         sdev->class_dev.class = &srpt_class;
2734         sdev->class_dev.dev = device->dma_device;
2735         snprintf(sdev->class_dev.class_id, BUS_ID_SIZE,
2736                  "srpt-%s", device->name);
2737 #else
2738         sdev->dev.class = &srpt_class;
2739         sdev->dev.parent = device->dma_device;
2740 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
2741         snprintf(sdev->dev.bus_id, BUS_ID_SIZE, "srpt-%s", device->name);
2742 #else
2743         snprintf(sdev->init_name, sizeof(sdev->init_name),
2744                  "srpt-%s", device->name);
2745         sdev->dev.init_name = sdev->init_name;
2746 #endif
2747 #endif
2748
2749 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2750         if (class_device_register(&sdev->class_dev))
2751                 goto free_dev;
2752         if (class_device_create_file(&sdev->class_dev,
2753                                      &class_device_attr_login_info))
2754                 goto err_dev;
2755 #else
2756         if (device_register(&sdev->dev))
2757                 goto free_dev;
2758         if (device_create_file(&sdev->dev, &dev_attr_login_info))
2759                 goto err_dev;
2760 #endif
2761
2762         if (ib_query_device(device, &sdev->dev_attr))
2763                 goto err_dev;
2764
2765         sdev->pd = ib_alloc_pd(device);
2766         if (IS_ERR(sdev->pd))
2767                 goto err_dev;
2768
2769         sdev->mr = ib_get_dma_mr(sdev->pd, IB_ACCESS_LOCAL_WRITE);
2770         if (IS_ERR(sdev->mr))
2771                 goto err_pd;
2772
2773         srq_attr.event_handler = srpt_srq_event;
2774         srq_attr.srq_context = (void *)sdev;
2775         srq_attr.attr.max_wr = min(SRPT_SRQ_SIZE, sdev->dev_attr.max_srq_wr);
2776         srq_attr.attr.max_sge = 1;
2777         srq_attr.attr.srq_limit = 0;
2778
2779         sdev->srq = ib_create_srq(sdev->pd, &srq_attr);
2780         if (IS_ERR(sdev->srq))
2781                 goto err_mr;
2782
2783         TRACE_DBG("%s: create SRQ #wr= %d max_allow=%d dev= %s",
2784                __func__, srq_attr.attr.max_wr,
2785               sdev->dev_attr.max_srq_wr, device->name);
2786
2787         if (!srpt_service_guid)
2788                 srpt_service_guid = be64_to_cpu(device->node_guid);
2789
2790         sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);
2791         if (IS_ERR(sdev->cm_id))
2792                 goto err_srq;
2793
2794         /* print out target login information */
2795         TRACE_DBG("Target login info: id_ext=%016llx,"
2796                   "ioc_guid=%016llx,pkey=ffff,service_id=%016llx",
2797                   (unsigned long long) srpt_service_guid,
2798                   (unsigned long long) srpt_service_guid,
2799                   (unsigned long long) srpt_service_guid);
2800
2801         /*
2802          * We do not have a consistent service_id (ie. also id_ext of target_id)
2803          * to identify this target. We currently use the guid of the first HCA
2804          * in the system as service_id; therefore, the target_id will change
2805          * if this HCA is gone bad and replaced by different HCA
2806          */
2807         if (ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0, NULL))
2808                 goto err_cm;
2809
2810         INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
2811                               srpt_event_handler);
2812         if (ib_register_event_handler(&sdev->event_handler))
2813                 goto err_cm;
2814
2815         if (srpt_alloc_ioctx_ring(sdev))
2816                 goto err_event;
2817
2818         INIT_LIST_HEAD(&sdev->rch_list);
2819         spin_lock_init(&sdev->spinlock);
2820
2821         for (i = 0; i < SRPT_SRQ_SIZE; ++i)
2822                 srpt_post_recv(sdev, sdev->ioctx_ring[i]);
2823
2824         ib_set_client_data(device, &srpt_client, sdev);
2825
2826         sdev->scst_tgt = scst_register(&srpt_template, NULL);
2827         if (!sdev->scst_tgt) {
2828                 PRINT_ERROR("SCST registration failed for %s.",
2829                             sdev->device->name);
2830                 goto err_ring;
2831         }
2832
2833         scst_tgt_set_tgt_priv(sdev->scst_tgt, sdev);
2834
2835         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
2836                 sport = &sdev->port[i - 1];
2837                 sport->sdev = sdev;
2838                 sport->port = i;
2839 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2840                 /*
2841                  * A vanilla 2.6.19 or older kernel without backported OFED
2842                  * kernel headers.
2843                  */
2844                 INIT_WORK(&sport->work, srpt_refresh_port_work, sport);
2845 #else
2846                 INIT_WORK(&sport->work, srpt_refresh_port_work);
2847 #endif
2848                 if (srpt_refresh_port(sport)) {
2849                         PRINT_ERROR("MAD registration failed for %s-%d.",
2850                                     sdev->device->name, i);
2851                         goto err_refresh_port;
2852                 }
2853         }
2854
2855         atomic_inc(&srpt_device_count);
2856
2857         TRACE_EXIT();
2858
2859         return;
2860
2861 err_refresh_port:
2862         scst_unregister(sdev->scst_tgt);
2863 err_ring:
2864         ib_set_client_data(device, &srpt_client, NULL);
2865         srpt_free_ioctx_ring(sdev);
2866 err_event:
2867         ib_unregister_event_handler(&sdev->event_handler);
2868 err_cm:
2869         ib_destroy_cm_id(sdev->cm_id);
2870 err_srq:
2871         ib_destroy_srq(sdev->srq);
2872 err_mr:
2873         ib_dereg_mr(sdev->mr);
2874 err_pd:
2875         ib_dealloc_pd(sdev->pd);
2876 err_dev:
2877 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2878         class_device_unregister(&sdev->class_dev);
2879 #else
2880         device_unregister(&sdev->dev);
2881 #endif
2882 free_dev:
2883         kfree(sdev);
2884
2885         TRACE_EXIT();
2886 }
2887
2888 /*
2889  * Callback function called by the InfiniBand core when either an InfiniBand
2890  * device has been removed or during the ib_unregister_client() call for each
2891  * registered InfiniBand device.
2892  */
2893 static void srpt_remove_one(struct ib_device *device)
2894 {
2895         int i;
2896         struct srpt_device *sdev;
2897
2898         TRACE_ENTRY();
2899
2900         sdev = ib_get_client_data(device, &srpt_client);
2901 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
2902         WARN_ON(!sdev);
2903         if (!sdev)
2904                 return;
2905 #else
2906         if (WARN_ON(!sdev))
2907                 return;
2908 #endif
2909
2910         /*
2911          * Cancel the work if it is queued. Wait until srpt_refresh_port_work()
2912          * finished if it is running.
2913          */
2914         for (i = 0; i < sdev->device->phys_port_cnt; i++)
2915 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
2916                 cancel_work_sync(&sdev->port[i].work);
2917 #else
2918                 /*
2919                  * cancel_work_sync() was introduced in kernel 2.6.22. Older
2920                  * kernels do not have a facility to cancel scheduled work.
2921                  */
2922                 PRINT_ERROR("%s",
2923                        "your kernel does not provide cancel_work_sync().");
2924 #endif
2925
2926         scst_unregister(sdev->scst_tgt);
2927         sdev->scst_tgt = NULL;
2928
2929         ib_unregister_event_handler(&sdev->event_handler);
2930         ib_destroy_cm_id(sdev->cm_id);
2931         ib_destroy_srq(sdev->srq);
2932         ib_dereg_mr(sdev->mr);
2933         ib_dealloc_pd(sdev->pd);
2934 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2935         class_device_unregister(&sdev->class_dev);
2936 #else
2937         device_unregister(&sdev->dev);
2938 #endif
2939
2940         srpt_free_ioctx_ring(sdev);
2941         kfree(sdev);
2942
2943         TRACE_EXIT();
2944 }
2945
2946 /**
2947  * Create procfs entries for srpt. Currently the only procfs entry created
2948  * by this function is the "trace_level" entry.
2949  */
2950 static int srpt_register_procfs_entry(struct scst_tgt_template *tgt)
2951 {
2952         int res = 0;
2953 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2954         struct proc_dir_entry *p, *root;
2955
2956         root = scst_proc_get_tgt_root(tgt);
2957         WARN_ON(!root);
2958         if (root) {
2959                 /*
2960                  * Fill in the scst_proc_data::data pointer, which is used in
2961                  * a printk(KERN_INFO ...) statement in
2962                  * scst_proc_log_entry_write() in scst_proc.c.
2963                  */
2964                 srpt_log_proc_data.data = (char *)tgt->name;
2965                 p = scst_create_proc_entry(root, SRPT_PROC_TRACE_LEVEL_NAME,
2966                                            &srpt_log_proc_data);
2967                 if (!p)
2968                         res = -ENOMEM;
2969         } else
2970                 res = -ENOMEM;
2971
2972 #endif
2973         return res;
2974 }
2975
2976 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt)
2977 {
2978 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2979         struct proc_dir_entry *root;
2980
2981         root = scst_proc_get_tgt_root(tgt);
2982         WARN_ON(!root);
2983         if (root)
2984                 remove_proc_entry(SRPT_PROC_TRACE_LEVEL_NAME, root);
2985 #endif
2986 }
2987
2988 /*
2989  * Module initialization.
2990  *
2991  * Note: since ib_register_client() registers callback functions, and since at
2992  * least one of these callback functions (srpt_add_one()) calls SCST functions,
2993  * the SCST target template must be registered before ib_register_client() is
2994  * called.
2995  */
2996 static int __init srpt_init_module(void)
2997 {
2998         int ret;
2999
3000         ret = class_register(&srpt_class);
3001         if (ret) {
3002                 PRINT_ERROR("%s", "couldn't register class ib_srpt");
3003                 goto out;
3004         }
3005
3006         ret = scst_register_target_template(&srpt_template);
3007         if (ret < 0) {
3008                 PRINT_ERROR("%s", "couldn't register with scst");
3009                 ret = -ENODEV;
3010                 goto out_unregister_class;
3011         }
3012
3013         ret = srpt_register_procfs_entry(&srpt_template);
3014         if (ret) {
3015                 PRINT_ERROR("%s", "couldn't register procfs entry");
3016                 goto out_unregister_target;
3017         }
3018
3019         ret = ib_register_client(&srpt_client);
3020         if (ret) {
3021                 PRINT_ERROR("%s", "couldn't register IB client");
3022                 goto out_unregister_target;
3023         }
3024
3025         if (thread) {
3026                 spin_lock_init(&srpt_thread.thread_lock);
3027                 INIT_LIST_HEAD(&srpt_thread.thread_ioctx_list);
3028                 srpt_thread.thread = kthread_run(srpt_ioctx_thread,
3029                                                  NULL, "srpt_thread");
3030                 if (IS_ERR(srpt_thread.thread)) {
3031                         srpt_thread.thread = NULL;
3032                         thread = 0;
3033                 }
3034         }
3035
3036         return 0;
3037
3038 out_unregister_target:
3039         /*
3040          * Note: the procfs entry is unregistered in srpt_release(), which is
3041          * called by scst_unregister_target_template().
3042          */
3043         scst_unregister_target_template(&srpt_template);
3044 out_unregister_class:
3045         class_unregister(&srpt_class);
3046 out:
3047         return ret;
3048 }
3049
3050 static void __exit srpt_cleanup_module(void)
3051 {
3052         TRACE_ENTRY();
3053
3054         if (srpt_thread.thread)
3055                 kthread_stop(srpt_thread.thread);
3056         ib_unregister_client(&srpt_client);
3057         scst_unregister_target_template(&srpt_template);
3058         class_unregister(&srpt_class);
3059
3060         TRACE_EXIT();
3061 }
3062
3063 module_init(srpt_init_module);
3064 module_exit(srpt_cleanup_module);