1df516ba999c338b4033d7995011f3752605266a
[mirror/scst/.git] / srpt / src / ib_srpt.c
1 /*
2  * Copyright (c) 2006 - 2009 Mellanox Technology Inc.  All rights reserved.
3  * Copyright (C) 2008 Vladislav Bolkhovitin <vst@vlnb.net>
4  * Copyright (C) 2008 - 2009 Bart Van Assche <bart.vanassche@gmail.com>
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  */
35
36 #include <linux/module.h>
37 #include <linux/init.h>
38 #include <linux/slab.h>
39 #include <linux/err.h>
40 #include <linux/ctype.h>
41 #include <linux/string.h>
42 #include <linux/kthread.h>
43 #include <asm/atomic.h>
44 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
45 #include <linux/proc_fs.h>
46 #include <linux/seq_file.h>
47 #endif
48 #include "ib_srpt.h"
49 #include "scst_debug.h"
50
51 /* Name of this kernel module. */
52 #define DRV_NAME                "ib_srpt"
53 /* Prefix for printk() kernel messages. */
54 #define LOG_PFX                 DRV_NAME ": "
55 #define DRV_VERSION             "1.0.1"
56 #define DRV_RELDATE             "July 10, 2008"
57 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
58 /* Flags to be used in SCST debug tracing statements. */
59 #define DEFAULT_SRPT_TRACE_FLAGS (TRACE_OUT_OF_MEM | TRACE_MINOR \
60                                   | TRACE_MGMT | TRACE_SPECIAL)
61 /* Name of the entry that will be created under /proc/scsi_tgt/ib_srpt. */
62 #define SRPT_PROC_TRACE_LEVEL_NAME      "trace_level"
63 #endif
64
65 #define MELLANOX_SRPT_ID_STRING "SCST SRP target"
66
67 MODULE_AUTHOR("Vu Pham");
68 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
69                    "v" DRV_VERSION " (" DRV_RELDATE ")");
70 MODULE_LICENSE("Dual BSD/GPL");
71
72 struct srpt_thread {
73         /* Protects thread_ioctx_list. */
74         spinlock_t thread_lock;
75         /* I/O contexts to be processed by the kernel thread. */
76         struct list_head thread_ioctx_list;
77         /* SRPT kernel thread. */
78         struct task_struct *thread;
79 };
80
81 /*
82  * Global Variables
83  */
84
85 static u64 srpt_service_guid;
86 /* List of srpt_device structures. */
87 static atomic_t srpt_device_count;
88 static int use_port_guid_in_session_name;
89 static int thread;
90 static struct srpt_thread srpt_thread;
91 static DECLARE_WAIT_QUEUE_HEAD(ioctx_list_waitQ);
92 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
93 static unsigned long trace_flag = DEFAULT_SRPT_TRACE_FLAGS;
94 module_param(trace_flag, long, 0644);
95 MODULE_PARM_DESC(trace_flag,
96                  "Trace flags for the ib_srpt kernel module.");
97 static unsigned long processing_delay_in_us;
98 module_param(processing_delay_in_us, long, 0744);
99 MODULE_PARM_DESC(processing_delay_in_us,
100                  "Per CQ event processing delay in microseconds.");
101 #endif
102
103 module_param(thread, int, 0444);
104 MODULE_PARM_DESC(thread,
105                  "Executing ioctx in thread context. Default 0, i.e. soft IRQ, "
106                  "where possible");
107
108 module_param(use_port_guid_in_session_name, bool, 0444);
109 MODULE_PARM_DESC(use_port_guid_in_session_name,
110                  "Use target port ID in the SCST session name such that"
111                  " redundant paths between multiport systems can be masked.");
112
113 static void srpt_add_one(struct ib_device *device);
114 static void srpt_remove_one(struct ib_device *device);
115 static void srpt_unregister_mad_agent(struct srpt_device *sdev);
116 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt);
117
118 static struct ib_client srpt_client = {
119         .name = DRV_NAME,
120         .add = srpt_add_one,
121         .remove = srpt_remove_one
122 };
123
124 /**
125  * Atomically test and set the channel state.
126  * @ch: RDMA channel.
127  * @old: channel state to compare with.
128  * @new: state to change the channel state to if the current state matches the
129  *       argument 'old'.
130  *
131  * Returns true if the channel state matched old upon entry of this function,
132  * and false otherwise.
133  */
134 static bool srpt_test_and_set_channel_state(struct srpt_rdma_ch *ch,
135                                             enum rdma_ch_state old,
136                                             enum rdma_ch_state new)
137 {
138         unsigned long flags;
139         enum rdma_ch_state cur;
140
141         spin_lock_irqsave(&ch->spinlock, flags);
142         cur = ch->state;
143         if (cur == old)
144                 ch->state = new;
145         spin_unlock_irqrestore(&ch->spinlock, flags);
146
147         return cur == old;
148 }
149
150 /*
151  * Callback function called by the InfiniBand core when an asynchronous IB
152  * event occurs. This callback may occur in interrupt context. See also
153  * section 11.5.2, Set Asynchronous Event Handler in the InfiniBand
154  * Architecture Specification.
155  */
156 static void srpt_event_handler(struct ib_event_handler *handler,
157                                struct ib_event *event)
158 {
159         struct srpt_device *sdev;
160         struct srpt_port *sport;
161
162         sdev = ib_get_client_data(event->device, &srpt_client);
163         if (!sdev || sdev->device != event->device)
164                 return;
165
166         TRACE_DBG("ASYNC event= %d on device= %s",
167                   event->event, sdev->device->name);
168
169         switch (event->event) {
170         case IB_EVENT_PORT_ERR:
171                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
172                         sport = &sdev->port[event->element.port_num - 1];
173                         sport->lid = 0;
174                         sport->sm_lid = 0;
175                 }
176                 break;
177         case IB_EVENT_PORT_ACTIVE:
178         case IB_EVENT_LID_CHANGE:
179         case IB_EVENT_PKEY_CHANGE:
180         case IB_EVENT_SM_CHANGE:
181         case IB_EVENT_CLIENT_REREGISTER:
182                 /*
183                  * Refresh port data asynchronously. Note: it is safe to call
184                  * schedule_work() even if &sport->work is already on the
185                  * global workqueue because schedule_work() tests for the
186                  * work_pending() condition before adding &sport->work to the
187                  * global work queue.
188                  */
189                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
190                         sport = &sdev->port[event->element.port_num - 1];
191                         if (!sport->lid && !sport->sm_lid)
192                                 schedule_work(&sport->work);
193                 }
194                 break;
195         default:
196                 break;
197         }
198
199 }
200
201 /*
202  * Callback function called by the InfiniBand core for SRQ (shared receive
203  * queue) events.
204  */
205 static void srpt_srq_event(struct ib_event *event, void *ctx)
206 {
207         TRACE_DBG("SRQ event %d", event->event);
208 }
209
210 /*
211  * Callback function called by the InfiniBand core for QP (queue pair) events.
212  */
213 static void srpt_qp_event(struct ib_event *event, void *ctx)
214 {
215         struct srpt_rdma_ch *ch = ctx;
216
217         TRACE_DBG("QP event %d on cm_id=%p sess_name=%s state=%d",
218                   event->event, ch->cm_id, ch->sess_name, ch->state);
219
220         switch (event->event) {
221         case IB_EVENT_COMM_EST:
222 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20) || defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
223                 ib_cm_notify(ch->cm_id, event->event);
224 #else
225                 /* Vanilla 2.6.19 kernel (or before) without OFED. */
226                 PRINT_ERROR("%s", "how to perform ib_cm_notify() on a"
227                             " vanilla 2.6.18 kernel ???");
228 #endif
229                 break;
230         case IB_EVENT_QP_LAST_WQE_REACHED:
231                 if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_LIVE,
232                                         RDMA_CHANNEL_DISCONNECTING)) {
233                         PRINT_INFO("disconnected session %s.", ch->sess_name);
234                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
235                 }
236                 break;
237         default:
238                 break;
239         }
240 }
241
242 /*
243  * Helper function for filling in an InfiniBand IOUnitInfo structure. Copies
244  * the lowest four bits of value in element slot of the array of four bit
245  * elements called c_list (controller list). The index slot is one-based.
246  *
247  * @pre 1 <= slot && 0 <= value && value < 16
248  */
249 static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
250 {
251         u16 id;
252         u8 tmp;
253
254         id = (slot - 1) / 2;
255         if (slot & 0x1) {
256                 tmp = c_list[id] & 0xf;
257                 c_list[id] = (value << 4) | tmp;
258         } else {
259                 tmp = c_list[id] & 0xf0;
260                 c_list[id] = (value & 0xf) | tmp;
261         }
262 }
263
264 /*
265  * Write InfiniBand ClassPortInfo to mad. See also section 16.3.3.1
266  * ClassPortInfo in the InfiniBand Architecture Specification.
267  */
268 static void srpt_get_class_port_info(struct ib_dm_mad *mad)
269 {
270         struct ib_class_port_info *cif;
271
272         cif = (struct ib_class_port_info *)mad->data;
273         memset(cif, 0, sizeof *cif);
274         cif->base_version = 1;
275         cif->class_version = 1;
276         cif->resp_time_value = 20;
277
278         mad->mad_hdr.status = 0;
279 }
280
281 /*
282  * Write IOUnitInfo to mad. See also section 16.3.3.3 IOUnitInfo in the
283  * InfiniBand Architecture Specification. See also section B.7,
284  * table B.6 in the T10 SRP r16a document.
285  */
286 static void srpt_get_iou(struct ib_dm_mad *mad)
287 {
288         struct ib_dm_iou_info *ioui;
289         u8 slot;
290         int i;
291
292         ioui = (struct ib_dm_iou_info *)mad->data;
293         ioui->change_id = 1;
294         ioui->max_controllers = 16;
295
296         /* set present for slot 1 and empty for the rest */
297         srpt_set_ioc(ioui->controller_list, 1, 1);
298         for (i = 1, slot = 2; i < 16; i++, slot++)
299                 srpt_set_ioc(ioui->controller_list, slot, 0);
300
301         mad->mad_hdr.status = 0;
302 }
303
304 /*
305  * Write IOControllerprofile to mad for I/O controller (sdev, slot). See also
306  * section 16.3.3.4 IOControllerProfile in the InfiniBand Architecture
307  * Specification. See also section B.7, table B.7 in the T10 SRP r16a
308  * document.
309  */
310 static void srpt_get_ioc(struct srpt_device *sdev, u32 slot,
311                          struct ib_dm_mad *mad)
312 {
313         struct ib_dm_ioc_profile *iocp;
314
315         iocp = (struct ib_dm_ioc_profile *)mad->data;
316
317         if (!slot || slot > 16) {
318                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
319                 return;
320         }
321
322         if (slot > 2) {
323                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
324                 return;
325         }
326
327         memset(iocp, 0, sizeof *iocp);
328         strcpy(iocp->id_string, MELLANOX_SRPT_ID_STRING);
329         iocp->guid = cpu_to_be64(srpt_service_guid);
330         iocp->vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
331         iocp->device_id = cpu_to_be32(sdev->dev_attr.vendor_part_id);
332         iocp->device_version = cpu_to_be16(sdev->dev_attr.hw_ver);
333         iocp->subsys_vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
334         iocp->subsys_device_id = 0x0;
335         iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
336         iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
337         iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
338         iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
339         iocp->send_queue_depth = cpu_to_be16(SRPT_SRQ_SIZE);
340         iocp->rdma_read_depth = 4;
341         iocp->send_size = cpu_to_be32(MAX_MESSAGE_SIZE);
342         iocp->rdma_size = cpu_to_be32(MAX_RDMA_SIZE);
343         iocp->num_svc_entries = 1;
344         iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
345                 SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
346
347         mad->mad_hdr.status = 0;
348 }
349
350 /*
351  * Device management: write ServiceEntries to mad for the given slot. See also
352  * section 16.3.3.5 ServiceEntries in the InfiniBand Architecture
353  * Specification. See also section B.7, table B.8 in the T10 SRP r16a document.
354  */
355 static void srpt_get_svc_entries(u64 ioc_guid,
356                                  u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
357 {
358         struct ib_dm_svc_entries *svc_entries;
359
360         WARN_ON(!ioc_guid);
361
362         if (!slot || slot > 16) {
363                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
364                 return;
365         }
366
367         if (slot > 2 || lo > hi || hi > 1) {
368                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
369                 return;
370         }
371
372         svc_entries = (struct ib_dm_svc_entries *)mad->data;
373         memset(svc_entries, 0, sizeof *svc_entries);
374         svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid);
375         snprintf(svc_entries->service_entries[0].name,
376                  sizeof(svc_entries->service_entries[0].name),
377                  "%s%016llx",
378                  SRP_SERVICE_NAME_PREFIX,
379                  (unsigned long long)ioc_guid);
380
381         mad->mad_hdr.status = 0;
382 }
383
384 /*
385  * Actual processing of a received MAD *rq_mad received through source port *sp
386  * (MAD = InfiniBand management datagram). The response to be sent back is
387  * written to *rsp_mad.
388  */
389 static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
390                                  struct ib_dm_mad *rsp_mad)
391 {
392         u16 attr_id;
393         u32 slot;
394         u8 hi, lo;
395
396         attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
397         switch (attr_id) {
398         case DM_ATTR_CLASS_PORT_INFO:
399                 srpt_get_class_port_info(rsp_mad);
400                 break;
401         case DM_ATTR_IOU_INFO:
402                 srpt_get_iou(rsp_mad);
403                 break;
404         case DM_ATTR_IOC_PROFILE:
405                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
406                 srpt_get_ioc(sp->sdev, slot, rsp_mad);
407                 break;
408         case DM_ATTR_SVC_ENTRIES:
409                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
410                 hi = (u8) ((slot >> 8) & 0xff);
411                 lo = (u8) (slot & 0xff);
412                 slot = (u16) ((slot >> 16) & 0xffff);
413                 srpt_get_svc_entries(srpt_service_guid,
414                                      slot, hi, lo, rsp_mad);
415                 break;
416         default:
417                 rsp_mad->mad_hdr.status =
418                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
419                 break;
420         }
421 }
422
423 /*
424  * Callback function that is called by the InfiniBand core after transmission of
425  * a MAD. (MAD = management datagram; AH = address handle.)
426  */
427 static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
428                                   struct ib_mad_send_wc *mad_wc)
429 {
430         ib_destroy_ah(mad_wc->send_buf->ah);
431         ib_free_send_mad(mad_wc->send_buf);
432 }
433
434 /*
435  * Callback function that is called by the InfiniBand core after reception of
436  * a MAD (management datagram).
437  */
438 static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
439                                   struct ib_mad_recv_wc *mad_wc)
440 {
441         struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
442         struct ib_ah *ah;
443         struct ib_mad_send_buf *rsp;
444         struct ib_dm_mad *dm_mad;
445
446         if (!mad_wc || !mad_wc->recv_buf.mad)
447                 return;
448
449         ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
450                                   mad_wc->recv_buf.grh, mad_agent->port_num);
451         if (IS_ERR(ah))
452                 goto err;
453
454         BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR);
455
456         rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
457                                  mad_wc->wc->pkey_index, 0,
458                                  IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA,
459                                  GFP_KERNEL);
460         if (IS_ERR(rsp))
461                 goto err_rsp;
462
463         rsp->ah = ah;
464
465         dm_mad = rsp->mad;
466         memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof *dm_mad);
467         dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
468         dm_mad->mad_hdr.status = 0;
469
470         switch (mad_wc->recv_buf.mad->mad_hdr.method) {
471         case IB_MGMT_METHOD_GET:
472                 srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
473                 break;
474         case IB_MGMT_METHOD_SET:
475                 dm_mad->mad_hdr.status =
476                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
477                 break;
478         default:
479                 dm_mad->mad_hdr.status =
480                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
481                 break;
482         }
483
484         if (!ib_post_send_mad(rsp, NULL)) {
485                 ib_free_recv_mad(mad_wc);
486                 /* will destroy_ah & free_send_mad in send completion */
487                 return;
488         }
489
490         ib_free_send_mad(rsp);
491
492 err_rsp:
493         ib_destroy_ah(ah);
494 err:
495         ib_free_recv_mad(mad_wc);
496 }
497
498 /*
499  * Enable InfiniBand management datagram processing, update the cached sm_lid,
500  * lid and gid values, and register a callback function for processing MADs
501  * on the specified port. It is safe to call this function more than once for
502  * the same port.
503  */
504 static int srpt_refresh_port(struct srpt_port *sport)
505 {
506         struct ib_mad_reg_req reg_req;
507         struct ib_port_modify port_modify;
508         struct ib_port_attr port_attr;
509         int ret;
510
511         memset(&port_modify, 0, sizeof port_modify);
512         port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
513         port_modify.clr_port_cap_mask = 0;
514
515         ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
516         if (ret)
517                 goto err_mod_port;
518
519         ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
520         if (ret)
521                 goto err_query_port;
522
523         sport->sm_lid = port_attr.sm_lid;
524         sport->lid = port_attr.lid;
525
526         ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid);
527         if (ret)
528                 goto err_query_port;
529
530         if (!sport->mad_agent) {
531                 memset(&reg_req, 0, sizeof reg_req);
532                 reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
533                 reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
534                 set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
535                 set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
536
537                 sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
538                                                          sport->port,
539                                                          IB_QPT_GSI,
540                                                          &reg_req, 0,
541                                                          srpt_mad_send_handler,
542                                                          srpt_mad_recv_handler,
543                                                          sport);
544                 if (IS_ERR(sport->mad_agent)) {
545                         ret = PTR_ERR(sport->mad_agent);
546                         sport->mad_agent = NULL;
547                         goto err_query_port;
548                 }
549         }
550
551         return 0;
552
553 err_query_port:
554
555         port_modify.set_port_cap_mask = 0;
556         port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
557         ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
558
559 err_mod_port:
560
561         return ret;
562 }
563
564 /*
565  * Unregister the callback function for processing MADs and disable MAD
566  * processing for all ports of the specified device. It is safe to call this
567  * function more than once for the same device.
568  */
569 static void srpt_unregister_mad_agent(struct srpt_device *sdev)
570 {
571         struct ib_port_modify port_modify = {
572                 .clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP,
573         };
574         struct srpt_port *sport;
575         int i;
576
577         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
578                 sport = &sdev->port[i - 1];
579                 WARN_ON(sport->port != i);
580                 if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
581                         PRINT_ERROR("%s", "disabling MAD processing failed.");
582                 if (sport->mad_agent) {
583                         ib_unregister_mad_agent(sport->mad_agent);
584                         sport->mad_agent = NULL;
585                 }
586         }
587 }
588
589 /*
590  * Allocate and initialize an SRPT I/O context structure.
591  */
592 static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev)
593 {
594         struct srpt_ioctx *ioctx;
595
596         ioctx = kmalloc(sizeof *ioctx, GFP_KERNEL);
597         if (!ioctx)
598                 goto out;
599
600         ioctx->buf = kzalloc(MAX_MESSAGE_SIZE, GFP_KERNEL);
601         if (!ioctx->buf)
602                 goto out_free_ioctx;
603
604         ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf,
605                                        MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
606         if (ib_dma_mapping_error(sdev->device, ioctx->dma))
607                 goto out_free_buf;
608
609         return ioctx;
610
611 out_free_buf:
612         kfree(ioctx->buf);
613 out_free_ioctx:
614         kfree(ioctx);
615 out:
616         return NULL;
617 }
618
619 /*
620  * Deallocate an SRPT I/O context structure.
621  */
622 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
623 {
624         if (!ioctx)
625                 return;
626
627         ib_dma_unmap_single(sdev->device, ioctx->dma,
628                             MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
629         kfree(ioctx->buf);
630         kfree(ioctx);
631 }
632
633 /*
634  * Associate a ring of SRPT I/O context structures with the specified device.
635  */
636 static int srpt_alloc_ioctx_ring(struct srpt_device *sdev)
637 {
638         int i;
639
640         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
641                 sdev->ioctx_ring[i] = srpt_alloc_ioctx(sdev);
642
643                 if (!sdev->ioctx_ring[i])
644                         goto err;
645
646                 sdev->ioctx_ring[i]->index = i;
647         }
648
649         return 0;
650
651 err:
652         while (--i > 0) {
653                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
654                 sdev->ioctx_ring[i] = NULL;
655         }
656         return -ENOMEM;
657 }
658
659 /* Free the ring of SRPT I/O context structures. */
660 static void srpt_free_ioctx_ring(struct srpt_device *sdev)
661 {
662         int i;
663
664         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
665                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
666                 sdev->ioctx_ring[i] = NULL;
667         }
668 }
669
670 /*
671  * Post a receive request on the work queue of InfiniBand device 'sdev'.
672  */
673 static int srpt_post_recv(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
674 {
675         struct ib_sge list;
676         struct ib_recv_wr wr, *bad_wr;
677
678         wr.wr_id = ioctx->index | SRPT_OP_RECV;
679
680         list.addr = ioctx->dma;
681         list.length = MAX_MESSAGE_SIZE;
682         list.lkey = sdev->mr->lkey;
683
684         wr.next = NULL;
685         wr.sg_list = &list;
686         wr.num_sge = 1;
687
688         return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);
689 }
690
691 /*
692  * Post an IB send request.
693  * @ch: RDMA channel to post the send request on.
694  * @ioctx: I/O context of the send request.
695  * @len: length of the request to be sent in bytes.
696  *
697  * Returns zero upon success and a non-zero value upon failure.
698  */
699 static int srpt_post_send(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
700                           int len)
701 {
702         struct ib_sge list;
703         struct ib_send_wr wr, *bad_wr;
704         struct srpt_device *sdev = ch->sport->sdev;
705
706         ib_dma_sync_single_for_device(sdev->device, ioctx->dma,
707                                       MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
708
709         list.addr = ioctx->dma;
710         list.length = len;
711         list.lkey = sdev->mr->lkey;
712
713         wr.next = NULL;
714         wr.wr_id = ioctx->index;
715         wr.sg_list = &list;
716         wr.num_sge = 1;
717         wr.opcode = IB_WR_SEND;
718         wr.send_flags = IB_SEND_SIGNALED;
719
720         return ib_post_send(ch->qp, &wr, &bad_wr);
721 }
722
723 static int srpt_get_desc_tbl(struct srpt_ioctx *ioctx, struct srp_cmd *srp_cmd,
724                              int *ind)
725 {
726         struct srp_indirect_buf *idb;
727         struct srp_direct_buf *db;
728
729         *ind = 0;
730         if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
731             ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
732                 ioctx->n_rbuf = 1;
733                 ioctx->rbufs = &ioctx->single_rbuf;
734
735                 db = (void *)srp_cmd->add_data;
736                 memcpy(ioctx->rbufs, db, sizeof *db);
737                 ioctx->data_len = be32_to_cpu(db->len);
738         } else {
739                 idb = (void *)srp_cmd->add_data;
740
741                 ioctx->n_rbuf = be32_to_cpu(idb->table_desc.len) / sizeof *db;
742
743                 if (ioctx->n_rbuf >
744                     (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
745                         *ind = 1;
746                         ioctx->n_rbuf = 0;
747                         goto out;
748                 }
749
750                 if (ioctx->n_rbuf == 1)
751                         ioctx->rbufs = &ioctx->single_rbuf;
752                 else
753                         ioctx->rbufs =
754                                 kmalloc(ioctx->n_rbuf * sizeof *db, GFP_ATOMIC);
755                 if (!ioctx->rbufs) {
756                         ioctx->n_rbuf = 0;
757                         return -ENOMEM;
758                 }
759
760                 db = idb->desc_list;
761                 memcpy(ioctx->rbufs, db, ioctx->n_rbuf * sizeof *db);
762                 ioctx->data_len = be32_to_cpu(idb->len);
763         }
764 out:
765         return 0;
766 }
767
768 /*
769  * Modify the attributes of queue pair 'qp': allow local write, remote read,
770  * and remote write. Also transition 'qp' to state IB_QPS_INIT.
771  */
772 static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
773 {
774         struct ib_qp_attr *attr;
775         int ret;
776
777         attr = kzalloc(sizeof *attr, GFP_KERNEL);
778         if (!attr)
779                 return -ENOMEM;
780
781         attr->qp_state = IB_QPS_INIT;
782         attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ |
783             IB_ACCESS_REMOTE_WRITE;
784         attr->port_num = ch->sport->port;
785         attr->pkey_index = 0;
786
787         ret = ib_modify_qp(qp, attr,
788                            IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
789                            IB_QP_PKEY_INDEX);
790
791         kfree(attr);
792         return ret;
793 }
794
795 /**
796  * Change the state of a channel to 'ready to receive' (RTR).
797  * @ch: channel of the queue pair.
798  * @qp: queue pair to change the state of.
799  *
800  * Returns zero upon success and a negative value upon failure.
801  *
802  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
803  * If this structure ever becomes larger, it might be necessary to allocate
804  * it dynamically instead of on the stack.
805  */
806 static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
807 {
808         struct ib_qp_attr qp_attr;
809         int attr_mask;
810         int ret;
811
812         qp_attr.qp_state = IB_QPS_RTR;
813         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
814         if (ret)
815                 goto out;
816
817         qp_attr.max_dest_rd_atomic = 4;
818
819         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
820
821 out:
822         return ret;
823 }
824
825 /**
826  * Change the state of a channel to 'ready to send' (RTS).
827  * @ch: channel of the queue pair.
828  * @qp: queue pair to change the state of.
829  *
830  * Returns zero upon success and a negative value upon failure.
831  *
832  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
833  * If this structure ever becomes larger, it might be necessary to allocate
834  * it dynamically instead of on the stack.
835  */
836 static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
837 {
838         struct ib_qp_attr qp_attr;
839         int attr_mask;
840         int ret;
841
842         qp_attr.qp_state = IB_QPS_RTS;
843         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
844         if (ret)
845                 goto out;
846
847         qp_attr.max_rd_atomic = 4;
848
849         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
850
851 out:
852         return ret;
853 }
854
855 static void srpt_reset_ioctx(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
856 {
857         int i;
858
859         if (ioctx->n_rdma_ius > 0 && ioctx->rdma_ius) {
860                 struct rdma_iu *riu = ioctx->rdma_ius;
861
862                 for (i = 0; i < ioctx->n_rdma_ius; ++i, ++riu)
863                         kfree(riu->sge);
864                 kfree(ioctx->rdma_ius);
865         }
866
867         if (ioctx->n_rbuf > 1)
868                 kfree(ioctx->rbufs);
869
870         if (srpt_post_recv(ch->sport->sdev, ioctx))
871                 PRINT_ERROR("%s", "SRQ post_recv failed - this is serious.");
872                 /* we should queue it back to free_ioctx queue */
873         else
874                 atomic_inc(&ch->req_lim_delta);
875 }
876
877 static void srpt_abort_scst_cmd(struct srpt_device *sdev,
878                                 struct scst_cmd *scmnd,
879                                 bool tell_initiator)
880 {
881         struct srpt_ioctx *ioctx;
882         scst_data_direction dir;
883         enum srpt_command_state orig_ioctx_state;
884
885         ioctx = scst_cmd_get_tgt_priv(scmnd);
886         BUG_ON(!ioctx);
887         dir = scst_cmd_get_data_direction(scmnd);
888         if (dir != SCST_DATA_NONE) {
889                 ib_dma_unmap_sg(sdev->device,
890                                 scst_cmd_get_sg(scmnd),
891                                 scst_cmd_get_sg_cnt(scmnd),
892                                 scst_to_tgt_dma_dir(dir));
893
894 #ifdef CONFIG_SCST_EXTRACHECKS
895                 switch (scmnd->state) {
896                 case SCST_CMD_STATE_DATA_WAIT:
897                         WARN_ON(ioctx->state != SRPT_STATE_NEED_DATA);
898                         break;
899                 case SCST_CMD_STATE_XMIT_WAIT:
900                         WARN_ON(ioctx->state != SRPT_STATE_PROCESSED);
901                         break;
902                 default:
903                         WARN_ON(ioctx->state == SRPT_STATE_NEED_DATA ||
904                                 ioctx->state == SRPT_STATE_PROCESSED);
905                 }
906 #endif
907         }
908
909         orig_ioctx_state = ioctx->state;
910         ioctx->state = SRPT_STATE_ABORTED;
911
912         if (orig_ioctx_state == SRPT_STATE_NEED_DATA) {
913                 WARN_ON(scst_cmd_get_data_direction(ioctx->scmnd)
914                         == SCST_DATA_READ);
915                 scst_rx_data(scmnd,
916                              tell_initiator ? SCST_RX_STATUS_ERROR
917                              : SCST_RX_STATUS_ERROR_FATAL,
918                              SCST_CONTEXT_THREAD);
919                 goto out;
920         } else if (ioctx->state == SRPT_STATE_PROCESSED)
921                 ;
922         else
923                 WARN_ON("unexpected cmd state");
924
925         scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_FAILED);
926         WARN_ON(scmnd->state != SCST_CMD_STATE_XMIT_WAIT);
927         scst_tgt_cmd_done(scmnd, scst_estimate_context());
928 out:
929         return;
930 }
931
932 static void srpt_handle_err_comp(struct srpt_rdma_ch *ch, struct ib_wc *wc)
933 {
934         struct srpt_ioctx *ioctx;
935         struct srpt_device *sdev = ch->sport->sdev;
936
937         if (wc->wr_id & SRPT_OP_RECV) {
938                 ioctx = sdev->ioctx_ring[wc->wr_id & ~SRPT_OP_RECV];
939                 PRINT_ERROR("%s", "This is serious - SRQ is in bad state.");
940         } else {
941                 ioctx = sdev->ioctx_ring[wc->wr_id];
942
943                 if (ioctx->scmnd)
944                         srpt_abort_scst_cmd(sdev, ioctx->scmnd, true);
945                 else
946                         srpt_reset_ioctx(ch, ioctx);
947         }
948 }
949
950 static void srpt_handle_send_comp(struct srpt_rdma_ch *ch,
951                                   struct srpt_ioctx *ioctx,
952                                   enum scst_exec_context context)
953 {
954         if (ioctx->scmnd) {
955                 scst_data_direction dir =
956                         scst_cmd_get_data_direction(ioctx->scmnd);
957
958                 if (dir != SCST_DATA_NONE)
959                         ib_dma_unmap_sg(ch->sport->sdev->device,
960                                         scst_cmd_get_sg(ioctx->scmnd),
961                                         scst_cmd_get_sg_cnt(ioctx->scmnd),
962                                         scst_to_tgt_dma_dir(dir));
963
964                 WARN_ON(ioctx->scmnd->state != SCST_CMD_STATE_XMIT_WAIT);
965                 scst_tgt_cmd_done(ioctx->scmnd, context);
966         } else
967                 srpt_reset_ioctx(ch, ioctx);
968 }
969
970 /** Process an RDMA completion notification. */
971 static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch,
972                                   struct srpt_ioctx *ioctx)
973 {
974         if (!ioctx->scmnd) {
975                 WARN_ON("ERROR: ioctx->scmnd == NULL");
976                 srpt_reset_ioctx(ch, ioctx);
977                 return;
978         }
979
980         /*
981          * If an RDMA completion notification has been received for a write
982          * command, tell SCST that processing can continue by calling
983          * scst_rx_data().
984          */
985         if (ioctx->state == SRPT_STATE_NEED_DATA) {
986                 WARN_ON(scst_cmd_get_data_direction(ioctx->scmnd)
987                         == SCST_DATA_READ);
988                 ioctx->state = SRPT_STATE_DATA_IN;
989                 scst_rx_data(ioctx->scmnd, SCST_RX_STATUS_SUCCESS,
990                              scst_estimate_context());
991         }
992 }
993
994 /**
995  * Build an SRP_RSP response.
996  * @ch: RDMA channel through which the request has been received.
997  * @ioctx: I/O context in which the SRP_RSP response will be built.
998  * @s_key: sense key that will be stored in the response.
999  * @s_code: value that will be stored in the asc_ascq field of the sense data.
1000  * @tag: tag of the request for which this response is being generated.
1001  *
1002  * Returns the size in bytes of the SRP_RSP response.
1003  *
1004  * An SRP_RSP response contains a SCSI status or service response. See also
1005  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
1006  * response. See also SPC-2 for more information about sense data.
1007  */
1008 static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
1009                               struct srpt_ioctx *ioctx, u8 s_key, u8 s_code,
1010                               u64 tag)
1011 {
1012         struct srp_rsp *srp_rsp;
1013         struct sense_data *sense;
1014         int limit_delta;
1015         int sense_data_len = 0;
1016
1017         srp_rsp = ioctx->buf;
1018         memset(srp_rsp, 0, sizeof *srp_rsp);
1019
1020         limit_delta = atomic_read(&ch->req_lim_delta);
1021         atomic_sub(limit_delta, &ch->req_lim_delta);
1022
1023         srp_rsp->opcode = SRP_RSP;
1024         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1025         srp_rsp->tag = tag;
1026
1027         if (s_key != NO_SENSE) {
1028                 sense_data_len = sizeof *sense + (sizeof *sense % 4);
1029                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1030                 srp_rsp->status = SAM_STAT_CHECK_CONDITION;
1031                 srp_rsp->sense_data_len = cpu_to_be32(sense_data_len);
1032
1033                 sense = (struct sense_data *)(srp_rsp + 1);
1034                 sense->err_code = 0x70;
1035                 sense->key = s_key;
1036                 sense->asc_ascq = s_code;
1037         }
1038
1039         return sizeof(*srp_rsp) + sense_data_len;
1040 }
1041
1042 /**
1043  * Build a task management response, which is a specific SRP_RSP response.
1044  * @ch: RDMA channel through which the request has been received.
1045  * @ioctx: I/O context in which the SRP_RSP response will be built.
1046  * @rsp_code: RSP_CODE that will be stored in the response.
1047  * @tag: tag of the request for which this response is being generated.
1048  *
1049  * Returns the size in bytes of the SRP_RSP response.
1050  *
1051  * An SRP_RSP response contains a SCSI status or service response. See also
1052  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
1053  * response.
1054  */
1055 static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
1056                                   struct srpt_ioctx *ioctx, u8 rsp_code,
1057                                   u64 tag)
1058 {
1059         struct srp_rsp *srp_rsp;
1060         int limit_delta;
1061         int resp_data_len = 0;
1062
1063         ib_dma_sync_single_for_cpu(ch->sport->sdev->device, ioctx->dma,
1064                                    MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
1065
1066         srp_rsp = ioctx->buf;
1067         memset(srp_rsp, 0, sizeof *srp_rsp);
1068
1069         limit_delta = atomic_read(&ch->req_lim_delta);
1070         atomic_sub(limit_delta, &ch->req_lim_delta);
1071
1072         srp_rsp->opcode = SRP_RSP;
1073         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1074         srp_rsp->tag = tag;
1075
1076         if (rsp_code != SRP_TSK_MGMT_SUCCESS) {
1077                 resp_data_len = 4;
1078                 srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
1079                 srp_rsp->resp_data_len = cpu_to_be32(resp_data_len);
1080                 srp_rsp->data[3] = rsp_code;
1081         }
1082
1083         return sizeof(*srp_rsp) + resp_data_len;
1084 }
1085
1086 /*
1087  * Process SRP_CMD.
1088  */
1089 static int srpt_handle_cmd(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
1090 {
1091         struct scst_cmd *scmnd;
1092         struct srp_cmd *srp_cmd;
1093         struct srp_rsp *srp_rsp;
1094         scst_data_direction dir;
1095         int indirect_desc = 0;
1096         int ret;
1097         unsigned long flags;
1098
1099         srp_cmd = ioctx->buf;
1100         srp_rsp = ioctx->buf;
1101
1102         dir = SCST_DATA_NONE;
1103         if (srp_cmd->buf_fmt) {
1104                 ret = srpt_get_desc_tbl(ioctx, srp_cmd, &indirect_desc);
1105                 if (ret) {
1106                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1107                                            NO_ADD_SENSE, srp_cmd->tag);
1108                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1109                         goto err;
1110                 }
1111
1112                 if (indirect_desc) {
1113                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1114                                            NO_ADD_SENSE, srp_cmd->tag);
1115                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1116                         goto err;
1117                 }
1118
1119                 /*
1120                  * The lower four bits of the buffer format field contain the
1121                  * DATA-IN buffer descriptor format, and the highest four bits
1122                  * contain the DATA-OUT buffer descriptor format.
1123                  */
1124                 if (srp_cmd->buf_fmt & 0xf)
1125                         /* DATA-IN: transfer data from target to initiator. */
1126                         dir = SCST_DATA_READ;
1127                 else if (srp_cmd->buf_fmt >> 4)
1128                         /* DATA-OUT: transfer data from initiator to target. */
1129                         dir = SCST_DATA_WRITE;
1130         }
1131
1132         scmnd = scst_rx_cmd(ch->scst_sess, (u8 *) &srp_cmd->lun,
1133                             sizeof srp_cmd->lun, srp_cmd->cdb, 16,
1134                             thread ? SCST_NON_ATOMIC : SCST_ATOMIC);
1135         if (!scmnd) {
1136                 srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1137                                    NO_ADD_SENSE, srp_cmd->tag);
1138                 srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1139                 goto err;
1140         }
1141
1142         ioctx->scmnd = scmnd;
1143
1144         switch (srp_cmd->task_attr) {
1145         case SRP_CMD_HEAD_OF_Q:
1146                 scmnd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1147                 break;
1148         case SRP_CMD_ORDERED_Q:
1149                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1150                 break;
1151         case SRP_CMD_SIMPLE_Q:
1152                 scmnd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1153                 break;
1154         case SRP_CMD_ACA:
1155                 scmnd->queue_type = SCST_CMD_QUEUE_ACA;
1156                 break;
1157         default:
1158                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1159                 break;
1160         }
1161
1162         scst_cmd_set_tag(scmnd, srp_cmd->tag);
1163         scst_cmd_set_tgt_priv(scmnd, ioctx);
1164         scst_cmd_set_expected(scmnd, dir, ioctx->data_len);
1165
1166         spin_lock_irqsave(&ch->spinlock, flags);
1167         list_add_tail(&ioctx->scmnd_list, &ch->active_scmnd_list);
1168         ch->active_scmnd_cnt++;
1169         spin_unlock_irqrestore(&ch->spinlock, flags);
1170
1171         scst_cmd_init_done(scmnd, scst_estimate_context());
1172
1173         return 0;
1174
1175 err:
1176         WARN_ON(srp_rsp->opcode != SRP_RSP);
1177
1178         return -1;
1179 }
1180
1181 /*
1182  * Process an SRP_TSK_MGMT request.
1183  *
1184  * Returns 0 upon success and -1 upon failure.
1185  *
1186  * Each task management function is performed by calling one of the
1187  * scst_rx_mgmt_fn*() functions. These functions will either report failure
1188  * or process the task management function asynchronously. The function
1189  * srpt_tsk_mgmt_done() will be called by the SCST core upon completion of the
1190  * task management function. When srpt_handle_tsk_mgmt() reports failure
1191  * (i.e. returns -1) a response will have been built in ioctx->buf. This
1192  * information unit has to be sent back by the caller.
1193  *
1194  * For more information about SRP_TSK_MGMT information units, see also section
1195  * 6.7 in the T10 SRP r16a document.
1196  */
1197 static int srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
1198                                 struct srpt_ioctx *ioctx)
1199 {
1200         struct srp_tsk_mgmt *srp_tsk;
1201         struct srpt_mgmt_ioctx *mgmt_ioctx;
1202         int ret;
1203
1204         srp_tsk = ioctx->buf;
1205
1206         TRACE_DBG("recv_tsk_mgmt= %d for task_tag= %lld"
1207                   " using tag= %lld cm_id= %p sess= %p",
1208                   srp_tsk->tsk_mgmt_func,
1209                   (unsigned long long) srp_tsk->task_tag,
1210                   (unsigned long long) srp_tsk->tag,
1211                   ch->cm_id, ch->scst_sess);
1212
1213         mgmt_ioctx = kmalloc(sizeof *mgmt_ioctx, GFP_ATOMIC);
1214         if (!mgmt_ioctx) {
1215                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1216                                        srp_tsk->tag);
1217                 goto err;
1218         }
1219
1220         mgmt_ioctx->ioctx = ioctx;
1221         mgmt_ioctx->ch = ch;
1222         mgmt_ioctx->tag = srp_tsk->tag;
1223
1224         switch (srp_tsk->tsk_mgmt_func) {
1225         case SRP_TSK_ABORT_TASK:
1226                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK");
1227                 ret = scst_rx_mgmt_fn_tag(ch->scst_sess,
1228                                           SCST_ABORT_TASK,
1229                                           srp_tsk->task_tag,
1230                                           thread ?
1231                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1232                                           mgmt_ioctx);
1233                 break;
1234         case SRP_TSK_ABORT_TASK_SET:
1235                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK_SET");
1236                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1237                                           SCST_ABORT_TASK_SET,
1238                                           (u8 *) &srp_tsk->lun,
1239                                           sizeof srp_tsk->lun,
1240                                           thread ?
1241                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1242                                           mgmt_ioctx);
1243                 break;
1244         case SRP_TSK_CLEAR_TASK_SET:
1245                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_TASK_SET");
1246                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1247                                           SCST_CLEAR_TASK_SET,
1248                                           (u8 *) &srp_tsk->lun,
1249                                           sizeof srp_tsk->lun,
1250                                           thread ?
1251                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1252                                           mgmt_ioctx);
1253                 break;
1254         case SRP_TSK_LUN_RESET:
1255                 TRACE_DBG("%s", "Processing SRP_TSK_LUN_RESET");
1256                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1257                                           SCST_LUN_RESET,
1258                                           (u8 *) &srp_tsk->lun,
1259                                           sizeof srp_tsk->lun,
1260                                           thread ?
1261                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1262                                           mgmt_ioctx);
1263                 break;
1264         case SRP_TSK_CLEAR_ACA:
1265                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_ACA");
1266                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1267                                           SCST_CLEAR_ACA,
1268                                           (u8 *) &srp_tsk->lun,
1269                                           sizeof srp_tsk->lun,
1270                                           thread ?
1271                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1272                                           mgmt_ioctx);
1273                 break;
1274         default:
1275                 TRACE_DBG("%s", "Unsupported task management function.");
1276                 srpt_build_tskmgmt_rsp(ch, ioctx,
1277                                        SRP_TSK_MGMT_FUNC_NOT_SUPP,
1278                                        srp_tsk->tag);
1279                 goto err;
1280         }
1281
1282         if (ret) {
1283                 TRACE_DBG("%s", "Processing task management function failed.");
1284                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1285                                        srp_tsk->tag);
1286                 goto err;
1287         }
1288
1289         WARN_ON(srp_tsk->opcode == SRP_RSP);
1290
1291         return 0;
1292
1293 err:
1294         WARN_ON(srp_tsk->opcode != SRP_RSP);
1295
1296         kfree(mgmt_ioctx);
1297         return -1;
1298 }
1299
1300 /**
1301  * Process a receive completion event.
1302  * @ch: RDMA channel for which the completion event has been received.
1303  * @ioctx: SRPT I/O context for which the completion event has been received.
1304  */
1305 static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
1306                                struct srpt_ioctx *ioctx)
1307 {
1308         struct srp_cmd *srp_cmd;
1309         struct srp_rsp *srp_rsp;
1310         unsigned long flags;
1311         int len;
1312
1313         spin_lock_irqsave(&ch->spinlock, flags);
1314         if (ch->state != RDMA_CHANNEL_LIVE) {
1315                 if (ch->state == RDMA_CHANNEL_CONNECTING) {
1316                         list_add_tail(&ioctx->wait_list, &ch->cmd_wait_list);
1317                         spin_unlock_irqrestore(&ch->spinlock, flags);
1318                         return;
1319                 } else {
1320                         spin_unlock_irqrestore(&ch->spinlock, flags);
1321                         srpt_reset_ioctx(ch, ioctx);
1322                         return;
1323                 }
1324         }
1325         spin_unlock_irqrestore(&ch->spinlock, flags);
1326
1327         ib_dma_sync_single_for_cpu(ch->sport->sdev->device, ioctx->dma,
1328                                    MAX_MESSAGE_SIZE, DMA_FROM_DEVICE);
1329
1330         ioctx->data_len = 0;
1331         ioctx->n_rbuf = 0;
1332         ioctx->rbufs = NULL;
1333         ioctx->n_rdma = 0;
1334         ioctx->n_rdma_ius = 0;
1335         ioctx->rdma_ius = NULL;
1336         ioctx->scmnd = NULL;
1337         ioctx->state = SRPT_STATE_NEW;
1338
1339         srp_cmd = ioctx->buf;
1340         srp_rsp = ioctx->buf;
1341
1342         switch (srp_cmd->opcode) {
1343         case SRP_CMD:
1344                 if (srpt_handle_cmd(ch, ioctx) < 0)
1345                         goto err;
1346                 break;
1347
1348         case SRP_TSK_MGMT:
1349                 if (srpt_handle_tsk_mgmt(ch, ioctx) < 0)
1350                         goto err;
1351                 break;
1352
1353         case SRP_I_LOGOUT:
1354         case SRP_AER_REQ:
1355         default:
1356                 srpt_build_cmd_rsp(ch, ioctx, ILLEGAL_REQUEST, INVALID_CDB,
1357                                    srp_cmd->tag);
1358                 goto err;
1359         }
1360
1361         ib_dma_sync_single_for_device(ch->sport->sdev->device,
1362                                    ioctx->dma, MAX_MESSAGE_SIZE,
1363                                    DMA_FROM_DEVICE);
1364
1365         return;
1366
1367 err:
1368         WARN_ON(srp_rsp->opcode != SRP_RSP);
1369         len = (sizeof *srp_rsp) + be32_to_cpu(srp_rsp->sense_data_len);
1370
1371         if (ch->state != RDMA_CHANNEL_LIVE) {
1372                 /* Give up if another thread modified the channel state. */
1373                 PRINT_ERROR("%s: channel is in state %d", __func__, ch->state);
1374                 srpt_reset_ioctx(ch, ioctx);
1375         } else if (srpt_post_send(ch, ioctx, len)) {
1376                 PRINT_ERROR("%s: sending SRP_RSP response failed", __func__);
1377                 srpt_reset_ioctx(ch, ioctx);
1378         }
1379 }
1380
1381 /*
1382  * Returns true if the ioctx list is non-empty or if the ib_srpt kernel thread
1383  * should stop.
1384  * @pre thread != 0
1385  */
1386 static inline int srpt_test_ioctx_list(void)
1387 {
1388         int res = (!list_empty(&srpt_thread.thread_ioctx_list) ||
1389                    unlikely(kthread_should_stop()));
1390         return res;
1391 }
1392
1393 /*
1394  * Add 'ioctx' to the tail of the ioctx list and wake up the kernel thread.
1395  *
1396  * @pre thread != 0
1397  */
1398 static inline void srpt_schedule_thread(struct srpt_ioctx *ioctx)
1399 {
1400         unsigned long flags;
1401
1402         spin_lock_irqsave(&srpt_thread.thread_lock, flags);
1403         list_add_tail(&ioctx->comp_list, &srpt_thread.thread_ioctx_list);
1404         spin_unlock_irqrestore(&srpt_thread.thread_lock, flags);
1405         wake_up(&ioctx_list_waitQ);
1406 }
1407
1408 /**
1409  * InfiniBand completion queue callback function.
1410  * @cq: completion queue.
1411  * @ctx: completion queue context, which was passed as the fourth argument of
1412  *       the function ib_create_cq().
1413  */
1414 static void srpt_completion(struct ib_cq *cq, void *ctx)
1415 {
1416         struct srpt_rdma_ch *ch = ctx;
1417         struct srpt_device *sdev = ch->sport->sdev;
1418         struct ib_wc wc;
1419         struct srpt_ioctx *ioctx;
1420
1421         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1422         while (ib_poll_cq(ch->cq, 1, &wc) > 0) {
1423                 if (wc.status) {
1424                         PRINT_ERROR("failed %s status= %d",
1425                                wc.wr_id & SRPT_OP_RECV ? "receive" : "send",
1426                                wc.status);
1427                         srpt_handle_err_comp(ch, &wc);
1428                         break;
1429                 }
1430
1431                 if (wc.wr_id & SRPT_OP_RECV) {
1432                         ioctx = sdev->ioctx_ring[wc.wr_id & ~SRPT_OP_RECV];
1433                         if (thread) {
1434                                 ioctx->ch = ch;
1435                                 ioctx->op = IB_WC_RECV;
1436                                 srpt_schedule_thread(ioctx);
1437                         } else
1438                                 srpt_handle_new_iu(ch, ioctx);
1439                         continue;
1440                 } else
1441                         ioctx = sdev->ioctx_ring[wc.wr_id];
1442
1443                 if (thread) {
1444                         ioctx->ch = ch;
1445                         ioctx->op = wc.opcode;
1446                         srpt_schedule_thread(ioctx);
1447                 } else {
1448                         switch (wc.opcode) {
1449                         case IB_WC_SEND:
1450                                 srpt_handle_send_comp(ch, ioctx,
1451                                         scst_estimate_context());
1452                                 break;
1453                         case IB_WC_RDMA_WRITE:
1454                         case IB_WC_RDMA_READ:
1455                                 srpt_handle_rdma_comp(ch, ioctx);
1456                                 break;
1457                         default:
1458                                 break;
1459                         }
1460                 }
1461
1462                 if (processing_delay_in_us <= MAX_UDELAY_MS * 1000)
1463                         udelay(processing_delay_in_us);
1464         }
1465 }
1466
1467 /*
1468  * Create a completion queue on the specified device.
1469  */
1470 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
1471 {
1472         struct ib_qp_init_attr *qp_init;
1473         struct srpt_device *sdev = ch->sport->sdev;
1474         int cqe;
1475         int ret;
1476
1477         qp_init = kzalloc(sizeof *qp_init, GFP_KERNEL);
1478         if (!qp_init)
1479                 return -ENOMEM;
1480
1481         /* Create a completion queue (CQ). */
1482
1483         cqe = SRPT_RQ_SIZE + SRPT_SQ_SIZE - 1;
1484 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(RHEL_RELEASE_CODE)
1485         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe);
1486 #else
1487         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe, 0);
1488 #endif
1489         if (IS_ERR(ch->cq)) {
1490                 ret = PTR_ERR(ch->cq);
1491                 PRINT_ERROR("failed to create_cq cqe= %d ret= %d", cqe, ret);
1492                 goto out;
1493         }
1494
1495         /* Request completion notification. */
1496
1497         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1498
1499         /* Create a queue pair (QP). */
1500
1501         qp_init->qp_context = (void *)ch;
1502         qp_init->event_handler = srpt_qp_event;
1503         qp_init->send_cq = ch->cq;
1504         qp_init->recv_cq = ch->cq;
1505         qp_init->srq = sdev->srq;
1506         qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
1507         qp_init->qp_type = IB_QPT_RC;
1508         qp_init->cap.max_send_wr = SRPT_SQ_SIZE;
1509         qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE;
1510
1511         ch->qp = ib_create_qp(sdev->pd, qp_init);
1512         if (IS_ERR(ch->qp)) {
1513                 ret = PTR_ERR(ch->qp);
1514                 ib_destroy_cq(ch->cq);
1515                 PRINT_ERROR("failed to create_qp ret= %d", ret);
1516                 goto out;
1517         }
1518
1519         TRACE_DBG("%s: max_cqe= %d max_sge= %d cm_id= %p",
1520                __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
1521                ch->cm_id);
1522
1523         /* Modify the attributes and the state of queue pair ch->qp. */
1524
1525         ret = srpt_init_ch_qp(ch, ch->qp);
1526         if (ret) {
1527                 ib_destroy_qp(ch->qp);
1528                 ib_destroy_cq(ch->cq);
1529                 goto out;
1530         }
1531
1532         atomic_set(&ch->req_lim_delta, SRPT_RQ_SIZE);
1533 out:
1534         kfree(qp_init);
1535         return ret;
1536 }
1537
1538 /**
1539  * Look up the RDMA channel that corresponds to the specified cm_id.
1540  *
1541  * Return NULL if no matching RDMA channel has been found.
1542  */
1543 static struct srpt_rdma_ch *srpt_find_channel(struct ib_cm_id *cm_id, bool del)
1544 {
1545         struct srpt_device *sdev = cm_id->context;
1546         struct srpt_rdma_ch *ch;
1547
1548         spin_lock_irq(&sdev->spinlock);
1549         list_for_each_entry(ch, &sdev->rch_list, list) {
1550                 if (ch->cm_id == cm_id) {
1551                         if (del)
1552                                 list_del(&ch->list);
1553                         spin_unlock_irq(&sdev->spinlock);
1554                         return ch;
1555                 }
1556         }
1557
1558         spin_unlock_irq(&sdev->spinlock);
1559
1560         return NULL;
1561 }
1562
1563 /**
1564  * Release all resources associated with the specified RDMA channel.
1565  *
1566  * Note: the caller must have removed the channel from the channel list
1567  * before calling this function.
1568  */
1569 static void srpt_release_channel(struct srpt_rdma_ch *ch, int destroy_cmid)
1570 {
1571         TRACE_ENTRY();
1572
1573         WARN_ON(srpt_find_channel(ch->cm_id, false) == ch);
1574
1575         if (ch->cm_id && destroy_cmid) {
1576                 TRACE_DBG("%s: destroy cm_id= %p", __func__, ch->cm_id);
1577                 ib_destroy_cm_id(ch->cm_id);
1578                 ch->cm_id = NULL;
1579         }
1580
1581         ib_destroy_qp(ch->qp);
1582         ib_destroy_cq(ch->cq);
1583
1584         if (ch->scst_sess) {
1585                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1586
1587                 if (ch->active_scmnd_cnt)
1588                         PRINT_INFO("Releasing session %s which still has %d"
1589                                    " active commands",
1590                                    ch->sess_name, ch->active_scmnd_cnt);
1591                 else
1592                         PRINT_INFO("Releasing session %s", ch->sess_name);
1593
1594                 spin_lock_irq(&ch->spinlock);
1595                 list_for_each_entry_safe(ioctx, ioctx_tmp,
1596                                          &ch->active_scmnd_list, scmnd_list) {
1597                         spin_unlock_irq(&ch->spinlock);
1598
1599                         if (ioctx->scmnd)
1600                                 srpt_abort_scst_cmd(ch->sport->sdev,
1601                                                     ioctx->scmnd, true);
1602
1603                         spin_lock_irq(&ch->spinlock);
1604                 }
1605                 WARN_ON(!list_empty(&ch->active_scmnd_list));
1606                 WARN_ON(ch->active_scmnd_cnt != 0);
1607                 spin_unlock_irq(&ch->spinlock);
1608
1609                 scst_unregister_session(ch->scst_sess, 0, NULL);
1610                 ch->scst_sess = NULL;
1611         }
1612
1613         kfree(ch);
1614
1615         TRACE_EXIT();
1616 }
1617
1618 static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
1619                             struct ib_cm_req_event_param *param,
1620                             void *private_data)
1621 {
1622         struct srpt_device *sdev = cm_id->context;
1623         struct srp_login_req *req;
1624         struct srp_login_rsp *rsp;
1625         struct srp_login_rej *rej;
1626         struct ib_cm_rep_param *rep_param;
1627         struct srpt_rdma_ch *ch, *tmp_ch;
1628         u32 it_iu_len;
1629         int ret = 0;
1630
1631 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
1632         WARN_ON(!sdev || !private_data);
1633         if (!sdev || !private_data)
1634                 return -EINVAL;
1635 #else
1636         if (WARN_ON(!sdev || !private_data))
1637                 return -EINVAL;
1638 #endif
1639
1640         req = (struct srp_login_req *)private_data;
1641
1642         it_iu_len = be32_to_cpu(req->req_it_iu_len);
1643
1644         PRINT_INFO("Received SRP_LOGIN_REQ with"
1645             " i_port_id 0x%llx:0x%llx, t_port_id 0x%llx:0x%llx and length %d"
1646             " on port %d (guid=0x%llx:0x%llx)",
1647             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[0]),
1648             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[8]),
1649             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[0]),
1650             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[8]),
1651             it_iu_len,
1652             param->port,
1653             (unsigned long long)be64_to_cpu(*(u64 *)
1654                                 &sdev->port[param->port - 1].gid.raw[0]),
1655             (unsigned long long)be64_to_cpu(*(u64 *)
1656                                 &sdev->port[param->port - 1].gid.raw[8]));
1657
1658         rsp = kzalloc(sizeof *rsp, GFP_KERNEL);
1659         rej = kzalloc(sizeof *rej, GFP_KERNEL);
1660         rep_param = kzalloc(sizeof *rep_param, GFP_KERNEL);
1661
1662         if (!rsp || !rej || !rep_param) {
1663                 ret = -ENOMEM;
1664                 goto out;
1665         }
1666
1667         if (it_iu_len > MAX_MESSAGE_SIZE || it_iu_len < 64) {
1668                 rej->reason =
1669                     cpu_to_be32(SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
1670                 ret = -EINVAL;
1671                 PRINT_ERROR("rejected SRP_LOGIN_REQ because its"
1672                             " length (%d bytes) is invalid", it_iu_len);
1673                 goto reject;
1674         }
1675
1676         if ((req->req_flags & 0x3) == SRP_MULTICHAN_SINGLE) {
1677                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
1678
1679                 spin_lock_irq(&sdev->spinlock);
1680
1681                 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
1682                         if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
1683                             && !memcmp(ch->t_port_id, req->target_port_id, 16)
1684                             && param->port == ch->sport->port
1685                             && param->listen_id == ch->sport->sdev->cm_id
1686                             && ch->cm_id) {
1687                                 enum rdma_ch_state prev_state;
1688
1689                                 /* found an existing channel */
1690                                 TRACE_DBG("Found existing channel name= %s"
1691                                           " cm_id= %p state= %d",
1692                                           ch->sess_name, ch->cm_id, ch->state);
1693
1694                                 prev_state = ch->state;
1695                                 if (ch->state == RDMA_CHANNEL_LIVE)
1696                                         ch->state = RDMA_CHANNEL_DISCONNECTING;
1697                                 else if (ch->state == RDMA_CHANNEL_CONNECTING)
1698                                         list_del(&ch->list);
1699
1700                                 spin_unlock_irq(&sdev->spinlock);
1701
1702                                 rsp->rsp_flags =
1703                                         SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
1704
1705                                 if (prev_state == RDMA_CHANNEL_LIVE) {
1706                                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
1707                                         PRINT_INFO("disconnected"
1708                                           " session %s because a new"
1709                                           " SRP_LOGIN_REQ has been received.",
1710                                           ch->sess_name);
1711                                 } else if (prev_state ==
1712                                          RDMA_CHANNEL_CONNECTING) {
1713                                         PRINT_ERROR("%s", "rejected"
1714                                           " SRP_LOGIN_REQ because another login"
1715                                           " request is being processed.");
1716                                         ib_send_cm_rej(ch->cm_id,
1717                                                        IB_CM_REJ_NO_RESOURCES,
1718                                                        NULL, 0, NULL, 0);
1719                                         srpt_release_channel(ch, 1);
1720                                 }
1721
1722                                 spin_lock_irq(&sdev->spinlock);
1723                         }
1724                 }
1725
1726                 spin_unlock_irq(&sdev->spinlock);
1727
1728         } else
1729                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
1730
1731         if (((u64) (*(u64 *) req->target_port_id) !=
1732              cpu_to_be64(srpt_service_guid)) ||
1733             ((u64) (*(u64 *) (req->target_port_id + 8)) !=
1734              cpu_to_be64(srpt_service_guid))) {
1735                 rej->reason =
1736                     cpu_to_be32(SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
1737                 ret = -ENOMEM;
1738                 PRINT_ERROR("%s", "rejected SRP_LOGIN_REQ because it"
1739                        " has an invalid target port identifier.");
1740                 goto reject;
1741         }
1742
1743         ch = kzalloc(sizeof *ch, GFP_KERNEL);
1744         if (!ch) {
1745                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1746                 PRINT_ERROR("%s",
1747                             "rejected SRP_LOGIN_REQ because out of memory.");
1748                 ret = -ENOMEM;
1749                 goto reject;
1750         }
1751
1752         spin_lock_init(&ch->spinlock);
1753         memcpy(ch->i_port_id, req->initiator_port_id, 16);
1754         memcpy(ch->t_port_id, req->target_port_id, 16);
1755         ch->sport = &sdev->port[param->port - 1];
1756         ch->cm_id = cm_id;
1757         ch->state = RDMA_CHANNEL_CONNECTING;
1758         INIT_LIST_HEAD(&ch->cmd_wait_list);
1759         INIT_LIST_HEAD(&ch->active_scmnd_list);
1760
1761         ret = srpt_create_ch_ib(ch);
1762         if (ret) {
1763                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1764                 PRINT_ERROR("%s", "rejected SRP_LOGIN_REQ because creating"
1765                             " a new RDMA channel failed.");
1766                 goto free_ch;
1767         }
1768
1769         ret = srpt_ch_qp_rtr(ch, ch->qp);
1770         if (ret) {
1771                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1772                 PRINT_ERROR("rejected SRP_LOGIN_REQ because enabling"
1773                        " RTR failed (error code = %d)", ret);
1774                 goto destroy_ib;
1775         }
1776
1777         if (use_port_guid_in_session_name) {
1778                 /*
1779                  * If the kernel module parameter use_port_guid_in_session_name
1780                  * has been specified, use a combination of the target port
1781                  * GUID and the initiator port ID as the session name. This
1782                  * was the original behavior of the SRP target implementation
1783                  * (i.e. before the SRPT was included in OFED 1.3).
1784                  */
1785                 snprintf(ch->sess_name, sizeof(ch->sess_name),
1786                          "0x%016llx%016llx",
1787                          (unsigned long long)be64_to_cpu(*(u64 *)
1788                                 &sdev->port[param->port - 1].gid.raw[8]),
1789                          (unsigned long long)be64_to_cpu(*(u64 *)
1790                                 (ch->i_port_id + 8)));
1791         } else {
1792                 /*
1793                  * Default behavior: use the initator port identifier as the
1794                  * session name.
1795                  */
1796                 snprintf(ch->sess_name, sizeof(ch->sess_name),
1797                          "0x%016llx%016llx",
1798                          (unsigned long long)be64_to_cpu(*(u64 *)ch->i_port_id),
1799                          (unsigned long long)be64_to_cpu(*(u64 *)
1800                                  (ch->i_port_id + 8)));
1801         }
1802
1803         TRACE_DBG("registering session %s", ch->sess_name);
1804
1805         BUG_ON(!sdev->scst_tgt);
1806         ch->scst_sess = scst_register_session(sdev->scst_tgt, 0, ch->sess_name,
1807                                               NULL, NULL);
1808         if (!ch->scst_sess) {
1809                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1810                 TRACE_DBG("%s", "Failed to create scst sess");
1811                 goto destroy_ib;
1812         }
1813
1814         TRACE_DBG("Establish connection sess=%p name=%s cm_id=%p",
1815                   ch->scst_sess, ch->sess_name, ch->cm_id);
1816
1817         scst_sess_set_tgt_priv(ch->scst_sess, ch);
1818
1819         /* create srp_login_response */
1820         rsp->opcode = SRP_LOGIN_RSP;
1821         rsp->tag = req->tag;
1822         rsp->max_it_iu_len = req->req_it_iu_len;
1823         rsp->max_ti_iu_len = req->req_it_iu_len;
1824         rsp->buf_fmt =
1825             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1826         rsp->req_lim_delta = cpu_to_be32(SRPT_RQ_SIZE);
1827         atomic_set(&ch->req_lim_delta, 0);
1828
1829         /* create cm reply */
1830         rep_param->qp_num = ch->qp->qp_num;
1831         rep_param->private_data = (void *)rsp;
1832         rep_param->private_data_len = sizeof *rsp;
1833         rep_param->rnr_retry_count = 7;
1834         rep_param->flow_control = 1;
1835         rep_param->failover_accepted = 0;
1836         rep_param->srq = 1;
1837         rep_param->responder_resources = 4;
1838         rep_param->initiator_depth = 4;
1839
1840         ret = ib_send_cm_rep(cm_id, rep_param);
1841         if (ret) {
1842                 PRINT_ERROR("sending SRP_LOGIN_REQ response failed"
1843                             " (error code = %d)", ret);
1844                 goto release_channel;
1845         }
1846
1847         spin_lock_irq(&sdev->spinlock);
1848         list_add_tail(&ch->list, &sdev->rch_list);
1849         spin_unlock_irq(&sdev->spinlock);
1850
1851         goto out;
1852
1853 release_channel:
1854         scst_unregister_session(ch->scst_sess, 0, NULL);
1855         ch->scst_sess = NULL;
1856
1857 destroy_ib:
1858         ib_destroy_qp(ch->qp);
1859         ib_destroy_cq(ch->cq);
1860
1861 free_ch:
1862         kfree(ch);
1863
1864 reject:
1865         rej->opcode = SRP_LOGIN_REJ;
1866         rej->tag = req->tag;
1867         rej->buf_fmt =
1868             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1869
1870         ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
1871                              (void *)rej, sizeof *rej);
1872
1873 out:
1874         kfree(rep_param);
1875         kfree(rsp);
1876         kfree(rej);
1877
1878         return ret;
1879 }
1880
1881 /**
1882  * Release the channel with the specified cm_id.
1883  *
1884  * Returns one to indicate that the caller of srpt_cm_handler() should destroy
1885  * the cm_id.
1886  */
1887 static void srpt_find_and_release_channel(struct ib_cm_id *cm_id)
1888 {
1889         struct srpt_rdma_ch *ch;
1890
1891         ch = srpt_find_channel(cm_id, true);
1892         if (ch)
1893                 srpt_release_channel(ch, 0);
1894 }
1895
1896 static void srpt_cm_rej_recv(struct ib_cm_id *cm_id)
1897 {
1898         PRINT_INFO("%s", "Received InfiniBand REJ packet.");
1899         srpt_find_and_release_channel(cm_id);
1900 }
1901
1902 /**
1903  * Process an IB_CM_RTU_RECEIVED or IB_CM_USER_ESTABLISHED event.
1904  *
1905  * An IB_CM_RTU_RECEIVED message indicates that the connection is established
1906  * and that the recipient may begin transmitting (RTU = ready to use).
1907  */
1908 static int srpt_cm_rtu_recv(struct ib_cm_id *cm_id)
1909 {
1910         struct srpt_rdma_ch *ch;
1911         int ret;
1912
1913         ch = srpt_find_channel(cm_id, false);
1914         if (!ch)
1915                 return -EINVAL;
1916
1917         if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_CONNECTING,
1918                                             RDMA_CHANNEL_LIVE)) {
1919                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1920
1921                 ret = srpt_ch_qp_rts(ch, ch->qp);
1922
1923                 list_for_each_entry_safe(ioctx, ioctx_tmp, &ch->cmd_wait_list,
1924                                          wait_list) {
1925                         list_del(&ioctx->wait_list);
1926                         srpt_handle_new_iu(ch, ioctx);
1927                 }
1928                 if (ret && srpt_test_and_set_channel_state(ch,
1929                                         RDMA_CHANNEL_LIVE,
1930                                         RDMA_CHANNEL_DISCONNECTING)) {
1931                         TRACE_DBG("cm_id=%p sess_name=%s state=%d",
1932                                   cm_id, ch->sess_name, ch->state);
1933                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
1934                 }
1935         } else if (ch->state == RDMA_CHANNEL_DISCONNECTING) {
1936                 TRACE_DBG("cm_id=%p sess_name=%s state=%d",
1937                           cm_id, ch->sess_name, ch->state);
1938                 ib_send_cm_dreq(ch->cm_id, NULL, 0);
1939                 ret = -EAGAIN;
1940         } else
1941                 ret = 0;
1942
1943         return ret;
1944 }
1945
1946 static void srpt_cm_timewait_exit(struct ib_cm_id *cm_id)
1947 {
1948         PRINT_INFO("%s", "Received InfiniBand TimeWait exit.");
1949         srpt_find_and_release_channel(cm_id);
1950 }
1951
1952 static void srpt_cm_rep_error(struct ib_cm_id *cm_id)
1953 {
1954         PRINT_INFO("%s", "Received InfiniBand REP error.");
1955         srpt_find_and_release_channel(cm_id);
1956 }
1957
1958 static int srpt_cm_dreq_recv(struct ib_cm_id *cm_id)
1959 {
1960         struct srpt_rdma_ch *ch;
1961
1962         ch = srpt_find_channel(cm_id, false);
1963         if (!ch)
1964                 return -EINVAL;
1965
1966         TRACE_DBG("%s: cm_id= %p ch->state= %d",
1967                  __func__, cm_id, ch->state);
1968
1969         switch (ch->state) {
1970         case RDMA_CHANNEL_LIVE:
1971         case RDMA_CHANNEL_CONNECTING:
1972                 ib_send_cm_drep(ch->cm_id, NULL, 0);
1973                 PRINT_INFO("Received DREQ and sent DREP for session %s.",
1974                            ch->sess_name);
1975                 break;
1976         case RDMA_CHANNEL_DISCONNECTING:
1977         default:
1978                 break;
1979         }
1980
1981         return 0;
1982 }
1983
1984 static void srpt_cm_drep_recv(struct ib_cm_id *cm_id)
1985 {
1986         PRINT_INFO("%s", "Received InfiniBand DREP message.");
1987         srpt_find_and_release_channel(cm_id);
1988 }
1989
1990 /**
1991  * IB connection manager callback function.
1992  *
1993  * A non-zero return value will make the caller destroy the CM ID.
1994  *
1995  * Note: srpt_add_one passes a struct srpt_device* as the third argument to
1996  * the ib_create_cm_id() call.
1997  */
1998 static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
1999 {
2000         int ret = 0;
2001
2002         switch (event->event) {
2003         case IB_CM_REQ_RECEIVED:
2004                 ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd,
2005                                        event->private_data);
2006                 break;
2007         case IB_CM_REJ_RECEIVED:
2008                 srpt_cm_rej_recv(cm_id);
2009                 ret = -EINVAL;
2010                 break;
2011         case IB_CM_RTU_RECEIVED:
2012         case IB_CM_USER_ESTABLISHED:
2013                 ret = srpt_cm_rtu_recv(cm_id);
2014                 break;
2015         case IB_CM_DREQ_RECEIVED:
2016                 ret = srpt_cm_dreq_recv(cm_id);
2017                 break;
2018         case IB_CM_DREP_RECEIVED:
2019                 srpt_cm_drep_recv(cm_id);
2020                 ret = -EINVAL;
2021                 break;
2022         case IB_CM_TIMEWAIT_EXIT:
2023                 srpt_cm_timewait_exit(cm_id);
2024                 ret = -EINVAL;
2025                 break;
2026         case IB_CM_REP_ERROR:
2027                 srpt_cm_rep_error(cm_id);
2028                 ret = -EINVAL;
2029                 break;
2030         default:
2031                 break;
2032         }
2033
2034         return ret;
2035 }
2036
2037 static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
2038                                  struct srpt_ioctx *ioctx,
2039                                  struct scst_cmd *scmnd)
2040 {
2041         struct scatterlist *scat;
2042         scst_data_direction dir;
2043         struct rdma_iu *riu;
2044         struct srp_direct_buf *db;
2045         dma_addr_t dma_addr;
2046         struct ib_sge *sge;
2047         u64 raddr;
2048         u32 rsize;
2049         u32 tsize;
2050         u32 dma_len;
2051         int count, nrdma;
2052         int i, j, k;
2053
2054         scat = scst_cmd_get_sg(scmnd);
2055         dir = scst_cmd_get_data_direction(scmnd);
2056         count = ib_dma_map_sg(ch->sport->sdev->device, scat,
2057                               scst_cmd_get_sg_cnt(scmnd),
2058                               scst_to_tgt_dma_dir(dir));
2059         if (unlikely(!count))
2060                 return -EBUSY;
2061
2062         if (ioctx->rdma_ius && ioctx->n_rdma_ius)
2063                 nrdma = ioctx->n_rdma_ius;
2064         else {
2065                 nrdma = count / SRPT_DEF_SG_PER_WQE + ioctx->n_rbuf;
2066
2067                 ioctx->rdma_ius = kzalloc(nrdma * sizeof *riu,
2068                                           scst_cmd_atomic(scmnd)
2069                                           ? GFP_ATOMIC : GFP_KERNEL);
2070                 if (!ioctx->rdma_ius) {
2071                         ib_dma_unmap_sg(ch->sport->sdev->device,
2072                                         scat, scst_cmd_get_sg_cnt(scmnd),
2073                                         scst_to_tgt_dma_dir(dir));
2074                         return -ENOMEM;
2075                 }
2076
2077                 ioctx->n_rdma_ius = nrdma;
2078         }
2079
2080         db = ioctx->rbufs;
2081         tsize = (dir == SCST_DATA_READ) ?
2082                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2083         dma_len = sg_dma_len(&scat[0]);
2084         riu = ioctx->rdma_ius;
2085
2086         /*
2087          * For each remote desc - calculate the #ib_sge.
2088          * If #ib_sge < SRPT_DEF_SG_PER_WQE per rdma operation then
2089          *      each remote desc rdma_iu is required a rdma wr;
2090          * else
2091          *      we need to allocate extra rdma_iu to carry extra #ib_sge in
2092          *      another rdma wr
2093          */
2094         for (i = 0, j = 0;
2095              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2096                 rsize = be32_to_cpu(db->len);
2097                 raddr = be64_to_cpu(db->va);
2098                 riu->raddr = raddr;
2099                 riu->rkey = be32_to_cpu(db->key);
2100                 riu->sge_cnt = 0;
2101
2102                 /* calculate how many sge required for this remote_buf */
2103                 while (rsize > 0 && tsize > 0) {
2104
2105                         if (rsize >= dma_len) {
2106                                 tsize -= dma_len;
2107                                 rsize -= dma_len;
2108                                 raddr += dma_len;
2109
2110                                 if (tsize > 0) {
2111                                         ++j;
2112                                         if (j < count)
2113                                                 dma_len = sg_dma_len(&scat[j]);
2114                                 }
2115                         } else {
2116                                 tsize -= rsize;
2117                                 dma_len -= rsize;
2118                                 rsize = 0;
2119                         }
2120
2121                         ++riu->sge_cnt;
2122
2123                         if (rsize > 0 && riu->sge_cnt == SRPT_DEF_SG_PER_WQE) {
2124                                 riu->sge =
2125                                     kmalloc(riu->sge_cnt * sizeof *riu->sge,
2126                                             scst_cmd_atomic(scmnd)
2127                                             ? GFP_ATOMIC : GFP_KERNEL);
2128                                 if (!riu->sge)
2129                                         goto free_mem;
2130
2131                                 ++ioctx->n_rdma;
2132                                 ++riu;
2133                                 riu->sge_cnt = 0;
2134                                 riu->raddr = raddr;
2135                                 riu->rkey = be32_to_cpu(db->key);
2136                         }
2137                 }
2138
2139                 riu->sge = kmalloc(riu->sge_cnt * sizeof *riu->sge,
2140                                    scst_cmd_atomic(scmnd)
2141                                    ? GFP_ATOMIC : GFP_KERNEL);
2142
2143                 if (!riu->sge)
2144                         goto free_mem;
2145
2146                 ++ioctx->n_rdma;
2147         }
2148
2149         db = ioctx->rbufs;
2150         scat = scst_cmd_get_sg(scmnd);
2151         tsize = (dir == SCST_DATA_READ) ?
2152                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2153         riu = ioctx->rdma_ius;
2154         dma_len = sg_dma_len(&scat[0]);
2155         dma_addr = sg_dma_address(&scat[0]);
2156
2157         /* this second loop is really mapped sg_addres to rdma_iu->ib_sge */
2158         for (i = 0, j = 0;
2159              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2160                 rsize = be32_to_cpu(db->len);
2161                 sge = riu->sge;
2162                 k = 0;
2163
2164                 while (rsize > 0 && tsize > 0) {
2165                         sge->addr = dma_addr;
2166                         sge->lkey = ch->sport->sdev->mr->lkey;
2167
2168                         if (rsize >= dma_len) {
2169                                 sge->length =
2170                                         (tsize < dma_len) ? tsize : dma_len;
2171                                 tsize -= dma_len;
2172                                 rsize -= dma_len;
2173
2174                                 if (tsize > 0) {
2175                                         ++j;
2176                                         if (j < count) {
2177                                                 dma_len = sg_dma_len(&scat[j]);
2178                                                 dma_addr =
2179                                                     sg_dma_address(&scat[j]);
2180                                         }
2181                                 }
2182                         } else {
2183                                 sge->length = (tsize < rsize) ? tsize : rsize;
2184                                 tsize -= rsize;
2185                                 dma_len -= rsize;
2186                                 dma_addr += rsize;
2187                                 rsize = 0;
2188                         }
2189
2190                         ++k;
2191                         if (k == riu->sge_cnt && rsize > 0) {
2192                                 ++riu;
2193                                 sge = riu->sge;
2194                                 k = 0;
2195                         } else if (rsize > 0)
2196                                 ++sge;
2197                 }
2198         }
2199
2200         return 0;
2201
2202 free_mem:
2203         while (ioctx->n_rdma)
2204                 kfree(ioctx->rdma_ius[ioctx->n_rdma--].sge);
2205
2206         kfree(ioctx->rdma_ius);
2207
2208         ib_dma_unmap_sg(ch->sport->sdev->device,
2209                         scat, scst_cmd_get_sg_cnt(scmnd),
2210                         scst_to_tgt_dma_dir(dir));
2211
2212         return -ENOMEM;
2213 }
2214
2215 static int srpt_perform_rdmas(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2216                               scst_data_direction dir)
2217 {
2218         struct ib_send_wr wr;
2219         struct ib_send_wr *bad_wr;
2220         struct rdma_iu *riu;
2221         int i;
2222         int ret = 0;
2223
2224         riu = ioctx->rdma_ius;
2225         memset(&wr, 0, sizeof wr);
2226
2227         for (i = 0; i < ioctx->n_rdma; ++i, ++riu) {
2228                 wr.opcode = (dir == SCST_DATA_READ) ?
2229                     IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;
2230                 wr.next = NULL;
2231                 wr.wr_id = ioctx->index;
2232                 wr.wr.rdma.remote_addr = riu->raddr;
2233                 wr.wr.rdma.rkey = riu->rkey;
2234                 wr.num_sge = riu->sge_cnt;
2235                 wr.sg_list = riu->sge;
2236
2237                 /* only get completion event for the last rdma wr */
2238                 if (i == (ioctx->n_rdma - 1) && dir == SCST_DATA_WRITE)
2239                         wr.send_flags = IB_SEND_SIGNALED;
2240
2241                 ret = ib_post_send(ch->qp, &wr, &bad_wr);
2242                 if (ret)
2243                         break;
2244         }
2245
2246         return ret;
2247 }
2248
2249 /*
2250  * Start data transfer between initiator and target. Must not block.
2251  */
2252 static int srpt_xfer_data(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2253                           struct scst_cmd *scmnd)
2254 {
2255         int ret;
2256
2257         ret = srpt_map_sg_to_ib_sge(ch, ioctx, scmnd);
2258         if (ret) {
2259                 PRINT_ERROR("%s[%d] ret=%d", __func__, __LINE__, ret);
2260                 ret = SCST_TGT_RES_QUEUE_FULL;
2261                 goto out;
2262         }
2263
2264         ret = srpt_perform_rdmas(ch, ioctx, scst_cmd_get_data_direction(scmnd));
2265         if (ret) {
2266                 PRINT_ERROR("%s[%d] ret=%d", __func__, __LINE__, ret);
2267                 if (ret == -EAGAIN || ret == -ENOMEM)
2268                         ret = SCST_TGT_RES_QUEUE_FULL;
2269                 else
2270                         ret = SCST_TGT_RES_FATAL_ERROR;
2271                 goto out;
2272         }
2273
2274         ret = SCST_TGT_RES_SUCCESS;
2275
2276 out:
2277         return ret;
2278 }
2279
2280 /*
2281  * Called by the SCST core to inform ib_srpt that data reception from the
2282  * initiator should start (SCST_DATA_WRITE). Must not block.
2283  */
2284 static int srpt_rdy_to_xfer(struct scst_cmd *scmnd)
2285 {
2286         struct srpt_rdma_ch *ch;
2287         struct srpt_ioctx *ioctx;
2288
2289         ioctx = scst_cmd_get_tgt_priv(scmnd);
2290         BUG_ON(!ioctx);
2291
2292         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2293         BUG_ON(!ch);
2294
2295         if (ch->state == RDMA_CHANNEL_DISCONNECTING)
2296                 return SCST_TGT_RES_FATAL_ERROR;
2297         else if (ch->state == RDMA_CHANNEL_CONNECTING)
2298                 return SCST_TGT_RES_QUEUE_FULL;
2299
2300         ioctx->state = SRPT_STATE_NEED_DATA;
2301
2302         return srpt_xfer_data(ch, ioctx, scmnd);
2303 }
2304
2305 /*
2306  * Called by the SCST core. Transmits the response buffer and status held in
2307  * 'scmnd'. Must not block.
2308  */
2309 static int srpt_xmit_response(struct scst_cmd *scmnd)
2310 {
2311         struct srpt_rdma_ch *ch;
2312         struct srpt_ioctx *ioctx;
2313         struct srp_rsp *srp_rsp;
2314         u64 tag;
2315         int ret = SCST_TGT_RES_SUCCESS;
2316         int dir;
2317         int status;
2318
2319         ioctx = scst_cmd_get_tgt_priv(scmnd);
2320         BUG_ON(!ioctx);
2321
2322         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2323         BUG_ON(!ch);
2324
2325         tag = scst_cmd_get_tag(scmnd);
2326
2327         ioctx->state = SRPT_STATE_PROCESSED;
2328
2329         if (ch->state != RDMA_CHANNEL_LIVE) {
2330                 PRINT_ERROR("%s: tag= %lld channel in bad state %d",
2331                        __func__, (unsigned long long)tag, ch->state);
2332
2333                 if (ch->state == RDMA_CHANNEL_DISCONNECTING)
2334                         ret = SCST_TGT_RES_FATAL_ERROR;
2335                 else if (ch->state == RDMA_CHANNEL_CONNECTING)
2336                         ret = SCST_TGT_RES_QUEUE_FULL;
2337
2338                 if (unlikely(scst_cmd_aborted(scmnd)))
2339                         goto out_aborted;
2340
2341                 goto out;
2342         }
2343
2344         ib_dma_sync_single_for_cpu(ch->sport->sdev->device, ioctx->dma,
2345                                    MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
2346
2347         srp_rsp = ioctx->buf;
2348
2349         if (unlikely(scst_cmd_aborted(scmnd))) {
2350                 PRINT_INFO("%s: tag= %lld already got aborted",
2351                            __func__, (unsigned long long)tag);
2352                 goto out_aborted;
2353         }
2354
2355         dir = scst_cmd_get_data_direction(scmnd);
2356         status = scst_cmd_get_status(scmnd) & 0xff;
2357
2358         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE, NO_ADD_SENSE, tag);
2359
2360         if (SCST_SENSE_VALID(scst_cmd_get_sense_buffer(scmnd))) {
2361                 srp_rsp->sense_data_len = scst_cmd_get_sense_buffer_len(scmnd);
2362                 if (srp_rsp->sense_data_len >
2363                     (MAX_MESSAGE_SIZE - sizeof *srp_rsp))
2364                         srp_rsp->sense_data_len =
2365                             MAX_MESSAGE_SIZE - sizeof *srp_rsp;
2366
2367                 memcpy((u8 *) (srp_rsp + 1), scst_cmd_get_sense_buffer(scmnd),
2368                        srp_rsp->sense_data_len);
2369
2370                 srp_rsp->sense_data_len = cpu_to_be32(srp_rsp->sense_data_len);
2371                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
2372
2373                 if (!status)
2374                         status = SAM_STAT_CHECK_CONDITION;
2375         }
2376
2377         srp_rsp->status = status;
2378
2379         /* For read commands, transfer the data to the initiator. */
2380         if (dir == SCST_DATA_READ && scst_cmd_get_resp_data_len(scmnd)) {
2381                 ret = srpt_xfer_data(ch, ioctx, scmnd);
2382                 if (ret != SCST_TGT_RES_SUCCESS) {
2383                         PRINT_ERROR("%s: tag= %lld xfer_data failed",
2384                                     __func__, (unsigned long long)tag);
2385                         goto out;
2386                 }
2387         }
2388
2389         if (srpt_post_send(ch, ioctx,
2390                            sizeof *srp_rsp +
2391                            be32_to_cpu(srp_rsp->sense_data_len))) {
2392                 PRINT_ERROR("%s: ch->state= %d tag= %lld",
2393                             __func__, ch->state,
2394                             (unsigned long long)tag);
2395                 ret = SCST_TGT_RES_FATAL_ERROR;
2396         }
2397
2398 out:
2399         return ret;
2400
2401 out_aborted:
2402         ret = SCST_TGT_RES_SUCCESS;
2403         scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_ABORTED);
2404         ioctx->state = SRPT_STATE_ABORTED;
2405         WARN_ON(scmnd->state != SCST_CMD_STATE_XMIT_WAIT);
2406         scst_tgt_cmd_done(scmnd, SCST_CONTEXT_SAME);
2407         goto out;
2408 }
2409
2410 /*
2411  * Called by the SCST core to inform ib_srpt that a received task management
2412  * function has been completed. Must not block.
2413  */
2414 static void srpt_tsk_mgmt_done(struct scst_mgmt_cmd *mcmnd)
2415 {
2416         struct srpt_rdma_ch *ch;
2417         struct srpt_mgmt_ioctx *mgmt_ioctx;
2418         struct srpt_ioctx *ioctx;
2419         int rsp_len;
2420
2421         mgmt_ioctx = scst_mgmt_cmd_get_tgt_priv(mcmnd);
2422         BUG_ON(!mgmt_ioctx);
2423
2424         ch = mgmt_ioctx->ch;
2425         BUG_ON(!ch);
2426
2427         ioctx = mgmt_ioctx->ioctx;
2428         BUG_ON(!ioctx);
2429
2430         TRACE_DBG("%s: tsk_mgmt_done for tag= %lld status=%d",
2431                   __func__, (unsigned long long)mgmt_ioctx->tag,
2432                   scst_mgmt_cmd_get_status(mcmnd));
2433
2434         rsp_len = srpt_build_tskmgmt_rsp(ch, ioctx,
2435                                          (scst_mgmt_cmd_get_status(mcmnd) ==
2436                                           SCST_MGMT_STATUS_SUCCESS) ?
2437                                          SRP_TSK_MGMT_SUCCESS :
2438                                          SRP_TSK_MGMT_FAILED,
2439                                          mgmt_ioctx->tag);
2440         srpt_post_send(ch, ioctx, rsp_len);
2441
2442         scst_mgmt_cmd_set_tgt_priv(mcmnd, NULL);
2443
2444         kfree(mgmt_ioctx);
2445 }
2446
2447 /*
2448  * Called by the SCST core to inform ib_srpt that the command 'scmnd' is about
2449  * to be freed. May be called in IRQ context.
2450  */
2451 static void srpt_on_free_cmd(struct scst_cmd *scmnd)
2452 {
2453         struct srpt_rdma_ch *ch;
2454         struct srpt_ioctx *ioctx;
2455
2456         ioctx = scst_cmd_get_tgt_priv(scmnd);
2457         BUG_ON(!ioctx);
2458
2459         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2460         BUG_ON(!ch);
2461
2462         spin_lock_irq(&ch->spinlock);
2463         list_del(&ioctx->scmnd_list);
2464         ch->active_scmnd_cnt--;
2465         spin_unlock_irq(&ch->spinlock);
2466
2467         srpt_reset_ioctx(ch, ioctx);
2468         scst_cmd_set_tgt_priv(scmnd, NULL);
2469 }
2470
2471 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2472 /* A vanilla 2.6.19 or older kernel without backported OFED kernel headers. */
2473 static void srpt_refresh_port_work(void *ctx)
2474 #else
2475 static void srpt_refresh_port_work(struct work_struct *work)
2476 #endif
2477 {
2478 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2479         struct srpt_port *sport = (struct srpt_port *)ctx;
2480 #else
2481         struct srpt_port *sport = container_of(work, struct srpt_port, work);
2482 #endif
2483
2484         srpt_refresh_port(sport);
2485 }
2486
2487 /*
2488  * Called by the SCST core to detect target adapters. Returns the number of
2489  * detected target adapters.
2490  */
2491 static int srpt_detect(struct scst_tgt_template *tp)
2492 {
2493         int device_count;
2494
2495         TRACE_ENTRY();
2496
2497         device_count = atomic_read(&srpt_device_count);
2498
2499         TRACE_EXIT_RES(device_count);
2500
2501         return device_count;
2502 }
2503
2504 /*
2505  * Callback function called by the SCST core from scst_unregister() to free up
2506  * the resources associated with device scst_tgt.
2507  */
2508 static int srpt_release(struct scst_tgt *scst_tgt)
2509 {
2510         struct srpt_device *sdev = scst_tgt_get_tgt_priv(scst_tgt);
2511         struct srpt_rdma_ch *ch, *tmp_ch;
2512
2513         TRACE_ENTRY();
2514
2515         BUG_ON(!scst_tgt);
2516 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
2517         WARN_ON(!sdev);
2518         if (!sdev)
2519                 return -ENODEV;
2520 #else
2521         if (WARN_ON(!sdev))
2522                 return -ENODEV;
2523 #endif
2524
2525         srpt_unregister_procfs_entry(scst_tgt->tgtt);
2526
2527         spin_lock_irq(&sdev->spinlock);
2528         list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
2529                 list_del(&ch->list);
2530                 spin_unlock_irq(&sdev->spinlock);
2531                 srpt_release_channel(ch, 1);
2532                 spin_lock_irq(&sdev->spinlock);
2533         }
2534         spin_unlock_irq(&sdev->spinlock);
2535
2536         srpt_unregister_mad_agent(sdev);
2537
2538         scst_tgt_set_tgt_priv(scst_tgt, NULL);
2539
2540         TRACE_EXIT();
2541
2542         return 0;
2543 }
2544
2545 /*
2546  * Entry point for ib_srpt's kernel thread. This kernel thread is only created
2547  * when the module parameter 'thread' is not zero (the default is zero).
2548  * This thread processes the ioctx list srpt_thread.thread_ioctx_list.
2549  *
2550  * @pre thread != 0
2551  */
2552 static int srpt_ioctx_thread(void *arg)
2553 {
2554         struct srpt_ioctx *ioctx;
2555
2556         /* Hibernation / freezing of the SRPT kernel thread is not supported. */
2557         current->flags |= PF_NOFREEZE;
2558
2559         spin_lock_irq(&srpt_thread.thread_lock);
2560         while (!kthread_should_stop()) {
2561                 wait_queue_t wait;
2562                 init_waitqueue_entry(&wait, current);
2563
2564                 if (!srpt_test_ioctx_list()) {
2565                         add_wait_queue_exclusive(&ioctx_list_waitQ, &wait);
2566
2567                         for (;;) {
2568                                 set_current_state(TASK_INTERRUPTIBLE);
2569                                 if (srpt_test_ioctx_list())
2570                                         break;
2571                                 spin_unlock_irq(&srpt_thread.thread_lock);
2572                                 schedule();
2573                                 spin_lock_irq(&srpt_thread.thread_lock);
2574                         }
2575                         set_current_state(TASK_RUNNING);
2576                         remove_wait_queue(&ioctx_list_waitQ, &wait);
2577                 }
2578
2579                 while (!list_empty(&srpt_thread.thread_ioctx_list)) {
2580                         ioctx = list_entry(srpt_thread.thread_ioctx_list.next,
2581                                            struct srpt_ioctx, comp_list);
2582
2583                         list_del(&ioctx->comp_list);
2584
2585                         spin_unlock_irq(&srpt_thread.thread_lock);
2586                         switch (ioctx->op) {
2587                         case IB_WC_SEND:
2588                                 srpt_handle_send_comp(ioctx->ch, ioctx,
2589                                         SCST_CONTEXT_DIRECT);
2590                                 break;
2591                         case IB_WC_RDMA_WRITE:
2592                         case IB_WC_RDMA_READ:
2593                                 srpt_handle_rdma_comp(ioctx->ch, ioctx);
2594                                 break;
2595                         case IB_WC_RECV:
2596                                 srpt_handle_new_iu(ioctx->ch, ioctx);
2597                                 break;
2598                         default:
2599                                 break;
2600                         }
2601                         spin_lock_irq(&srpt_thread.thread_lock);
2602                 }
2603         }
2604         spin_unlock_irq(&srpt_thread.thread_lock);
2605
2606         return 0;
2607 }
2608
2609 /* SCST target template for the SRP target implementation. */
2610 static struct scst_tgt_template srpt_template = {
2611         .name = DRV_NAME,
2612         .sg_tablesize = SRPT_DEF_SG_TABLESIZE,
2613         .xmit_response_atomic = 1,
2614         .rdy_to_xfer_atomic = 1,
2615         .no_proc_entry = 0,
2616         .detect = srpt_detect,
2617         .release = srpt_release,
2618         .xmit_response = srpt_xmit_response,
2619         .rdy_to_xfer = srpt_rdy_to_xfer,
2620         .on_free_cmd = srpt_on_free_cmd,
2621         .task_mgmt_fn_done = srpt_tsk_mgmt_done
2622 };
2623
2624 /*
2625  * The callback function srpt_release_class_dev() is called whenever a
2626  * device is removed from the /sys/class/infiniband_srpt device class.
2627  */
2628 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2629 static void srpt_release_class_dev(struct class_device *class_dev)
2630 #else
2631 static void srpt_release_class_dev(struct device *dev)
2632 #endif
2633 {
2634 }
2635
2636 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2637 static int srpt_trace_level_show(struct seq_file *seq, void *v)
2638 {
2639         return scst_proc_log_entry_read(seq, trace_flag, NULL);
2640 }
2641
2642 static ssize_t srpt_proc_trace_level_write(struct file *file,
2643         const char __user *buf, size_t length, loff_t *off)
2644 {
2645         return scst_proc_log_entry_write(file, buf, length, &trace_flag,
2646                 DEFAULT_SRPT_TRACE_FLAGS, NULL);
2647 }
2648
2649 static struct scst_proc_data srpt_log_proc_data = {
2650         SCST_DEF_RW_SEQ_OP(srpt_proc_trace_level_write)
2651         .show = srpt_trace_level_show,
2652 };
2653 #endif
2654
2655 static struct class_attribute srpt_class_attrs[] = {
2656         __ATTR_NULL,
2657 };
2658
2659 static struct class srpt_class = {
2660         .name = "infiniband_srpt",
2661 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2662         .release = srpt_release_class_dev,
2663 #else
2664         .dev_release = srpt_release_class_dev,
2665 #endif
2666         .class_attrs = srpt_class_attrs,
2667 };
2668
2669 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2670 static ssize_t show_login_info(struct class_device *class_dev, char *buf)
2671 #else
2672 static ssize_t show_login_info(struct device *dev,
2673                                struct device_attribute *attr, char *buf)
2674 #endif
2675 {
2676         struct srpt_device *sdev =
2677 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2678                 container_of(class_dev, struct srpt_device, class_dev);
2679 #else
2680                 container_of(dev, struct srpt_device, dev);
2681 #endif
2682         struct srpt_port *sport;
2683         int i;
2684         int len = 0;
2685
2686         for (i = 0; i < sdev->device->phys_port_cnt; i++) {
2687                 sport = &sdev->port[i];
2688
2689                 len += sprintf(buf + len,
2690                                "tid_ext=%016llx,ioc_guid=%016llx,pkey=ffff,"
2691                                "dgid=%04x%04x%04x%04x%04x%04x%04x%04x,"
2692                                "service_id=%016llx\n",
2693                                (unsigned long long) srpt_service_guid,
2694                                (unsigned long long) srpt_service_guid,
2695                                be16_to_cpu(((__be16 *) sport->gid.raw)[0]),
2696                                be16_to_cpu(((__be16 *) sport->gid.raw)[1]),
2697                                be16_to_cpu(((__be16 *) sport->gid.raw)[2]),
2698                                be16_to_cpu(((__be16 *) sport->gid.raw)[3]),
2699                                be16_to_cpu(((__be16 *) sport->gid.raw)[4]),
2700                                be16_to_cpu(((__be16 *) sport->gid.raw)[5]),
2701                                be16_to_cpu(((__be16 *) sport->gid.raw)[6]),
2702                                be16_to_cpu(((__be16 *) sport->gid.raw)[7]),
2703                                (unsigned long long) srpt_service_guid);
2704         }
2705
2706         return len;
2707 }
2708
2709 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2710 static CLASS_DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2711 #else
2712 static DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2713 #endif
2714
2715 /*
2716  * Callback function called by the InfiniBand core when either an InfiniBand
2717  * device has been added or during the ib_register_client() call for each
2718  * registered InfiniBand device.
2719  */
2720 static void srpt_add_one(struct ib_device *device)
2721 {
2722         struct srpt_device *sdev;
2723         struct srpt_port *sport;
2724         struct ib_srq_init_attr srq_attr;
2725         int i;
2726
2727         TRACE_ENTRY();
2728
2729         sdev = kzalloc(sizeof *sdev, GFP_KERNEL);
2730         if (!sdev)
2731                 return;
2732
2733         sdev->device = device;
2734
2735 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2736         sdev->class_dev.class = &srpt_class;
2737         sdev->class_dev.dev = device->dma_device;
2738         snprintf(sdev->class_dev.class_id, BUS_ID_SIZE,
2739                  "srpt-%s", device->name);
2740 #else
2741         sdev->dev.class = &srpt_class;
2742         sdev->dev.parent = device->dma_device;
2743 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
2744         snprintf(sdev->dev.bus_id, BUS_ID_SIZE, "srpt-%s", device->name);
2745 #else
2746         dev_set_name(&sdev->dev, "srpt-%s", device->name);
2747 #endif
2748 #endif
2749
2750 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2751         if (class_device_register(&sdev->class_dev))
2752                 goto free_dev;
2753         if (class_device_create_file(&sdev->class_dev,
2754                                      &class_device_attr_login_info))
2755                 goto err_dev;
2756 #else
2757         if (device_register(&sdev->dev))
2758                 goto free_dev;
2759         if (device_create_file(&sdev->dev, &dev_attr_login_info))
2760                 goto err_dev;
2761 #endif
2762
2763         if (ib_query_device(device, &sdev->dev_attr))
2764                 goto err_dev;
2765
2766         sdev->pd = ib_alloc_pd(device);
2767         if (IS_ERR(sdev->pd))
2768                 goto err_dev;
2769
2770         sdev->mr = ib_get_dma_mr(sdev->pd, IB_ACCESS_LOCAL_WRITE);
2771         if (IS_ERR(sdev->mr))
2772                 goto err_pd;
2773
2774         srq_attr.event_handler = srpt_srq_event;
2775         srq_attr.srq_context = (void *)sdev;
2776         srq_attr.attr.max_wr = min(SRPT_SRQ_SIZE, sdev->dev_attr.max_srq_wr);
2777         srq_attr.attr.max_sge = 1;
2778         srq_attr.attr.srq_limit = 0;
2779
2780         sdev->srq = ib_create_srq(sdev->pd, &srq_attr);
2781         if (IS_ERR(sdev->srq))
2782                 goto err_mr;
2783
2784         TRACE_DBG("%s: create SRQ #wr= %d max_allow=%d dev= %s",
2785                __func__, srq_attr.attr.max_wr,
2786               sdev->dev_attr.max_srq_wr, device->name);
2787
2788         if (!srpt_service_guid)
2789                 srpt_service_guid = be64_to_cpu(device->node_guid);
2790
2791         sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);
2792         if (IS_ERR(sdev->cm_id))
2793                 goto err_srq;
2794
2795         /* print out target login information */
2796         TRACE_DBG("Target login info: id_ext=%016llx,"
2797                   "ioc_guid=%016llx,pkey=ffff,service_id=%016llx",
2798                   (unsigned long long) srpt_service_guid,
2799                   (unsigned long long) srpt_service_guid,
2800                   (unsigned long long) srpt_service_guid);
2801
2802         /*
2803          * We do not have a consistent service_id (ie. also id_ext of target_id)
2804          * to identify this target. We currently use the guid of the first HCA
2805          * in the system as service_id; therefore, the target_id will change
2806          * if this HCA is gone bad and replaced by different HCA
2807          */
2808         if (ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0, NULL))
2809                 goto err_cm;
2810
2811         INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
2812                               srpt_event_handler);
2813         if (ib_register_event_handler(&sdev->event_handler))
2814                 goto err_cm;
2815
2816         if (srpt_alloc_ioctx_ring(sdev))
2817                 goto err_event;
2818
2819         INIT_LIST_HEAD(&sdev->rch_list);
2820         spin_lock_init(&sdev->spinlock);
2821
2822         for (i = 0; i < SRPT_SRQ_SIZE; ++i)
2823                 srpt_post_recv(sdev, sdev->ioctx_ring[i]);
2824
2825         ib_set_client_data(device, &srpt_client, sdev);
2826
2827         sdev->scst_tgt = scst_register(&srpt_template, NULL);
2828         if (!sdev->scst_tgt) {
2829                 PRINT_ERROR("SCST registration failed for %s.",
2830                             sdev->device->name);
2831                 goto err_ring;
2832         }
2833
2834         scst_tgt_set_tgt_priv(sdev->scst_tgt, sdev);
2835
2836         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
2837                 sport = &sdev->port[i - 1];
2838                 sport->sdev = sdev;
2839                 sport->port = i;
2840 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2841                 /*
2842                  * A vanilla 2.6.19 or older kernel without backported OFED
2843                  * kernel headers.
2844                  */
2845                 INIT_WORK(&sport->work, srpt_refresh_port_work, sport);
2846 #else
2847                 INIT_WORK(&sport->work, srpt_refresh_port_work);
2848 #endif
2849                 if (srpt_refresh_port(sport)) {
2850                         PRINT_ERROR("MAD registration failed for %s-%d.",
2851                                     sdev->device->name, i);
2852                         goto err_refresh_port;
2853                 }
2854         }
2855
2856         atomic_inc(&srpt_device_count);
2857
2858         TRACE_EXIT();
2859
2860         return;
2861
2862 err_refresh_port:
2863         scst_unregister(sdev->scst_tgt);
2864 err_ring:
2865         ib_set_client_data(device, &srpt_client, NULL);
2866         srpt_free_ioctx_ring(sdev);
2867 err_event:
2868         ib_unregister_event_handler(&sdev->event_handler);
2869 err_cm:
2870         ib_destroy_cm_id(sdev->cm_id);
2871 err_srq:
2872         ib_destroy_srq(sdev->srq);
2873 err_mr:
2874         ib_dereg_mr(sdev->mr);
2875 err_pd:
2876         ib_dealloc_pd(sdev->pd);
2877 err_dev:
2878 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2879         class_device_unregister(&sdev->class_dev);
2880 #else
2881         device_unregister(&sdev->dev);
2882 #endif
2883 free_dev:
2884         kfree(sdev);
2885
2886         TRACE_EXIT();
2887 }
2888
2889 /*
2890  * Callback function called by the InfiniBand core when either an InfiniBand
2891  * device has been removed or during the ib_unregister_client() call for each
2892  * registered InfiniBand device.
2893  */
2894 static void srpt_remove_one(struct ib_device *device)
2895 {
2896         int i;
2897         struct srpt_device *sdev;
2898
2899         TRACE_ENTRY();
2900
2901         sdev = ib_get_client_data(device, &srpt_client);
2902 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
2903         WARN_ON(!sdev);
2904         if (!sdev)
2905                 return;
2906 #else
2907         if (WARN_ON(!sdev))
2908                 return;
2909 #endif
2910
2911         /*
2912          * Cancel the work if it is queued. Wait until srpt_refresh_port_work()
2913          * finished if it is running.
2914          */
2915         for (i = 0; i < sdev->device->phys_port_cnt; i++)
2916 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
2917                 cancel_work_sync(&sdev->port[i].work);
2918 #else
2919                 /*
2920                  * cancel_work_sync() was introduced in kernel 2.6.22. Older
2921                  * kernels do not have a facility to cancel scheduled work.
2922                  */
2923                 PRINT_ERROR("%s", 
2924                        "your kernel does not provide cancel_work_sync().");
2925 #endif
2926
2927         scst_unregister(sdev->scst_tgt);
2928         sdev->scst_tgt = NULL;
2929
2930         ib_unregister_event_handler(&sdev->event_handler);
2931         ib_destroy_cm_id(sdev->cm_id);
2932         ib_destroy_srq(sdev->srq);
2933         ib_dereg_mr(sdev->mr);
2934         ib_dealloc_pd(sdev->pd);
2935 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2936         class_device_unregister(&sdev->class_dev);
2937 #else
2938         device_unregister(&sdev->dev);
2939 #endif
2940
2941         srpt_free_ioctx_ring(sdev);
2942         kfree(sdev);
2943
2944         TRACE_EXIT();
2945 }
2946
2947 /**
2948  * Create procfs entries for srpt. Currently the only procfs entry created
2949  * by this function is the "trace_level" entry.
2950  */
2951 static int srpt_register_procfs_entry(struct scst_tgt_template *tgt)
2952 {
2953         int res = 0;
2954 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2955         struct proc_dir_entry *p, *root;
2956
2957         root = scst_proc_get_tgt_root(tgt);
2958         WARN_ON(!root);
2959         if (root) {
2960                 /*
2961                  * Fill in the scst_proc_data::data pointer, which is used in
2962                  * a printk(KERN_INFO ...) statement in
2963                  * scst_proc_log_entry_write() in scst_proc.c.
2964                  */
2965                 srpt_log_proc_data.data = (char *)tgt->name;
2966                 p = scst_create_proc_entry(root, SRPT_PROC_TRACE_LEVEL_NAME,
2967                                            &srpt_log_proc_data);
2968                 if (!p)
2969                         res = -ENOMEM;
2970         } else
2971                 res = -ENOMEM;
2972
2973 #endif
2974         return res;
2975 }
2976
2977 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt)
2978 {
2979 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2980         struct proc_dir_entry *root;
2981
2982         root = scst_proc_get_tgt_root(tgt);
2983         WARN_ON(!root);
2984         if (root)
2985                 remove_proc_entry(SRPT_PROC_TRACE_LEVEL_NAME, root);
2986 #endif
2987 }
2988
2989 /*
2990  * Module initialization.
2991  *
2992  * Note: since ib_register_client() registers callback functions, and since at
2993  * least one of these callback functions (srpt_add_one()) calls SCST functions,
2994  * the SCST target template must be registered before ib_register_client() is
2995  * called.
2996  */
2997 static int __init srpt_init_module(void)
2998 {
2999         int ret;
3000
3001         ret = class_register(&srpt_class);
3002         if (ret) {
3003                 PRINT_ERROR("%s", "couldn't register class ib_srpt");
3004                 goto out;
3005         }
3006
3007         ret = scst_register_target_template(&srpt_template);
3008         if (ret < 0) {
3009                 PRINT_ERROR("%s", "couldn't register with scst");
3010                 ret = -ENODEV;
3011                 goto out_unregister_class;
3012         }
3013
3014         ret = srpt_register_procfs_entry(&srpt_template);
3015         if (ret) {
3016                 PRINT_ERROR("%s", "couldn't register procfs entry");
3017                 goto out_unregister_target;
3018         }
3019
3020         ret = ib_register_client(&srpt_client);
3021         if (ret) {
3022                 PRINT_ERROR("%s", "couldn't register IB client");
3023                 goto out_unregister_target;
3024         }
3025
3026         if (thread) {
3027                 spin_lock_init(&srpt_thread.thread_lock);
3028                 INIT_LIST_HEAD(&srpt_thread.thread_ioctx_list);
3029                 srpt_thread.thread = kthread_run(srpt_ioctx_thread,
3030                                                  NULL, "srpt_thread");
3031                 if (IS_ERR(srpt_thread.thread)) {
3032                         srpt_thread.thread = NULL;
3033                         thread = 0;
3034                 }
3035         }
3036
3037         return 0;
3038
3039 out_unregister_target:
3040         /*
3041          * Note: the procfs entry is unregistered in srpt_release(), which is
3042          * called by scst_unregister_target_template().
3043          */
3044         scst_unregister_target_template(&srpt_template);
3045 out_unregister_class:
3046         class_unregister(&srpt_class);
3047 out:
3048         return ret;
3049 }
3050
3051 static void __exit srpt_cleanup_module(void)
3052 {
3053         TRACE_ENTRY();
3054
3055         if (srpt_thread.thread)
3056                 kthread_stop(srpt_thread.thread);
3057         ib_unregister_client(&srpt_client);
3058         scst_unregister_target_template(&srpt_template);
3059         class_unregister(&srpt_class);
3060
3061         TRACE_EXIT();
3062 }
3063
3064 module_init(srpt_init_module);
3065 module_exit(srpt_cleanup_module);