- Changed the identification string displayed by the ibsrpdm tool from
[mirror/scst/.git] / srpt / src / ib_srpt.c
1 /*
2  * Copyright (c) 2006 - 2009 Mellanox Technology Inc.  All rights reserved.
3  * Copyright (C) 2008 Vladislav Bolkhovitin <vst@vlnb.net>
4  * Copyright (C) 2008 - 2009 Bart Van Assche <bart.vanassche@gmail.com>
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  */
35
36 #include <linux/module.h>
37 #include <linux/init.h>
38 #include <linux/slab.h>
39 #include <linux/err.h>
40 #include <linux/ctype.h>
41 #include <linux/string.h>
42 #include <linux/kthread.h>
43 #include <asm/atomic.h>
44 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
45 #include <linux/proc_fs.h>
46 #include <linux/seq_file.h>
47 #endif
48 #include "ib_srpt.h"
49 #include "scst_debug.h"
50
51 /* Name of this kernel module. */
52 #define DRV_NAME                "ib_srpt"
53 /* Prefix for printk() kernel messages. */
54 #define PFX                     DRV_NAME ": "
55 #define DRV_VERSION             "1.0.1"
56 #define DRV_RELDATE             "July 10, 2008"
57 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
58 /* Flags to be used in SCST debug tracing statements. */
59 #define DEFAULT_SRPT_TRACE_FLAGS (TRACE_OUT_OF_MEM | TRACE_MINOR \
60                                   | TRACE_MGMT | TRACE_SPECIAL)
61 /* Name of the entry that will be created under /proc/scsi_tgt/ib_srpt. */
62 #define SRPT_PROC_TRACE_LEVEL_NAME      "trace_level"
63 #endif
64
65 #define MELLANOX_SRPT_ID_STRING "SCST SRP target"
66
67 MODULE_AUTHOR("Vu Pham");
68 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
69                    "v" DRV_VERSION " (" DRV_RELDATE ")");
70 MODULE_LICENSE("Dual BSD/GPL");
71
72 struct srpt_thread {
73         /* Protects thread_ioctx_list. */
74         spinlock_t thread_lock;
75         /* I/O contexts to be processed by the kernel thread. */
76         struct list_head thread_ioctx_list;
77         /* SRPT kernel thread. */
78         struct task_struct *thread;
79 };
80
81 /*
82  * Global Variables
83  */
84
85 static u64 srpt_service_guid;
86 /* List of srpt_device structures. */
87 static atomic_t srpt_device_count;
88 static int use_port_guid_in_session_name;
89 static int thread;
90 static struct srpt_thread srpt_thread;
91 static DECLARE_WAIT_QUEUE_HEAD(ioctx_list_waitQ);
92 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
93 static unsigned long trace_flag = DEFAULT_SRPT_TRACE_FLAGS;
94 module_param(trace_flag, long, 0644);
95 MODULE_PARM_DESC(trace_flag,
96                  "Trace flags for the ib_srpt kernel module.");
97 #endif
98
99 module_param(thread, int, 0444);
100 MODULE_PARM_DESC(thread,
101                  "Executing ioctx in thread context. Default 0, i.e. soft IRQ, "
102                  "where possible");
103
104 module_param(use_port_guid_in_session_name, bool, 0444);
105 MODULE_PARM_DESC(use_port_guid_in_session_name,
106                  "Use target port ID in the SCST session name such that"
107                  " redundant paths between multiport systems can be masked.");
108
109 static void srpt_add_one(struct ib_device *device);
110 static void srpt_remove_one(struct ib_device *device);
111 static void srpt_unregister_mad_agent(struct srpt_device *sdev);
112 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt);
113
114 static struct ib_client srpt_client = {
115         .name = DRV_NAME,
116         .add = srpt_add_one,
117         .remove = srpt_remove_one
118 };
119
120 /**
121  * Atomically test and set the channel state.
122  * @ch: RDMA channel.
123  * @old: channel state to compare with.
124  * @new: state to change the channel state to if the current state matches the
125  *       argument 'old'.
126  *
127  * Returns true if the channel state matched old upon entry of this function,
128  * and false otherwise.
129  */
130 static bool srpt_test_and_set_channel_state(struct srpt_rdma_ch *ch,
131                                             enum rdma_ch_state old,
132                                             enum rdma_ch_state new)
133 {
134         unsigned long flags;
135         enum rdma_ch_state cur;
136
137         spin_lock_irqsave(&ch->spinlock, flags);
138         cur = ch->state;
139         if (cur == old)
140                 ch->state = new;
141         spin_unlock_irqrestore(&ch->spinlock, flags);
142
143         return cur == old;
144 }
145
146 /*
147  * Callback function called by the InfiniBand core when an asynchronous IB
148  * event occurs. This callback may occur in interrupt context. See also
149  * section 11.5.2, Set Asynchronous Event Handler in the InfiniBand
150  * Architecture Specification.
151  */
152 static void srpt_event_handler(struct ib_event_handler *handler,
153                                struct ib_event *event)
154 {
155         struct srpt_device *sdev;
156         struct srpt_port *sport;
157
158         sdev = ib_get_client_data(event->device, &srpt_client);
159         if (!sdev || sdev->device != event->device)
160                 return;
161
162         TRACE_DBG("ASYNC event= %d on device= %s",
163                   event->event, sdev->device->name);
164
165         switch (event->event) {
166         case IB_EVENT_PORT_ERR:
167                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
168                         sport = &sdev->port[event->element.port_num - 1];
169                         sport->lid = 0;
170                         sport->sm_lid = 0;
171                 }
172                 break;
173         case IB_EVENT_PORT_ACTIVE:
174         case IB_EVENT_LID_CHANGE:
175         case IB_EVENT_PKEY_CHANGE:
176         case IB_EVENT_SM_CHANGE:
177         case IB_EVENT_CLIENT_REREGISTER:
178                 /*
179                  * Refresh port data asynchronously. Note: it is safe to call
180                  * schedule_work() even if &sport->work is already on the
181                  * global workqueue because schedule_work() tests for the
182                  * work_pending() condition before adding &sport->work to the
183                  * global work queue.
184                  */
185                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
186                         sport = &sdev->port[event->element.port_num - 1];
187                         if (!sport->lid && !sport->sm_lid)
188                                 schedule_work(&sport->work);
189                 }
190                 break;
191         default:
192                 break;
193         }
194
195 }
196
197 /*
198  * Callback function called by the InfiniBand core for SRQ (shared receive
199  * queue) events.
200  */
201 static void srpt_srq_event(struct ib_event *event, void *ctx)
202 {
203         TRACE_DBG("SRQ event %d", event->event);
204 }
205
206 /*
207  * Callback function called by the InfiniBand core for QP (queue pair) events.
208  */
209 static void srpt_qp_event(struct ib_event *event, void *ctx)
210 {
211         struct srpt_rdma_ch *ch = ctx;
212
213         TRACE_DBG("QP event %d on cm_id=%p sess_name=%s state=%d",
214                   event->event, ch->cm_id, ch->sess_name, ch->state);
215
216         switch (event->event) {
217         case IB_EVENT_COMM_EST:
218 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20) || defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
219                 ib_cm_notify(ch->cm_id, event->event);
220 #else
221                 /* Vanilla 2.6.19 kernel (or before) without OFED. */
222                 printk(KERN_ERR PFX "how to perform ib_cm_notify() on a"
223                         " vanilla 2.6.18 kernel ???\n");
224 #endif
225                 break;
226         case IB_EVENT_QP_LAST_WQE_REACHED:
227                 if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_LIVE,
228                                         RDMA_CHANNEL_DISCONNECTING)) {
229                         TRACE_DBG("%s", "Disconnecting channel.");
230                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
231                 }
232                 break;
233         default:
234                 break;
235         }
236 }
237
238 /*
239  * Helper function for filling in an InfiniBand IOUnitInfo structure. Copies
240  * the lowest four bits of value in element slot of the array of four bit
241  * elements called c_list (controller list). The index slot is one-based.
242  *
243  * @pre 1 <= slot && 0 <= value && value < 16
244  */
245 static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
246 {
247         u16 id;
248         u8 tmp;
249
250         id = (slot - 1) / 2;
251         if (slot & 0x1) {
252                 tmp = c_list[id] & 0xf;
253                 c_list[id] = (value << 4) | tmp;
254         } else {
255                 tmp = c_list[id] & 0xf0;
256                 c_list[id] = (value & 0xf) | tmp;
257         }
258 }
259
260 /*
261  * Write InfiniBand ClassPortInfo to mad. See also section 16.3.3.1
262  * ClassPortInfo in the InfiniBand Architecture Specification.
263  */
264 static void srpt_get_class_port_info(struct ib_dm_mad *mad)
265 {
266         struct ib_class_port_info *cif;
267
268         cif = (struct ib_class_port_info *)mad->data;
269         memset(cif, 0, sizeof *cif);
270         cif->base_version = 1;
271         cif->class_version = 1;
272         cif->resp_time_value = 20;
273
274         mad->mad_hdr.status = 0;
275 }
276
277 /*
278  * Write IOUnitInfo to mad. See also section 16.3.3.3 IOUnitInfo in the
279  * InfiniBand Architecture Specification. See also section B.7,
280  * table B.6 in the T10 SRP r16a document.
281  */
282 static void srpt_get_iou(struct ib_dm_mad *mad)
283 {
284         struct ib_dm_iou_info *ioui;
285         u8 slot;
286         int i;
287
288         ioui = (struct ib_dm_iou_info *)mad->data;
289         ioui->change_id = 1;
290         ioui->max_controllers = 16;
291
292         /* set present for slot 1 and empty for the rest */
293         srpt_set_ioc(ioui->controller_list, 1, 1);
294         for (i = 1, slot = 2; i < 16; i++, slot++)
295                 srpt_set_ioc(ioui->controller_list, slot, 0);
296
297         mad->mad_hdr.status = 0;
298 }
299
300 /*
301  * Write IOControllerprofile to mad for I/O controller (sdev, slot). See also
302  * section 16.3.3.4 IOControllerProfile in the InfiniBand Architecture
303  * Specification. See also section B.7, table B.7 in the T10 SRP r16a
304  * document.
305  */
306 static void srpt_get_ioc(struct srpt_device *sdev, u32 slot,
307                          struct ib_dm_mad *mad)
308 {
309         struct ib_dm_ioc_profile *iocp;
310
311         iocp = (struct ib_dm_ioc_profile *)mad->data;
312
313         if (!slot || slot > 16) {
314                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
315                 return;
316         }
317
318         if (slot > 2) {
319                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
320                 return;
321         }
322
323         memset(iocp, 0, sizeof *iocp);
324         strcpy(iocp->id_string, MELLANOX_SRPT_ID_STRING);
325         iocp->guid = cpu_to_be64(srpt_service_guid);
326         iocp->vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
327         iocp->device_id = cpu_to_be32(sdev->dev_attr.vendor_part_id);
328         iocp->device_version = cpu_to_be16(sdev->dev_attr.hw_ver);
329         iocp->subsys_vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
330         iocp->subsys_device_id = 0x0;
331         iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
332         iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
333         iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
334         iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
335         iocp->send_queue_depth = cpu_to_be16(SRPT_SRQ_SIZE);
336         iocp->rdma_read_depth = 4;
337         iocp->send_size = cpu_to_be32(MAX_MESSAGE_SIZE);
338         iocp->rdma_size = cpu_to_be32(MAX_RDMA_SIZE);
339         iocp->num_svc_entries = 1;
340         iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
341                 SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
342
343         mad->mad_hdr.status = 0;
344 }
345
346 /*
347  * Device management: write ServiceEntries to mad for the given slot. See also
348  * section 16.3.3.5 ServiceEntries in the InfiniBand Architecture
349  * Specification. See also section B.7, table B.8 in the T10 SRP r16a document.
350  */
351 static void srpt_get_svc_entries(u64 ioc_guid,
352                                  u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
353 {
354         struct ib_dm_svc_entries *svc_entries;
355
356         WARN_ON(!ioc_guid);
357
358         if (!slot || slot > 16) {
359                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
360                 return;
361         }
362
363         if (slot > 2 || lo > hi || hi > 1) {
364                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
365                 return;
366         }
367
368         svc_entries = (struct ib_dm_svc_entries *)mad->data;
369         memset(svc_entries, 0, sizeof *svc_entries);
370         svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid);
371         snprintf(svc_entries->service_entries[0].name,
372                  sizeof(svc_entries->service_entries[0].name),
373                  "%s%016llx",
374                  SRP_SERVICE_NAME_PREFIX,
375                  (unsigned long long)ioc_guid);
376
377         mad->mad_hdr.status = 0;
378 }
379
380 /*
381  * Actual processing of a received MAD *rq_mad received through source port *sp
382  * (MAD = InfiniBand management datagram). The response to be sent back is
383  * written to *rsp_mad.
384  */
385 static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
386                                  struct ib_dm_mad *rsp_mad)
387 {
388         u16 attr_id;
389         u32 slot;
390         u8 hi, lo;
391
392         attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
393         switch (attr_id) {
394         case DM_ATTR_CLASS_PORT_INFO:
395                 srpt_get_class_port_info(rsp_mad);
396                 break;
397         case DM_ATTR_IOU_INFO:
398                 srpt_get_iou(rsp_mad);
399                 break;
400         case DM_ATTR_IOC_PROFILE:
401                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
402                 srpt_get_ioc(sp->sdev, slot, rsp_mad);
403                 break;
404         case DM_ATTR_SVC_ENTRIES:
405                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
406                 hi = (u8) ((slot >> 8) & 0xff);
407                 lo = (u8) (slot & 0xff);
408                 slot = (u16) ((slot >> 16) & 0xffff);
409                 srpt_get_svc_entries(srpt_service_guid,
410                                      slot, hi, lo, rsp_mad);
411                 break;
412         default:
413                 rsp_mad->mad_hdr.status =
414                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
415                 break;
416         }
417 }
418
419 /*
420  * Callback function that is called by the InfiniBand core after transmission of
421  * a MAD. (MAD = management datagram; AH = address handle.)
422  */
423 static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
424                                   struct ib_mad_send_wc *mad_wc)
425 {
426         ib_destroy_ah(mad_wc->send_buf->ah);
427         ib_free_send_mad(mad_wc->send_buf);
428 }
429
430 /*
431  * Callback function that is called by the InfiniBand core after reception of
432  * a MAD (management datagram).
433  */
434 static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
435                                   struct ib_mad_recv_wc *mad_wc)
436 {
437         struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
438         struct ib_ah *ah;
439         struct ib_mad_send_buf *rsp;
440         struct ib_dm_mad *dm_mad;
441
442         if (!mad_wc || !mad_wc->recv_buf.mad)
443                 return;
444
445         ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
446                                   mad_wc->recv_buf.grh, mad_agent->port_num);
447         if (IS_ERR(ah))
448                 goto err;
449
450         BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR);
451
452         rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
453                                  mad_wc->wc->pkey_index, 0,
454                                  IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA,
455                                  GFP_KERNEL);
456         if (IS_ERR(rsp))
457                 goto err_rsp;
458
459         rsp->ah = ah;
460
461         dm_mad = rsp->mad;
462         memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof *dm_mad);
463         dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
464         dm_mad->mad_hdr.status = 0;
465
466         switch (mad_wc->recv_buf.mad->mad_hdr.method) {
467         case IB_MGMT_METHOD_GET:
468                 srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
469                 break;
470         case IB_MGMT_METHOD_SET:
471                 dm_mad->mad_hdr.status =
472                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
473                 break;
474         default:
475                 dm_mad->mad_hdr.status =
476                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
477                 break;
478         }
479
480         if (!ib_post_send_mad(rsp, NULL)) {
481                 ib_free_recv_mad(mad_wc);
482                 /* will destroy_ah & free_send_mad in send completion */
483                 return;
484         }
485
486         ib_free_send_mad(rsp);
487
488 err_rsp:
489         ib_destroy_ah(ah);
490 err:
491         ib_free_recv_mad(mad_wc);
492 }
493
494 /*
495  * Enable InfiniBand management datagram processing, update the cached sm_lid,
496  * lid and gid values, and register a callback function for processing MADs
497  * on the specified port. It is safe to call this function more than once for
498  * the same port.
499  */
500 static int srpt_refresh_port(struct srpt_port *sport)
501 {
502         struct ib_mad_reg_req reg_req;
503         struct ib_port_modify port_modify;
504         struct ib_port_attr port_attr;
505         int ret;
506
507         memset(&port_modify, 0, sizeof port_modify);
508         port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
509         port_modify.clr_port_cap_mask = 0;
510
511         ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
512         if (ret)
513                 goto err_mod_port;
514
515         ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
516         if (ret)
517                 goto err_query_port;
518
519         sport->sm_lid = port_attr.sm_lid;
520         sport->lid = port_attr.lid;
521
522         ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid);
523         if (ret)
524                 goto err_query_port;
525
526         if (!sport->mad_agent) {
527                 memset(&reg_req, 0, sizeof reg_req);
528                 reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
529                 reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
530                 set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
531                 set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
532
533                 sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
534                                                          sport->port,
535                                                          IB_QPT_GSI,
536                                                          &reg_req, 0,
537                                                          srpt_mad_send_handler,
538                                                          srpt_mad_recv_handler,
539                                                          sport);
540                 if (IS_ERR(sport->mad_agent)) {
541                         ret = PTR_ERR(sport->mad_agent);
542                         sport->mad_agent = NULL;
543                         goto err_query_port;
544                 }
545         }
546
547         return 0;
548
549 err_query_port:
550
551         port_modify.set_port_cap_mask = 0;
552         port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
553         ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
554
555 err_mod_port:
556
557         return ret;
558 }
559
560 /*
561  * Unregister the callback function for processing MADs and disable MAD
562  * processing for all ports of the specified device. It is safe to call this
563  * function more than once for the same device.
564  */
565 static void srpt_unregister_mad_agent(struct srpt_device *sdev)
566 {
567         struct ib_port_modify port_modify = {
568                 .clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP,
569         };
570         struct srpt_port *sport;
571         int i;
572
573         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
574                 sport = &sdev->port[i - 1];
575                 WARN_ON(sport->port != i);
576                 if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
577                         printk(KERN_ERR PFX "disabling MAD processing"
578                                " failed.\n");
579                 if (sport->mad_agent) {
580                         ib_unregister_mad_agent(sport->mad_agent);
581                         sport->mad_agent = NULL;
582                 }
583         }
584 }
585
586 /*
587  * Allocate and initialize an SRPT I/O context structure.
588  */
589 static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev)
590 {
591         struct srpt_ioctx *ioctx;
592
593         ioctx = kmalloc(sizeof *ioctx, GFP_KERNEL);
594         if (!ioctx)
595                 goto out;
596
597         ioctx->buf = kzalloc(MAX_MESSAGE_SIZE, GFP_KERNEL);
598         if (!ioctx->buf)
599                 goto out_free_ioctx;
600
601         ioctx->dma = dma_map_single(sdev->device->dma_device, ioctx->buf,
602                                     MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
603 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
604         if (dma_mapping_error(sdev->device->dma_device, ioctx->dma))
605 #else
606         if (dma_mapping_error(ioctx->dma))
607 #endif
608                 goto out_free_buf;
609
610         return ioctx;
611
612 out_free_buf:
613         kfree(ioctx->buf);
614 out_free_ioctx:
615         kfree(ioctx);
616 out:
617         return NULL;
618 }
619
620 /*
621  * Deallocate an SRPT I/O context structure.
622  */
623 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
624 {
625         if (!ioctx)
626                 return;
627
628         dma_unmap_single(sdev->device->dma_device, ioctx->dma,
629                          MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
630         kfree(ioctx->buf);
631         kfree(ioctx);
632 }
633
634 /*
635  * Associate a ring of SRPT I/O context structures with the specified device.
636  */
637 static int srpt_alloc_ioctx_ring(struct srpt_device *sdev)
638 {
639         int i;
640
641         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
642                 sdev->ioctx_ring[i] = srpt_alloc_ioctx(sdev);
643
644                 if (!sdev->ioctx_ring[i])
645                         goto err;
646
647                 sdev->ioctx_ring[i]->index = i;
648         }
649
650         return 0;
651
652 err:
653         while (--i > 0) {
654                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
655                 sdev->ioctx_ring[i] = NULL;
656         }
657         return -ENOMEM;
658 }
659
660 /* Free the ring of SRPT I/O context structures. */
661 static void srpt_free_ioctx_ring(struct srpt_device *sdev)
662 {
663         int i;
664
665         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
666                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
667                 sdev->ioctx_ring[i] = NULL;
668         }
669 }
670
671 /*
672  * Post a receive request on the work queue of InfiniBand device 'sdev'.
673  */
674 static int srpt_post_recv(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
675 {
676         struct ib_sge list;
677         struct ib_recv_wr wr, *bad_wr;
678
679         wr.wr_id = ioctx->index | SRPT_OP_RECV;
680
681         list.addr = ioctx->dma;
682         list.length = MAX_MESSAGE_SIZE;
683         list.lkey = sdev->mr->lkey;
684
685         wr.next = NULL;
686         wr.sg_list = &list;
687         wr.num_sge = 1;
688
689         return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);
690 }
691
692 /*
693  * Post an IB send request.
694  * @ch: RDMA channel to post the send request on.
695  * @ioctx: I/O context of the send request.
696  * @len: length of the request to be sent in bytes.
697  *
698  * Returns zero upon success and a non-zero value upon failure.
699  */
700 static int srpt_post_send(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
701                           int len)
702 {
703         struct ib_sge list;
704         struct ib_send_wr wr, *bad_wr;
705         struct srpt_device *sdev = ch->sport->sdev;
706
707         dma_sync_single_for_device(sdev->device->dma_device, ioctx->dma,
708                                    MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
709
710         list.addr = ioctx->dma;
711         list.length = len;
712         list.lkey = sdev->mr->lkey;
713
714         wr.next = NULL;
715         wr.wr_id = ioctx->index;
716         wr.sg_list = &list;
717         wr.num_sge = 1;
718         wr.opcode = IB_WR_SEND;
719         wr.send_flags = IB_SEND_SIGNALED;
720
721         return ib_post_send(ch->qp, &wr, &bad_wr);
722 }
723
724 static int srpt_get_desc_tbl(struct srpt_ioctx *ioctx, struct srp_cmd *srp_cmd,
725                              int *ind)
726 {
727         struct srp_indirect_buf *idb;
728         struct srp_direct_buf *db;
729
730         *ind = 0;
731         if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
732             ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
733                 ioctx->n_rbuf = 1;
734                 ioctx->rbufs = &ioctx->single_rbuf;
735
736                 db = (void *)srp_cmd->add_data;
737                 memcpy(ioctx->rbufs, db, sizeof *db);
738                 ioctx->data_len = be32_to_cpu(db->len);
739         } else {
740                 idb = (void *)srp_cmd->add_data;
741
742                 ioctx->n_rbuf = be32_to_cpu(idb->table_desc.len) / sizeof *db;
743
744                 if (ioctx->n_rbuf >
745                     (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
746                         *ind = 1;
747                         ioctx->n_rbuf = 0;
748                         goto out;
749                 }
750
751                 if (ioctx->n_rbuf == 1)
752                         ioctx->rbufs = &ioctx->single_rbuf;
753                 else
754                         ioctx->rbufs =
755                                 kmalloc(ioctx->n_rbuf * sizeof *db, GFP_ATOMIC);
756                 if (!ioctx->rbufs) {
757                         ioctx->n_rbuf = 0;
758                         return -ENOMEM;
759                 }
760
761                 db = idb->desc_list;
762                 memcpy(ioctx->rbufs, db, ioctx->n_rbuf * sizeof *db);
763                 ioctx->data_len = be32_to_cpu(idb->len);
764         }
765 out:
766         return 0;
767 }
768
769 /*
770  * Modify the attributes of queue pair 'qp': allow local write, remote read,
771  * and remote write. Also transition 'qp' to state IB_QPS_INIT.
772  */
773 static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
774 {
775         struct ib_qp_attr *attr;
776         int ret;
777
778         attr = kzalloc(sizeof *attr, GFP_KERNEL);
779         if (!attr)
780                 return -ENOMEM;
781
782         attr->qp_state = IB_QPS_INIT;
783         attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ |
784             IB_ACCESS_REMOTE_WRITE;
785         attr->port_num = ch->sport->port;
786         attr->pkey_index = 0;
787
788         ret = ib_modify_qp(qp, attr,
789                            IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
790                            IB_QP_PKEY_INDEX);
791
792         kfree(attr);
793         return ret;
794 }
795
796 /**
797  * Change the state of a channel to 'ready to receive' (RTR).
798  * @ch: channel of the queue pair.
799  * @qp: queue pair to change the state of.
800  *
801  * Returns zero upon success and a negative value upon failure.
802  *
803  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
804  * If this structure ever becomes larger, it might be necessary to allocate
805  * it dynamically instead of on the stack.
806  */
807 static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
808 {
809         struct ib_qp_attr qp_attr;
810         int attr_mask;
811         int ret;
812
813         qp_attr.qp_state = IB_QPS_RTR;
814         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
815         if (ret)
816                 goto out;
817
818         qp_attr.max_dest_rd_atomic = 4;
819
820         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
821
822 out:
823         return ret;
824 }
825
826 /**
827  * Change the state of a channel to 'ready to send' (RTS).
828  * @ch: channel of the queue pair.
829  * @qp: queue pair to change the state of.
830  *
831  * Returns zero upon success and a negative value upon failure.
832  *
833  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
834  * If this structure ever becomes larger, it might be necessary to allocate
835  * it dynamically instead of on the stack.
836  */
837 static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
838 {
839         struct ib_qp_attr qp_attr;
840         int attr_mask;
841         int ret;
842
843         qp_attr.qp_state = IB_QPS_RTS;
844         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
845         if (ret)
846                 goto out;
847
848         qp_attr.max_rd_atomic = 4;
849
850         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
851
852 out:
853         return ret;
854 }
855
856 static void srpt_reset_ioctx(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
857 {
858         int i;
859
860         if (ioctx->n_rdma_ius > 0 && ioctx->rdma_ius) {
861                 struct rdma_iu *riu = ioctx->rdma_ius;
862
863                 for (i = 0; i < ioctx->n_rdma_ius; ++i, ++riu)
864                         kfree(riu->sge);
865                 kfree(ioctx->rdma_ius);
866         }
867
868         if (ioctx->n_rbuf > 1)
869                 kfree(ioctx->rbufs);
870
871         if (srpt_post_recv(ch->sport->sdev, ioctx))
872                 printk(KERN_ERR PFX "SRQ post_recv failed - this is serious\n");
873                 /* we should queue it back to free_ioctx queue */
874         else
875                 atomic_inc(&ch->req_lim_delta);
876 }
877
878 static void srpt_abort_scst_cmd(struct srpt_device *sdev,
879                                 struct scst_cmd *scmnd,
880                                 bool tell_initiator)
881 {
882         struct srpt_ioctx *ioctx;
883         scst_data_direction dir;
884
885         ioctx = scst_cmd_get_tgt_priv(scmnd);
886         BUG_ON(!ioctx);
887         dir = scst_cmd_get_data_direction(scmnd);
888         if (dir != SCST_DATA_NONE) {
889                 dma_unmap_sg(sdev->device->dma_device,
890                              scst_cmd_get_sg(scmnd),
891                              scst_cmd_get_sg_cnt(scmnd),
892                              scst_to_tgt_dma_dir(dir));
893
894 #if 1
895                 switch (scmnd->state) {
896                 case SCST_CMD_STATE_DATA_WAIT:
897                         WARN_ON(ioctx->state != SRPT_STATE_NEED_DATA);
898                         break;
899                 case SCST_CMD_STATE_XMIT_WAIT:
900                         WARN_ON(ioctx->state != SRPT_STATE_PROCESSED);
901                         break;
902                 default:
903                         WARN_ON(ioctx->state == SRPT_STATE_NEED_DATA ||
904                                 ioctx->state == SRPT_STATE_PROCESSED);
905                 }
906 #endif
907
908                 if (ioctx->state == SRPT_STATE_NEED_DATA) {
909                         scst_rx_data(scmnd,
910                                      tell_initiator ? SCST_RX_STATUS_ERROR
911                                      : SCST_RX_STATUS_ERROR_FATAL,
912                                      SCST_CONTEXT_THREAD);
913                         goto out;
914                 } else if (ioctx->state == SRPT_STATE_PROCESSED)
915                         ;
916                 else {
917                         printk(KERN_ERR PFX
918                                "unexpected cmd state %d (SCST) %d (SRPT)\n",
919                                scmnd->state, ioctx->state);
920                         WARN_ON("unexpected cmd state");
921                 }
922         }
923
924         scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_FAILED);
925         scst_tgt_cmd_done(scmnd, scst_estimate_context());
926 out:
927         return;
928 }
929
930 static void srpt_handle_err_comp(struct srpt_rdma_ch *ch, struct ib_wc *wc)
931 {
932         struct srpt_ioctx *ioctx;
933         struct srpt_device *sdev = ch->sport->sdev;
934
935         if (wc->wr_id & SRPT_OP_RECV) {
936                 ioctx = sdev->ioctx_ring[wc->wr_id & ~SRPT_OP_RECV];
937                 printk(KERN_ERR PFX "This is serious - SRQ is in bad state\n");
938         } else {
939                 ioctx = sdev->ioctx_ring[wc->wr_id];
940
941                 if (ioctx->scmnd)
942                         srpt_abort_scst_cmd(sdev, ioctx->scmnd, true);
943                 else
944                         srpt_reset_ioctx(ch, ioctx);
945         }
946 }
947
948 static void srpt_handle_send_comp(struct srpt_rdma_ch *ch,
949                                   struct srpt_ioctx *ioctx,
950                                   enum scst_exec_context context)
951 {
952         if (ioctx->scmnd) {
953                 scst_data_direction dir =
954                         scst_cmd_get_data_direction(ioctx->scmnd);
955
956                 if (dir != SCST_DATA_NONE)
957                         dma_unmap_sg(ch->sport->sdev->device->dma_device,
958                                      scst_cmd_get_sg(ioctx->scmnd),
959                                      scst_cmd_get_sg_cnt(ioctx->scmnd),
960                                      scst_to_tgt_dma_dir(dir));
961
962                 scst_tgt_cmd_done(ioctx->scmnd, context);
963         } else
964                 srpt_reset_ioctx(ch, ioctx);
965 }
966
967 static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch,
968                                   struct srpt_ioctx *ioctx)
969 {
970         if (!ioctx->scmnd) {
971                 srpt_reset_ioctx(ch, ioctx);
972                 return;
973         }
974
975         if (scst_cmd_get_data_direction(ioctx->scmnd) == SCST_DATA_WRITE)
976                 scst_rx_data(ioctx->scmnd, SCST_RX_STATUS_SUCCESS,
977                         scst_estimate_context());
978 }
979
980 /**
981  * Build an SRP_RSP response.
982  * @ch: RDMA channel through which the request has been received.
983  * @ioctx: I/O context in which the SRP_RSP response will be built.
984  * @s_key: sense key that will be stored in the response.
985  * @s_code: value that will be stored in the asc_ascq field of the sense data.
986  * @tag: tag of the request for which this response is being generated.
987  *
988  * Returns the size in bytes of the SRP_RSP response.
989  *
990  * An SRP_RSP response contains a SCSI status or service response. See also
991  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
992  * response. See also SPC-2 for more information about sense data.
993  */
994 static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
995                               struct srpt_ioctx *ioctx, u8 s_key, u8 s_code,
996                               u64 tag)
997 {
998         struct srp_rsp *srp_rsp;
999         struct sense_data *sense;
1000         int limit_delta;
1001         int sense_data_len = 0;
1002
1003         srp_rsp = ioctx->buf;
1004         memset(srp_rsp, 0, sizeof *srp_rsp);
1005
1006         limit_delta = atomic_read(&ch->req_lim_delta);
1007         atomic_sub(limit_delta, &ch->req_lim_delta);
1008
1009         srp_rsp->opcode = SRP_RSP;
1010         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1011         srp_rsp->tag = tag;
1012
1013         if (s_key != NO_SENSE) {
1014                 sense_data_len = sizeof *sense + (sizeof *sense % 4);
1015                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1016                 srp_rsp->status = SAM_STAT_CHECK_CONDITION;
1017                 srp_rsp->sense_data_len = cpu_to_be32(sense_data_len);
1018
1019                 sense = (struct sense_data *)(srp_rsp + 1);
1020                 sense->err_code = 0x70;
1021                 sense->key = s_key;
1022                 sense->asc_ascq = s_code;
1023         }
1024
1025         return sizeof(*srp_rsp) + sense_data_len;
1026 }
1027
1028 /**
1029  * Build a task management response, which is a specific SRP_RSP response.
1030  * @ch: RDMA channel through which the request has been received.
1031  * @ioctx: I/O context in which the SRP_RSP response will be built.
1032  * @rsp_code: RSP_CODE that will be stored in the response.
1033  * @tag: tag of the request for which this response is being generated.
1034  *
1035  * Returns the size in bytes of the SRP_RSP response.
1036  *
1037  * An SRP_RSP response contains a SCSI status or service response. See also
1038  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
1039  * response.
1040  */
1041 static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
1042                                   struct srpt_ioctx *ioctx, u8 rsp_code,
1043                                   u64 tag)
1044 {
1045         struct srp_rsp *srp_rsp;
1046         int limit_delta;
1047         int resp_data_len = 0;
1048
1049         dma_sync_single_for_cpu(ch->sport->sdev->device->dma_device, ioctx->dma,
1050                                 MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
1051
1052         srp_rsp = ioctx->buf;
1053         memset(srp_rsp, 0, sizeof *srp_rsp);
1054
1055         limit_delta = atomic_read(&ch->req_lim_delta);
1056         atomic_sub(limit_delta, &ch->req_lim_delta);
1057
1058         srp_rsp->opcode = SRP_RSP;
1059         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1060         srp_rsp->tag = tag;
1061
1062         if (rsp_code != SRP_TSK_MGMT_SUCCESS) {
1063                 resp_data_len = 4;
1064                 srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
1065                 srp_rsp->resp_data_len = cpu_to_be32(resp_data_len);
1066                 srp_rsp->data[3] = rsp_code;
1067         }
1068
1069         return sizeof(*srp_rsp) + resp_data_len;
1070 }
1071
1072 /*
1073  * Process SRP_CMD.
1074  */
1075 static int srpt_handle_cmd(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
1076 {
1077         struct scst_cmd *scmnd;
1078         struct srp_cmd *srp_cmd;
1079         struct srp_rsp *srp_rsp;
1080         scst_data_direction dir = SCST_DATA_NONE;
1081         int indirect_desc = 0;
1082         int ret;
1083         unsigned long flags;
1084
1085         srp_cmd = ioctx->buf;
1086         srp_rsp = ioctx->buf;
1087
1088         if (srp_cmd->buf_fmt) {
1089                 ret = srpt_get_desc_tbl(ioctx, srp_cmd, &indirect_desc);
1090                 if (ret) {
1091                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1092                                            NO_ADD_SENSE, srp_cmd->tag);
1093                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1094                         goto err;
1095                 }
1096
1097                 if (indirect_desc) {
1098                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1099                                            NO_ADD_SENSE, srp_cmd->tag);
1100                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1101                         goto err;
1102                 }
1103
1104                 if (srp_cmd->buf_fmt & 0xf)
1105                         dir = SCST_DATA_READ;
1106                 else if (srp_cmd->buf_fmt >> 4)
1107                         dir = SCST_DATA_WRITE;
1108                 else
1109                         dir = SCST_DATA_NONE;
1110         } else
1111                 dir = SCST_DATA_NONE;
1112
1113         scmnd = scst_rx_cmd(ch->scst_sess, (u8 *) &srp_cmd->lun,
1114                             sizeof srp_cmd->lun, srp_cmd->cdb, 16,
1115                             thread ? SCST_NON_ATOMIC : SCST_ATOMIC);
1116         if (!scmnd) {
1117                 srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1118                                    NO_ADD_SENSE, srp_cmd->tag);
1119                 srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1120                 goto err;
1121         }
1122
1123         ioctx->scmnd = scmnd;
1124
1125         switch (srp_cmd->task_attr) {
1126         case SRP_CMD_HEAD_OF_Q:
1127                 scmnd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1128                 break;
1129         case SRP_CMD_ORDERED_Q:
1130                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1131                 break;
1132         case SRP_CMD_SIMPLE_Q:
1133                 scmnd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1134                 break;
1135         case SRP_CMD_ACA:
1136                 scmnd->queue_type = SCST_CMD_QUEUE_ACA;
1137                 break;
1138         default:
1139                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1140                 break;
1141         }
1142
1143         scst_cmd_set_tag(scmnd, srp_cmd->tag);
1144         scst_cmd_set_tgt_priv(scmnd, ioctx);
1145         scst_cmd_set_expected(scmnd, dir, ioctx->data_len);
1146
1147         spin_lock_irqsave(&ch->spinlock, flags);
1148         list_add_tail(&ioctx->scmnd_list, &ch->active_scmnd_list);
1149         ch->active_scmnd_cnt++;
1150         spin_unlock_irqrestore(&ch->spinlock, flags);
1151
1152         scst_cmd_init_done(scmnd, scst_estimate_context());
1153
1154         return 0;
1155
1156 err:
1157         WARN_ON(srp_rsp->opcode != SRP_RSP);
1158
1159         return -1;
1160 }
1161
1162 /*
1163  * Process an SRP_TSK_MGMT request.
1164  *
1165  * Returns 0 upon success and -1 upon failure.
1166  *
1167  * Each task management function is performed by calling one of the
1168  * scst_rx_mgmt_fn*() functions. These functions will either report failure
1169  * or process the task management function asynchronously. The function
1170  * srpt_tsk_mgmt_done() will be called by the SCST core upon completion of the
1171  * task management function. When srpt_handle_tsk_mgmt() reports failure
1172  * (i.e. returns -1) a response will have been built in ioctx->buf. This
1173  * information unit has to be sent back by the caller.
1174  *
1175  * For more information about SRP_TSK_MGMT information units, see also section
1176  * 6.7 in the T10 SRP r16a document.
1177  */
1178 static int srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
1179                                 struct srpt_ioctx *ioctx)
1180 {
1181         struct srp_tsk_mgmt *srp_tsk;
1182         struct srpt_mgmt_ioctx *mgmt_ioctx;
1183         int ret;
1184
1185         srp_tsk = ioctx->buf;
1186
1187         TRACE_DBG("recv_tsk_mgmt= %d for task_tag= %lld"
1188                   " using tag= %lld cm_id= %p sess= %p",
1189                   srp_tsk->tsk_mgmt_func,
1190                   (unsigned long long) srp_tsk->task_tag,
1191                   (unsigned long long) srp_tsk->tag,
1192                   ch->cm_id, ch->scst_sess);
1193
1194         mgmt_ioctx = kmalloc(sizeof *mgmt_ioctx, GFP_ATOMIC);
1195         if (!mgmt_ioctx) {
1196                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1197                                        srp_tsk->tag);
1198                 goto err;
1199         }
1200
1201         mgmt_ioctx->ioctx = ioctx;
1202         mgmt_ioctx->ch = ch;
1203         mgmt_ioctx->tag = srp_tsk->tag;
1204
1205         switch (srp_tsk->tsk_mgmt_func) {
1206         case SRP_TSK_ABORT_TASK:
1207                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK");
1208                 ret = scst_rx_mgmt_fn_tag(ch->scst_sess,
1209                                           SCST_ABORT_TASK,
1210                                           srp_tsk->task_tag,
1211                                           thread ?
1212                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1213                                           mgmt_ioctx);
1214                 break;
1215         case SRP_TSK_ABORT_TASK_SET:
1216                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK_SET");
1217                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1218                                           SCST_ABORT_TASK_SET,
1219                                           (u8 *) &srp_tsk->lun,
1220                                           sizeof srp_tsk->lun,
1221                                           thread ?
1222                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1223                                           mgmt_ioctx);
1224                 break;
1225         case SRP_TSK_CLEAR_TASK_SET:
1226                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_TASK_SET");
1227                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1228                                           SCST_CLEAR_TASK_SET,
1229                                           (u8 *) &srp_tsk->lun,
1230                                           sizeof srp_tsk->lun,
1231                                           thread ?
1232                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1233                                           mgmt_ioctx);
1234                 break;
1235         case SRP_TSK_LUN_RESET:
1236                 TRACE_DBG("%s", "Processing SRP_TSK_LUN_RESET");
1237                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1238                                           SCST_LUN_RESET,
1239                                           (u8 *) &srp_tsk->lun,
1240                                           sizeof srp_tsk->lun,
1241                                           thread ?
1242                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1243                                           mgmt_ioctx);
1244                 break;
1245         case SRP_TSK_CLEAR_ACA:
1246                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_ACA");
1247                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1248                                           SCST_CLEAR_ACA,
1249                                           (u8 *) &srp_tsk->lun,
1250                                           sizeof srp_tsk->lun,
1251                                           thread ?
1252                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1253                                           mgmt_ioctx);
1254                 break;
1255         default:
1256                 TRACE_DBG("%s", "Unsupported task management function.");
1257                 srpt_build_tskmgmt_rsp(ch, ioctx,
1258                                        SRP_TSK_MGMT_FUNC_NOT_SUPP,
1259                                        srp_tsk->tag);
1260                 goto err;
1261         }
1262
1263         if (ret) {
1264                 TRACE_DBG("%s", "Processing task management function failed.");
1265                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1266                                        srp_tsk->tag);
1267                 goto err;
1268         }
1269
1270         WARN_ON(srp_tsk->opcode == SRP_RSP);
1271
1272         return 0;
1273
1274 err:
1275         WARN_ON(srp_tsk->opcode != SRP_RSP);
1276
1277         kfree(mgmt_ioctx);
1278         return -1;
1279 }
1280
1281 /**
1282  * Process a receive completion event.
1283  * @ch: RDMA channel for which the completion event has been received.
1284  * @ioctx: SRPT I/O context for which the completion event has been received.
1285  */
1286 static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
1287                                struct srpt_ioctx *ioctx)
1288 {
1289         struct srp_cmd *srp_cmd;
1290         struct srp_rsp *srp_rsp;
1291         unsigned long flags;
1292         int len;
1293
1294         spin_lock_irqsave(&ch->spinlock, flags);
1295         if (ch->state != RDMA_CHANNEL_LIVE) {
1296                 if (ch->state == RDMA_CHANNEL_CONNECTING) {
1297                         list_add_tail(&ioctx->wait_list, &ch->cmd_wait_list);
1298                         spin_unlock_irqrestore(&ch->spinlock, flags);
1299                         return;
1300                 } else {
1301                         spin_unlock_irqrestore(&ch->spinlock, flags);
1302                         srpt_reset_ioctx(ch, ioctx);
1303                         return;
1304                 }
1305         }
1306         spin_unlock_irqrestore(&ch->spinlock, flags);
1307
1308         dma_sync_single_for_cpu(ch->sport->sdev->device->dma_device, ioctx->dma,
1309                                 MAX_MESSAGE_SIZE, DMA_FROM_DEVICE);
1310
1311         ioctx->data_len = 0;
1312         ioctx->n_rbuf = 0;
1313         ioctx->rbufs = NULL;
1314         ioctx->n_rdma = 0;
1315         ioctx->n_rdma_ius = 0;
1316         ioctx->rdma_ius = NULL;
1317         ioctx->scmnd = NULL;
1318         ioctx->state = SRPT_STATE_NEW;
1319
1320         srp_cmd = ioctx->buf;
1321         srp_rsp = ioctx->buf;
1322
1323         switch (srp_cmd->opcode) {
1324         case SRP_CMD:
1325                 if (srpt_handle_cmd(ch, ioctx) < 0)
1326                         goto err;
1327                 break;
1328
1329         case SRP_TSK_MGMT:
1330                 if (srpt_handle_tsk_mgmt(ch, ioctx) < 0)
1331                         goto err;
1332                 break;
1333
1334         case SRP_I_LOGOUT:
1335         case SRP_AER_REQ:
1336         default:
1337                 srpt_build_cmd_rsp(ch, ioctx, ILLEGAL_REQUEST, INVALID_CDB,
1338                                    srp_cmd->tag);
1339                 goto err;
1340         }
1341
1342         dma_sync_single_for_device(ch->sport->sdev->device->dma_device,
1343                                    ioctx->dma, MAX_MESSAGE_SIZE,
1344                                    DMA_FROM_DEVICE);
1345
1346         return;
1347
1348 err:
1349         WARN_ON(srp_rsp->opcode != SRP_RSP);
1350         len = (sizeof *srp_rsp) + be32_to_cpu(srp_rsp->sense_data_len);
1351
1352         if (ch->state != RDMA_CHANNEL_LIVE) {
1353                 /* Give up if another thread modified the channel state. */
1354                 printk(KERN_ERR PFX "%s: channel is in state %d",
1355                        __func__, ch->state);
1356                 srpt_reset_ioctx(ch, ioctx);
1357         } else if (srpt_post_send(ch, ioctx, len)) {
1358                 printk(KERN_ERR PFX "%s: sending SRP_RSP response failed",
1359                        __func__);
1360                 srpt_reset_ioctx(ch, ioctx);
1361         }
1362 }
1363
1364 /*
1365  * Returns true if the ioctx list is non-empty or if the ib_srpt kernel thread
1366  * should stop.
1367  * @pre thread != 0
1368  */
1369 static inline int srpt_test_ioctx_list(void)
1370 {
1371         int res = (!list_empty(&srpt_thread.thread_ioctx_list) ||
1372                    unlikely(kthread_should_stop()));
1373         return res;
1374 }
1375
1376 /*
1377  * Add 'ioctx' to the tail of the ioctx list and wake up the kernel thread.
1378  *
1379  * @pre thread != 0
1380  */
1381 static inline void srpt_schedule_thread(struct srpt_ioctx *ioctx)
1382 {
1383         unsigned long flags;
1384
1385         spin_lock_irqsave(&srpt_thread.thread_lock, flags);
1386         list_add_tail(&ioctx->comp_list, &srpt_thread.thread_ioctx_list);
1387         spin_unlock_irqrestore(&srpt_thread.thread_lock, flags);
1388         wake_up(&ioctx_list_waitQ);
1389 }
1390
1391 /**
1392  * InfiniBand completion queue callback function.
1393  * @cq: completion queue.
1394  * @ctx: completion queue context, which was passed as the fourth argument of
1395  *       the function ib_create_cq().
1396  */
1397 static void srpt_completion(struct ib_cq *cq, void *ctx)
1398 {
1399         struct srpt_rdma_ch *ch = ctx;
1400         struct srpt_device *sdev = ch->sport->sdev;
1401         struct ib_wc wc;
1402         struct srpt_ioctx *ioctx;
1403
1404         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1405         while (ib_poll_cq(ch->cq, 1, &wc) > 0) {
1406                 if (wc.status) {
1407                         printk(KERN_ERR PFX "failed %s status= %d\n",
1408                                wc.wr_id & SRPT_OP_RECV ? "receive" : "send",
1409                                wc.status);
1410                         srpt_handle_err_comp(ch, &wc);
1411                         break;
1412                 }
1413
1414                 if (wc.wr_id & SRPT_OP_RECV) {
1415                         ioctx = sdev->ioctx_ring[wc.wr_id & ~SRPT_OP_RECV];
1416                         if (thread) {
1417                                 ioctx->ch = ch;
1418                                 ioctx->op = IB_WC_RECV;
1419                                 srpt_schedule_thread(ioctx);
1420                         } else
1421                                 srpt_handle_new_iu(ch, ioctx);
1422                         continue;
1423                 } else
1424                         ioctx = sdev->ioctx_ring[wc.wr_id];
1425
1426                 if (thread) {
1427                         ioctx->ch = ch;
1428                         ioctx->op = wc.opcode;
1429                         srpt_schedule_thread(ioctx);
1430                 } else {
1431                         switch (wc.opcode) {
1432                         case IB_WC_SEND:
1433                                 srpt_handle_send_comp(ch, ioctx,
1434                                         scst_estimate_context());
1435                                 break;
1436                         case IB_WC_RDMA_WRITE:
1437                         case IB_WC_RDMA_READ:
1438                                 srpt_handle_rdma_comp(ch, ioctx);
1439                                 break;
1440                         default:
1441                                 break;
1442                         }
1443                 }
1444         }
1445 }
1446
1447 /*
1448  * Create a completion queue on the specified device.
1449  */
1450 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
1451 {
1452         struct ib_qp_init_attr *qp_init;
1453         struct srpt_device *sdev = ch->sport->sdev;
1454         int cqe;
1455         int ret;
1456
1457         qp_init = kzalloc(sizeof *qp_init, GFP_KERNEL);
1458         if (!qp_init)
1459                 return -ENOMEM;
1460
1461         /* Create a completion queue (CQ). */
1462
1463         cqe = SRPT_RQ_SIZE + SRPT_SQ_SIZE - 1;
1464 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(RHEL_RELEASE_CODE)
1465         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe);
1466 #else
1467         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe, 0);
1468 #endif
1469         if (IS_ERR(ch->cq)) {
1470                 ret = PTR_ERR(ch->cq);
1471                 printk(KERN_ERR PFX "failed to create_cq cqe= %d ret= %d\n",
1472                         cqe, ret);
1473                 goto out;
1474         }
1475
1476         /* Request completion notification. */
1477
1478         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1479
1480         /* Create a queue pair (QP). */
1481
1482         qp_init->qp_context = (void *)ch;
1483         qp_init->event_handler = srpt_qp_event;
1484         qp_init->send_cq = ch->cq;
1485         qp_init->recv_cq = ch->cq;
1486         qp_init->srq = sdev->srq;
1487         qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
1488         qp_init->qp_type = IB_QPT_RC;
1489         qp_init->cap.max_send_wr = SRPT_SQ_SIZE;
1490         qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE;
1491
1492         ch->qp = ib_create_qp(sdev->pd, qp_init);
1493         if (IS_ERR(ch->qp)) {
1494                 ret = PTR_ERR(ch->qp);
1495                 ib_destroy_cq(ch->cq);
1496                 printk(KERN_ERR PFX "failed to create_qp ret= %d\n", ret);
1497                 goto out;
1498         }
1499
1500         TRACE_DBG("%s: max_cqe= %d max_sge= %d cm_id= %p",
1501                __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
1502                ch->cm_id);
1503
1504         /* Modify the attributes and the state of queue pair ch->qp. */
1505
1506         ret = srpt_init_ch_qp(ch, ch->qp);
1507         if (ret) {
1508                 ib_destroy_qp(ch->qp);
1509                 ib_destroy_cq(ch->cq);
1510                 goto out;
1511         }
1512
1513         atomic_set(&ch->req_lim_delta, SRPT_RQ_SIZE);
1514 out:
1515         kfree(qp_init);
1516         return ret;
1517 }
1518
1519 /**
1520  * Look up the RDMA channel that corresponds to the specified cm_id.
1521  *
1522  * Return NULL if no matching RDMA channel has been found.
1523  */
1524 static struct srpt_rdma_ch *srpt_find_channel(struct ib_cm_id *cm_id, bool del)
1525 {
1526         struct srpt_device *sdev = cm_id->context;
1527         struct srpt_rdma_ch *ch;
1528
1529         spin_lock_irq(&sdev->spinlock);
1530         list_for_each_entry(ch, &sdev->rch_list, list) {
1531                 if (ch->cm_id == cm_id) {
1532                         if (del)
1533                                 list_del(&ch->list);
1534                         spin_unlock_irq(&sdev->spinlock);
1535                         return ch;
1536                 }
1537         }
1538
1539         spin_unlock_irq(&sdev->spinlock);
1540
1541         return NULL;
1542 }
1543
1544 /**
1545  * Release all resources associated with the specified RDMA channel.
1546  *
1547  * Note: the caller must have removed the channel from the channel list
1548  * before calling this function.
1549  */
1550 static void srpt_release_channel(struct srpt_rdma_ch *ch, int destroy_cmid)
1551 {
1552         TRACE_ENTRY();
1553
1554         WARN_ON(srpt_find_channel(ch->cm_id, false) == ch);
1555
1556         if (ch->cm_id && destroy_cmid) {
1557                 TRACE_DBG("%s: destroy cm_id= %p", __func__, ch->cm_id);
1558                 ib_destroy_cm_id(ch->cm_id);
1559                 ch->cm_id = NULL;
1560         }
1561
1562         ib_destroy_qp(ch->qp);
1563         ib_destroy_cq(ch->cq);
1564
1565         if (ch->scst_sess) {
1566                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1567
1568                 TRACE_DBG("%s: release sess= %p sess_name= %s active_cmd= %d",
1569                           __func__, ch->scst_sess, ch->sess_name,
1570                           ch->active_scmnd_cnt);
1571
1572                 spin_lock_irq(&ch->spinlock);
1573                 list_for_each_entry_safe(ioctx, ioctx_tmp,
1574                                          &ch->active_scmnd_list, scmnd_list) {
1575                         spin_unlock_irq(&ch->spinlock);
1576
1577                         if (ioctx->scmnd)
1578                                 srpt_abort_scst_cmd(ch->sport->sdev,
1579                                                     ioctx->scmnd, true);
1580
1581                         spin_lock_irq(&ch->spinlock);
1582                 }
1583                 WARN_ON(!list_empty(&ch->active_scmnd_list));
1584                 WARN_ON(ch->active_scmnd_cnt != 0);
1585                 spin_unlock_irq(&ch->spinlock);
1586
1587                 scst_unregister_session(ch->scst_sess, 0, NULL);
1588                 ch->scst_sess = NULL;
1589         }
1590
1591         kfree(ch);
1592
1593         TRACE_EXIT();
1594 }
1595
1596 static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
1597                             struct ib_cm_req_event_param *param,
1598                             void *private_data)
1599 {
1600         struct srpt_device *sdev = cm_id->context;
1601         struct srp_login_req *req;
1602         struct srp_login_rsp *rsp;
1603         struct srp_login_rej *rej;
1604         struct ib_cm_rep_param *rep_param;
1605         struct srpt_rdma_ch *ch, *tmp_ch;
1606         u32 it_iu_len;
1607         int ret = 0;
1608
1609 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
1610         WARN_ON(!sdev || !private_data);
1611         if (!sdev || !private_data)
1612                 return -EINVAL;
1613 #else
1614         if (WARN_ON(!sdev || !private_data))
1615                 return -EINVAL;
1616 #endif
1617
1618         req = (struct srp_login_req *)private_data;
1619
1620         it_iu_len = be32_to_cpu(req->req_it_iu_len);
1621
1622         printk(KERN_INFO PFX "received SRP_LOGIN_REQ with"
1623             " i_port_id 0x%llx:0x%llx, t_port_id 0x%llx:0x%llx and length %d"
1624             " on port %d (guid=0x%llx:0x%llx)\n",
1625             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[0]),
1626             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[8]),
1627             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[0]),
1628             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[8]),
1629             it_iu_len,
1630             param->port,
1631             (unsigned long long)be64_to_cpu(*(u64 *)
1632                                 &sdev->port[param->port - 1].gid.raw[0]),
1633             (unsigned long long)be64_to_cpu(*(u64 *)
1634                                 &sdev->port[param->port - 1].gid.raw[8]));
1635
1636         rsp = kzalloc(sizeof *rsp, GFP_KERNEL);
1637         rej = kzalloc(sizeof *rej, GFP_KERNEL);
1638         rep_param = kzalloc(sizeof *rep_param, GFP_KERNEL);
1639
1640         if (!rsp || !rej || !rep_param) {
1641                 ret = -ENOMEM;
1642                 goto out;
1643         }
1644
1645         if (it_iu_len > MAX_MESSAGE_SIZE || it_iu_len < 64) {
1646                 rej->reason =
1647                     cpu_to_be32(SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
1648                 ret = -EINVAL;
1649                 printk(KERN_ERR PFX "rejected SRP_LOGIN_REQ because its"
1650                        " length (%d bytes) is invalid\n", it_iu_len);
1651                 goto reject;
1652         }
1653
1654         if ((req->req_flags & 0x3) == SRP_MULTICHAN_SINGLE) {
1655                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
1656
1657                 spin_lock_irq(&sdev->spinlock);
1658
1659                 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
1660                         if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
1661                             && !memcmp(ch->t_port_id, req->target_port_id, 16)
1662                             && param->port == ch->sport->port
1663                             && param->listen_id == ch->sport->sdev->cm_id
1664                             && ch->cm_id) {
1665                                 enum rdma_ch_state prev_state;
1666
1667                                 /* found an existing channel */
1668                                 TRACE_DBG("Found existing channel name= %s"
1669                                           " cm_id= %p state= %d",
1670                                           ch->sess_name, ch->cm_id, ch->state);
1671
1672                                 prev_state = ch->state;
1673                                 if (ch->state == RDMA_CHANNEL_LIVE)
1674                                         ch->state = RDMA_CHANNEL_DISCONNECTING;
1675                                 else if (ch->state == RDMA_CHANNEL_CONNECTING)
1676                                         list_del(&ch->list);
1677
1678                                 spin_unlock_irq(&sdev->spinlock);
1679
1680                                 rsp->rsp_flags =
1681                                         SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
1682
1683                                 if (prev_state == RDMA_CHANNEL_LIVE) {
1684                                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
1685                                         printk(KERN_ERR PFX "disconnected"
1686                                           " session %s because a new"
1687                                           " SRP_LOGIN_REQ has been received.\n",
1688                                           ch->sess_name);
1689                                 } else if (prev_state ==
1690                                          RDMA_CHANNEL_CONNECTING) {
1691                                         printk(KERN_ERR PFX "rejected"
1692                                           " SRP_LOGIN_REQ because another login"
1693                                           " request is being processed.\n");
1694                                         ib_send_cm_rej(ch->cm_id,
1695                                                        IB_CM_REJ_NO_RESOURCES,
1696                                                        NULL, 0, NULL, 0);
1697                                         srpt_release_channel(ch, 1);
1698                                 }
1699
1700                                 spin_lock_irq(&sdev->spinlock);
1701                         }
1702                 }
1703
1704                 spin_unlock_irq(&sdev->spinlock);
1705
1706         } else
1707                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
1708
1709         if (((u64) (*(u64 *) req->target_port_id) !=
1710              cpu_to_be64(srpt_service_guid)) ||
1711             ((u64) (*(u64 *) (req->target_port_id + 8)) !=
1712              cpu_to_be64(srpt_service_guid))) {
1713                 rej->reason =
1714                     cpu_to_be32(SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
1715                 ret = -ENOMEM;
1716                 printk(KERN_ERR PFX "rejected SRP_LOGIN_REQ because it"
1717                        " has an invalid target port identifier\n");
1718                 goto reject;
1719         }
1720
1721         ch = kzalloc(sizeof *ch, GFP_KERNEL);
1722         if (!ch) {
1723                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1724                 printk(KERN_ERR PFX "rejected SRP_LOGIN_REQ because out of"
1725                        " memory\n");
1726                 ret = -ENOMEM;
1727                 goto reject;
1728         }
1729
1730         spin_lock_init(&ch->spinlock);
1731         memcpy(ch->i_port_id, req->initiator_port_id, 16);
1732         memcpy(ch->t_port_id, req->target_port_id, 16);
1733         ch->sport = &sdev->port[param->port - 1];
1734         ch->cm_id = cm_id;
1735         ch->state = RDMA_CHANNEL_CONNECTING;
1736         INIT_LIST_HEAD(&ch->cmd_wait_list);
1737         INIT_LIST_HEAD(&ch->active_scmnd_list);
1738
1739         ret = srpt_create_ch_ib(ch);
1740         if (ret) {
1741                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1742                 printk(KERN_ERR PFX "rejected SRP_LOGIN_REQ because creating"
1743                        " a new RDMA channel failed\n");
1744                 goto free_ch;
1745         }
1746
1747         ret = srpt_ch_qp_rtr(ch, ch->qp);
1748         if (ret) {
1749                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1750                 printk(KERN_ERR PFX "rejected SRP_LOGIN_REQ because enabling"
1751                        " RTR failed (error code = %d)\n", ret);
1752                 goto destroy_ib;
1753         }
1754
1755         if (use_port_guid_in_session_name) {
1756                 /*
1757                  * If the kernel module parameter use_port_guid_in_session_name
1758                  * has been specified, use a combination of the target port
1759                  * GUID and the initiator port ID as the session name. This
1760                  * was the original behavior of the SRP target implementation
1761                  * (i.e. before the SRPT was included in OFED 1.3).
1762                  */
1763                 snprintf(ch->sess_name, sizeof(ch->sess_name),
1764                          "0x%016llx%016llx",
1765                          (unsigned long long)be64_to_cpu(*(u64 *)
1766                                 &sdev->port[param->port - 1].gid.raw[8]),
1767                          (unsigned long long)be64_to_cpu(*(u64 *)
1768                                 (ch->i_port_id + 8)));
1769         } else {
1770                 /*
1771                  * Default behavior: use the initator port identifier as the
1772                  * session name.
1773                  */
1774                 snprintf(ch->sess_name, sizeof(ch->sess_name),
1775                          "0x%016llx%016llx",
1776                          (unsigned long long)be64_to_cpu(*(u64 *)ch->i_port_id),
1777                          (unsigned long long)be64_to_cpu(*(u64 *)
1778                                  (ch->i_port_id + 8)));
1779         }
1780
1781         TRACE_DBG("registering session %s", ch->sess_name);
1782
1783         BUG_ON(!sdev->scst_tgt);
1784         ch->scst_sess = scst_register_session(sdev->scst_tgt, 0, ch->sess_name,
1785                                               NULL, NULL);
1786         if (!ch->scst_sess) {
1787                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1788                 TRACE_DBG("%s", "Failed to create scst sess");
1789                 goto destroy_ib;
1790         }
1791
1792         TRACE_DBG("Establish connection sess=%p name=%s cm_id=%p",
1793                   ch->scst_sess, ch->sess_name, ch->cm_id);
1794
1795         scst_sess_set_tgt_priv(ch->scst_sess, ch);
1796
1797         /* create srp_login_response */
1798         rsp->opcode = SRP_LOGIN_RSP;
1799         rsp->tag = req->tag;
1800         rsp->max_it_iu_len = req->req_it_iu_len;
1801         rsp->max_ti_iu_len = req->req_it_iu_len;
1802         rsp->buf_fmt =
1803             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1804         rsp->req_lim_delta = cpu_to_be32(SRPT_RQ_SIZE);
1805         atomic_set(&ch->req_lim_delta, 0);
1806
1807         /* create cm reply */
1808         rep_param->qp_num = ch->qp->qp_num;
1809         rep_param->private_data = (void *)rsp;
1810         rep_param->private_data_len = sizeof *rsp;
1811         rep_param->rnr_retry_count = 7;
1812         rep_param->flow_control = 1;
1813         rep_param->failover_accepted = 0;
1814         rep_param->srq = 1;
1815         rep_param->responder_resources = 4;
1816         rep_param->initiator_depth = 4;
1817
1818         ret = ib_send_cm_rep(cm_id, rep_param);
1819         if (ret) {
1820                 printk(KERN_ERR PFX "sending SRP_LOGIN_REQ response failed"
1821                        " (error code = %d)\n", ret);
1822                 goto release_channel;
1823         }
1824
1825         spin_lock_irq(&sdev->spinlock);
1826         list_add_tail(&ch->list, &sdev->rch_list);
1827         spin_unlock_irq(&sdev->spinlock);
1828
1829         goto out;
1830
1831 release_channel:
1832         scst_unregister_session(ch->scst_sess, 0, NULL);
1833         ch->scst_sess = NULL;
1834
1835 destroy_ib:
1836         ib_destroy_qp(ch->qp);
1837         ib_destroy_cq(ch->cq);
1838
1839 free_ch:
1840         kfree(ch);
1841
1842 reject:
1843         rej->opcode = SRP_LOGIN_REJ;
1844         rej->tag = req->tag;
1845         rej->buf_fmt =
1846             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1847
1848         ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
1849                              (void *)rej, sizeof *rej);
1850
1851 out:
1852         kfree(rep_param);
1853         kfree(rsp);
1854         kfree(rej);
1855
1856         return ret;
1857 }
1858
1859 /**
1860  * Release the channel with the specified cm_id.
1861  *
1862  * Returns one to indicate that the caller of srpt_cm_handler() should destroy
1863  * the cm_id.
1864  */
1865 static void srpt_find_and_release_channel(struct ib_cm_id *cm_id)
1866 {
1867         struct srpt_rdma_ch *ch;
1868
1869         ch = srpt_find_channel(cm_id, true);
1870         if (ch)
1871                 srpt_release_channel(ch, 0);
1872 }
1873
1874 static void srpt_cm_rej_recv(struct ib_cm_id *cm_id)
1875 {
1876         TRACE_DBG("%s: cm_id=%p", __func__, cm_id);
1877         srpt_find_and_release_channel(cm_id);
1878 }
1879
1880 /**
1881  * Process an IB_CM_RTU_RECEIVED or IB_CM_USER_ESTABLISHED event.
1882  *
1883  * An IB_CM_RTU_RECEIVED message indicates that the connection is established
1884  * and that the recipient may begin transmitting (RTU = ready to use).
1885  */
1886 static int srpt_cm_rtu_recv(struct ib_cm_id *cm_id)
1887 {
1888         struct srpt_rdma_ch *ch;
1889         int ret;
1890
1891         ch = srpt_find_channel(cm_id, false);
1892         if (!ch)
1893                 return -EINVAL;
1894
1895         if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_CONNECTING,
1896                                             RDMA_CHANNEL_LIVE)) {
1897                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1898
1899                 ret = srpt_ch_qp_rts(ch, ch->qp);
1900
1901                 list_for_each_entry_safe(ioctx, ioctx_tmp, &ch->cmd_wait_list,
1902                                          wait_list) {
1903                         list_del(&ioctx->wait_list);
1904                         srpt_handle_new_iu(ch, ioctx);
1905                 }
1906                 if (ret && srpt_test_and_set_channel_state(ch,
1907                                         RDMA_CHANNEL_LIVE,
1908                                         RDMA_CHANNEL_DISCONNECTING)) {
1909                         TRACE_DBG("cm_id=%p sess_name=%s state=%d",
1910                                   cm_id, ch->sess_name, ch->state);
1911                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
1912                 }
1913         } else if (ch->state == RDMA_CHANNEL_DISCONNECTING) {
1914                 TRACE_DBG("cm_id=%p sess_name=%s state=%d",
1915                           cm_id, ch->sess_name, ch->state);
1916                 ib_send_cm_dreq(ch->cm_id, NULL, 0);
1917                 ret = -EAGAIN;
1918         } else
1919                 ret = 0;
1920
1921         return ret;
1922 }
1923
1924 static void srpt_cm_timewait_exit(struct ib_cm_id *cm_id)
1925 {
1926         TRACE_DBG("%s: cm_id=%p", __func__, cm_id);
1927         srpt_find_and_release_channel(cm_id);
1928 }
1929
1930 static void srpt_cm_rep_error(struct ib_cm_id *cm_id)
1931 {
1932         TRACE_DBG("%s: cm_id=%p", __func__, cm_id);
1933         srpt_find_and_release_channel(cm_id);
1934 }
1935
1936 static int srpt_cm_dreq_recv(struct ib_cm_id *cm_id)
1937 {
1938         struct srpt_rdma_ch *ch;
1939
1940         ch = srpt_find_channel(cm_id, false);
1941         if (!ch)
1942                 return -EINVAL;
1943
1944         TRACE_DBG("%s: cm_id= %p ch->state= %d",
1945                  __func__, cm_id, ch->state);
1946
1947         switch (ch->state) {
1948         case RDMA_CHANNEL_LIVE:
1949         case RDMA_CHANNEL_CONNECTING:
1950                 ib_send_cm_drep(ch->cm_id, NULL, 0);
1951                 break;
1952         case RDMA_CHANNEL_DISCONNECTING:
1953         default:
1954                 break;
1955         }
1956
1957         return 0;
1958 }
1959
1960 static void srpt_cm_drep_recv(struct ib_cm_id *cm_id)
1961 {
1962         TRACE_DBG("%s: cm_id=%p", __func__, cm_id);
1963         srpt_find_and_release_channel(cm_id);
1964 }
1965
1966 /**
1967  * IB connection manager callback function.
1968  *
1969  * A non-zero return value will make the caller destroy the CM ID.
1970  *
1971  * Note: srpt_add_one passes a struct srpt_device* as the third argument to
1972  * the ib_create_cm_id() call.
1973  */
1974 static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
1975 {
1976         int ret = 0;
1977
1978         switch (event->event) {
1979         case IB_CM_REQ_RECEIVED:
1980                 ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd,
1981                                        event->private_data);
1982                 break;
1983         case IB_CM_REJ_RECEIVED:
1984                 srpt_cm_rej_recv(cm_id);
1985                 ret = -EINVAL;
1986                 break;
1987         case IB_CM_RTU_RECEIVED:
1988         case IB_CM_USER_ESTABLISHED:
1989                 ret = srpt_cm_rtu_recv(cm_id);
1990                 break;
1991         case IB_CM_DREQ_RECEIVED:
1992                 ret = srpt_cm_dreq_recv(cm_id);
1993                 break;
1994         case IB_CM_DREP_RECEIVED:
1995                 srpt_cm_drep_recv(cm_id);
1996                 ret = -EINVAL;
1997                 break;
1998         case IB_CM_TIMEWAIT_EXIT:
1999                 srpt_cm_timewait_exit(cm_id);
2000                 ret = -EINVAL;
2001                 break;
2002         case IB_CM_REP_ERROR:
2003                 srpt_cm_rep_error(cm_id);
2004                 ret = -EINVAL;
2005                 break;
2006         default:
2007                 break;
2008         }
2009
2010         return ret;
2011 }
2012
2013 static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
2014                                  struct srpt_ioctx *ioctx,
2015                                  struct scst_cmd *scmnd)
2016 {
2017         struct scatterlist *scat;
2018         scst_data_direction dir;
2019         struct rdma_iu *riu;
2020         struct srp_direct_buf *db;
2021         dma_addr_t dma_addr;
2022         struct ib_sge *sge;
2023         u64 raddr;
2024         u32 rsize;
2025         u32 tsize;
2026         u32 dma_len;
2027         int count, nrdma;
2028         int i, j, k;
2029
2030         scat = scst_cmd_get_sg(scmnd);
2031         dir = scst_cmd_get_data_direction(scmnd);
2032         count = dma_map_sg(ch->sport->sdev->device->dma_device, scat,
2033                            scst_cmd_get_sg_cnt(scmnd),
2034                            scst_to_tgt_dma_dir(dir));
2035         if (unlikely(!count))
2036                 return -EBUSY;
2037
2038         if (ioctx->rdma_ius && ioctx->n_rdma_ius)
2039                 nrdma = ioctx->n_rdma_ius;
2040         else {
2041                 nrdma = count / SRPT_DEF_SG_PER_WQE + ioctx->n_rbuf;
2042
2043                 ioctx->rdma_ius = kzalloc(nrdma * sizeof *riu,
2044                                           scst_cmd_atomic(scmnd)
2045                                           ? GFP_ATOMIC : GFP_KERNEL);
2046                 if (!ioctx->rdma_ius) {
2047                         dma_unmap_sg(ch->sport->sdev->device->dma_device,
2048                                      scat, scst_cmd_get_sg_cnt(scmnd),
2049                                      scst_to_tgt_dma_dir(dir));
2050                         return -ENOMEM;
2051                 }
2052
2053                 ioctx->n_rdma_ius = nrdma;
2054         }
2055
2056         db = ioctx->rbufs;
2057         tsize = (dir == SCST_DATA_READ) ?
2058                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2059         dma_len = sg_dma_len(&scat[0]);
2060         riu = ioctx->rdma_ius;
2061
2062         /*
2063          * For each remote desc - calculate the #ib_sge.
2064          * If #ib_sge < SRPT_DEF_SG_PER_WQE per rdma operation then
2065          *      each remote desc rdma_iu is required a rdma wr;
2066          * else
2067          *      we need to allocate extra rdma_iu to carry extra #ib_sge in
2068          *      another rdma wr
2069          */
2070         for (i = 0, j = 0;
2071              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2072                 rsize = be32_to_cpu(db->len);
2073                 raddr = be64_to_cpu(db->va);
2074                 riu->raddr = raddr;
2075                 riu->rkey = be32_to_cpu(db->key);
2076                 riu->sge_cnt = 0;
2077
2078                 /* calculate how many sge required for this remote_buf */
2079                 while (rsize > 0 && tsize > 0) {
2080
2081                         if (rsize >= dma_len) {
2082                                 tsize -= dma_len;
2083                                 rsize -= dma_len;
2084                                 raddr += dma_len;
2085
2086                                 if (tsize > 0) {
2087                                         ++j;
2088                                         if (j < count)
2089                                                 dma_len = sg_dma_len(&scat[j]);
2090                                 }
2091                         } else {
2092                                 tsize -= rsize;
2093                                 dma_len -= rsize;
2094                                 rsize = 0;
2095                         }
2096
2097                         ++riu->sge_cnt;
2098
2099                         if (rsize > 0 && riu->sge_cnt == SRPT_DEF_SG_PER_WQE) {
2100                                 riu->sge =
2101                                     kmalloc(riu->sge_cnt * sizeof *riu->sge,
2102                                             scst_cmd_atomic(scmnd)
2103                                             ? GFP_ATOMIC : GFP_KERNEL);
2104                                 if (!riu->sge)
2105                                         goto free_mem;
2106
2107                                 ++ioctx->n_rdma;
2108                                 ++riu;
2109                                 riu->sge_cnt = 0;
2110                                 riu->raddr = raddr;
2111                                 riu->rkey = be32_to_cpu(db->key);
2112                         }
2113                 }
2114
2115                 riu->sge = kmalloc(riu->sge_cnt * sizeof *riu->sge,
2116                                    scst_cmd_atomic(scmnd)
2117                                    ? GFP_ATOMIC : GFP_KERNEL);
2118
2119                 if (!riu->sge)
2120                         goto free_mem;
2121
2122                 ++ioctx->n_rdma;
2123         }
2124
2125         db = ioctx->rbufs;
2126         scat = scst_cmd_get_sg(scmnd);
2127         tsize = (dir == SCST_DATA_READ) ?
2128                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2129         riu = ioctx->rdma_ius;
2130         dma_len = sg_dma_len(&scat[0]);
2131         dma_addr = sg_dma_address(&scat[0]);
2132
2133         /* this second loop is really mapped sg_addres to rdma_iu->ib_sge */
2134         for (i = 0, j = 0;
2135              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2136                 rsize = be32_to_cpu(db->len);
2137                 sge = riu->sge;
2138                 k = 0;
2139
2140                 while (rsize > 0 && tsize > 0) {
2141                         sge->addr = dma_addr;
2142                         sge->lkey = ch->sport->sdev->mr->lkey;
2143
2144                         if (rsize >= dma_len) {
2145                                 sge->length =
2146                                         (tsize < dma_len) ? tsize : dma_len;
2147                                 tsize -= dma_len;
2148                                 rsize -= dma_len;
2149
2150                                 if (tsize > 0) {
2151                                         ++j;
2152                                         if (j < count) {
2153                                                 dma_len = sg_dma_len(&scat[j]);
2154                                                 dma_addr =
2155                                                     sg_dma_address(&scat[j]);
2156                                         }
2157                                 }
2158                         } else {
2159                                 sge->length = (tsize < rsize) ? tsize : rsize;
2160                                 tsize -= rsize;
2161                                 dma_len -= rsize;
2162                                 dma_addr += rsize;
2163                                 rsize = 0;
2164                         }
2165
2166                         ++k;
2167                         if (k == riu->sge_cnt && rsize > 0) {
2168                                 ++riu;
2169                                 sge = riu->sge;
2170                                 k = 0;
2171                         } else if (rsize > 0)
2172                                 ++sge;
2173                 }
2174         }
2175
2176         return 0;
2177
2178 free_mem:
2179         while (ioctx->n_rdma)
2180                 kfree(ioctx->rdma_ius[ioctx->n_rdma--].sge);
2181
2182         kfree(ioctx->rdma_ius);
2183
2184         dma_unmap_sg(ch->sport->sdev->device->dma_device,
2185                      scat, scst_cmd_get_sg_cnt(scmnd),
2186                      scst_to_tgt_dma_dir(dir));
2187
2188         return -ENOMEM;
2189 }
2190
2191 static int srpt_perform_rdmas(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2192                               scst_data_direction dir)
2193 {
2194         struct ib_send_wr wr;
2195         struct ib_send_wr *bad_wr;
2196         struct rdma_iu *riu;
2197         int i;
2198         int ret = 0;
2199
2200         riu = ioctx->rdma_ius;
2201         memset(&wr, 0, sizeof wr);
2202
2203         for (i = 0; i < ioctx->n_rdma; ++i, ++riu) {
2204                 wr.opcode = (dir == SCST_DATA_READ) ?
2205                     IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;
2206                 wr.next = NULL;
2207                 wr.wr_id = ioctx->index;
2208                 wr.wr.rdma.remote_addr = riu->raddr;
2209                 wr.wr.rdma.rkey = riu->rkey;
2210                 wr.num_sge = riu->sge_cnt;
2211                 wr.sg_list = riu->sge;
2212
2213                 /* only get completion event for the last rdma wr */
2214                 if (i == (ioctx->n_rdma - 1) && dir == SCST_DATA_WRITE)
2215                         wr.send_flags = IB_SEND_SIGNALED;
2216
2217                 ret = ib_post_send(ch->qp, &wr, &bad_wr);
2218                 if (ret)
2219                         break;
2220         }
2221
2222         return ret;
2223 }
2224
2225 /*
2226  * Start data reception. Must not block.
2227  */
2228 static int srpt_xfer_data(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2229                           struct scst_cmd *scmnd)
2230 {
2231         int ret;
2232
2233         ret = srpt_map_sg_to_ib_sge(ch, ioctx, scmnd);
2234         if (ret) {
2235                 printk(KERN_ERR PFX "%s[%d] ret=%d\n", __func__, __LINE__, ret);
2236                 ret = SCST_TGT_RES_QUEUE_FULL;
2237                 goto out;
2238         }
2239
2240         ret = srpt_perform_rdmas(ch, ioctx, scst_cmd_get_data_direction(scmnd));
2241         if (ret) {
2242                 printk(KERN_ERR PFX "%s[%d] ret=%d\n", __func__, __LINE__, ret);
2243                 if (ret == -EAGAIN || ret == -ENOMEM)
2244                         ret = SCST_TGT_RES_QUEUE_FULL;
2245                 else
2246                         ret = SCST_TGT_RES_FATAL_ERROR;
2247                 goto out;
2248         }
2249
2250         ret = SCST_TGT_RES_SUCCESS;
2251
2252 out:
2253         return ret;
2254 }
2255
2256 /*
2257  * Called by the SCST core to inform ib_srpt that data reception should start.
2258  * Must not block.
2259  */
2260 static int srpt_rdy_to_xfer(struct scst_cmd *scmnd)
2261 {
2262         struct srpt_rdma_ch *ch;
2263         struct srpt_ioctx *ioctx;
2264
2265         ioctx = scst_cmd_get_tgt_priv(scmnd);
2266         BUG_ON(!ioctx);
2267
2268         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2269         BUG_ON(!ch);
2270
2271         if (ch->state == RDMA_CHANNEL_DISCONNECTING)
2272                 return SCST_TGT_RES_FATAL_ERROR;
2273         else if (ch->state == RDMA_CHANNEL_CONNECTING)
2274                 return SCST_TGT_RES_QUEUE_FULL;
2275
2276         ioctx->state = SRPT_STATE_NEED_DATA;
2277
2278         return srpt_xfer_data(ch, ioctx, scmnd);
2279 }
2280
2281 /*
2282  * Called by the SCST core. Transmits the response buffer and status held in
2283  * 'scmnd'. Must not block.
2284  */
2285 static int srpt_xmit_response(struct scst_cmd *scmnd)
2286 {
2287         struct srpt_rdma_ch *ch;
2288         struct srpt_ioctx *ioctx;
2289         struct srp_rsp *srp_rsp;
2290         u64 tag;
2291         int ret = SCST_TGT_RES_SUCCESS;
2292         int dir;
2293         int status;
2294
2295         ioctx = scst_cmd_get_tgt_priv(scmnd);
2296         BUG_ON(!ioctx);
2297
2298         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2299         BUG_ON(!ch);
2300
2301         tag = scst_cmd_get_tag(scmnd);
2302
2303         if (ch->state != RDMA_CHANNEL_LIVE) {
2304                 printk(KERN_ERR PFX
2305                        "%s: tag= %lld channel in bad state %d\n",
2306                        __func__, (unsigned long long)tag, ch->state);
2307
2308                 if (ch->state == RDMA_CHANNEL_DISCONNECTING)
2309                         ret = SCST_TGT_RES_FATAL_ERROR;
2310                 else if (ch->state == RDMA_CHANNEL_CONNECTING)
2311                         ret = SCST_TGT_RES_QUEUE_FULL;
2312
2313                 if (unlikely(scst_cmd_aborted(scmnd)))
2314                         goto out_aborted;
2315
2316                 goto out;
2317         }
2318
2319         dma_sync_single_for_cpu(ch->sport->sdev->device->dma_device, ioctx->dma,
2320                                 MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
2321
2322         srp_rsp = ioctx->buf;
2323
2324         if (unlikely(scst_cmd_aborted(scmnd))) {
2325                 printk(KERN_ERR PFX
2326                        "%s: tag= %lld already get aborted\n",
2327                        __func__, (unsigned long long)tag);
2328                 goto out_aborted;
2329         }
2330
2331         dir = scst_cmd_get_data_direction(scmnd);
2332         status = scst_cmd_get_status(scmnd) & 0xff;
2333
2334         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE, NO_ADD_SENSE, tag);
2335
2336         if (SCST_SENSE_VALID(scst_cmd_get_sense_buffer(scmnd))) {
2337                 srp_rsp->sense_data_len = scst_cmd_get_sense_buffer_len(scmnd);
2338                 if (srp_rsp->sense_data_len >
2339                     (MAX_MESSAGE_SIZE - sizeof *srp_rsp))
2340                         srp_rsp->sense_data_len =
2341                             MAX_MESSAGE_SIZE - sizeof *srp_rsp;
2342
2343                 memcpy((u8 *) (srp_rsp + 1), scst_cmd_get_sense_buffer(scmnd),
2344                        srp_rsp->sense_data_len);
2345
2346                 srp_rsp->sense_data_len = cpu_to_be32(srp_rsp->sense_data_len);
2347                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
2348
2349                 if (!status)
2350                         status = SAM_STAT_CHECK_CONDITION;
2351         }
2352
2353         srp_rsp->status = status;
2354
2355         /* transfer read data if any */
2356         if (dir == SCST_DATA_READ && scst_cmd_get_resp_data_len(scmnd)) {
2357                 ret = srpt_xfer_data(ch, ioctx, scmnd);
2358                 if (ret != SCST_TGT_RES_SUCCESS) {
2359                         printk(KERN_ERR PFX
2360                                "%s: tag= %lld xfer_data failed\n",
2361                                __func__, (unsigned long long)tag);
2362                         goto out;
2363                 }
2364         }
2365
2366         ioctx->state = SRPT_STATE_PROCESSED;
2367
2368         if (srpt_post_send(ch, ioctx,
2369                            sizeof *srp_rsp +
2370                            be32_to_cpu(srp_rsp->sense_data_len))) {
2371                 printk(KERN_ERR PFX "%s: ch->state= %d tag= %lld\n",
2372                        __func__, ch->state,
2373                        (unsigned long long)tag);
2374                 ret = SCST_TGT_RES_FATAL_ERROR;
2375         }
2376
2377 out:
2378         return ret;
2379
2380 out_aborted:
2381         ret = SCST_TGT_RES_SUCCESS;
2382         scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_ABORTED);
2383         ioctx->state = SRPT_STATE_ABORTED;
2384         scst_tgt_cmd_done(scmnd, SCST_CONTEXT_SAME);
2385         goto out;
2386 }
2387
2388 /*
2389  * Called by the SCST core to inform ib_srpt that a received task management
2390  * function has been completed. Must not block.
2391  */
2392 static void srpt_tsk_mgmt_done(struct scst_mgmt_cmd *mcmnd)
2393 {
2394         struct srpt_rdma_ch *ch;
2395         struct srpt_mgmt_ioctx *mgmt_ioctx;
2396         struct srpt_ioctx *ioctx;
2397         int rsp_len;
2398
2399         mgmt_ioctx = scst_mgmt_cmd_get_tgt_priv(mcmnd);
2400         BUG_ON(!mgmt_ioctx);
2401
2402         ch = mgmt_ioctx->ch;
2403         BUG_ON(!ch);
2404
2405         ioctx = mgmt_ioctx->ioctx;
2406         BUG_ON(!ioctx);
2407
2408         TRACE_DBG("%s: tsk_mgmt_done for tag= %lld status=%d\n",
2409                   __func__, (unsigned long long)mgmt_ioctx->tag,
2410                   scst_mgmt_cmd_get_status(mcmnd));
2411
2412         rsp_len = srpt_build_tskmgmt_rsp(ch, ioctx,
2413                                          (scst_mgmt_cmd_get_status(mcmnd) ==
2414                                           SCST_MGMT_STATUS_SUCCESS) ?
2415                                          SRP_TSK_MGMT_SUCCESS :
2416                                          SRP_TSK_MGMT_FAILED,
2417                                          mgmt_ioctx->tag);
2418         srpt_post_send(ch, ioctx, rsp_len);
2419
2420         scst_mgmt_cmd_set_tgt_priv(mcmnd, NULL);
2421
2422         kfree(mgmt_ioctx);
2423 }
2424
2425 /*
2426  * Called by the SCST core to inform ib_srpt that the command 'scmnd' is about
2427  * to be freed. May be called in IRQ context.
2428  */
2429 static void srpt_on_free_cmd(struct scst_cmd *scmnd)
2430 {
2431         struct srpt_rdma_ch *ch;
2432         struct srpt_ioctx *ioctx;
2433
2434         ioctx = scst_cmd_get_tgt_priv(scmnd);
2435         BUG_ON(!ioctx);
2436
2437         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2438         BUG_ON(!ch);
2439
2440         spin_lock_irq(&ch->spinlock);
2441         list_del(&ioctx->scmnd_list);
2442         ch->active_scmnd_cnt--;
2443         spin_unlock_irq(&ch->spinlock);
2444
2445         srpt_reset_ioctx(ch, ioctx);
2446         scst_cmd_set_tgt_priv(scmnd, NULL);
2447 }
2448
2449 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2450 /* A vanilla 2.6.19 or older kernel without backported OFED kernel headers. */
2451 static void srpt_refresh_port_work(void *ctx)
2452 #else
2453 static void srpt_refresh_port_work(struct work_struct *work)
2454 #endif
2455 {
2456 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2457         struct srpt_port *sport = (struct srpt_port *)ctx;
2458 #else
2459         struct srpt_port *sport = container_of(work, struct srpt_port, work);
2460 #endif
2461
2462         srpt_refresh_port(sport);
2463 }
2464
2465 /*
2466  * Called by the SCST core to detect target adapters. Returns the number of
2467  * detected target adapters.
2468  */
2469 static int srpt_detect(struct scst_tgt_template *tp)
2470 {
2471         int device_count;
2472
2473         TRACE_ENTRY();
2474
2475         device_count = atomic_read(&srpt_device_count);
2476
2477         TRACE_EXIT_RES(device_count);
2478
2479         return device_count;
2480 }
2481
2482 /*
2483  * Callback function called by the SCST core from scst_unregister() to free up
2484  * the resources associated with device scst_tgt.
2485  */
2486 static int srpt_release(struct scst_tgt *scst_tgt)
2487 {
2488         struct srpt_device *sdev = scst_tgt_get_tgt_priv(scst_tgt);
2489         struct srpt_rdma_ch *ch, *tmp_ch;
2490
2491         TRACE_ENTRY();
2492
2493         BUG_ON(!scst_tgt);
2494 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
2495         WARN_ON(!sdev);
2496         if (!sdev)
2497                 return -ENODEV;
2498 #else
2499         if (WARN_ON(!sdev))
2500                 return -ENODEV;
2501 #endif
2502
2503         srpt_unregister_procfs_entry(scst_tgt->tgtt);
2504
2505         spin_lock_irq(&sdev->spinlock);
2506         list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
2507                 list_del(&ch->list);
2508                 spin_unlock_irq(&sdev->spinlock);
2509                 srpt_release_channel(ch, 1);
2510                 spin_lock_irq(&sdev->spinlock);
2511         }
2512         spin_unlock_irq(&sdev->spinlock);
2513
2514         srpt_unregister_mad_agent(sdev);
2515
2516         scst_tgt_set_tgt_priv(scst_tgt, NULL);
2517
2518         TRACE_EXIT();
2519
2520         return 0;
2521 }
2522
2523 /*
2524  * Entry point for ib_srpt's kernel thread. This kernel thread is only created
2525  * when the module parameter 'thread' is not zero (the default is zero).
2526  * This thread processes the ioctx list srpt_thread.thread_ioctx_list.
2527  *
2528  * @pre thread != 0
2529  */
2530 static int srpt_ioctx_thread(void *arg)
2531 {
2532         struct srpt_ioctx *ioctx;
2533
2534         /* Hibernation / freezing of the SRPT kernel thread is not supported. */
2535         current->flags |= PF_NOFREEZE;
2536
2537         spin_lock_irq(&srpt_thread.thread_lock);
2538         while (!kthread_should_stop()) {
2539                 wait_queue_t wait;
2540                 init_waitqueue_entry(&wait, current);
2541
2542                 if (!srpt_test_ioctx_list()) {
2543                         add_wait_queue_exclusive(&ioctx_list_waitQ, &wait);
2544
2545                         for (;;) {
2546                                 set_current_state(TASK_INTERRUPTIBLE);
2547                                 if (srpt_test_ioctx_list())
2548                                         break;
2549                                 spin_unlock_irq(&srpt_thread.thread_lock);
2550                                 schedule();
2551                                 spin_lock_irq(&srpt_thread.thread_lock);
2552                         }
2553                         set_current_state(TASK_RUNNING);
2554                         remove_wait_queue(&ioctx_list_waitQ, &wait);
2555                 }
2556
2557                 while (!list_empty(&srpt_thread.thread_ioctx_list)) {
2558                         ioctx = list_entry(srpt_thread.thread_ioctx_list.next,
2559                                            struct srpt_ioctx, comp_list);
2560
2561                         list_del(&ioctx->comp_list);
2562
2563                         spin_unlock_irq(&srpt_thread.thread_lock);
2564                         switch (ioctx->op) {
2565                         case IB_WC_SEND:
2566                                 srpt_handle_send_comp(ioctx->ch, ioctx,
2567                                         SCST_CONTEXT_DIRECT);
2568                                 break;
2569                         case IB_WC_RDMA_WRITE:
2570                         case IB_WC_RDMA_READ:
2571                                 srpt_handle_rdma_comp(ioctx->ch, ioctx);
2572                                 break;
2573                         case IB_WC_RECV:
2574                                 srpt_handle_new_iu(ioctx->ch, ioctx);
2575                                 break;
2576                         default:
2577                                 break;
2578                         }
2579                         spin_lock_irq(&srpt_thread.thread_lock);
2580                 }
2581         }
2582         spin_unlock_irq(&srpt_thread.thread_lock);
2583
2584         return 0;
2585 }
2586
2587 /* SCST target template for the SRP target implementation. */
2588 static struct scst_tgt_template srpt_template = {
2589         .name = DRV_NAME,
2590         .sg_tablesize = SRPT_DEF_SG_TABLESIZE,
2591         .xmit_response_atomic = 1,
2592         .rdy_to_xfer_atomic = 1,
2593         .no_proc_entry = 0,
2594         .detect = srpt_detect,
2595         .release = srpt_release,
2596         .xmit_response = srpt_xmit_response,
2597         .rdy_to_xfer = srpt_rdy_to_xfer,
2598         .on_free_cmd = srpt_on_free_cmd,
2599         .task_mgmt_fn_done = srpt_tsk_mgmt_done
2600 };
2601
2602 /*
2603  * The callback function srpt_release_class_dev() is called whenever a
2604  * device is removed from the /sys/class/infiniband_srpt device class.
2605  */
2606 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2607 static void srpt_release_class_dev(struct class_device *class_dev)
2608 #else
2609 static void srpt_release_class_dev(struct device *dev)
2610 #endif
2611 {
2612 }
2613
2614 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2615 static int srpt_trace_level_show(struct seq_file *seq, void *v)
2616 {
2617         return scst_proc_log_entry_read(seq, trace_flag, NULL);
2618 }
2619
2620 static ssize_t srpt_proc_trace_level_write(struct file *file,
2621         const char __user *buf, size_t length, loff_t *off)
2622 {
2623         return scst_proc_log_entry_write(file, buf, length, &trace_flag,
2624                 DEFAULT_SRPT_TRACE_FLAGS, NULL);
2625 }
2626
2627 static struct scst_proc_data srpt_log_proc_data = {
2628         SCST_DEF_RW_SEQ_OP(srpt_proc_trace_level_write)
2629         .show = srpt_trace_level_show,
2630 };
2631 #endif
2632
2633 static struct class_attribute srpt_class_attrs[] = {
2634         __ATTR_NULL,
2635 };
2636
2637 static struct class srpt_class = {
2638         .name = "infiniband_srpt",
2639 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2640         .release = srpt_release_class_dev,
2641 #else
2642         .dev_release = srpt_release_class_dev,
2643 #endif
2644         .class_attrs = srpt_class_attrs,
2645 };
2646
2647 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2648 static ssize_t show_login_info(struct class_device *class_dev, char *buf)
2649 #else
2650 static ssize_t show_login_info(struct device *dev,
2651                                struct device_attribute *attr, char *buf)
2652 #endif
2653 {
2654         struct srpt_device *sdev =
2655 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2656                 container_of(class_dev, struct srpt_device, class_dev);
2657 #else
2658                 container_of(dev, struct srpt_device, dev);
2659 #endif
2660         struct srpt_port *sport;
2661         int i;
2662         int len = 0;
2663
2664         for (i = 0; i < sdev->device->phys_port_cnt; i++) {
2665                 sport = &sdev->port[i];
2666
2667                 len += sprintf(buf + len,
2668                                "tid_ext=%016llx,ioc_guid=%016llx,pkey=ffff,"
2669                                "dgid=%04x%04x%04x%04x%04x%04x%04x%04x,"
2670                                "service_id=%016llx\n",
2671                                (unsigned long long) srpt_service_guid,
2672                                (unsigned long long) srpt_service_guid,
2673                                be16_to_cpu(((__be16 *) sport->gid.raw)[0]),
2674                                be16_to_cpu(((__be16 *) sport->gid.raw)[1]),
2675                                be16_to_cpu(((__be16 *) sport->gid.raw)[2]),
2676                                be16_to_cpu(((__be16 *) sport->gid.raw)[3]),
2677                                be16_to_cpu(((__be16 *) sport->gid.raw)[4]),
2678                                be16_to_cpu(((__be16 *) sport->gid.raw)[5]),
2679                                be16_to_cpu(((__be16 *) sport->gid.raw)[6]),
2680                                be16_to_cpu(((__be16 *) sport->gid.raw)[7]),
2681                                (unsigned long long) srpt_service_guid);
2682         }
2683
2684         return len;
2685 }
2686
2687 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2688 static CLASS_DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2689 #else
2690 static DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2691 #endif
2692
2693 /*
2694  * Callback function called by the InfiniBand core when either an InfiniBand
2695  * device has been added or during the ib_register_client() call for each
2696  * registered InfiniBand device.
2697  */
2698 static void srpt_add_one(struct ib_device *device)
2699 {
2700         struct srpt_device *sdev;
2701         struct srpt_port *sport;
2702         struct ib_srq_init_attr srq_attr;
2703         int i;
2704
2705         TRACE_ENTRY();
2706
2707         sdev = kzalloc(sizeof *sdev, GFP_KERNEL);
2708         if (!sdev)
2709                 return;
2710
2711         sdev->device = device;
2712
2713 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2714         sdev->class_dev.class = &srpt_class;
2715         sdev->class_dev.dev = device->dma_device;
2716         snprintf(sdev->class_dev.class_id, BUS_ID_SIZE,
2717                  "srpt-%s", device->name);
2718 #else
2719         sdev->dev.class = &srpt_class;
2720         sdev->dev.parent = device->dma_device;
2721 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
2722         snprintf(sdev->dev.bus_id, BUS_ID_SIZE, "srpt-%s", device->name);
2723 #else
2724         snprintf(sdev->init_name, sizeof(sdev->init_name),
2725                  "srpt-%s", device->name);
2726         sdev->dev.init_name = sdev->init_name;
2727 #endif
2728 #endif
2729
2730 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2731         if (class_device_register(&sdev->class_dev))
2732                 goto free_dev;
2733         if (class_device_create_file(&sdev->class_dev,
2734                                      &class_device_attr_login_info))
2735                 goto err_dev;
2736 #else
2737         if (device_register(&sdev->dev))
2738                 goto free_dev;
2739         if (device_create_file(&sdev->dev, &dev_attr_login_info))
2740                 goto err_dev;
2741 #endif
2742
2743         if (ib_query_device(device, &sdev->dev_attr))
2744                 goto err_dev;
2745
2746         sdev->pd = ib_alloc_pd(device);
2747         if (IS_ERR(sdev->pd))
2748                 goto err_dev;
2749
2750         sdev->mr = ib_get_dma_mr(sdev->pd, IB_ACCESS_LOCAL_WRITE);
2751         if (IS_ERR(sdev->mr))
2752                 goto err_pd;
2753
2754         srq_attr.event_handler = srpt_srq_event;
2755         srq_attr.srq_context = (void *)sdev;
2756         srq_attr.attr.max_wr = min(SRPT_SRQ_SIZE, sdev->dev_attr.max_srq_wr);
2757         srq_attr.attr.max_sge = 1;
2758         srq_attr.attr.srq_limit = 0;
2759
2760         sdev->srq = ib_create_srq(sdev->pd, &srq_attr);
2761         if (IS_ERR(sdev->srq))
2762                 goto err_mr;
2763
2764         TRACE_DBG("%s: create SRQ #wr= %d max_allow=%d dev= %s",
2765                __func__, srq_attr.attr.max_wr,
2766               sdev->dev_attr.max_srq_wr, device->name);
2767
2768         if (!srpt_service_guid)
2769                 srpt_service_guid = be64_to_cpu(device->node_guid);
2770
2771         sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);
2772         if (IS_ERR(sdev->cm_id))
2773                 goto err_srq;
2774
2775         /* print out target login information */
2776         TRACE_DBG("Target login info: id_ext=%016llx,"
2777                   "ioc_guid=%016llx,pkey=ffff,service_id=%016llx",
2778                   (unsigned long long) srpt_service_guid,
2779                   (unsigned long long) srpt_service_guid,
2780                   (unsigned long long) srpt_service_guid);
2781
2782         /*
2783          * We do not have a consistent service_id (ie. also id_ext of target_id)
2784          * to identify this target. We currently use the guid of the first HCA
2785          * in the system as service_id; therefore, the target_id will change
2786          * if this HCA is gone bad and replaced by different HCA
2787          */
2788         if (ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0, NULL))
2789                 goto err_cm;
2790
2791         INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
2792                               srpt_event_handler);
2793         if (ib_register_event_handler(&sdev->event_handler))
2794                 goto err_cm;
2795
2796         if (srpt_alloc_ioctx_ring(sdev))
2797                 goto err_event;
2798
2799         INIT_LIST_HEAD(&sdev->rch_list);
2800         spin_lock_init(&sdev->spinlock);
2801
2802         for (i = 0; i < SRPT_SRQ_SIZE; ++i)
2803                 srpt_post_recv(sdev, sdev->ioctx_ring[i]);
2804
2805         ib_set_client_data(device, &srpt_client, sdev);
2806
2807         sdev->scst_tgt = scst_register(&srpt_template, NULL);
2808         if (!sdev->scst_tgt) {
2809                 printk(KERN_ERR PFX "SCST registration failed for %s.\n",
2810                         sdev->device->name);
2811                 goto err_ring;
2812         }
2813
2814         scst_tgt_set_tgt_priv(sdev->scst_tgt, sdev);
2815
2816         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
2817                 sport = &sdev->port[i - 1];
2818                 sport->sdev = sdev;
2819                 sport->port = i;
2820 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2821                 /*
2822                  * A vanilla 2.6.19 or older kernel without backported OFED
2823                  * kernel headers.
2824                  */
2825                 INIT_WORK(&sport->work, srpt_refresh_port_work, sport);
2826 #else
2827                 INIT_WORK(&sport->work, srpt_refresh_port_work);
2828 #endif
2829                 if (srpt_refresh_port(sport)) {
2830                         printk(KERN_ERR PFX "MAD registration failed"
2831                                " for %s-%d.\n", sdev->device->name, i);
2832                         goto err_refresh_port;
2833                 }
2834         }
2835
2836         atomic_inc(&srpt_device_count);
2837
2838         TRACE_EXIT();
2839
2840         return;
2841
2842 err_refresh_port:
2843         scst_unregister(sdev->scst_tgt);
2844 err_ring:
2845         ib_set_client_data(device, &srpt_client, NULL);
2846         srpt_free_ioctx_ring(sdev);
2847 err_event:
2848         ib_unregister_event_handler(&sdev->event_handler);
2849 err_cm:
2850         ib_destroy_cm_id(sdev->cm_id);
2851 err_srq:
2852         ib_destroy_srq(sdev->srq);
2853 err_mr:
2854         ib_dereg_mr(sdev->mr);
2855 err_pd:
2856         ib_dealloc_pd(sdev->pd);
2857 err_dev:
2858 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2859         class_device_unregister(&sdev->class_dev);
2860 #else
2861         device_unregister(&sdev->dev);
2862 #endif
2863 free_dev:
2864         kfree(sdev);
2865
2866         TRACE_EXIT();
2867 }
2868
2869 /*
2870  * Callback function called by the InfiniBand core when either an InfiniBand
2871  * device has been removed or during the ib_unregister_client() call for each
2872  * registered InfiniBand device.
2873  */
2874 static void srpt_remove_one(struct ib_device *device)
2875 {
2876         int i;
2877         struct srpt_device *sdev;
2878
2879         TRACE_ENTRY();
2880
2881         sdev = ib_get_client_data(device, &srpt_client);
2882 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
2883         WARN_ON(!sdev);
2884         if (!sdev)
2885                 return;
2886 #else
2887         if (WARN_ON(!sdev))
2888                 return;
2889 #endif
2890
2891         /*
2892          * Cancel the work if it is queued. Wait until srpt_refresh_port_work()
2893          * finished if it is running.
2894          */
2895         for (i = 0; i < sdev->device->phys_port_cnt; i++)
2896 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
2897                 cancel_work_sync(&sdev->port[i].work);
2898 #else
2899                 /*
2900                  * cancel_work_sync() was introduced in kernel 2.6.22. Older
2901                  * kernels do not have a facility to cancel scheduled work.
2902                  */
2903                 printk(KERN_ERR PFX
2904                        "your kernel does not provide cancel_work_sync().\n");
2905 #endif
2906
2907         scst_unregister(sdev->scst_tgt);
2908         sdev->scst_tgt = NULL;
2909
2910         ib_unregister_event_handler(&sdev->event_handler);
2911         ib_destroy_cm_id(sdev->cm_id);
2912         ib_destroy_srq(sdev->srq);
2913         ib_dereg_mr(sdev->mr);
2914         ib_dealloc_pd(sdev->pd);
2915 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2916         class_device_unregister(&sdev->class_dev);
2917 #else
2918         device_unregister(&sdev->dev);
2919 #endif
2920
2921         srpt_free_ioctx_ring(sdev);
2922         kfree(sdev);
2923
2924         TRACE_EXIT();
2925 }
2926
2927 /**
2928  * Create procfs entries for srpt. Currently the only procfs entry created
2929  * by this function is the "trace_level" entry.
2930  */
2931 static int srpt_register_procfs_entry(struct scst_tgt_template *tgt)
2932 {
2933         int res = 0;
2934 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2935         struct proc_dir_entry *p, *root;
2936
2937         root = scst_proc_get_tgt_root(tgt);
2938         WARN_ON(!root);
2939         if (root) {
2940                 /*
2941                  * Fill in the scst_proc_data::data pointer, which is used in
2942                  * a printk(KERN_INFO ...) statement in
2943                  * scst_proc_log_entry_write() in scst_proc.c.
2944                  */
2945                 srpt_log_proc_data.data = (char *)tgt->name;
2946                 p = scst_create_proc_entry(root, SRPT_PROC_TRACE_LEVEL_NAME,
2947                                            &srpt_log_proc_data);
2948                 if (!p)
2949                         res = -ENOMEM;
2950         } else
2951                 res = -ENOMEM;
2952
2953 #endif
2954         return res;
2955 }
2956
2957 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt)
2958 {
2959 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2960         struct proc_dir_entry *root;
2961
2962         root = scst_proc_get_tgt_root(tgt);
2963         WARN_ON(!root);
2964         if (root)
2965                 remove_proc_entry(SRPT_PROC_TRACE_LEVEL_NAME, root);
2966 #endif
2967 }
2968
2969 /*
2970  * Module initialization.
2971  *
2972  * Note: since ib_register_client() registers callback functions, and since at
2973  * least one of these callback functions (srpt_add_one()) calls SCST functions,
2974  * the SCST target template must be registered before ib_register_client() is
2975  * called.
2976  */
2977 static int __init srpt_init_module(void)
2978 {
2979         int ret;
2980
2981         ret = class_register(&srpt_class);
2982         if (ret) {
2983                 printk(KERN_ERR PFX "couldn't register class ib_srpt\n");
2984                 goto out;
2985         }
2986
2987         ret = scst_register_target_template(&srpt_template);
2988         if (ret < 0) {
2989                 printk(KERN_ERR PFX "couldn't register with scst\n");
2990                 ret = -ENODEV;
2991                 goto out_unregister_class;
2992         }
2993
2994         ret = srpt_register_procfs_entry(&srpt_template);
2995         if (ret) {
2996                 printk(KERN_ERR PFX "couldn't register procfs entry\n");
2997                 goto out_unregister_target;
2998         }
2999
3000         ret = ib_register_client(&srpt_client);
3001         if (ret) {
3002                 printk(KERN_ERR PFX "couldn't register IB client\n");
3003                 goto out_unregister_target;
3004         }
3005
3006         if (thread) {
3007                 spin_lock_init(&srpt_thread.thread_lock);
3008                 INIT_LIST_HEAD(&srpt_thread.thread_ioctx_list);
3009                 srpt_thread.thread = kthread_run(srpt_ioctx_thread,
3010                                                  NULL, "srpt_thread");
3011                 if (IS_ERR(srpt_thread.thread)) {
3012                         srpt_thread.thread = NULL;
3013                         thread = 0;
3014                 }
3015         }
3016
3017         return 0;
3018
3019 out_unregister_target:
3020         /*
3021          * Note: the procfs entry is unregistered in srpt_release(), which is
3022          * called by scst_unregister_target_template().
3023          */
3024         scst_unregister_target_template(&srpt_template);
3025 out_unregister_class:
3026         class_unregister(&srpt_class);
3027 out:
3028         return ret;
3029 }
3030
3031 static void __exit srpt_cleanup_module(void)
3032 {
3033         TRACE_ENTRY();
3034
3035         if (srpt_thread.thread)
3036                 kthread_stop(srpt_thread.thread);
3037         ib_unregister_client(&srpt_client);
3038         scst_unregister_target_template(&srpt_template);
3039         class_unregister(&srpt_class);
3040
3041         TRACE_EXIT();
3042 }
3043
3044 module_init(srpt_init_module);
3045 module_exit(srpt_cleanup_module);