bbe99390874b2992ec8488e0b90975afcb129cc1
[mirror/scst/.git] / srpt / src / ib_srpt.c
1 /*
2  * Copyright (c) 2006 - 2009 Mellanox Technology Inc.  All rights reserved.
3  * Copyright (C) 2008 Vladislav Bolkhovitin <vst@vlnb.net>
4  * Copyright (C) 2008 - 2009 Bart Van Assche <bart.vanassche@gmail.com>
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  */
35
36 #include <linux/module.h>
37 #include <linux/init.h>
38 #include <linux/slab.h>
39 #include <linux/err.h>
40 #include <linux/ctype.h>
41 #include <linux/string.h>
42 #include <linux/kthread.h>
43 #include <asm/atomic.h>
44 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
45 #include <linux/proc_fs.h>
46 #include <linux/seq_file.h>
47 #endif
48 #include "ib_srpt.h"
49 #include "scst_debug.h"
50
51 /* Name of this kernel module. */
52 #define DRV_NAME                "ib_srpt"
53 /* Prefix for printk() kernel messages. */
54 #define LOG_PFX                 DRV_NAME ": "
55 #define DRV_VERSION             "1.0.1"
56 #define DRV_RELDATE             "July 10, 2008"
57 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
58 /* Flags to be used in SCST debug tracing statements. */
59 #define DEFAULT_SRPT_TRACE_FLAGS (TRACE_OUT_OF_MEM | TRACE_MINOR \
60                                   | TRACE_MGMT | TRACE_SPECIAL)
61 /* Name of the entry that will be created under /proc/scsi_tgt/ib_srpt. */
62 #define SRPT_PROC_TRACE_LEVEL_NAME      "trace_level"
63 #endif
64
65 #define MELLANOX_SRPT_ID_STRING "SCST SRP target"
66
67 MODULE_AUTHOR("Vu Pham");
68 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
69                    "v" DRV_VERSION " (" DRV_RELDATE ")");
70 MODULE_LICENSE("Dual BSD/GPL");
71
72 struct srpt_thread {
73         /* Protects thread_ioctx_list. */
74         spinlock_t thread_lock;
75         /* I/O contexts to be processed by the kernel thread. */
76         struct list_head thread_ioctx_list;
77         /* SRPT kernel thread. */
78         struct task_struct *thread;
79 };
80
81 /*
82  * Global Variables
83  */
84
85 static u64 srpt_service_guid;
86 /* List of srpt_device structures. */
87 static atomic_t srpt_device_count;
88 static int use_port_guid_in_session_name;
89 static int thread;
90 static struct srpt_thread srpt_thread;
91 static DECLARE_WAIT_QUEUE_HEAD(ioctx_list_waitQ);
92 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
93 static unsigned long trace_flag = DEFAULT_SRPT_TRACE_FLAGS;
94 module_param(trace_flag, long, 0644);
95 MODULE_PARM_DESC(trace_flag,
96                  "Trace flags for the ib_srpt kernel module.");
97 #endif
98
99 module_param(thread, int, 0444);
100 MODULE_PARM_DESC(thread,
101                  "Executing ioctx in thread context. Default 0, i.e. soft IRQ, "
102                  "where possible");
103
104 module_param(use_port_guid_in_session_name, bool, 0444);
105 MODULE_PARM_DESC(use_port_guid_in_session_name,
106                  "Use target port ID in the SCST session name such that"
107                  " redundant paths between multiport systems can be masked.");
108
109 static void srpt_add_one(struct ib_device *device);
110 static void srpt_remove_one(struct ib_device *device);
111 static void srpt_unregister_mad_agent(struct srpt_device *sdev);
112 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt);
113
114 static struct ib_client srpt_client = {
115         .name = DRV_NAME,
116         .add = srpt_add_one,
117         .remove = srpt_remove_one
118 };
119
120 /**
121  * Atomically test and set the channel state.
122  * @ch: RDMA channel.
123  * @old: channel state to compare with.
124  * @new: state to change the channel state to if the current state matches the
125  *       argument 'old'.
126  *
127  * Returns true if the channel state matched old upon entry of this function,
128  * and false otherwise.
129  */
130 static bool srpt_test_and_set_channel_state(struct srpt_rdma_ch *ch,
131                                             enum rdma_ch_state old,
132                                             enum rdma_ch_state new)
133 {
134         unsigned long flags;
135         enum rdma_ch_state cur;
136
137         spin_lock_irqsave(&ch->spinlock, flags);
138         cur = ch->state;
139         if (cur == old)
140                 ch->state = new;
141         spin_unlock_irqrestore(&ch->spinlock, flags);
142
143         return cur == old;
144 }
145
146 /*
147  * Callback function called by the InfiniBand core when an asynchronous IB
148  * event occurs. This callback may occur in interrupt context. See also
149  * section 11.5.2, Set Asynchronous Event Handler in the InfiniBand
150  * Architecture Specification.
151  */
152 static void srpt_event_handler(struct ib_event_handler *handler,
153                                struct ib_event *event)
154 {
155         struct srpt_device *sdev;
156         struct srpt_port *sport;
157
158         sdev = ib_get_client_data(event->device, &srpt_client);
159         if (!sdev || sdev->device != event->device)
160                 return;
161
162         TRACE_DBG("ASYNC event= %d on device= %s",
163                   event->event, sdev->device->name);
164
165         switch (event->event) {
166         case IB_EVENT_PORT_ERR:
167                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
168                         sport = &sdev->port[event->element.port_num - 1];
169                         sport->lid = 0;
170                         sport->sm_lid = 0;
171                 }
172                 break;
173         case IB_EVENT_PORT_ACTIVE:
174         case IB_EVENT_LID_CHANGE:
175         case IB_EVENT_PKEY_CHANGE:
176         case IB_EVENT_SM_CHANGE:
177         case IB_EVENT_CLIENT_REREGISTER:
178                 /*
179                  * Refresh port data asynchronously. Note: it is safe to call
180                  * schedule_work() even if &sport->work is already on the
181                  * global workqueue because schedule_work() tests for the
182                  * work_pending() condition before adding &sport->work to the
183                  * global work queue.
184                  */
185                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
186                         sport = &sdev->port[event->element.port_num - 1];
187                         if (!sport->lid && !sport->sm_lid)
188                                 schedule_work(&sport->work);
189                 }
190                 break;
191         default:
192                 break;
193         }
194
195 }
196
197 /*
198  * Callback function called by the InfiniBand core for SRQ (shared receive
199  * queue) events.
200  */
201 static void srpt_srq_event(struct ib_event *event, void *ctx)
202 {
203         TRACE_DBG("SRQ event %d", event->event);
204 }
205
206 /*
207  * Callback function called by the InfiniBand core for QP (queue pair) events.
208  */
209 static void srpt_qp_event(struct ib_event *event, void *ctx)
210 {
211         struct srpt_rdma_ch *ch = ctx;
212
213         TRACE_DBG("QP event %d on cm_id=%p sess_name=%s state=%d",
214                   event->event, ch->cm_id, ch->sess_name, ch->state);
215
216         switch (event->event) {
217         case IB_EVENT_COMM_EST:
218 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20) || defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
219                 ib_cm_notify(ch->cm_id, event->event);
220 #else
221                 /* Vanilla 2.6.19 kernel (or before) without OFED. */
222                 PRINT_ERROR("%s", "how to perform ib_cm_notify() on a"
223                             " vanilla 2.6.18 kernel ???");
224 #endif
225                 break;
226         case IB_EVENT_QP_LAST_WQE_REACHED:
227                 if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_LIVE,
228                                         RDMA_CHANNEL_DISCONNECTING)) {
229                         PRINT_INFO("disconnected session %s.", ch->sess_name);
230                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
231                 }
232                 break;
233         default:
234                 break;
235         }
236 }
237
238 /*
239  * Helper function for filling in an InfiniBand IOUnitInfo structure. Copies
240  * the lowest four bits of value in element slot of the array of four bit
241  * elements called c_list (controller list). The index slot is one-based.
242  *
243  * @pre 1 <= slot && 0 <= value && value < 16
244  */
245 static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
246 {
247         u16 id;
248         u8 tmp;
249
250         id = (slot - 1) / 2;
251         if (slot & 0x1) {
252                 tmp = c_list[id] & 0xf;
253                 c_list[id] = (value << 4) | tmp;
254         } else {
255                 tmp = c_list[id] & 0xf0;
256                 c_list[id] = (value & 0xf) | tmp;
257         }
258 }
259
260 /*
261  * Write InfiniBand ClassPortInfo to mad. See also section 16.3.3.1
262  * ClassPortInfo in the InfiniBand Architecture Specification.
263  */
264 static void srpt_get_class_port_info(struct ib_dm_mad *mad)
265 {
266         struct ib_class_port_info *cif;
267
268         cif = (struct ib_class_port_info *)mad->data;
269         memset(cif, 0, sizeof *cif);
270         cif->base_version = 1;
271         cif->class_version = 1;
272         cif->resp_time_value = 20;
273
274         mad->mad_hdr.status = 0;
275 }
276
277 /*
278  * Write IOUnitInfo to mad. See also section 16.3.3.3 IOUnitInfo in the
279  * InfiniBand Architecture Specification. See also section B.7,
280  * table B.6 in the T10 SRP r16a document.
281  */
282 static void srpt_get_iou(struct ib_dm_mad *mad)
283 {
284         struct ib_dm_iou_info *ioui;
285         u8 slot;
286         int i;
287
288         ioui = (struct ib_dm_iou_info *)mad->data;
289         ioui->change_id = 1;
290         ioui->max_controllers = 16;
291
292         /* set present for slot 1 and empty for the rest */
293         srpt_set_ioc(ioui->controller_list, 1, 1);
294         for (i = 1, slot = 2; i < 16; i++, slot++)
295                 srpt_set_ioc(ioui->controller_list, slot, 0);
296
297         mad->mad_hdr.status = 0;
298 }
299
300 /*
301  * Write IOControllerprofile to mad for I/O controller (sdev, slot). See also
302  * section 16.3.3.4 IOControllerProfile in the InfiniBand Architecture
303  * Specification. See also section B.7, table B.7 in the T10 SRP r16a
304  * document.
305  */
306 static void srpt_get_ioc(struct srpt_device *sdev, u32 slot,
307                          struct ib_dm_mad *mad)
308 {
309         struct ib_dm_ioc_profile *iocp;
310
311         iocp = (struct ib_dm_ioc_profile *)mad->data;
312
313         if (!slot || slot > 16) {
314                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
315                 return;
316         }
317
318         if (slot > 2) {
319                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
320                 return;
321         }
322
323         memset(iocp, 0, sizeof *iocp);
324         strcpy(iocp->id_string, MELLANOX_SRPT_ID_STRING);
325         iocp->guid = cpu_to_be64(srpt_service_guid);
326         iocp->vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
327         iocp->device_id = cpu_to_be32(sdev->dev_attr.vendor_part_id);
328         iocp->device_version = cpu_to_be16(sdev->dev_attr.hw_ver);
329         iocp->subsys_vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
330         iocp->subsys_device_id = 0x0;
331         iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
332         iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
333         iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
334         iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
335         iocp->send_queue_depth = cpu_to_be16(SRPT_SRQ_SIZE);
336         iocp->rdma_read_depth = 4;
337         iocp->send_size = cpu_to_be32(MAX_MESSAGE_SIZE);
338         iocp->rdma_size = cpu_to_be32(MAX_RDMA_SIZE);
339         iocp->num_svc_entries = 1;
340         iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
341                 SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
342
343         mad->mad_hdr.status = 0;
344 }
345
346 /*
347  * Device management: write ServiceEntries to mad for the given slot. See also
348  * section 16.3.3.5 ServiceEntries in the InfiniBand Architecture
349  * Specification. See also section B.7, table B.8 in the T10 SRP r16a document.
350  */
351 static void srpt_get_svc_entries(u64 ioc_guid,
352                                  u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
353 {
354         struct ib_dm_svc_entries *svc_entries;
355
356         WARN_ON(!ioc_guid);
357
358         if (!slot || slot > 16) {
359                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
360                 return;
361         }
362
363         if (slot > 2 || lo > hi || hi > 1) {
364                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
365                 return;
366         }
367
368         svc_entries = (struct ib_dm_svc_entries *)mad->data;
369         memset(svc_entries, 0, sizeof *svc_entries);
370         svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid);
371         snprintf(svc_entries->service_entries[0].name,
372                  sizeof(svc_entries->service_entries[0].name),
373                  "%s%016llx",
374                  SRP_SERVICE_NAME_PREFIX,
375                  (unsigned long long)ioc_guid);
376
377         mad->mad_hdr.status = 0;
378 }
379
380 /*
381  * Actual processing of a received MAD *rq_mad received through source port *sp
382  * (MAD = InfiniBand management datagram). The response to be sent back is
383  * written to *rsp_mad.
384  */
385 static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
386                                  struct ib_dm_mad *rsp_mad)
387 {
388         u16 attr_id;
389         u32 slot;
390         u8 hi, lo;
391
392         attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
393         switch (attr_id) {
394         case DM_ATTR_CLASS_PORT_INFO:
395                 srpt_get_class_port_info(rsp_mad);
396                 break;
397         case DM_ATTR_IOU_INFO:
398                 srpt_get_iou(rsp_mad);
399                 break;
400         case DM_ATTR_IOC_PROFILE:
401                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
402                 srpt_get_ioc(sp->sdev, slot, rsp_mad);
403                 break;
404         case DM_ATTR_SVC_ENTRIES:
405                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
406                 hi = (u8) ((slot >> 8) & 0xff);
407                 lo = (u8) (slot & 0xff);
408                 slot = (u16) ((slot >> 16) & 0xffff);
409                 srpt_get_svc_entries(srpt_service_guid,
410                                      slot, hi, lo, rsp_mad);
411                 break;
412         default:
413                 rsp_mad->mad_hdr.status =
414                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
415                 break;
416         }
417 }
418
419 /*
420  * Callback function that is called by the InfiniBand core after transmission of
421  * a MAD. (MAD = management datagram; AH = address handle.)
422  */
423 static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
424                                   struct ib_mad_send_wc *mad_wc)
425 {
426         ib_destroy_ah(mad_wc->send_buf->ah);
427         ib_free_send_mad(mad_wc->send_buf);
428 }
429
430 /*
431  * Callback function that is called by the InfiniBand core after reception of
432  * a MAD (management datagram).
433  */
434 static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
435                                   struct ib_mad_recv_wc *mad_wc)
436 {
437         struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
438         struct ib_ah *ah;
439         struct ib_mad_send_buf *rsp;
440         struct ib_dm_mad *dm_mad;
441
442         if (!mad_wc || !mad_wc->recv_buf.mad)
443                 return;
444
445         ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
446                                   mad_wc->recv_buf.grh, mad_agent->port_num);
447         if (IS_ERR(ah))
448                 goto err;
449
450         BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR);
451
452         rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
453                                  mad_wc->wc->pkey_index, 0,
454                                  IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA,
455                                  GFP_KERNEL);
456         if (IS_ERR(rsp))
457                 goto err_rsp;
458
459         rsp->ah = ah;
460
461         dm_mad = rsp->mad;
462         memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof *dm_mad);
463         dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
464         dm_mad->mad_hdr.status = 0;
465
466         switch (mad_wc->recv_buf.mad->mad_hdr.method) {
467         case IB_MGMT_METHOD_GET:
468                 srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
469                 break;
470         case IB_MGMT_METHOD_SET:
471                 dm_mad->mad_hdr.status =
472                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
473                 break;
474         default:
475                 dm_mad->mad_hdr.status =
476                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
477                 break;
478         }
479
480         if (!ib_post_send_mad(rsp, NULL)) {
481                 ib_free_recv_mad(mad_wc);
482                 /* will destroy_ah & free_send_mad in send completion */
483                 return;
484         }
485
486         ib_free_send_mad(rsp);
487
488 err_rsp:
489         ib_destroy_ah(ah);
490 err:
491         ib_free_recv_mad(mad_wc);
492 }
493
494 /*
495  * Enable InfiniBand management datagram processing, update the cached sm_lid,
496  * lid and gid values, and register a callback function for processing MADs
497  * on the specified port. It is safe to call this function more than once for
498  * the same port.
499  */
500 static int srpt_refresh_port(struct srpt_port *sport)
501 {
502         struct ib_mad_reg_req reg_req;
503         struct ib_port_modify port_modify;
504         struct ib_port_attr port_attr;
505         int ret;
506
507         memset(&port_modify, 0, sizeof port_modify);
508         port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
509         port_modify.clr_port_cap_mask = 0;
510
511         ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
512         if (ret)
513                 goto err_mod_port;
514
515         ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
516         if (ret)
517                 goto err_query_port;
518
519         sport->sm_lid = port_attr.sm_lid;
520         sport->lid = port_attr.lid;
521
522         ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid);
523         if (ret)
524                 goto err_query_port;
525
526         if (!sport->mad_agent) {
527                 memset(&reg_req, 0, sizeof reg_req);
528                 reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
529                 reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
530                 set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
531                 set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
532
533                 sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
534                                                          sport->port,
535                                                          IB_QPT_GSI,
536                                                          &reg_req, 0,
537                                                          srpt_mad_send_handler,
538                                                          srpt_mad_recv_handler,
539                                                          sport);
540                 if (IS_ERR(sport->mad_agent)) {
541                         ret = PTR_ERR(sport->mad_agent);
542                         sport->mad_agent = NULL;
543                         goto err_query_port;
544                 }
545         }
546
547         return 0;
548
549 err_query_port:
550
551         port_modify.set_port_cap_mask = 0;
552         port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
553         ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
554
555 err_mod_port:
556
557         return ret;
558 }
559
560 /*
561  * Unregister the callback function for processing MADs and disable MAD
562  * processing for all ports of the specified device. It is safe to call this
563  * function more than once for the same device.
564  */
565 static void srpt_unregister_mad_agent(struct srpt_device *sdev)
566 {
567         struct ib_port_modify port_modify = {
568                 .clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP,
569         };
570         struct srpt_port *sport;
571         int i;
572
573         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
574                 sport = &sdev->port[i - 1];
575                 WARN_ON(sport->port != i);
576                 if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
577                         PRINT_ERROR("%s", "disabling MAD processing failed.");
578                 if (sport->mad_agent) {
579                         ib_unregister_mad_agent(sport->mad_agent);
580                         sport->mad_agent = NULL;
581                 }
582         }
583 }
584
585 /*
586  * Allocate and initialize an SRPT I/O context structure.
587  */
588 static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev)
589 {
590         struct srpt_ioctx *ioctx;
591
592         ioctx = kmalloc(sizeof *ioctx, GFP_KERNEL);
593         if (!ioctx)
594                 goto out;
595
596         ioctx->buf = kzalloc(MAX_MESSAGE_SIZE, GFP_KERNEL);
597         if (!ioctx->buf)
598                 goto out_free_ioctx;
599
600         ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf,
601                                        MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
602         if (ib_dma_mapping_error(sdev->device, ioctx->dma))
603                 goto out_free_buf;
604
605         return ioctx;
606
607 out_free_buf:
608         kfree(ioctx->buf);
609 out_free_ioctx:
610         kfree(ioctx);
611 out:
612         return NULL;
613 }
614
615 /*
616  * Deallocate an SRPT I/O context structure.
617  */
618 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
619 {
620         if (!ioctx)
621                 return;
622
623         ib_dma_unmap_single(sdev->device, ioctx->dma,
624                             MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
625         kfree(ioctx->buf);
626         kfree(ioctx);
627 }
628
629 /*
630  * Associate a ring of SRPT I/O context structures with the specified device.
631  */
632 static int srpt_alloc_ioctx_ring(struct srpt_device *sdev)
633 {
634         int i;
635
636         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
637                 sdev->ioctx_ring[i] = srpt_alloc_ioctx(sdev);
638
639                 if (!sdev->ioctx_ring[i])
640                         goto err;
641
642                 sdev->ioctx_ring[i]->index = i;
643         }
644
645         return 0;
646
647 err:
648         while (--i > 0) {
649                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
650                 sdev->ioctx_ring[i] = NULL;
651         }
652         return -ENOMEM;
653 }
654
655 /* Free the ring of SRPT I/O context structures. */
656 static void srpt_free_ioctx_ring(struct srpt_device *sdev)
657 {
658         int i;
659
660         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
661                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
662                 sdev->ioctx_ring[i] = NULL;
663         }
664 }
665
666 /*
667  * Post a receive request on the work queue of InfiniBand device 'sdev'.
668  */
669 static int srpt_post_recv(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
670 {
671         struct ib_sge list;
672         struct ib_recv_wr wr, *bad_wr;
673
674         wr.wr_id = ioctx->index | SRPT_OP_RECV;
675
676         list.addr = ioctx->dma;
677         list.length = MAX_MESSAGE_SIZE;
678         list.lkey = sdev->mr->lkey;
679
680         wr.next = NULL;
681         wr.sg_list = &list;
682         wr.num_sge = 1;
683
684         return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);
685 }
686
687 /*
688  * Post an IB send request.
689  * @ch: RDMA channel to post the send request on.
690  * @ioctx: I/O context of the send request.
691  * @len: length of the request to be sent in bytes.
692  *
693  * Returns zero upon success and a non-zero value upon failure.
694  */
695 static int srpt_post_send(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
696                           int len)
697 {
698         struct ib_sge list;
699         struct ib_send_wr wr, *bad_wr;
700         struct srpt_device *sdev = ch->sport->sdev;
701
702         ib_dma_sync_single_for_device(sdev->device, ioctx->dma,
703                                       MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
704
705         list.addr = ioctx->dma;
706         list.length = len;
707         list.lkey = sdev->mr->lkey;
708
709         wr.next = NULL;
710         wr.wr_id = ioctx->index;
711         wr.sg_list = &list;
712         wr.num_sge = 1;
713         wr.opcode = IB_WR_SEND;
714         wr.send_flags = IB_SEND_SIGNALED;
715
716         return ib_post_send(ch->qp, &wr, &bad_wr);
717 }
718
719 static int srpt_get_desc_tbl(struct srpt_ioctx *ioctx, struct srp_cmd *srp_cmd,
720                              int *ind)
721 {
722         struct srp_indirect_buf *idb;
723         struct srp_direct_buf *db;
724
725         *ind = 0;
726         if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
727             ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
728                 ioctx->n_rbuf = 1;
729                 ioctx->rbufs = &ioctx->single_rbuf;
730
731                 db = (void *)srp_cmd->add_data;
732                 memcpy(ioctx->rbufs, db, sizeof *db);
733                 ioctx->data_len = be32_to_cpu(db->len);
734         } else {
735                 idb = (void *)srp_cmd->add_data;
736
737                 ioctx->n_rbuf = be32_to_cpu(idb->table_desc.len) / sizeof *db;
738
739                 if (ioctx->n_rbuf >
740                     (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
741                         *ind = 1;
742                         ioctx->n_rbuf = 0;
743                         goto out;
744                 }
745
746                 if (ioctx->n_rbuf == 1)
747                         ioctx->rbufs = &ioctx->single_rbuf;
748                 else
749                         ioctx->rbufs =
750                                 kmalloc(ioctx->n_rbuf * sizeof *db, GFP_ATOMIC);
751                 if (!ioctx->rbufs) {
752                         ioctx->n_rbuf = 0;
753                         return -ENOMEM;
754                 }
755
756                 db = idb->desc_list;
757                 memcpy(ioctx->rbufs, db, ioctx->n_rbuf * sizeof *db);
758                 ioctx->data_len = be32_to_cpu(idb->len);
759         }
760 out:
761         return 0;
762 }
763
764 /*
765  * Modify the attributes of queue pair 'qp': allow local write, remote read,
766  * and remote write. Also transition 'qp' to state IB_QPS_INIT.
767  */
768 static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
769 {
770         struct ib_qp_attr *attr;
771         int ret;
772
773         attr = kzalloc(sizeof *attr, GFP_KERNEL);
774         if (!attr)
775                 return -ENOMEM;
776
777         attr->qp_state = IB_QPS_INIT;
778         attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ |
779             IB_ACCESS_REMOTE_WRITE;
780         attr->port_num = ch->sport->port;
781         attr->pkey_index = 0;
782
783         ret = ib_modify_qp(qp, attr,
784                            IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
785                            IB_QP_PKEY_INDEX);
786
787         kfree(attr);
788         return ret;
789 }
790
791 /**
792  * Change the state of a channel to 'ready to receive' (RTR).
793  * @ch: channel of the queue pair.
794  * @qp: queue pair to change the state of.
795  *
796  * Returns zero upon success and a negative value upon failure.
797  *
798  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
799  * If this structure ever becomes larger, it might be necessary to allocate
800  * it dynamically instead of on the stack.
801  */
802 static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
803 {
804         struct ib_qp_attr qp_attr;
805         int attr_mask;
806         int ret;
807
808         qp_attr.qp_state = IB_QPS_RTR;
809         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
810         if (ret)
811                 goto out;
812
813         qp_attr.max_dest_rd_atomic = 4;
814
815         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
816
817 out:
818         return ret;
819 }
820
821 /**
822  * Change the state of a channel to 'ready to send' (RTS).
823  * @ch: channel of the queue pair.
824  * @qp: queue pair to change the state of.
825  *
826  * Returns zero upon success and a negative value upon failure.
827  *
828  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
829  * If this structure ever becomes larger, it might be necessary to allocate
830  * it dynamically instead of on the stack.
831  */
832 static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
833 {
834         struct ib_qp_attr qp_attr;
835         int attr_mask;
836         int ret;
837
838         qp_attr.qp_state = IB_QPS_RTS;
839         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
840         if (ret)
841                 goto out;
842
843         qp_attr.max_rd_atomic = 4;
844
845         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
846
847 out:
848         return ret;
849 }
850
851 static void srpt_reset_ioctx(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
852 {
853         int i;
854
855         if (ioctx->n_rdma_ius > 0 && ioctx->rdma_ius) {
856                 struct rdma_iu *riu = ioctx->rdma_ius;
857
858                 for (i = 0; i < ioctx->n_rdma_ius; ++i, ++riu)
859                         kfree(riu->sge);
860                 kfree(ioctx->rdma_ius);
861         }
862
863         if (ioctx->n_rbuf > 1)
864                 kfree(ioctx->rbufs);
865
866         if (srpt_post_recv(ch->sport->sdev, ioctx))
867                 PRINT_ERROR("%s", "SRQ post_recv failed - this is serious.");
868                 /* we should queue it back to free_ioctx queue */
869         else
870                 atomic_inc(&ch->req_lim_delta);
871 }
872
873 static void srpt_abort_scst_cmd(struct srpt_device *sdev,
874                                 struct scst_cmd *scmnd,
875                                 bool tell_initiator)
876 {
877         struct srpt_ioctx *ioctx;
878         scst_data_direction dir;
879         enum srpt_command_state orig_ioctx_state;
880
881         ioctx = scst_cmd_get_tgt_priv(scmnd);
882         BUG_ON(!ioctx);
883         dir = scst_cmd_get_data_direction(scmnd);
884         if (dir != SCST_DATA_NONE) {
885                 ib_dma_unmap_sg(sdev->device,
886                                 scst_cmd_get_sg(scmnd),
887                                 scst_cmd_get_sg_cnt(scmnd),
888                                 scst_to_tgt_dma_dir(dir));
889
890 #ifdef CONFIG_SCST_EXTRACHECKS
891                 switch (scmnd->state) {
892                 case SCST_CMD_STATE_DATA_WAIT:
893                         WARN_ON(ioctx->state != SRPT_STATE_NEED_DATA);
894                         break;
895                 case SCST_CMD_STATE_XMIT_WAIT:
896                         WARN_ON(ioctx->state != SRPT_STATE_PROCESSED);
897                         break;
898                 default:
899                         WARN_ON(ioctx->state == SRPT_STATE_NEED_DATA ||
900                                 ioctx->state == SRPT_STATE_PROCESSED);
901                 }
902 #endif
903         }
904
905         orig_ioctx_state = ioctx->state;
906         ioctx->state = SRPT_STATE_ABORTED;
907
908         if (orig_ioctx_state == SRPT_STATE_NEED_DATA) {
909                 WARN_ON(scst_cmd_get_data_direction(ioctx->scmnd)
910                         == SCST_DATA_READ);
911                 scst_rx_data(scmnd,
912                              tell_initiator ? SCST_RX_STATUS_ERROR
913                              : SCST_RX_STATUS_ERROR_FATAL,
914                              SCST_CONTEXT_THREAD);
915                 goto out;
916         } else if (ioctx->state == SRPT_STATE_PROCESSED)
917                 ;
918         else
919                 WARN_ON("unexpected cmd state");
920
921         scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_FAILED);
922         WARN_ON(scmnd->state != SCST_CMD_STATE_XMIT_WAIT);
923         scst_tgt_cmd_done(scmnd, scst_estimate_context());
924 out:
925         return;
926 }
927
928 static void srpt_handle_err_comp(struct srpt_rdma_ch *ch, struct ib_wc *wc)
929 {
930         struct srpt_ioctx *ioctx;
931         struct srpt_device *sdev = ch->sport->sdev;
932
933         if (wc->wr_id & SRPT_OP_RECV) {
934                 ioctx = sdev->ioctx_ring[wc->wr_id & ~SRPT_OP_RECV];
935                 PRINT_ERROR("%s", "This is serious - SRQ is in bad state.");
936         } else {
937                 ioctx = sdev->ioctx_ring[wc->wr_id];
938
939                 if (ioctx->scmnd)
940                         srpt_abort_scst_cmd(sdev, ioctx->scmnd, true);
941                 else
942                         srpt_reset_ioctx(ch, ioctx);
943         }
944 }
945
946 static void srpt_handle_send_comp(struct srpt_rdma_ch *ch,
947                                   struct srpt_ioctx *ioctx,
948                                   enum scst_exec_context context)
949 {
950         if (ioctx->scmnd) {
951                 scst_data_direction dir =
952                         scst_cmd_get_data_direction(ioctx->scmnd);
953
954                 if (dir != SCST_DATA_NONE)
955                         ib_dma_unmap_sg(ch->sport->sdev->device,
956                                         scst_cmd_get_sg(ioctx->scmnd),
957                                         scst_cmd_get_sg_cnt(ioctx->scmnd),
958                                         scst_to_tgt_dma_dir(dir));
959
960                 WARN_ON(ioctx->scmnd->state != SCST_CMD_STATE_XMIT_WAIT);
961                 scst_tgt_cmd_done(ioctx->scmnd, context);
962         } else
963                 srpt_reset_ioctx(ch, ioctx);
964 }
965
966 /** Process an RDMA completion notification. */
967 static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch,
968                                   struct srpt_ioctx *ioctx)
969 {
970         if (!ioctx->scmnd) {
971                 WARN_ON("ERROR: ioctx->scmnd == NULL");
972                 srpt_reset_ioctx(ch, ioctx);
973                 return;
974         }
975
976         /*
977          * If an RDMA completion notification has been received for a write
978          * command, tell SCST that processing can continue by calling
979          * scst_rx_data().
980          */
981         if (ioctx->state == SRPT_STATE_NEED_DATA) {
982                 WARN_ON(scst_cmd_get_data_direction(ioctx->scmnd)
983                         == SCST_DATA_READ);
984                 ioctx->state = SRPT_STATE_DATA_IN;
985                 scst_rx_data(ioctx->scmnd, SCST_RX_STATUS_SUCCESS,
986                              scst_estimate_context());
987         }
988 }
989
990 /**
991  * Build an SRP_RSP response.
992  * @ch: RDMA channel through which the request has been received.
993  * @ioctx: I/O context in which the SRP_RSP response will be built.
994  * @s_key: sense key that will be stored in the response.
995  * @s_code: value that will be stored in the asc_ascq field of the sense data.
996  * @tag: tag of the request for which this response is being generated.
997  *
998  * Returns the size in bytes of the SRP_RSP response.
999  *
1000  * An SRP_RSP response contains a SCSI status or service response. See also
1001  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
1002  * response. See also SPC-2 for more information about sense data.
1003  */
1004 static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
1005                               struct srpt_ioctx *ioctx, u8 s_key, u8 s_code,
1006                               u64 tag)
1007 {
1008         struct srp_rsp *srp_rsp;
1009         struct sense_data *sense;
1010         int limit_delta;
1011         int sense_data_len = 0;
1012
1013         srp_rsp = ioctx->buf;
1014         memset(srp_rsp, 0, sizeof *srp_rsp);
1015
1016         limit_delta = atomic_read(&ch->req_lim_delta);
1017         atomic_sub(limit_delta, &ch->req_lim_delta);
1018
1019         srp_rsp->opcode = SRP_RSP;
1020         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1021         srp_rsp->tag = tag;
1022
1023         if (s_key != NO_SENSE) {
1024                 sense_data_len = sizeof *sense + (sizeof *sense % 4);
1025                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1026                 srp_rsp->status = SAM_STAT_CHECK_CONDITION;
1027                 srp_rsp->sense_data_len = cpu_to_be32(sense_data_len);
1028
1029                 sense = (struct sense_data *)(srp_rsp + 1);
1030                 sense->err_code = 0x70;
1031                 sense->key = s_key;
1032                 sense->asc_ascq = s_code;
1033         }
1034
1035         return sizeof(*srp_rsp) + sense_data_len;
1036 }
1037
1038 /**
1039  * Build a task management response, which is a specific SRP_RSP response.
1040  * @ch: RDMA channel through which the request has been received.
1041  * @ioctx: I/O context in which the SRP_RSP response will be built.
1042  * @rsp_code: RSP_CODE that will be stored in the response.
1043  * @tag: tag of the request for which this response is being generated.
1044  *
1045  * Returns the size in bytes of the SRP_RSP response.
1046  *
1047  * An SRP_RSP response contains a SCSI status or service response. See also
1048  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
1049  * response.
1050  */
1051 static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
1052                                   struct srpt_ioctx *ioctx, u8 rsp_code,
1053                                   u64 tag)
1054 {
1055         struct srp_rsp *srp_rsp;
1056         int limit_delta;
1057         int resp_data_len = 0;
1058
1059         ib_dma_sync_single_for_cpu(ch->sport->sdev->device, ioctx->dma,
1060                                    MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
1061
1062         srp_rsp = ioctx->buf;
1063         memset(srp_rsp, 0, sizeof *srp_rsp);
1064
1065         limit_delta = atomic_read(&ch->req_lim_delta);
1066         atomic_sub(limit_delta, &ch->req_lim_delta);
1067
1068         srp_rsp->opcode = SRP_RSP;
1069         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1070         srp_rsp->tag = tag;
1071
1072         if (rsp_code != SRP_TSK_MGMT_SUCCESS) {
1073                 resp_data_len = 4;
1074                 srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
1075                 srp_rsp->resp_data_len = cpu_to_be32(resp_data_len);
1076                 srp_rsp->data[3] = rsp_code;
1077         }
1078
1079         return sizeof(*srp_rsp) + resp_data_len;
1080 }
1081
1082 /*
1083  * Process SRP_CMD.
1084  */
1085 static int srpt_handle_cmd(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
1086 {
1087         struct scst_cmd *scmnd;
1088         struct srp_cmd *srp_cmd;
1089         struct srp_rsp *srp_rsp;
1090         scst_data_direction dir;
1091         int indirect_desc = 0;
1092         int ret;
1093         unsigned long flags;
1094
1095         srp_cmd = ioctx->buf;
1096         srp_rsp = ioctx->buf;
1097
1098         dir = SCST_DATA_NONE;
1099         if (srp_cmd->buf_fmt) {
1100                 ret = srpt_get_desc_tbl(ioctx, srp_cmd, &indirect_desc);
1101                 if (ret) {
1102                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1103                                            NO_ADD_SENSE, srp_cmd->tag);
1104                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1105                         goto err;
1106                 }
1107
1108                 if (indirect_desc) {
1109                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1110                                            NO_ADD_SENSE, srp_cmd->tag);
1111                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1112                         goto err;
1113                 }
1114
1115                 /*
1116                  * The lower four bits of the buffer format field contain the
1117                  * DATA-IN buffer descriptor format, and the highest four bits
1118                  * contain the DATA-OUT buffer descriptor format.
1119                  */
1120                 if (srp_cmd->buf_fmt & 0xf)
1121                         /* DATA-IN: transfer data from target to initiator. */
1122                         dir = SCST_DATA_READ;
1123                 else if (srp_cmd->buf_fmt >> 4)
1124                         /* DATA-OUT: transfer data from initiator to target. */
1125                         dir = SCST_DATA_WRITE;
1126         }
1127
1128         scmnd = scst_rx_cmd(ch->scst_sess, (u8 *) &srp_cmd->lun,
1129                             sizeof srp_cmd->lun, srp_cmd->cdb, 16,
1130                             thread ? SCST_NON_ATOMIC : SCST_ATOMIC);
1131         if (!scmnd) {
1132                 srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1133                                    NO_ADD_SENSE, srp_cmd->tag);
1134                 srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1135                 goto err;
1136         }
1137
1138         ioctx->scmnd = scmnd;
1139
1140         switch (srp_cmd->task_attr) {
1141         case SRP_CMD_HEAD_OF_Q:
1142                 scmnd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1143                 break;
1144         case SRP_CMD_ORDERED_Q:
1145                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1146                 break;
1147         case SRP_CMD_SIMPLE_Q:
1148                 scmnd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1149                 break;
1150         case SRP_CMD_ACA:
1151                 scmnd->queue_type = SCST_CMD_QUEUE_ACA;
1152                 break;
1153         default:
1154                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1155                 break;
1156         }
1157
1158         scst_cmd_set_tag(scmnd, srp_cmd->tag);
1159         scst_cmd_set_tgt_priv(scmnd, ioctx);
1160         scst_cmd_set_expected(scmnd, dir, ioctx->data_len);
1161
1162         spin_lock_irqsave(&ch->spinlock, flags);
1163         list_add_tail(&ioctx->scmnd_list, &ch->active_scmnd_list);
1164         ch->active_scmnd_cnt++;
1165         spin_unlock_irqrestore(&ch->spinlock, flags);
1166
1167         scst_cmd_init_done(scmnd, scst_estimate_context());
1168
1169         return 0;
1170
1171 err:
1172         WARN_ON(srp_rsp->opcode != SRP_RSP);
1173
1174         return -1;
1175 }
1176
1177 /*
1178  * Process an SRP_TSK_MGMT request.
1179  *
1180  * Returns 0 upon success and -1 upon failure.
1181  *
1182  * Each task management function is performed by calling one of the
1183  * scst_rx_mgmt_fn*() functions. These functions will either report failure
1184  * or process the task management function asynchronously. The function
1185  * srpt_tsk_mgmt_done() will be called by the SCST core upon completion of the
1186  * task management function. When srpt_handle_tsk_mgmt() reports failure
1187  * (i.e. returns -1) a response will have been built in ioctx->buf. This
1188  * information unit has to be sent back by the caller.
1189  *
1190  * For more information about SRP_TSK_MGMT information units, see also section
1191  * 6.7 in the T10 SRP r16a document.
1192  */
1193 static int srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
1194                                 struct srpt_ioctx *ioctx)
1195 {
1196         struct srp_tsk_mgmt *srp_tsk;
1197         struct srpt_mgmt_ioctx *mgmt_ioctx;
1198         int ret;
1199
1200         srp_tsk = ioctx->buf;
1201
1202         TRACE_DBG("recv_tsk_mgmt= %d for task_tag= %lld"
1203                   " using tag= %lld cm_id= %p sess= %p",
1204                   srp_tsk->tsk_mgmt_func,
1205                   (unsigned long long) srp_tsk->task_tag,
1206                   (unsigned long long) srp_tsk->tag,
1207                   ch->cm_id, ch->scst_sess);
1208
1209         mgmt_ioctx = kmalloc(sizeof *mgmt_ioctx, GFP_ATOMIC);
1210         if (!mgmt_ioctx) {
1211                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1212                                        srp_tsk->tag);
1213                 goto err;
1214         }
1215
1216         mgmt_ioctx->ioctx = ioctx;
1217         mgmt_ioctx->ch = ch;
1218         mgmt_ioctx->tag = srp_tsk->tag;
1219
1220         switch (srp_tsk->tsk_mgmt_func) {
1221         case SRP_TSK_ABORT_TASK:
1222                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK");
1223                 ret = scst_rx_mgmt_fn_tag(ch->scst_sess,
1224                                           SCST_ABORT_TASK,
1225                                           srp_tsk->task_tag,
1226                                           thread ?
1227                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1228                                           mgmt_ioctx);
1229                 break;
1230         case SRP_TSK_ABORT_TASK_SET:
1231                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK_SET");
1232                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1233                                           SCST_ABORT_TASK_SET,
1234                                           (u8 *) &srp_tsk->lun,
1235                                           sizeof srp_tsk->lun,
1236                                           thread ?
1237                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1238                                           mgmt_ioctx);
1239                 break;
1240         case SRP_TSK_CLEAR_TASK_SET:
1241                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_TASK_SET");
1242                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1243                                           SCST_CLEAR_TASK_SET,
1244                                           (u8 *) &srp_tsk->lun,
1245                                           sizeof srp_tsk->lun,
1246                                           thread ?
1247                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1248                                           mgmt_ioctx);
1249                 break;
1250         case SRP_TSK_LUN_RESET:
1251                 TRACE_DBG("%s", "Processing SRP_TSK_LUN_RESET");
1252                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1253                                           SCST_LUN_RESET,
1254                                           (u8 *) &srp_tsk->lun,
1255                                           sizeof srp_tsk->lun,
1256                                           thread ?
1257                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1258                                           mgmt_ioctx);
1259                 break;
1260         case SRP_TSK_CLEAR_ACA:
1261                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_ACA");
1262                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1263                                           SCST_CLEAR_ACA,
1264                                           (u8 *) &srp_tsk->lun,
1265                                           sizeof srp_tsk->lun,
1266                                           thread ?
1267                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1268                                           mgmt_ioctx);
1269                 break;
1270         default:
1271                 TRACE_DBG("%s", "Unsupported task management function.");
1272                 srpt_build_tskmgmt_rsp(ch, ioctx,
1273                                        SRP_TSK_MGMT_FUNC_NOT_SUPP,
1274                                        srp_tsk->tag);
1275                 goto err;
1276         }
1277
1278         if (ret) {
1279                 TRACE_DBG("%s", "Processing task management function failed.");
1280                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1281                                        srp_tsk->tag);
1282                 goto err;
1283         }
1284
1285         WARN_ON(srp_tsk->opcode == SRP_RSP);
1286
1287         return 0;
1288
1289 err:
1290         WARN_ON(srp_tsk->opcode != SRP_RSP);
1291
1292         kfree(mgmt_ioctx);
1293         return -1;
1294 }
1295
1296 /**
1297  * Process a receive completion event.
1298  * @ch: RDMA channel for which the completion event has been received.
1299  * @ioctx: SRPT I/O context for which the completion event has been received.
1300  */
1301 static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
1302                                struct srpt_ioctx *ioctx)
1303 {
1304         struct srp_cmd *srp_cmd;
1305         struct srp_rsp *srp_rsp;
1306         unsigned long flags;
1307         int len;
1308
1309         spin_lock_irqsave(&ch->spinlock, flags);
1310         if (ch->state != RDMA_CHANNEL_LIVE) {
1311                 if (ch->state == RDMA_CHANNEL_CONNECTING) {
1312                         list_add_tail(&ioctx->wait_list, &ch->cmd_wait_list);
1313                         spin_unlock_irqrestore(&ch->spinlock, flags);
1314                         return;
1315                 } else {
1316                         spin_unlock_irqrestore(&ch->spinlock, flags);
1317                         srpt_reset_ioctx(ch, ioctx);
1318                         return;
1319                 }
1320         }
1321         spin_unlock_irqrestore(&ch->spinlock, flags);
1322
1323         ib_dma_sync_single_for_cpu(ch->sport->sdev->device, ioctx->dma,
1324                                    MAX_MESSAGE_SIZE, DMA_FROM_DEVICE);
1325
1326         ioctx->data_len = 0;
1327         ioctx->n_rbuf = 0;
1328         ioctx->rbufs = NULL;
1329         ioctx->n_rdma = 0;
1330         ioctx->n_rdma_ius = 0;
1331         ioctx->rdma_ius = NULL;
1332         ioctx->scmnd = NULL;
1333         ioctx->state = SRPT_STATE_NEW;
1334
1335         srp_cmd = ioctx->buf;
1336         srp_rsp = ioctx->buf;
1337
1338         switch (srp_cmd->opcode) {
1339         case SRP_CMD:
1340                 if (srpt_handle_cmd(ch, ioctx) < 0)
1341                         goto err;
1342                 break;
1343
1344         case SRP_TSK_MGMT:
1345                 if (srpt_handle_tsk_mgmt(ch, ioctx) < 0)
1346                         goto err;
1347                 break;
1348
1349         case SRP_I_LOGOUT:
1350         case SRP_AER_REQ:
1351         default:
1352                 srpt_build_cmd_rsp(ch, ioctx, ILLEGAL_REQUEST, INVALID_CDB,
1353                                    srp_cmd->tag);
1354                 goto err;
1355         }
1356
1357         ib_dma_sync_single_for_device(ch->sport->sdev->device,
1358                                    ioctx->dma, MAX_MESSAGE_SIZE,
1359                                    DMA_FROM_DEVICE);
1360
1361         return;
1362
1363 err:
1364         WARN_ON(srp_rsp->opcode != SRP_RSP);
1365         len = (sizeof *srp_rsp) + be32_to_cpu(srp_rsp->sense_data_len);
1366
1367         if (ch->state != RDMA_CHANNEL_LIVE) {
1368                 /* Give up if another thread modified the channel state. */
1369                 PRINT_ERROR("%s: channel is in state %d", __func__, ch->state);
1370                 srpt_reset_ioctx(ch, ioctx);
1371         } else if (srpt_post_send(ch, ioctx, len)) {
1372                 PRINT_ERROR("%s: sending SRP_RSP response failed", __func__);
1373                 srpt_reset_ioctx(ch, ioctx);
1374         }
1375 }
1376
1377 /*
1378  * Returns true if the ioctx list is non-empty or if the ib_srpt kernel thread
1379  * should stop.
1380  * @pre thread != 0
1381  */
1382 static inline int srpt_test_ioctx_list(void)
1383 {
1384         int res = (!list_empty(&srpt_thread.thread_ioctx_list) ||
1385                    unlikely(kthread_should_stop()));
1386         return res;
1387 }
1388
1389 /*
1390  * Add 'ioctx' to the tail of the ioctx list and wake up the kernel thread.
1391  *
1392  * @pre thread != 0
1393  */
1394 static inline void srpt_schedule_thread(struct srpt_ioctx *ioctx)
1395 {
1396         unsigned long flags;
1397
1398         spin_lock_irqsave(&srpt_thread.thread_lock, flags);
1399         list_add_tail(&ioctx->comp_list, &srpt_thread.thread_ioctx_list);
1400         spin_unlock_irqrestore(&srpt_thread.thread_lock, flags);
1401         wake_up(&ioctx_list_waitQ);
1402 }
1403
1404 /**
1405  * InfiniBand completion queue callback function.
1406  * @cq: completion queue.
1407  * @ctx: completion queue context, which was passed as the fourth argument of
1408  *       the function ib_create_cq().
1409  */
1410 static void srpt_completion(struct ib_cq *cq, void *ctx)
1411 {
1412         struct srpt_rdma_ch *ch = ctx;
1413         struct srpt_device *sdev = ch->sport->sdev;
1414         struct ib_wc wc;
1415         struct srpt_ioctx *ioctx;
1416
1417         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1418         while (ib_poll_cq(ch->cq, 1, &wc) > 0) {
1419                 if (wc.status) {
1420                         PRINT_ERROR("failed %s status= %d",
1421                                wc.wr_id & SRPT_OP_RECV ? "receive" : "send",
1422                                wc.status);
1423                         srpt_handle_err_comp(ch, &wc);
1424                         break;
1425                 }
1426
1427                 if (wc.wr_id & SRPT_OP_RECV) {
1428                         ioctx = sdev->ioctx_ring[wc.wr_id & ~SRPT_OP_RECV];
1429                         if (thread) {
1430                                 ioctx->ch = ch;
1431                                 ioctx->op = IB_WC_RECV;
1432                                 srpt_schedule_thread(ioctx);
1433                         } else
1434                                 srpt_handle_new_iu(ch, ioctx);
1435                         continue;
1436                 } else
1437                         ioctx = sdev->ioctx_ring[wc.wr_id];
1438
1439                 if (thread) {
1440                         ioctx->ch = ch;
1441                         ioctx->op = wc.opcode;
1442                         srpt_schedule_thread(ioctx);
1443                 } else {
1444                         switch (wc.opcode) {
1445                         case IB_WC_SEND:
1446                                 srpt_handle_send_comp(ch, ioctx,
1447                                         scst_estimate_context());
1448                                 break;
1449                         case IB_WC_RDMA_WRITE:
1450                         case IB_WC_RDMA_READ:
1451                                 srpt_handle_rdma_comp(ch, ioctx);
1452                                 break;
1453                         default:
1454                                 break;
1455                         }
1456                 }
1457         }
1458 }
1459
1460 /*
1461  * Create a completion queue on the specified device.
1462  */
1463 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
1464 {
1465         struct ib_qp_init_attr *qp_init;
1466         struct srpt_device *sdev = ch->sport->sdev;
1467         int cqe;
1468         int ret;
1469
1470         qp_init = kzalloc(sizeof *qp_init, GFP_KERNEL);
1471         if (!qp_init)
1472                 return -ENOMEM;
1473
1474         /* Create a completion queue (CQ). */
1475
1476         cqe = SRPT_RQ_SIZE + SRPT_SQ_SIZE - 1;
1477 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(RHEL_RELEASE_CODE)
1478         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe);
1479 #else
1480         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe, 0);
1481 #endif
1482         if (IS_ERR(ch->cq)) {
1483                 ret = PTR_ERR(ch->cq);
1484                 PRINT_ERROR("failed to create_cq cqe= %d ret= %d", cqe, ret);
1485                 goto out;
1486         }
1487
1488         /* Request completion notification. */
1489
1490         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1491
1492         /* Create a queue pair (QP). */
1493
1494         qp_init->qp_context = (void *)ch;
1495         qp_init->event_handler = srpt_qp_event;
1496         qp_init->send_cq = ch->cq;
1497         qp_init->recv_cq = ch->cq;
1498         qp_init->srq = sdev->srq;
1499         qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
1500         qp_init->qp_type = IB_QPT_RC;
1501         qp_init->cap.max_send_wr = SRPT_SQ_SIZE;
1502         qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE;
1503
1504         ch->qp = ib_create_qp(sdev->pd, qp_init);
1505         if (IS_ERR(ch->qp)) {
1506                 ret = PTR_ERR(ch->qp);
1507                 ib_destroy_cq(ch->cq);
1508                 PRINT_ERROR("failed to create_qp ret= %d", ret);
1509                 goto out;
1510         }
1511
1512         TRACE_DBG("%s: max_cqe= %d max_sge= %d cm_id= %p",
1513                __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
1514                ch->cm_id);
1515
1516         /* Modify the attributes and the state of queue pair ch->qp. */
1517
1518         ret = srpt_init_ch_qp(ch, ch->qp);
1519         if (ret) {
1520                 ib_destroy_qp(ch->qp);
1521                 ib_destroy_cq(ch->cq);
1522                 goto out;
1523         }
1524
1525         atomic_set(&ch->req_lim_delta, SRPT_RQ_SIZE);
1526 out:
1527         kfree(qp_init);
1528         return ret;
1529 }
1530
1531 /**
1532  * Look up the RDMA channel that corresponds to the specified cm_id.
1533  *
1534  * Return NULL if no matching RDMA channel has been found.
1535  */
1536 static struct srpt_rdma_ch *srpt_find_channel(struct ib_cm_id *cm_id, bool del)
1537 {
1538         struct srpt_device *sdev = cm_id->context;
1539         struct srpt_rdma_ch *ch;
1540
1541         spin_lock_irq(&sdev->spinlock);
1542         list_for_each_entry(ch, &sdev->rch_list, list) {
1543                 if (ch->cm_id == cm_id) {
1544                         if (del)
1545                                 list_del(&ch->list);
1546                         spin_unlock_irq(&sdev->spinlock);
1547                         return ch;
1548                 }
1549         }
1550
1551         spin_unlock_irq(&sdev->spinlock);
1552
1553         return NULL;
1554 }
1555
1556 /**
1557  * Release all resources associated with the specified RDMA channel.
1558  *
1559  * Note: the caller must have removed the channel from the channel list
1560  * before calling this function.
1561  */
1562 static void srpt_release_channel(struct srpt_rdma_ch *ch, int destroy_cmid)
1563 {
1564         TRACE_ENTRY();
1565
1566         WARN_ON(srpt_find_channel(ch->cm_id, false) == ch);
1567
1568         if (ch->cm_id && destroy_cmid) {
1569                 TRACE_DBG("%s: destroy cm_id= %p", __func__, ch->cm_id);
1570                 ib_destroy_cm_id(ch->cm_id);
1571                 ch->cm_id = NULL;
1572         }
1573
1574         ib_destroy_qp(ch->qp);
1575         ib_destroy_cq(ch->cq);
1576
1577         if (ch->scst_sess) {
1578                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1579
1580                 if (ch->active_scmnd_cnt)
1581                         PRINT_INFO("Releasing session %s which still has %d"
1582                                    " active commands",
1583                                    ch->sess_name, ch->active_scmnd_cnt);
1584                 else
1585                         PRINT_INFO("Releasing session %s", ch->sess_name);
1586
1587                 spin_lock_irq(&ch->spinlock);
1588                 list_for_each_entry_safe(ioctx, ioctx_tmp,
1589                                          &ch->active_scmnd_list, scmnd_list) {
1590                         spin_unlock_irq(&ch->spinlock);
1591
1592                         if (ioctx->scmnd)
1593                                 srpt_abort_scst_cmd(ch->sport->sdev,
1594                                                     ioctx->scmnd, true);
1595
1596                         spin_lock_irq(&ch->spinlock);
1597                 }
1598                 WARN_ON(!list_empty(&ch->active_scmnd_list));
1599                 WARN_ON(ch->active_scmnd_cnt != 0);
1600                 spin_unlock_irq(&ch->spinlock);
1601
1602                 scst_unregister_session(ch->scst_sess, 0, NULL);
1603                 ch->scst_sess = NULL;
1604         }
1605
1606         kfree(ch);
1607
1608         TRACE_EXIT();
1609 }
1610
1611 static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
1612                             struct ib_cm_req_event_param *param,
1613                             void *private_data)
1614 {
1615         struct srpt_device *sdev = cm_id->context;
1616         struct srp_login_req *req;
1617         struct srp_login_rsp *rsp;
1618         struct srp_login_rej *rej;
1619         struct ib_cm_rep_param *rep_param;
1620         struct srpt_rdma_ch *ch, *tmp_ch;
1621         u32 it_iu_len;
1622         int ret = 0;
1623
1624 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
1625         WARN_ON(!sdev || !private_data);
1626         if (!sdev || !private_data)
1627                 return -EINVAL;
1628 #else
1629         if (WARN_ON(!sdev || !private_data))
1630                 return -EINVAL;
1631 #endif
1632
1633         req = (struct srp_login_req *)private_data;
1634
1635         it_iu_len = be32_to_cpu(req->req_it_iu_len);
1636
1637         PRINT_INFO("Received SRP_LOGIN_REQ with"
1638             " i_port_id 0x%llx:0x%llx, t_port_id 0x%llx:0x%llx and length %d"
1639             " on port %d (guid=0x%llx:0x%llx)",
1640             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[0]),
1641             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[8]),
1642             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[0]),
1643             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[8]),
1644             it_iu_len,
1645             param->port,
1646             (unsigned long long)be64_to_cpu(*(u64 *)
1647                                 &sdev->port[param->port - 1].gid.raw[0]),
1648             (unsigned long long)be64_to_cpu(*(u64 *)
1649                                 &sdev->port[param->port - 1].gid.raw[8]));
1650
1651         rsp = kzalloc(sizeof *rsp, GFP_KERNEL);
1652         rej = kzalloc(sizeof *rej, GFP_KERNEL);
1653         rep_param = kzalloc(sizeof *rep_param, GFP_KERNEL);
1654
1655         if (!rsp || !rej || !rep_param) {
1656                 ret = -ENOMEM;
1657                 goto out;
1658         }
1659
1660         if (it_iu_len > MAX_MESSAGE_SIZE || it_iu_len < 64) {
1661                 rej->reason =
1662                     cpu_to_be32(SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
1663                 ret = -EINVAL;
1664                 PRINT_ERROR("rejected SRP_LOGIN_REQ because its"
1665                             " length (%d bytes) is invalid", it_iu_len);
1666                 goto reject;
1667         }
1668
1669         if ((req->req_flags & 0x3) == SRP_MULTICHAN_SINGLE) {
1670                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
1671
1672                 spin_lock_irq(&sdev->spinlock);
1673
1674                 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
1675                         if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
1676                             && !memcmp(ch->t_port_id, req->target_port_id, 16)
1677                             && param->port == ch->sport->port
1678                             && param->listen_id == ch->sport->sdev->cm_id
1679                             && ch->cm_id) {
1680                                 enum rdma_ch_state prev_state;
1681
1682                                 /* found an existing channel */
1683                                 TRACE_DBG("Found existing channel name= %s"
1684                                           " cm_id= %p state= %d",
1685                                           ch->sess_name, ch->cm_id, ch->state);
1686
1687                                 prev_state = ch->state;
1688                                 if (ch->state == RDMA_CHANNEL_LIVE)
1689                                         ch->state = RDMA_CHANNEL_DISCONNECTING;
1690                                 else if (ch->state == RDMA_CHANNEL_CONNECTING)
1691                                         list_del(&ch->list);
1692
1693                                 spin_unlock_irq(&sdev->spinlock);
1694
1695                                 rsp->rsp_flags =
1696                                         SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
1697
1698                                 if (prev_state == RDMA_CHANNEL_LIVE) {
1699                                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
1700                                         PRINT_INFO("disconnected"
1701                                           " session %s because a new"
1702                                           " SRP_LOGIN_REQ has been received.",
1703                                           ch->sess_name);
1704                                 } else if (prev_state ==
1705                                          RDMA_CHANNEL_CONNECTING) {
1706                                         PRINT_ERROR("%s", "rejected"
1707                                           " SRP_LOGIN_REQ because another login"
1708                                           " request is being processed.");
1709                                         ib_send_cm_rej(ch->cm_id,
1710                                                        IB_CM_REJ_NO_RESOURCES,
1711                                                        NULL, 0, NULL, 0);
1712                                         srpt_release_channel(ch, 1);
1713                                 }
1714
1715                                 spin_lock_irq(&sdev->spinlock);
1716                         }
1717                 }
1718
1719                 spin_unlock_irq(&sdev->spinlock);
1720
1721         } else
1722                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
1723
1724         if (((u64) (*(u64 *) req->target_port_id) !=
1725              cpu_to_be64(srpt_service_guid)) ||
1726             ((u64) (*(u64 *) (req->target_port_id + 8)) !=
1727              cpu_to_be64(srpt_service_guid))) {
1728                 rej->reason =
1729                     cpu_to_be32(SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
1730                 ret = -ENOMEM;
1731                 PRINT_ERROR("%s", "rejected SRP_LOGIN_REQ because it"
1732                        " has an invalid target port identifier.");
1733                 goto reject;
1734         }
1735
1736         ch = kzalloc(sizeof *ch, GFP_KERNEL);
1737         if (!ch) {
1738                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1739                 PRINT_ERROR("%s",
1740                             "rejected SRP_LOGIN_REQ because out of memory.");
1741                 ret = -ENOMEM;
1742                 goto reject;
1743         }
1744
1745         spin_lock_init(&ch->spinlock);
1746         memcpy(ch->i_port_id, req->initiator_port_id, 16);
1747         memcpy(ch->t_port_id, req->target_port_id, 16);
1748         ch->sport = &sdev->port[param->port - 1];
1749         ch->cm_id = cm_id;
1750         ch->state = RDMA_CHANNEL_CONNECTING;
1751         INIT_LIST_HEAD(&ch->cmd_wait_list);
1752         INIT_LIST_HEAD(&ch->active_scmnd_list);
1753
1754         ret = srpt_create_ch_ib(ch);
1755         if (ret) {
1756                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1757                 PRINT_ERROR("%s", "rejected SRP_LOGIN_REQ because creating"
1758                             " a new RDMA channel failed.");
1759                 goto free_ch;
1760         }
1761
1762         ret = srpt_ch_qp_rtr(ch, ch->qp);
1763         if (ret) {
1764                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1765                 PRINT_ERROR("rejected SRP_LOGIN_REQ because enabling"
1766                        " RTR failed (error code = %d)", ret);
1767                 goto destroy_ib;
1768         }
1769
1770         if (use_port_guid_in_session_name) {
1771                 /*
1772                  * If the kernel module parameter use_port_guid_in_session_name
1773                  * has been specified, use a combination of the target port
1774                  * GUID and the initiator port ID as the session name. This
1775                  * was the original behavior of the SRP target implementation
1776                  * (i.e. before the SRPT was included in OFED 1.3).
1777                  */
1778                 snprintf(ch->sess_name, sizeof(ch->sess_name),
1779                          "0x%016llx%016llx",
1780                          (unsigned long long)be64_to_cpu(*(u64 *)
1781                                 &sdev->port[param->port - 1].gid.raw[8]),
1782                          (unsigned long long)be64_to_cpu(*(u64 *)
1783                                 (ch->i_port_id + 8)));
1784         } else {
1785                 /*
1786                  * Default behavior: use the initator port identifier as the
1787                  * session name.
1788                  */
1789                 snprintf(ch->sess_name, sizeof(ch->sess_name),
1790                          "0x%016llx%016llx",
1791                          (unsigned long long)be64_to_cpu(*(u64 *)ch->i_port_id),
1792                          (unsigned long long)be64_to_cpu(*(u64 *)
1793                                  (ch->i_port_id + 8)));
1794         }
1795
1796         TRACE_DBG("registering session %s", ch->sess_name);
1797
1798         BUG_ON(!sdev->scst_tgt);
1799         ch->scst_sess = scst_register_session(sdev->scst_tgt, 0, ch->sess_name,
1800                                               NULL, NULL);
1801         if (!ch->scst_sess) {
1802                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1803                 TRACE_DBG("%s", "Failed to create scst sess");
1804                 goto destroy_ib;
1805         }
1806
1807         TRACE_DBG("Establish connection sess=%p name=%s cm_id=%p",
1808                   ch->scst_sess, ch->sess_name, ch->cm_id);
1809
1810         scst_sess_set_tgt_priv(ch->scst_sess, ch);
1811
1812         /* create srp_login_response */
1813         rsp->opcode = SRP_LOGIN_RSP;
1814         rsp->tag = req->tag;
1815         rsp->max_it_iu_len = req->req_it_iu_len;
1816         rsp->max_ti_iu_len = req->req_it_iu_len;
1817         rsp->buf_fmt =
1818             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1819         rsp->req_lim_delta = cpu_to_be32(SRPT_RQ_SIZE);
1820         atomic_set(&ch->req_lim_delta, 0);
1821
1822         /* create cm reply */
1823         rep_param->qp_num = ch->qp->qp_num;
1824         rep_param->private_data = (void *)rsp;
1825         rep_param->private_data_len = sizeof *rsp;
1826         rep_param->rnr_retry_count = 7;
1827         rep_param->flow_control = 1;
1828         rep_param->failover_accepted = 0;
1829         rep_param->srq = 1;
1830         rep_param->responder_resources = 4;
1831         rep_param->initiator_depth = 4;
1832
1833         ret = ib_send_cm_rep(cm_id, rep_param);
1834         if (ret) {
1835                 PRINT_ERROR("sending SRP_LOGIN_REQ response failed"
1836                             " (error code = %d)", ret);
1837                 goto release_channel;
1838         }
1839
1840         spin_lock_irq(&sdev->spinlock);
1841         list_add_tail(&ch->list, &sdev->rch_list);
1842         spin_unlock_irq(&sdev->spinlock);
1843
1844         goto out;
1845
1846 release_channel:
1847         scst_unregister_session(ch->scst_sess, 0, NULL);
1848         ch->scst_sess = NULL;
1849
1850 destroy_ib:
1851         ib_destroy_qp(ch->qp);
1852         ib_destroy_cq(ch->cq);
1853
1854 free_ch:
1855         kfree(ch);
1856
1857 reject:
1858         rej->opcode = SRP_LOGIN_REJ;
1859         rej->tag = req->tag;
1860         rej->buf_fmt =
1861             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1862
1863         ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
1864                              (void *)rej, sizeof *rej);
1865
1866 out:
1867         kfree(rep_param);
1868         kfree(rsp);
1869         kfree(rej);
1870
1871         return ret;
1872 }
1873
1874 /**
1875  * Release the channel with the specified cm_id.
1876  *
1877  * Returns one to indicate that the caller of srpt_cm_handler() should destroy
1878  * the cm_id.
1879  */
1880 static void srpt_find_and_release_channel(struct ib_cm_id *cm_id)
1881 {
1882         struct srpt_rdma_ch *ch;
1883
1884         ch = srpt_find_channel(cm_id, true);
1885         if (ch)
1886                 srpt_release_channel(ch, 0);
1887 }
1888
1889 static void srpt_cm_rej_recv(struct ib_cm_id *cm_id)
1890 {
1891         PRINT_INFO("%s", "Received InfiniBand REJ packet.");
1892         srpt_find_and_release_channel(cm_id);
1893 }
1894
1895 /**
1896  * Process an IB_CM_RTU_RECEIVED or IB_CM_USER_ESTABLISHED event.
1897  *
1898  * An IB_CM_RTU_RECEIVED message indicates that the connection is established
1899  * and that the recipient may begin transmitting (RTU = ready to use).
1900  */
1901 static int srpt_cm_rtu_recv(struct ib_cm_id *cm_id)
1902 {
1903         struct srpt_rdma_ch *ch;
1904         int ret;
1905
1906         ch = srpt_find_channel(cm_id, false);
1907         if (!ch)
1908                 return -EINVAL;
1909
1910         if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_CONNECTING,
1911                                             RDMA_CHANNEL_LIVE)) {
1912                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1913
1914                 ret = srpt_ch_qp_rts(ch, ch->qp);
1915
1916                 list_for_each_entry_safe(ioctx, ioctx_tmp, &ch->cmd_wait_list,
1917                                          wait_list) {
1918                         list_del(&ioctx->wait_list);
1919                         srpt_handle_new_iu(ch, ioctx);
1920                 }
1921                 if (ret && srpt_test_and_set_channel_state(ch,
1922                                         RDMA_CHANNEL_LIVE,
1923                                         RDMA_CHANNEL_DISCONNECTING)) {
1924                         TRACE_DBG("cm_id=%p sess_name=%s state=%d",
1925                                   cm_id, ch->sess_name, ch->state);
1926                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
1927                 }
1928         } else if (ch->state == RDMA_CHANNEL_DISCONNECTING) {
1929                 TRACE_DBG("cm_id=%p sess_name=%s state=%d",
1930                           cm_id, ch->sess_name, ch->state);
1931                 ib_send_cm_dreq(ch->cm_id, NULL, 0);
1932                 ret = -EAGAIN;
1933         } else
1934                 ret = 0;
1935
1936         return ret;
1937 }
1938
1939 static void srpt_cm_timewait_exit(struct ib_cm_id *cm_id)
1940 {
1941         PRINT_INFO("%s", "Received InfiniBand TimeWait exit.");
1942         srpt_find_and_release_channel(cm_id);
1943 }
1944
1945 static void srpt_cm_rep_error(struct ib_cm_id *cm_id)
1946 {
1947         PRINT_INFO("%s", "Received InfiniBand REP error.");
1948         srpt_find_and_release_channel(cm_id);
1949 }
1950
1951 static int srpt_cm_dreq_recv(struct ib_cm_id *cm_id)
1952 {
1953         struct srpt_rdma_ch *ch;
1954
1955         ch = srpt_find_channel(cm_id, false);
1956         if (!ch)
1957                 return -EINVAL;
1958
1959         TRACE_DBG("%s: cm_id= %p ch->state= %d",
1960                  __func__, cm_id, ch->state);
1961
1962         switch (ch->state) {
1963         case RDMA_CHANNEL_LIVE:
1964         case RDMA_CHANNEL_CONNECTING:
1965                 ib_send_cm_drep(ch->cm_id, NULL, 0);
1966                 PRINT_INFO("Received DREQ and sent DREP for session %s.",
1967                            ch->sess_name);
1968                 break;
1969         case RDMA_CHANNEL_DISCONNECTING:
1970         default:
1971                 break;
1972         }
1973
1974         return 0;
1975 }
1976
1977 static void srpt_cm_drep_recv(struct ib_cm_id *cm_id)
1978 {
1979         PRINT_INFO("%s", "Received InfiniBand DREP message.");
1980         srpt_find_and_release_channel(cm_id);
1981 }
1982
1983 /**
1984  * IB connection manager callback function.
1985  *
1986  * A non-zero return value will make the caller destroy the CM ID.
1987  *
1988  * Note: srpt_add_one passes a struct srpt_device* as the third argument to
1989  * the ib_create_cm_id() call.
1990  */
1991 static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
1992 {
1993         int ret = 0;
1994
1995         switch (event->event) {
1996         case IB_CM_REQ_RECEIVED:
1997                 ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd,
1998                                        event->private_data);
1999                 break;
2000         case IB_CM_REJ_RECEIVED:
2001                 srpt_cm_rej_recv(cm_id);
2002                 ret = -EINVAL;
2003                 break;
2004         case IB_CM_RTU_RECEIVED:
2005         case IB_CM_USER_ESTABLISHED:
2006                 ret = srpt_cm_rtu_recv(cm_id);
2007                 break;
2008         case IB_CM_DREQ_RECEIVED:
2009                 ret = srpt_cm_dreq_recv(cm_id);
2010                 break;
2011         case IB_CM_DREP_RECEIVED:
2012                 srpt_cm_drep_recv(cm_id);
2013                 ret = -EINVAL;
2014                 break;
2015         case IB_CM_TIMEWAIT_EXIT:
2016                 srpt_cm_timewait_exit(cm_id);
2017                 ret = -EINVAL;
2018                 break;
2019         case IB_CM_REP_ERROR:
2020                 srpt_cm_rep_error(cm_id);
2021                 ret = -EINVAL;
2022                 break;
2023         default:
2024                 break;
2025         }
2026
2027         return ret;
2028 }
2029
2030 static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
2031                                  struct srpt_ioctx *ioctx,
2032                                  struct scst_cmd *scmnd)
2033 {
2034         struct scatterlist *scat;
2035         scst_data_direction dir;
2036         struct rdma_iu *riu;
2037         struct srp_direct_buf *db;
2038         dma_addr_t dma_addr;
2039         struct ib_sge *sge;
2040         u64 raddr;
2041         u32 rsize;
2042         u32 tsize;
2043         u32 dma_len;
2044         int count, nrdma;
2045         int i, j, k;
2046
2047         scat = scst_cmd_get_sg(scmnd);
2048         dir = scst_cmd_get_data_direction(scmnd);
2049         count = ib_dma_map_sg(ch->sport->sdev->device, scat,
2050                               scst_cmd_get_sg_cnt(scmnd),
2051                               scst_to_tgt_dma_dir(dir));
2052         if (unlikely(!count))
2053                 return -EBUSY;
2054
2055         if (ioctx->rdma_ius && ioctx->n_rdma_ius)
2056                 nrdma = ioctx->n_rdma_ius;
2057         else {
2058                 nrdma = count / SRPT_DEF_SG_PER_WQE + ioctx->n_rbuf;
2059
2060                 ioctx->rdma_ius = kzalloc(nrdma * sizeof *riu,
2061                                           scst_cmd_atomic(scmnd)
2062                                           ? GFP_ATOMIC : GFP_KERNEL);
2063                 if (!ioctx->rdma_ius) {
2064                         ib_dma_unmap_sg(ch->sport->sdev->device,
2065                                         scat, scst_cmd_get_sg_cnt(scmnd),
2066                                         scst_to_tgt_dma_dir(dir));
2067                         return -ENOMEM;
2068                 }
2069
2070                 ioctx->n_rdma_ius = nrdma;
2071         }
2072
2073         db = ioctx->rbufs;
2074         tsize = (dir == SCST_DATA_READ) ?
2075                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2076         dma_len = sg_dma_len(&scat[0]);
2077         riu = ioctx->rdma_ius;
2078
2079         /*
2080          * For each remote desc - calculate the #ib_sge.
2081          * If #ib_sge < SRPT_DEF_SG_PER_WQE per rdma operation then
2082          *      each remote desc rdma_iu is required a rdma wr;
2083          * else
2084          *      we need to allocate extra rdma_iu to carry extra #ib_sge in
2085          *      another rdma wr
2086          */
2087         for (i = 0, j = 0;
2088              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2089                 rsize = be32_to_cpu(db->len);
2090                 raddr = be64_to_cpu(db->va);
2091                 riu->raddr = raddr;
2092                 riu->rkey = be32_to_cpu(db->key);
2093                 riu->sge_cnt = 0;
2094
2095                 /* calculate how many sge required for this remote_buf */
2096                 while (rsize > 0 && tsize > 0) {
2097
2098                         if (rsize >= dma_len) {
2099                                 tsize -= dma_len;
2100                                 rsize -= dma_len;
2101                                 raddr += dma_len;
2102
2103                                 if (tsize > 0) {
2104                                         ++j;
2105                                         if (j < count)
2106                                                 dma_len = sg_dma_len(&scat[j]);
2107                                 }
2108                         } else {
2109                                 tsize -= rsize;
2110                                 dma_len -= rsize;
2111                                 rsize = 0;
2112                         }
2113
2114                         ++riu->sge_cnt;
2115
2116                         if (rsize > 0 && riu->sge_cnt == SRPT_DEF_SG_PER_WQE) {
2117                                 riu->sge =
2118                                     kmalloc(riu->sge_cnt * sizeof *riu->sge,
2119                                             scst_cmd_atomic(scmnd)
2120                                             ? GFP_ATOMIC : GFP_KERNEL);
2121                                 if (!riu->sge)
2122                                         goto free_mem;
2123
2124                                 ++ioctx->n_rdma;
2125                                 ++riu;
2126                                 riu->sge_cnt = 0;
2127                                 riu->raddr = raddr;
2128                                 riu->rkey = be32_to_cpu(db->key);
2129                         }
2130                 }
2131
2132                 riu->sge = kmalloc(riu->sge_cnt * sizeof *riu->sge,
2133                                    scst_cmd_atomic(scmnd)
2134                                    ? GFP_ATOMIC : GFP_KERNEL);
2135
2136                 if (!riu->sge)
2137                         goto free_mem;
2138
2139                 ++ioctx->n_rdma;
2140         }
2141
2142         db = ioctx->rbufs;
2143         scat = scst_cmd_get_sg(scmnd);
2144         tsize = (dir == SCST_DATA_READ) ?
2145                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2146         riu = ioctx->rdma_ius;
2147         dma_len = sg_dma_len(&scat[0]);
2148         dma_addr = sg_dma_address(&scat[0]);
2149
2150         /* this second loop is really mapped sg_addres to rdma_iu->ib_sge */
2151         for (i = 0, j = 0;
2152              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2153                 rsize = be32_to_cpu(db->len);
2154                 sge = riu->sge;
2155                 k = 0;
2156
2157                 while (rsize > 0 && tsize > 0) {
2158                         sge->addr = dma_addr;
2159                         sge->lkey = ch->sport->sdev->mr->lkey;
2160
2161                         if (rsize >= dma_len) {
2162                                 sge->length =
2163                                         (tsize < dma_len) ? tsize : dma_len;
2164                                 tsize -= dma_len;
2165                                 rsize -= dma_len;
2166
2167                                 if (tsize > 0) {
2168                                         ++j;
2169                                         if (j < count) {
2170                                                 dma_len = sg_dma_len(&scat[j]);
2171                                                 dma_addr =
2172                                                     sg_dma_address(&scat[j]);
2173                                         }
2174                                 }
2175                         } else {
2176                                 sge->length = (tsize < rsize) ? tsize : rsize;
2177                                 tsize -= rsize;
2178                                 dma_len -= rsize;
2179                                 dma_addr += rsize;
2180                                 rsize = 0;
2181                         }
2182
2183                         ++k;
2184                         if (k == riu->sge_cnt && rsize > 0) {
2185                                 ++riu;
2186                                 sge = riu->sge;
2187                                 k = 0;
2188                         } else if (rsize > 0)
2189                                 ++sge;
2190                 }
2191         }
2192
2193         return 0;
2194
2195 free_mem:
2196         while (ioctx->n_rdma)
2197                 kfree(ioctx->rdma_ius[ioctx->n_rdma--].sge);
2198
2199         kfree(ioctx->rdma_ius);
2200
2201         ib_dma_unmap_sg(ch->sport->sdev->device,
2202                         scat, scst_cmd_get_sg_cnt(scmnd),
2203                         scst_to_tgt_dma_dir(dir));
2204
2205         return -ENOMEM;
2206 }
2207
2208 static int srpt_perform_rdmas(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2209                               scst_data_direction dir)
2210 {
2211         struct ib_send_wr wr;
2212         struct ib_send_wr *bad_wr;
2213         struct rdma_iu *riu;
2214         int i;
2215         int ret = 0;
2216
2217         riu = ioctx->rdma_ius;
2218         memset(&wr, 0, sizeof wr);
2219
2220         for (i = 0; i < ioctx->n_rdma; ++i, ++riu) {
2221                 wr.opcode = (dir == SCST_DATA_READ) ?
2222                     IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;
2223                 wr.next = NULL;
2224                 wr.wr_id = ioctx->index;
2225                 wr.wr.rdma.remote_addr = riu->raddr;
2226                 wr.wr.rdma.rkey = riu->rkey;
2227                 wr.num_sge = riu->sge_cnt;
2228                 wr.sg_list = riu->sge;
2229
2230                 /* only get completion event for the last rdma wr */
2231                 if (i == (ioctx->n_rdma - 1) && dir == SCST_DATA_WRITE)
2232                         wr.send_flags = IB_SEND_SIGNALED;
2233
2234                 ret = ib_post_send(ch->qp, &wr, &bad_wr);
2235                 if (ret)
2236                         break;
2237         }
2238
2239         return ret;
2240 }
2241
2242 /*
2243  * Start data transfer between initiator and target. Must not block.
2244  */
2245 static int srpt_xfer_data(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2246                           struct scst_cmd *scmnd)
2247 {
2248         int ret;
2249
2250         ret = srpt_map_sg_to_ib_sge(ch, ioctx, scmnd);
2251         if (ret) {
2252                 PRINT_ERROR("%s[%d] ret=%d", __func__, __LINE__, ret);
2253                 ret = SCST_TGT_RES_QUEUE_FULL;
2254                 goto out;
2255         }
2256
2257         ret = srpt_perform_rdmas(ch, ioctx, scst_cmd_get_data_direction(scmnd));
2258         if (ret) {
2259                 PRINT_ERROR("%s[%d] ret=%d", __func__, __LINE__, ret);
2260                 if (ret == -EAGAIN || ret == -ENOMEM)
2261                         ret = SCST_TGT_RES_QUEUE_FULL;
2262                 else
2263                         ret = SCST_TGT_RES_FATAL_ERROR;
2264                 goto out;
2265         }
2266
2267         ret = SCST_TGT_RES_SUCCESS;
2268
2269 out:
2270         return ret;
2271 }
2272
2273 /*
2274  * Called by the SCST core to inform ib_srpt that data reception from the
2275  * initiator should start (SCST_DATA_WRITE). Must not block.
2276  */
2277 static int srpt_rdy_to_xfer(struct scst_cmd *scmnd)
2278 {
2279         struct srpt_rdma_ch *ch;
2280         struct srpt_ioctx *ioctx;
2281
2282         ioctx = scst_cmd_get_tgt_priv(scmnd);
2283         BUG_ON(!ioctx);
2284
2285         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2286         BUG_ON(!ch);
2287
2288         if (ch->state == RDMA_CHANNEL_DISCONNECTING)
2289                 return SCST_TGT_RES_FATAL_ERROR;
2290         else if (ch->state == RDMA_CHANNEL_CONNECTING)
2291                 return SCST_TGT_RES_QUEUE_FULL;
2292
2293         ioctx->state = SRPT_STATE_NEED_DATA;
2294
2295         return srpt_xfer_data(ch, ioctx, scmnd);
2296 }
2297
2298 /*
2299  * Called by the SCST core. Transmits the response buffer and status held in
2300  * 'scmnd'. Must not block.
2301  */
2302 static int srpt_xmit_response(struct scst_cmd *scmnd)
2303 {
2304         struct srpt_rdma_ch *ch;
2305         struct srpt_ioctx *ioctx;
2306         struct srp_rsp *srp_rsp;
2307         u64 tag;
2308         int ret = SCST_TGT_RES_SUCCESS;
2309         int dir;
2310         int status;
2311
2312         ioctx = scst_cmd_get_tgt_priv(scmnd);
2313         BUG_ON(!ioctx);
2314
2315         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2316         BUG_ON(!ch);
2317
2318         tag = scst_cmd_get_tag(scmnd);
2319
2320         ioctx->state = SRPT_STATE_PROCESSED;
2321
2322         if (ch->state != RDMA_CHANNEL_LIVE) {
2323                 PRINT_ERROR("%s: tag= %lld channel in bad state %d",
2324                        __func__, (unsigned long long)tag, ch->state);
2325
2326                 if (ch->state == RDMA_CHANNEL_DISCONNECTING)
2327                         ret = SCST_TGT_RES_FATAL_ERROR;
2328                 else if (ch->state == RDMA_CHANNEL_CONNECTING)
2329                         ret = SCST_TGT_RES_QUEUE_FULL;
2330
2331                 if (unlikely(scst_cmd_aborted(scmnd)))
2332                         goto out_aborted;
2333
2334                 goto out;
2335         }
2336
2337         ib_dma_sync_single_for_cpu(ch->sport->sdev->device, ioctx->dma,
2338                                    MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
2339
2340         srp_rsp = ioctx->buf;
2341
2342         if (unlikely(scst_cmd_aborted(scmnd))) {
2343                 PRINT_ERROR("%s: tag= %lld already get aborted",
2344                        __func__, (unsigned long long)tag);
2345                 goto out_aborted;
2346         }
2347
2348         dir = scst_cmd_get_data_direction(scmnd);
2349         status = scst_cmd_get_status(scmnd) & 0xff;
2350
2351         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE, NO_ADD_SENSE, tag);
2352
2353         if (SCST_SENSE_VALID(scst_cmd_get_sense_buffer(scmnd))) {
2354                 srp_rsp->sense_data_len = scst_cmd_get_sense_buffer_len(scmnd);
2355                 if (srp_rsp->sense_data_len >
2356                     (MAX_MESSAGE_SIZE - sizeof *srp_rsp))
2357                         srp_rsp->sense_data_len =
2358                             MAX_MESSAGE_SIZE - sizeof *srp_rsp;
2359
2360                 memcpy((u8 *) (srp_rsp + 1), scst_cmd_get_sense_buffer(scmnd),
2361                        srp_rsp->sense_data_len);
2362
2363                 srp_rsp->sense_data_len = cpu_to_be32(srp_rsp->sense_data_len);
2364                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
2365
2366                 if (!status)
2367                         status = SAM_STAT_CHECK_CONDITION;
2368         }
2369
2370         srp_rsp->status = status;
2371
2372         /* For read commands, transfer the data to the initiator. */
2373         if (dir == SCST_DATA_READ && scst_cmd_get_resp_data_len(scmnd)) {
2374                 ret = srpt_xfer_data(ch, ioctx, scmnd);
2375                 if (ret != SCST_TGT_RES_SUCCESS) {
2376                         PRINT_ERROR("%s: tag= %lld xfer_data failed",
2377                                     __func__, (unsigned long long)tag);
2378                         goto out;
2379                 }
2380         }
2381
2382         if (srpt_post_send(ch, ioctx,
2383                            sizeof *srp_rsp +
2384                            be32_to_cpu(srp_rsp->sense_data_len))) {
2385                 PRINT_ERROR("%s: ch->state= %d tag= %lld",
2386                             __func__, ch->state,
2387                             (unsigned long long)tag);
2388                 ret = SCST_TGT_RES_FATAL_ERROR;
2389         }
2390
2391 out:
2392         return ret;
2393
2394 out_aborted:
2395         ret = SCST_TGT_RES_SUCCESS;
2396         scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_ABORTED);
2397         ioctx->state = SRPT_STATE_ABORTED;
2398         WARN_ON(scmnd->state != SCST_CMD_STATE_XMIT_WAIT);
2399         scst_tgt_cmd_done(scmnd, SCST_CONTEXT_SAME);
2400         goto out;
2401 }
2402
2403 /*
2404  * Called by the SCST core to inform ib_srpt that a received task management
2405  * function has been completed. Must not block.
2406  */
2407 static void srpt_tsk_mgmt_done(struct scst_mgmt_cmd *mcmnd)
2408 {
2409         struct srpt_rdma_ch *ch;
2410         struct srpt_mgmt_ioctx *mgmt_ioctx;
2411         struct srpt_ioctx *ioctx;
2412         int rsp_len;
2413
2414         mgmt_ioctx = scst_mgmt_cmd_get_tgt_priv(mcmnd);
2415         BUG_ON(!mgmt_ioctx);
2416
2417         ch = mgmt_ioctx->ch;
2418         BUG_ON(!ch);
2419
2420         ioctx = mgmt_ioctx->ioctx;
2421         BUG_ON(!ioctx);
2422
2423         TRACE_DBG("%s: tsk_mgmt_done for tag= %lld status=%d",
2424                   __func__, (unsigned long long)mgmt_ioctx->tag,
2425                   scst_mgmt_cmd_get_status(mcmnd));
2426
2427         rsp_len = srpt_build_tskmgmt_rsp(ch, ioctx,
2428                                          (scst_mgmt_cmd_get_status(mcmnd) ==
2429                                           SCST_MGMT_STATUS_SUCCESS) ?
2430                                          SRP_TSK_MGMT_SUCCESS :
2431                                          SRP_TSK_MGMT_FAILED,
2432                                          mgmt_ioctx->tag);
2433         srpt_post_send(ch, ioctx, rsp_len);
2434
2435         scst_mgmt_cmd_set_tgt_priv(mcmnd, NULL);
2436
2437         kfree(mgmt_ioctx);
2438 }
2439
2440 /*
2441  * Called by the SCST core to inform ib_srpt that the command 'scmnd' is about
2442  * to be freed. May be called in IRQ context.
2443  */
2444 static void srpt_on_free_cmd(struct scst_cmd *scmnd)
2445 {
2446         struct srpt_rdma_ch *ch;
2447         struct srpt_ioctx *ioctx;
2448
2449         ioctx = scst_cmd_get_tgt_priv(scmnd);
2450         BUG_ON(!ioctx);
2451
2452         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2453         BUG_ON(!ch);
2454
2455         spin_lock_irq(&ch->spinlock);
2456         list_del(&ioctx->scmnd_list);
2457         ch->active_scmnd_cnt--;
2458         spin_unlock_irq(&ch->spinlock);
2459
2460         srpt_reset_ioctx(ch, ioctx);
2461         scst_cmd_set_tgt_priv(scmnd, NULL);
2462 }
2463
2464 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2465 /* A vanilla 2.6.19 or older kernel without backported OFED kernel headers. */
2466 static void srpt_refresh_port_work(void *ctx)
2467 #else
2468 static void srpt_refresh_port_work(struct work_struct *work)
2469 #endif
2470 {
2471 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2472         struct srpt_port *sport = (struct srpt_port *)ctx;
2473 #else
2474         struct srpt_port *sport = container_of(work, struct srpt_port, work);
2475 #endif
2476
2477         srpt_refresh_port(sport);
2478 }
2479
2480 /*
2481  * Called by the SCST core to detect target adapters. Returns the number of
2482  * detected target adapters.
2483  */
2484 static int srpt_detect(struct scst_tgt_template *tp)
2485 {
2486         int device_count;
2487
2488         TRACE_ENTRY();
2489
2490         device_count = atomic_read(&srpt_device_count);
2491
2492         TRACE_EXIT_RES(device_count);
2493
2494         return device_count;
2495 }
2496
2497 /*
2498  * Callback function called by the SCST core from scst_unregister() to free up
2499  * the resources associated with device scst_tgt.
2500  */
2501 static int srpt_release(struct scst_tgt *scst_tgt)
2502 {
2503         struct srpt_device *sdev = scst_tgt_get_tgt_priv(scst_tgt);
2504         struct srpt_rdma_ch *ch, *tmp_ch;
2505
2506         TRACE_ENTRY();
2507
2508         BUG_ON(!scst_tgt);
2509 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
2510         WARN_ON(!sdev);
2511         if (!sdev)
2512                 return -ENODEV;
2513 #else
2514         if (WARN_ON(!sdev))
2515                 return -ENODEV;
2516 #endif
2517
2518         srpt_unregister_procfs_entry(scst_tgt->tgtt);
2519
2520         spin_lock_irq(&sdev->spinlock);
2521         list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
2522                 list_del(&ch->list);
2523                 spin_unlock_irq(&sdev->spinlock);
2524                 srpt_release_channel(ch, 1);
2525                 spin_lock_irq(&sdev->spinlock);
2526         }
2527         spin_unlock_irq(&sdev->spinlock);
2528
2529         srpt_unregister_mad_agent(sdev);
2530
2531         scst_tgt_set_tgt_priv(scst_tgt, NULL);
2532
2533         TRACE_EXIT();
2534
2535         return 0;
2536 }
2537
2538 /*
2539  * Entry point for ib_srpt's kernel thread. This kernel thread is only created
2540  * when the module parameter 'thread' is not zero (the default is zero).
2541  * This thread processes the ioctx list srpt_thread.thread_ioctx_list.
2542  *
2543  * @pre thread != 0
2544  */
2545 static int srpt_ioctx_thread(void *arg)
2546 {
2547         struct srpt_ioctx *ioctx;
2548
2549         /* Hibernation / freezing of the SRPT kernel thread is not supported. */
2550         current->flags |= PF_NOFREEZE;
2551
2552         spin_lock_irq(&srpt_thread.thread_lock);
2553         while (!kthread_should_stop()) {
2554                 wait_queue_t wait;
2555                 init_waitqueue_entry(&wait, current);
2556
2557                 if (!srpt_test_ioctx_list()) {
2558                         add_wait_queue_exclusive(&ioctx_list_waitQ, &wait);
2559
2560                         for (;;) {
2561                                 set_current_state(TASK_INTERRUPTIBLE);
2562                                 if (srpt_test_ioctx_list())
2563                                         break;
2564                                 spin_unlock_irq(&srpt_thread.thread_lock);
2565                                 schedule();
2566                                 spin_lock_irq(&srpt_thread.thread_lock);
2567                         }
2568                         set_current_state(TASK_RUNNING);
2569                         remove_wait_queue(&ioctx_list_waitQ, &wait);
2570                 }
2571
2572                 while (!list_empty(&srpt_thread.thread_ioctx_list)) {
2573                         ioctx = list_entry(srpt_thread.thread_ioctx_list.next,
2574                                            struct srpt_ioctx, comp_list);
2575
2576                         list_del(&ioctx->comp_list);
2577
2578                         spin_unlock_irq(&srpt_thread.thread_lock);
2579                         switch (ioctx->op) {
2580                         case IB_WC_SEND:
2581                                 srpt_handle_send_comp(ioctx->ch, ioctx,
2582                                         SCST_CONTEXT_DIRECT);
2583                                 break;
2584                         case IB_WC_RDMA_WRITE:
2585                         case IB_WC_RDMA_READ:
2586                                 srpt_handle_rdma_comp(ioctx->ch, ioctx);
2587                                 break;
2588                         case IB_WC_RECV:
2589                                 srpt_handle_new_iu(ioctx->ch, ioctx);
2590                                 break;
2591                         default:
2592                                 break;
2593                         }
2594                         spin_lock_irq(&srpt_thread.thread_lock);
2595                 }
2596         }
2597         spin_unlock_irq(&srpt_thread.thread_lock);
2598
2599         return 0;
2600 }
2601
2602 /* SCST target template for the SRP target implementation. */
2603 static struct scst_tgt_template srpt_template = {
2604         .name = DRV_NAME,
2605         .sg_tablesize = SRPT_DEF_SG_TABLESIZE,
2606         .xmit_response_atomic = 1,
2607         .rdy_to_xfer_atomic = 1,
2608         .no_proc_entry = 0,
2609         .detect = srpt_detect,
2610         .release = srpt_release,
2611         .xmit_response = srpt_xmit_response,
2612         .rdy_to_xfer = srpt_rdy_to_xfer,
2613         .on_free_cmd = srpt_on_free_cmd,
2614         .task_mgmt_fn_done = srpt_tsk_mgmt_done
2615 };
2616
2617 /*
2618  * The callback function srpt_release_class_dev() is called whenever a
2619  * device is removed from the /sys/class/infiniband_srpt device class.
2620  */
2621 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2622 static void srpt_release_class_dev(struct class_device *class_dev)
2623 #else
2624 static void srpt_release_class_dev(struct device *dev)
2625 #endif
2626 {
2627 }
2628
2629 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2630 static int srpt_trace_level_show(struct seq_file *seq, void *v)
2631 {
2632         return scst_proc_log_entry_read(seq, trace_flag, NULL);
2633 }
2634
2635 static ssize_t srpt_proc_trace_level_write(struct file *file,
2636         const char __user *buf, size_t length, loff_t *off)
2637 {
2638         return scst_proc_log_entry_write(file, buf, length, &trace_flag,
2639                 DEFAULT_SRPT_TRACE_FLAGS, NULL);
2640 }
2641
2642 static struct scst_proc_data srpt_log_proc_data = {
2643         SCST_DEF_RW_SEQ_OP(srpt_proc_trace_level_write)
2644         .show = srpt_trace_level_show,
2645 };
2646 #endif
2647
2648 static struct class_attribute srpt_class_attrs[] = {
2649         __ATTR_NULL,
2650 };
2651
2652 static struct class srpt_class = {
2653         .name = "infiniband_srpt",
2654 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2655         .release = srpt_release_class_dev,
2656 #else
2657         .dev_release = srpt_release_class_dev,
2658 #endif
2659         .class_attrs = srpt_class_attrs,
2660 };
2661
2662 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2663 static ssize_t show_login_info(struct class_device *class_dev, char *buf)
2664 #else
2665 static ssize_t show_login_info(struct device *dev,
2666                                struct device_attribute *attr, char *buf)
2667 #endif
2668 {
2669         struct srpt_device *sdev =
2670 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2671                 container_of(class_dev, struct srpt_device, class_dev);
2672 #else
2673                 container_of(dev, struct srpt_device, dev);
2674 #endif
2675         struct srpt_port *sport;
2676         int i;
2677         int len = 0;
2678
2679         for (i = 0; i < sdev->device->phys_port_cnt; i++) {
2680                 sport = &sdev->port[i];
2681
2682                 len += sprintf(buf + len,
2683                                "tid_ext=%016llx,ioc_guid=%016llx,pkey=ffff,"
2684                                "dgid=%04x%04x%04x%04x%04x%04x%04x%04x,"
2685                                "service_id=%016llx\n",
2686                                (unsigned long long) srpt_service_guid,
2687                                (unsigned long long) srpt_service_guid,
2688                                be16_to_cpu(((__be16 *) sport->gid.raw)[0]),
2689                                be16_to_cpu(((__be16 *) sport->gid.raw)[1]),
2690                                be16_to_cpu(((__be16 *) sport->gid.raw)[2]),
2691                                be16_to_cpu(((__be16 *) sport->gid.raw)[3]),
2692                                be16_to_cpu(((__be16 *) sport->gid.raw)[4]),
2693                                be16_to_cpu(((__be16 *) sport->gid.raw)[5]),
2694                                be16_to_cpu(((__be16 *) sport->gid.raw)[6]),
2695                                be16_to_cpu(((__be16 *) sport->gid.raw)[7]),
2696                                (unsigned long long) srpt_service_guid);
2697         }
2698
2699         return len;
2700 }
2701
2702 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2703 static CLASS_DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2704 #else
2705 static DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2706 #endif
2707
2708 /*
2709  * Callback function called by the InfiniBand core when either an InfiniBand
2710  * device has been added or during the ib_register_client() call for each
2711  * registered InfiniBand device.
2712  */
2713 static void srpt_add_one(struct ib_device *device)
2714 {
2715         struct srpt_device *sdev;
2716         struct srpt_port *sport;
2717         struct ib_srq_init_attr srq_attr;
2718         int i;
2719
2720         TRACE_ENTRY();
2721
2722         sdev = kzalloc(sizeof *sdev, GFP_KERNEL);
2723         if (!sdev)
2724                 return;
2725
2726         sdev->device = device;
2727
2728 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2729         sdev->class_dev.class = &srpt_class;
2730         sdev->class_dev.dev = device->dma_device;
2731         snprintf(sdev->class_dev.class_id, BUS_ID_SIZE,
2732                  "srpt-%s", device->name);
2733 #else
2734         sdev->dev.class = &srpt_class;
2735         sdev->dev.parent = device->dma_device;
2736 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
2737         snprintf(sdev->dev.bus_id, BUS_ID_SIZE, "srpt-%s", device->name);
2738 #else
2739         dev_set_name(&sdev->dev, "srpt-%s", device->name);
2740 #endif
2741 #endif
2742
2743 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2744         if (class_device_register(&sdev->class_dev))
2745                 goto free_dev;
2746         if (class_device_create_file(&sdev->class_dev,
2747                                      &class_device_attr_login_info))
2748                 goto err_dev;
2749 #else
2750         if (device_register(&sdev->dev))
2751                 goto free_dev;
2752         if (device_create_file(&sdev->dev, &dev_attr_login_info))
2753                 goto err_dev;
2754 #endif
2755
2756         if (ib_query_device(device, &sdev->dev_attr))
2757                 goto err_dev;
2758
2759         sdev->pd = ib_alloc_pd(device);
2760         if (IS_ERR(sdev->pd))
2761                 goto err_dev;
2762
2763         sdev->mr = ib_get_dma_mr(sdev->pd, IB_ACCESS_LOCAL_WRITE);
2764         if (IS_ERR(sdev->mr))
2765                 goto err_pd;
2766
2767         srq_attr.event_handler = srpt_srq_event;
2768         srq_attr.srq_context = (void *)sdev;
2769         srq_attr.attr.max_wr = min(SRPT_SRQ_SIZE, sdev->dev_attr.max_srq_wr);
2770         srq_attr.attr.max_sge = 1;
2771         srq_attr.attr.srq_limit = 0;
2772
2773         sdev->srq = ib_create_srq(sdev->pd, &srq_attr);
2774         if (IS_ERR(sdev->srq))
2775                 goto err_mr;
2776
2777         TRACE_DBG("%s: create SRQ #wr= %d max_allow=%d dev= %s",
2778                __func__, srq_attr.attr.max_wr,
2779               sdev->dev_attr.max_srq_wr, device->name);
2780
2781         if (!srpt_service_guid)
2782                 srpt_service_guid = be64_to_cpu(device->node_guid);
2783
2784         sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);
2785         if (IS_ERR(sdev->cm_id))
2786                 goto err_srq;
2787
2788         /* print out target login information */
2789         TRACE_DBG("Target login info: id_ext=%016llx,"
2790                   "ioc_guid=%016llx,pkey=ffff,service_id=%016llx",
2791                   (unsigned long long) srpt_service_guid,
2792                   (unsigned long long) srpt_service_guid,
2793                   (unsigned long long) srpt_service_guid);
2794
2795         /*
2796          * We do not have a consistent service_id (ie. also id_ext of target_id)
2797          * to identify this target. We currently use the guid of the first HCA
2798          * in the system as service_id; therefore, the target_id will change
2799          * if this HCA is gone bad and replaced by different HCA
2800          */
2801         if (ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0, NULL))
2802                 goto err_cm;
2803
2804         INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
2805                               srpt_event_handler);
2806         if (ib_register_event_handler(&sdev->event_handler))
2807                 goto err_cm;
2808
2809         if (srpt_alloc_ioctx_ring(sdev))
2810                 goto err_event;
2811
2812         INIT_LIST_HEAD(&sdev->rch_list);
2813         spin_lock_init(&sdev->spinlock);
2814
2815         for (i = 0; i < SRPT_SRQ_SIZE; ++i)
2816                 srpt_post_recv(sdev, sdev->ioctx_ring[i]);
2817
2818         ib_set_client_data(device, &srpt_client, sdev);
2819
2820         sdev->scst_tgt = scst_register(&srpt_template, NULL);
2821         if (!sdev->scst_tgt) {
2822                 PRINT_ERROR("SCST registration failed for %s.",
2823                             sdev->device->name);
2824                 goto err_ring;
2825         }
2826
2827         scst_tgt_set_tgt_priv(sdev->scst_tgt, sdev);
2828
2829         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
2830                 sport = &sdev->port[i - 1];
2831                 sport->sdev = sdev;
2832                 sport->port = i;
2833 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2834                 /*
2835                  * A vanilla 2.6.19 or older kernel without backported OFED
2836                  * kernel headers.
2837                  */
2838                 INIT_WORK(&sport->work, srpt_refresh_port_work, sport);
2839 #else
2840                 INIT_WORK(&sport->work, srpt_refresh_port_work);
2841 #endif
2842                 if (srpt_refresh_port(sport)) {
2843                         PRINT_ERROR("MAD registration failed for %s-%d.",
2844                                     sdev->device->name, i);
2845                         goto err_refresh_port;
2846                 }
2847         }
2848
2849         atomic_inc(&srpt_device_count);
2850
2851         TRACE_EXIT();
2852
2853         return;
2854
2855 err_refresh_port:
2856         scst_unregister(sdev->scst_tgt);
2857 err_ring:
2858         ib_set_client_data(device, &srpt_client, NULL);
2859         srpt_free_ioctx_ring(sdev);
2860 err_event:
2861         ib_unregister_event_handler(&sdev->event_handler);
2862 err_cm:
2863         ib_destroy_cm_id(sdev->cm_id);
2864 err_srq:
2865         ib_destroy_srq(sdev->srq);
2866 err_mr:
2867         ib_dereg_mr(sdev->mr);
2868 err_pd:
2869         ib_dealloc_pd(sdev->pd);
2870 err_dev:
2871 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2872         class_device_unregister(&sdev->class_dev);
2873 #else
2874         device_unregister(&sdev->dev);
2875 #endif
2876 free_dev:
2877         kfree(sdev);
2878
2879         TRACE_EXIT();
2880 }
2881
2882 /*
2883  * Callback function called by the InfiniBand core when either an InfiniBand
2884  * device has been removed or during the ib_unregister_client() call for each
2885  * registered InfiniBand device.
2886  */
2887 static void srpt_remove_one(struct ib_device *device)
2888 {
2889         int i;
2890         struct srpt_device *sdev;
2891
2892         TRACE_ENTRY();
2893
2894         sdev = ib_get_client_data(device, &srpt_client);
2895 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
2896         WARN_ON(!sdev);
2897         if (!sdev)
2898                 return;
2899 #else
2900         if (WARN_ON(!sdev))
2901                 return;
2902 #endif
2903
2904         /*
2905          * Cancel the work if it is queued. Wait until srpt_refresh_port_work()
2906          * finished if it is running.
2907          */
2908         for (i = 0; i < sdev->device->phys_port_cnt; i++)
2909 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
2910                 cancel_work_sync(&sdev->port[i].work);
2911 #else
2912                 /*
2913                  * cancel_work_sync() was introduced in kernel 2.6.22. Older
2914                  * kernels do not have a facility to cancel scheduled work.
2915                  */
2916                 PRINT_ERROR("%s", 
2917                        "your kernel does not provide cancel_work_sync().");
2918 #endif
2919
2920         scst_unregister(sdev->scst_tgt);
2921         sdev->scst_tgt = NULL;
2922
2923         ib_unregister_event_handler(&sdev->event_handler);
2924         ib_destroy_cm_id(sdev->cm_id);
2925         ib_destroy_srq(sdev->srq);
2926         ib_dereg_mr(sdev->mr);
2927         ib_dealloc_pd(sdev->pd);
2928 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2929         class_device_unregister(&sdev->class_dev);
2930 #else
2931         device_unregister(&sdev->dev);
2932 #endif
2933
2934         srpt_free_ioctx_ring(sdev);
2935         kfree(sdev);
2936
2937         TRACE_EXIT();
2938 }
2939
2940 /**
2941  * Create procfs entries for srpt. Currently the only procfs entry created
2942  * by this function is the "trace_level" entry.
2943  */
2944 static int srpt_register_procfs_entry(struct scst_tgt_template *tgt)
2945 {
2946         int res = 0;
2947 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2948         struct proc_dir_entry *p, *root;
2949
2950         root = scst_proc_get_tgt_root(tgt);
2951         WARN_ON(!root);
2952         if (root) {
2953                 /*
2954                  * Fill in the scst_proc_data::data pointer, which is used in
2955                  * a printk(KERN_INFO ...) statement in
2956                  * scst_proc_log_entry_write() in scst_proc.c.
2957                  */
2958                 srpt_log_proc_data.data = (char *)tgt->name;
2959                 p = scst_create_proc_entry(root, SRPT_PROC_TRACE_LEVEL_NAME,
2960                                            &srpt_log_proc_data);
2961                 if (!p)
2962                         res = -ENOMEM;
2963         } else
2964                 res = -ENOMEM;
2965
2966 #endif
2967         return res;
2968 }
2969
2970 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt)
2971 {
2972 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2973         struct proc_dir_entry *root;
2974
2975         root = scst_proc_get_tgt_root(tgt);
2976         WARN_ON(!root);
2977         if (root)
2978                 remove_proc_entry(SRPT_PROC_TRACE_LEVEL_NAME, root);
2979 #endif
2980 }
2981
2982 /*
2983  * Module initialization.
2984  *
2985  * Note: since ib_register_client() registers callback functions, and since at
2986  * least one of these callback functions (srpt_add_one()) calls SCST functions,
2987  * the SCST target template must be registered before ib_register_client() is
2988  * called.
2989  */
2990 static int __init srpt_init_module(void)
2991 {
2992         int ret;
2993
2994         ret = class_register(&srpt_class);
2995         if (ret) {
2996                 PRINT_ERROR("%s", "couldn't register class ib_srpt");
2997                 goto out;
2998         }
2999
3000         ret = scst_register_target_template(&srpt_template);
3001         if (ret < 0) {
3002                 PRINT_ERROR("%s", "couldn't register with scst");
3003                 ret = -ENODEV;
3004                 goto out_unregister_class;
3005         }
3006
3007         ret = srpt_register_procfs_entry(&srpt_template);
3008         if (ret) {
3009                 PRINT_ERROR("%s", "couldn't register procfs entry");
3010                 goto out_unregister_target;
3011         }
3012
3013         ret = ib_register_client(&srpt_client);
3014         if (ret) {
3015                 PRINT_ERROR("%s", "couldn't register IB client");
3016                 goto out_unregister_target;
3017         }
3018
3019         if (thread) {
3020                 spin_lock_init(&srpt_thread.thread_lock);
3021                 INIT_LIST_HEAD(&srpt_thread.thread_ioctx_list);
3022                 srpt_thread.thread = kthread_run(srpt_ioctx_thread,
3023                                                  NULL, "srpt_thread");
3024                 if (IS_ERR(srpt_thread.thread)) {
3025                         srpt_thread.thread = NULL;
3026                         thread = 0;
3027                 }
3028         }
3029
3030         return 0;
3031
3032 out_unregister_target:
3033         /*
3034          * Note: the procfs entry is unregistered in srpt_release(), which is
3035          * called by scst_unregister_target_template().
3036          */
3037         scst_unregister_target_template(&srpt_template);
3038 out_unregister_class:
3039         class_unregister(&srpt_class);
3040 out:
3041         return ret;
3042 }
3043
3044 static void __exit srpt_cleanup_module(void)
3045 {
3046         TRACE_ENTRY();
3047
3048         if (srpt_thread.thread)
3049                 kthread_stop(srpt_thread.thread);
3050         ib_unregister_client(&srpt_client);
3051         scst_unregister_target_template(&srpt_template);
3052         class_unregister(&srpt_class);
3053
3054         TRACE_EXIT();
3055 }
3056
3057 module_init(srpt_init_module);
3058 module_exit(srpt_cleanup_module);