514746e7d8b9da0692566fac1a5b78fa581cf432
[mirror/scst/.git] / srpt / src / ib_srpt.c
1 /*
2  * Copyright (c) 2006 - 2009 Mellanox Technology Inc.  All rights reserved.
3  * Copyright (C) 2008 Vladislav Bolkhovitin <vst@vlnb.net>
4  * Copyright (C) 2008 - 2009 Bart Van Assche <bart.vanassche@gmail.com>
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  */
35
36 #include <linux/module.h>
37 #include <linux/init.h>
38 #include <linux/slab.h>
39 #include <linux/err.h>
40 #include <linux/ctype.h>
41 #include <linux/string.h>
42 #include <linux/kthread.h>
43 #include <asm/atomic.h>
44 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
45 #include <linux/proc_fs.h>
46 #include <linux/seq_file.h>
47 #endif
48 #include "ib_srpt.h"
49 #include "scst_debug.h"
50
51 /* Name of this kernel module. */
52 #define DRV_NAME                "ib_srpt"
53 /* Prefix for printk() kernel messages. */
54 #define LOG_PFX                 DRV_NAME ": "
55 #define DRV_VERSION             "1.0.1"
56 #define DRV_RELDATE             "July 10, 2008"
57 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
58 /* Flags to be used in SCST debug tracing statements. */
59 #define DEFAULT_SRPT_TRACE_FLAGS (TRACE_OUT_OF_MEM | TRACE_MINOR \
60                                   | TRACE_MGMT | TRACE_SPECIAL)
61 /* Name of the entry that will be created under /proc/scsi_tgt/ib_srpt. */
62 #define SRPT_PROC_TRACE_LEVEL_NAME      "trace_level"
63 #endif
64
65 #define MELLANOX_SRPT_ID_STRING "SCST SRP target"
66
67 MODULE_AUTHOR("Vu Pham");
68 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
69                    "v" DRV_VERSION " (" DRV_RELDATE ")");
70 MODULE_LICENSE("Dual BSD/GPL");
71
72 struct srpt_thread {
73         /* Protects thread_ioctx_list. */
74         spinlock_t thread_lock;
75         /* I/O contexts to be processed by the kernel thread. */
76         struct list_head thread_ioctx_list;
77         /* SRPT kernel thread. */
78         struct task_struct *thread;
79 };
80
81 /*
82  * Global Variables
83  */
84
85 static u64 srpt_service_guid;
86 /* List of srpt_device structures. */
87 static atomic_t srpt_device_count;
88 static int use_port_guid_in_session_name;
89 static int thread;
90 static struct srpt_thread srpt_thread;
91 static DECLARE_WAIT_QUEUE_HEAD(ioctx_list_waitQ);
92 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
93 static unsigned long trace_flag = DEFAULT_SRPT_TRACE_FLAGS;
94 module_param(trace_flag, long, 0644);
95 MODULE_PARM_DESC(trace_flag,
96                  "Trace flags for the ib_srpt kernel module.");
97 #endif
98
99 module_param(thread, int, 0444);
100 MODULE_PARM_DESC(thread,
101                  "Executing ioctx in thread context. Default 0, i.e. soft IRQ, "
102                  "where possible");
103
104 module_param(use_port_guid_in_session_name, bool, 0444);
105 MODULE_PARM_DESC(use_port_guid_in_session_name,
106                  "Use target port ID in the SCST session name such that"
107                  " redundant paths between multiport systems can be masked.");
108
109 static void srpt_add_one(struct ib_device *device);
110 static void srpt_remove_one(struct ib_device *device);
111 static void srpt_unregister_mad_agent(struct srpt_device *sdev);
112 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt);
113
114 static struct ib_client srpt_client = {
115         .name = DRV_NAME,
116         .add = srpt_add_one,
117         .remove = srpt_remove_one
118 };
119
120 /**
121  * Atomically test and set the channel state.
122  * @ch: RDMA channel.
123  * @old: channel state to compare with.
124  * @new: state to change the channel state to if the current state matches the
125  *       argument 'old'.
126  *
127  * Returns true if the channel state matched old upon entry of this function,
128  * and false otherwise.
129  */
130 static bool srpt_test_and_set_channel_state(struct srpt_rdma_ch *ch,
131                                             enum rdma_ch_state old,
132                                             enum rdma_ch_state new)
133 {
134         unsigned long flags;
135         enum rdma_ch_state cur;
136
137         spin_lock_irqsave(&ch->spinlock, flags);
138         cur = ch->state;
139         if (cur == old)
140                 ch->state = new;
141         spin_unlock_irqrestore(&ch->spinlock, flags);
142
143         return cur == old;
144 }
145
146 /*
147  * Callback function called by the InfiniBand core when an asynchronous IB
148  * event occurs. This callback may occur in interrupt context. See also
149  * section 11.5.2, Set Asynchronous Event Handler in the InfiniBand
150  * Architecture Specification.
151  */
152 static void srpt_event_handler(struct ib_event_handler *handler,
153                                struct ib_event *event)
154 {
155         struct srpt_device *sdev;
156         struct srpt_port *sport;
157
158         sdev = ib_get_client_data(event->device, &srpt_client);
159         if (!sdev || sdev->device != event->device)
160                 return;
161
162         TRACE_DBG("ASYNC event= %d on device= %s",
163                   event->event, sdev->device->name);
164
165         switch (event->event) {
166         case IB_EVENT_PORT_ERR:
167                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
168                         sport = &sdev->port[event->element.port_num - 1];
169                         sport->lid = 0;
170                         sport->sm_lid = 0;
171                 }
172                 break;
173         case IB_EVENT_PORT_ACTIVE:
174         case IB_EVENT_LID_CHANGE:
175         case IB_EVENT_PKEY_CHANGE:
176         case IB_EVENT_SM_CHANGE:
177         case IB_EVENT_CLIENT_REREGISTER:
178                 /*
179                  * Refresh port data asynchronously. Note: it is safe to call
180                  * schedule_work() even if &sport->work is already on the
181                  * global workqueue because schedule_work() tests for the
182                  * work_pending() condition before adding &sport->work to the
183                  * global work queue.
184                  */
185                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
186                         sport = &sdev->port[event->element.port_num - 1];
187                         if (!sport->lid && !sport->sm_lid)
188                                 schedule_work(&sport->work);
189                 }
190                 break;
191         default:
192                 break;
193         }
194
195 }
196
197 /*
198  * Callback function called by the InfiniBand core for SRQ (shared receive
199  * queue) events.
200  */
201 static void srpt_srq_event(struct ib_event *event, void *ctx)
202 {
203         TRACE_DBG("SRQ event %d", event->event);
204 }
205
206 /*
207  * Callback function called by the InfiniBand core for QP (queue pair) events.
208  */
209 static void srpt_qp_event(struct ib_event *event, void *ctx)
210 {
211         struct srpt_rdma_ch *ch = ctx;
212
213         TRACE_DBG("QP event %d on cm_id=%p sess_name=%s state=%d",
214                   event->event, ch->cm_id, ch->sess_name, ch->state);
215
216         switch (event->event) {
217         case IB_EVENT_COMM_EST:
218 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20) || defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
219                 ib_cm_notify(ch->cm_id, event->event);
220 #else
221                 /* Vanilla 2.6.19 kernel (or before) without OFED. */
222                 PRINT_ERROR("%s", "how to perform ib_cm_notify() on a"
223                         " vanilla 2.6.18 kernel ???\n");
224 #endif
225                 break;
226         case IB_EVENT_QP_LAST_WQE_REACHED:
227                 if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_LIVE,
228                                         RDMA_CHANNEL_DISCONNECTING)) {
229                         PRINT_INFO("disconnected session %s.\n",
230                                ch->sess_name);
231                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
232                 }
233                 break;
234         default:
235                 break;
236         }
237 }
238
239 /*
240  * Helper function for filling in an InfiniBand IOUnitInfo structure. Copies
241  * the lowest four bits of value in element slot of the array of four bit
242  * elements called c_list (controller list). The index slot is one-based.
243  *
244  * @pre 1 <= slot && 0 <= value && value < 16
245  */
246 static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
247 {
248         u16 id;
249         u8 tmp;
250
251         id = (slot - 1) / 2;
252         if (slot & 0x1) {
253                 tmp = c_list[id] & 0xf;
254                 c_list[id] = (value << 4) | tmp;
255         } else {
256                 tmp = c_list[id] & 0xf0;
257                 c_list[id] = (value & 0xf) | tmp;
258         }
259 }
260
261 /*
262  * Write InfiniBand ClassPortInfo to mad. See also section 16.3.3.1
263  * ClassPortInfo in the InfiniBand Architecture Specification.
264  */
265 static void srpt_get_class_port_info(struct ib_dm_mad *mad)
266 {
267         struct ib_class_port_info *cif;
268
269         cif = (struct ib_class_port_info *)mad->data;
270         memset(cif, 0, sizeof *cif);
271         cif->base_version = 1;
272         cif->class_version = 1;
273         cif->resp_time_value = 20;
274
275         mad->mad_hdr.status = 0;
276 }
277
278 /*
279  * Write IOUnitInfo to mad. See also section 16.3.3.3 IOUnitInfo in the
280  * InfiniBand Architecture Specification. See also section B.7,
281  * table B.6 in the T10 SRP r16a document.
282  */
283 static void srpt_get_iou(struct ib_dm_mad *mad)
284 {
285         struct ib_dm_iou_info *ioui;
286         u8 slot;
287         int i;
288
289         ioui = (struct ib_dm_iou_info *)mad->data;
290         ioui->change_id = 1;
291         ioui->max_controllers = 16;
292
293         /* set present for slot 1 and empty for the rest */
294         srpt_set_ioc(ioui->controller_list, 1, 1);
295         for (i = 1, slot = 2; i < 16; i++, slot++)
296                 srpt_set_ioc(ioui->controller_list, slot, 0);
297
298         mad->mad_hdr.status = 0;
299 }
300
301 /*
302  * Write IOControllerprofile to mad for I/O controller (sdev, slot). See also
303  * section 16.3.3.4 IOControllerProfile in the InfiniBand Architecture
304  * Specification. See also section B.7, table B.7 in the T10 SRP r16a
305  * document.
306  */
307 static void srpt_get_ioc(struct srpt_device *sdev, u32 slot,
308                          struct ib_dm_mad *mad)
309 {
310         struct ib_dm_ioc_profile *iocp;
311
312         iocp = (struct ib_dm_ioc_profile *)mad->data;
313
314         if (!slot || slot > 16) {
315                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
316                 return;
317         }
318
319         if (slot > 2) {
320                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
321                 return;
322         }
323
324         memset(iocp, 0, sizeof *iocp);
325         strcpy(iocp->id_string, MELLANOX_SRPT_ID_STRING);
326         iocp->guid = cpu_to_be64(srpt_service_guid);
327         iocp->vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
328         iocp->device_id = cpu_to_be32(sdev->dev_attr.vendor_part_id);
329         iocp->device_version = cpu_to_be16(sdev->dev_attr.hw_ver);
330         iocp->subsys_vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
331         iocp->subsys_device_id = 0x0;
332         iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
333         iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
334         iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
335         iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
336         iocp->send_queue_depth = cpu_to_be16(SRPT_SRQ_SIZE);
337         iocp->rdma_read_depth = 4;
338         iocp->send_size = cpu_to_be32(MAX_MESSAGE_SIZE);
339         iocp->rdma_size = cpu_to_be32(MAX_RDMA_SIZE);
340         iocp->num_svc_entries = 1;
341         iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
342                 SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
343
344         mad->mad_hdr.status = 0;
345 }
346
347 /*
348  * Device management: write ServiceEntries to mad for the given slot. See also
349  * section 16.3.3.5 ServiceEntries in the InfiniBand Architecture
350  * Specification. See also section B.7, table B.8 in the T10 SRP r16a document.
351  */
352 static void srpt_get_svc_entries(u64 ioc_guid,
353                                  u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
354 {
355         struct ib_dm_svc_entries *svc_entries;
356
357         WARN_ON(!ioc_guid);
358
359         if (!slot || slot > 16) {
360                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
361                 return;
362         }
363
364         if (slot > 2 || lo > hi || hi > 1) {
365                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
366                 return;
367         }
368
369         svc_entries = (struct ib_dm_svc_entries *)mad->data;
370         memset(svc_entries, 0, sizeof *svc_entries);
371         svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid);
372         snprintf(svc_entries->service_entries[0].name,
373                  sizeof(svc_entries->service_entries[0].name),
374                  "%s%016llx",
375                  SRP_SERVICE_NAME_PREFIX,
376                  (unsigned long long)ioc_guid);
377
378         mad->mad_hdr.status = 0;
379 }
380
381 /*
382  * Actual processing of a received MAD *rq_mad received through source port *sp
383  * (MAD = InfiniBand management datagram). The response to be sent back is
384  * written to *rsp_mad.
385  */
386 static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
387                                  struct ib_dm_mad *rsp_mad)
388 {
389         u16 attr_id;
390         u32 slot;
391         u8 hi, lo;
392
393         attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
394         switch (attr_id) {
395         case DM_ATTR_CLASS_PORT_INFO:
396                 srpt_get_class_port_info(rsp_mad);
397                 break;
398         case DM_ATTR_IOU_INFO:
399                 srpt_get_iou(rsp_mad);
400                 break;
401         case DM_ATTR_IOC_PROFILE:
402                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
403                 srpt_get_ioc(sp->sdev, slot, rsp_mad);
404                 break;
405         case DM_ATTR_SVC_ENTRIES:
406                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
407                 hi = (u8) ((slot >> 8) & 0xff);
408                 lo = (u8) (slot & 0xff);
409                 slot = (u16) ((slot >> 16) & 0xffff);
410                 srpt_get_svc_entries(srpt_service_guid,
411                                      slot, hi, lo, rsp_mad);
412                 break;
413         default:
414                 rsp_mad->mad_hdr.status =
415                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
416                 break;
417         }
418 }
419
420 /*
421  * Callback function that is called by the InfiniBand core after transmission of
422  * a MAD. (MAD = management datagram; AH = address handle.)
423  */
424 static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
425                                   struct ib_mad_send_wc *mad_wc)
426 {
427         ib_destroy_ah(mad_wc->send_buf->ah);
428         ib_free_send_mad(mad_wc->send_buf);
429 }
430
431 /*
432  * Callback function that is called by the InfiniBand core after reception of
433  * a MAD (management datagram).
434  */
435 static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
436                                   struct ib_mad_recv_wc *mad_wc)
437 {
438         struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
439         struct ib_ah *ah;
440         struct ib_mad_send_buf *rsp;
441         struct ib_dm_mad *dm_mad;
442
443         if (!mad_wc || !mad_wc->recv_buf.mad)
444                 return;
445
446         ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
447                                   mad_wc->recv_buf.grh, mad_agent->port_num);
448         if (IS_ERR(ah))
449                 goto err;
450
451         BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR);
452
453         rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
454                                  mad_wc->wc->pkey_index, 0,
455                                  IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA,
456                                  GFP_KERNEL);
457         if (IS_ERR(rsp))
458                 goto err_rsp;
459
460         rsp->ah = ah;
461
462         dm_mad = rsp->mad;
463         memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof *dm_mad);
464         dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
465         dm_mad->mad_hdr.status = 0;
466
467         switch (mad_wc->recv_buf.mad->mad_hdr.method) {
468         case IB_MGMT_METHOD_GET:
469                 srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
470                 break;
471         case IB_MGMT_METHOD_SET:
472                 dm_mad->mad_hdr.status =
473                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
474                 break;
475         default:
476                 dm_mad->mad_hdr.status =
477                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
478                 break;
479         }
480
481         if (!ib_post_send_mad(rsp, NULL)) {
482                 ib_free_recv_mad(mad_wc);
483                 /* will destroy_ah & free_send_mad in send completion */
484                 return;
485         }
486
487         ib_free_send_mad(rsp);
488
489 err_rsp:
490         ib_destroy_ah(ah);
491 err:
492         ib_free_recv_mad(mad_wc);
493 }
494
495 /*
496  * Enable InfiniBand management datagram processing, update the cached sm_lid,
497  * lid and gid values, and register a callback function for processing MADs
498  * on the specified port. It is safe to call this function more than once for
499  * the same port.
500  */
501 static int srpt_refresh_port(struct srpt_port *sport)
502 {
503         struct ib_mad_reg_req reg_req;
504         struct ib_port_modify port_modify;
505         struct ib_port_attr port_attr;
506         int ret;
507
508         memset(&port_modify, 0, sizeof port_modify);
509         port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
510         port_modify.clr_port_cap_mask = 0;
511
512         ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
513         if (ret)
514                 goto err_mod_port;
515
516         ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
517         if (ret)
518                 goto err_query_port;
519
520         sport->sm_lid = port_attr.sm_lid;
521         sport->lid = port_attr.lid;
522
523         ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid);
524         if (ret)
525                 goto err_query_port;
526
527         if (!sport->mad_agent) {
528                 memset(&reg_req, 0, sizeof reg_req);
529                 reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
530                 reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
531                 set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
532                 set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
533
534                 sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
535                                                          sport->port,
536                                                          IB_QPT_GSI,
537                                                          &reg_req, 0,
538                                                          srpt_mad_send_handler,
539                                                          srpt_mad_recv_handler,
540                                                          sport);
541                 if (IS_ERR(sport->mad_agent)) {
542                         ret = PTR_ERR(sport->mad_agent);
543                         sport->mad_agent = NULL;
544                         goto err_query_port;
545                 }
546         }
547
548         return 0;
549
550 err_query_port:
551
552         port_modify.set_port_cap_mask = 0;
553         port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
554         ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
555
556 err_mod_port:
557
558         return ret;
559 }
560
561 /*
562  * Unregister the callback function for processing MADs and disable MAD
563  * processing for all ports of the specified device. It is safe to call this
564  * function more than once for the same device.
565  */
566 static void srpt_unregister_mad_agent(struct srpt_device *sdev)
567 {
568         struct ib_port_modify port_modify = {
569                 .clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP,
570         };
571         struct srpt_port *sport;
572         int i;
573
574         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
575                 sport = &sdev->port[i - 1];
576                 WARN_ON(sport->port != i);
577                 if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
578                         PRINT_ERROR("%s", "disabling MAD processing failed.\n");
579                 if (sport->mad_agent) {
580                         ib_unregister_mad_agent(sport->mad_agent);
581                         sport->mad_agent = NULL;
582                 }
583         }
584 }
585
586 /*
587  * Allocate and initialize an SRPT I/O context structure.
588  */
589 static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev)
590 {
591         struct srpt_ioctx *ioctx;
592
593         ioctx = kmalloc(sizeof *ioctx, GFP_KERNEL);
594         if (!ioctx)
595                 goto out;
596
597         ioctx->buf = kzalloc(MAX_MESSAGE_SIZE, GFP_KERNEL);
598         if (!ioctx->buf)
599                 goto out_free_ioctx;
600
601         ioctx->dma = dma_map_single(sdev->device->dma_device, ioctx->buf,
602                                     MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
603 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
604         if (dma_mapping_error(sdev->device->dma_device, ioctx->dma))
605 #else
606         if (dma_mapping_error(ioctx->dma))
607 #endif
608                 goto out_free_buf;
609
610         return ioctx;
611
612 out_free_buf:
613         kfree(ioctx->buf);
614 out_free_ioctx:
615         kfree(ioctx);
616 out:
617         return NULL;
618 }
619
620 /*
621  * Deallocate an SRPT I/O context structure.
622  */
623 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
624 {
625         if (!ioctx)
626                 return;
627
628         dma_unmap_single(sdev->device->dma_device, ioctx->dma,
629                          MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
630         kfree(ioctx->buf);
631         kfree(ioctx);
632 }
633
634 /*
635  * Associate a ring of SRPT I/O context structures with the specified device.
636  */
637 static int srpt_alloc_ioctx_ring(struct srpt_device *sdev)
638 {
639         int i;
640
641         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
642                 sdev->ioctx_ring[i] = srpt_alloc_ioctx(sdev);
643
644                 if (!sdev->ioctx_ring[i])
645                         goto err;
646
647                 sdev->ioctx_ring[i]->index = i;
648         }
649
650         return 0;
651
652 err:
653         while (--i > 0) {
654                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
655                 sdev->ioctx_ring[i] = NULL;
656         }
657         return -ENOMEM;
658 }
659
660 /* Free the ring of SRPT I/O context structures. */
661 static void srpt_free_ioctx_ring(struct srpt_device *sdev)
662 {
663         int i;
664
665         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
666                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
667                 sdev->ioctx_ring[i] = NULL;
668         }
669 }
670
671 /*
672  * Post a receive request on the work queue of InfiniBand device 'sdev'.
673  */
674 static int srpt_post_recv(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
675 {
676         struct ib_sge list;
677         struct ib_recv_wr wr, *bad_wr;
678
679         wr.wr_id = ioctx->index | SRPT_OP_RECV;
680
681         list.addr = ioctx->dma;
682         list.length = MAX_MESSAGE_SIZE;
683         list.lkey = sdev->mr->lkey;
684
685         wr.next = NULL;
686         wr.sg_list = &list;
687         wr.num_sge = 1;
688
689         return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);
690 }
691
692 /*
693  * Post an IB send request.
694  * @ch: RDMA channel to post the send request on.
695  * @ioctx: I/O context of the send request.
696  * @len: length of the request to be sent in bytes.
697  *
698  * Returns zero upon success and a non-zero value upon failure.
699  */
700 static int srpt_post_send(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
701                           int len)
702 {
703         struct ib_sge list;
704         struct ib_send_wr wr, *bad_wr;
705         struct srpt_device *sdev = ch->sport->sdev;
706
707         dma_sync_single_for_device(sdev->device->dma_device, ioctx->dma,
708                                    MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
709
710         list.addr = ioctx->dma;
711         list.length = len;
712         list.lkey = sdev->mr->lkey;
713
714         wr.next = NULL;
715         wr.wr_id = ioctx->index;
716         wr.sg_list = &list;
717         wr.num_sge = 1;
718         wr.opcode = IB_WR_SEND;
719         wr.send_flags = IB_SEND_SIGNALED;
720
721         return ib_post_send(ch->qp, &wr, &bad_wr);
722 }
723
724 static int srpt_get_desc_tbl(struct srpt_ioctx *ioctx, struct srp_cmd *srp_cmd,
725                              int *ind)
726 {
727         struct srp_indirect_buf *idb;
728         struct srp_direct_buf *db;
729
730         *ind = 0;
731         if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
732             ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
733                 ioctx->n_rbuf = 1;
734                 ioctx->rbufs = &ioctx->single_rbuf;
735
736                 db = (void *)srp_cmd->add_data;
737                 memcpy(ioctx->rbufs, db, sizeof *db);
738                 ioctx->data_len = be32_to_cpu(db->len);
739         } else {
740                 idb = (void *)srp_cmd->add_data;
741
742                 ioctx->n_rbuf = be32_to_cpu(idb->table_desc.len) / sizeof *db;
743
744                 if (ioctx->n_rbuf >
745                     (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
746                         *ind = 1;
747                         ioctx->n_rbuf = 0;
748                         goto out;
749                 }
750
751                 if (ioctx->n_rbuf == 1)
752                         ioctx->rbufs = &ioctx->single_rbuf;
753                 else
754                         ioctx->rbufs =
755                                 kmalloc(ioctx->n_rbuf * sizeof *db, GFP_ATOMIC);
756                 if (!ioctx->rbufs) {
757                         ioctx->n_rbuf = 0;
758                         return -ENOMEM;
759                 }
760
761                 db = idb->desc_list;
762                 memcpy(ioctx->rbufs, db, ioctx->n_rbuf * sizeof *db);
763                 ioctx->data_len = be32_to_cpu(idb->len);
764         }
765 out:
766         return 0;
767 }
768
769 /*
770  * Modify the attributes of queue pair 'qp': allow local write, remote read,
771  * and remote write. Also transition 'qp' to state IB_QPS_INIT.
772  */
773 static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
774 {
775         struct ib_qp_attr *attr;
776         int ret;
777
778         attr = kzalloc(sizeof *attr, GFP_KERNEL);
779         if (!attr)
780                 return -ENOMEM;
781
782         attr->qp_state = IB_QPS_INIT;
783         attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ |
784             IB_ACCESS_REMOTE_WRITE;
785         attr->port_num = ch->sport->port;
786         attr->pkey_index = 0;
787
788         ret = ib_modify_qp(qp, attr,
789                            IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
790                            IB_QP_PKEY_INDEX);
791
792         kfree(attr);
793         return ret;
794 }
795
796 /**
797  * Change the state of a channel to 'ready to receive' (RTR).
798  * @ch: channel of the queue pair.
799  * @qp: queue pair to change the state of.
800  *
801  * Returns zero upon success and a negative value upon failure.
802  *
803  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
804  * If this structure ever becomes larger, it might be necessary to allocate
805  * it dynamically instead of on the stack.
806  */
807 static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
808 {
809         struct ib_qp_attr qp_attr;
810         int attr_mask;
811         int ret;
812
813         qp_attr.qp_state = IB_QPS_RTR;
814         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
815         if (ret)
816                 goto out;
817
818         qp_attr.max_dest_rd_atomic = 4;
819
820         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
821
822 out:
823         return ret;
824 }
825
826 /**
827  * Change the state of a channel to 'ready to send' (RTS).
828  * @ch: channel of the queue pair.
829  * @qp: queue pair to change the state of.
830  *
831  * Returns zero upon success and a negative value upon failure.
832  *
833  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
834  * If this structure ever becomes larger, it might be necessary to allocate
835  * it dynamically instead of on the stack.
836  */
837 static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
838 {
839         struct ib_qp_attr qp_attr;
840         int attr_mask;
841         int ret;
842
843         qp_attr.qp_state = IB_QPS_RTS;
844         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
845         if (ret)
846                 goto out;
847
848         qp_attr.max_rd_atomic = 4;
849
850         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
851
852 out:
853         return ret;
854 }
855
856 static void srpt_reset_ioctx(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
857 {
858         int i;
859
860         if (ioctx->n_rdma_ius > 0 && ioctx->rdma_ius) {
861                 struct rdma_iu *riu = ioctx->rdma_ius;
862
863                 for (i = 0; i < ioctx->n_rdma_ius; ++i, ++riu)
864                         kfree(riu->sge);
865                 kfree(ioctx->rdma_ius);
866         }
867
868         if (ioctx->n_rbuf > 1)
869                 kfree(ioctx->rbufs);
870
871         if (srpt_post_recv(ch->sport->sdev, ioctx))
872                 PRINT_ERROR("%s", "SRQ post_recv failed - this is serious\n");
873                 /* we should queue it back to free_ioctx queue */
874         else
875                 atomic_inc(&ch->req_lim_delta);
876 }
877
878 static void srpt_abort_scst_cmd(struct srpt_device *sdev,
879                                 struct scst_cmd *scmnd,
880                                 bool tell_initiator)
881 {
882         struct srpt_ioctx *ioctx;
883         scst_data_direction dir;
884         enum srpt_command_state orig_ioctx_state;
885
886         ioctx = scst_cmd_get_tgt_priv(scmnd);
887         BUG_ON(!ioctx);
888         dir = scst_cmd_get_data_direction(scmnd);
889         if (dir != SCST_DATA_NONE) {
890                 dma_unmap_sg(sdev->device->dma_device,
891                              scst_cmd_get_sg(scmnd),
892                              scst_cmd_get_sg_cnt(scmnd),
893                              scst_to_tgt_dma_dir(dir));
894
895 #ifdef CONFIG_SCST_EXTRACHECKS
896                 switch (scmnd->state) {
897                 case SCST_CMD_STATE_DATA_WAIT:
898                         WARN_ON(ioctx->state != SRPT_STATE_NEED_DATA);
899                         break;
900                 case SCST_CMD_STATE_XMIT_WAIT:
901                         WARN_ON(ioctx->state != SRPT_STATE_PROCESSED);
902                         break;
903                 default:
904                         WARN_ON(ioctx->state == SRPT_STATE_NEED_DATA ||
905                                 ioctx->state == SRPT_STATE_PROCESSED);
906                 }
907 #endif
908
909                 orig_ioctx_state = ioctx->state;
910                 ioctx->state = SRPT_STATE_ABORTED;
911
912                 if (orig_ioctx_state == SRPT_STATE_NEED_DATA) {
913                         WARN_ON(scst_cmd_get_data_direction(ioctx->scmnd)
914                                 == SCST_DATA_READ);
915                         scst_rx_data(scmnd,
916                                      tell_initiator ? SCST_RX_STATUS_ERROR
917                                      : SCST_RX_STATUS_ERROR_FATAL,
918                                      SCST_CONTEXT_THREAD);
919                         goto out;
920                 } else if (ioctx->state == SRPT_STATE_PROCESSED)
921                         ;
922                 else
923                         WARN_ON("unexpected cmd state");
924         }
925
926         scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_FAILED);
927         scst_tgt_cmd_done(scmnd, scst_estimate_context());
928 out:
929         return;
930 }
931
932 static void srpt_handle_err_comp(struct srpt_rdma_ch *ch, struct ib_wc *wc)
933 {
934         struct srpt_ioctx *ioctx;
935         struct srpt_device *sdev = ch->sport->sdev;
936
937         if (wc->wr_id & SRPT_OP_RECV) {
938                 ioctx = sdev->ioctx_ring[wc->wr_id & ~SRPT_OP_RECV];
939                 PRINT_ERROR("%s", "This is serious - SRQ is in bad state\n");
940         } else {
941                 ioctx = sdev->ioctx_ring[wc->wr_id];
942
943                 if (ioctx->scmnd)
944                         srpt_abort_scst_cmd(sdev, ioctx->scmnd, true);
945                 else
946                         srpt_reset_ioctx(ch, ioctx);
947         }
948 }
949
950 static void srpt_handle_send_comp(struct srpt_rdma_ch *ch,
951                                   struct srpt_ioctx *ioctx,
952                                   enum scst_exec_context context)
953 {
954         if (ioctx->scmnd) {
955                 scst_data_direction dir =
956                         scst_cmd_get_data_direction(ioctx->scmnd);
957
958                 if (dir != SCST_DATA_NONE)
959                         dma_unmap_sg(ch->sport->sdev->device->dma_device,
960                                      scst_cmd_get_sg(ioctx->scmnd),
961                                      scst_cmd_get_sg_cnt(ioctx->scmnd),
962                                      scst_to_tgt_dma_dir(dir));
963
964                 scst_tgt_cmd_done(ioctx->scmnd, context);
965         } else
966                 srpt_reset_ioctx(ch, ioctx);
967 }
968
969 /** Process an RDMA completion notification. */
970 static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch,
971                                   struct srpt_ioctx *ioctx)
972 {
973         if (!ioctx->scmnd) {
974                 WARN_ON("ERROR: ioctx->scmnd == NULL");
975                 srpt_reset_ioctx(ch, ioctx);
976                 return;
977         }
978
979         /*
980          * If an RDMA completion notification has been received for a write
981          * command, tell SCST that processing can continue by calling
982          * scst_rx_data().
983          */
984         if (ioctx->state == SRPT_STATE_NEED_DATA) {
985                 WARN_ON(scst_cmd_get_data_direction(ioctx->scmnd)
986                         == SCST_DATA_READ);
987                 ioctx->state = SRPT_STATE_DATA_IN;
988                 scst_rx_data(ioctx->scmnd, SCST_RX_STATUS_SUCCESS,
989                              scst_estimate_context());
990         }
991 }
992
993 /**
994  * Build an SRP_RSP response.
995  * @ch: RDMA channel through which the request has been received.
996  * @ioctx: I/O context in which the SRP_RSP response will be built.
997  * @s_key: sense key that will be stored in the response.
998  * @s_code: value that will be stored in the asc_ascq field of the sense data.
999  * @tag: tag of the request for which this response is being generated.
1000  *
1001  * Returns the size in bytes of the SRP_RSP response.
1002  *
1003  * An SRP_RSP response contains a SCSI status or service response. See also
1004  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
1005  * response. See also SPC-2 for more information about sense data.
1006  */
1007 static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
1008                               struct srpt_ioctx *ioctx, u8 s_key, u8 s_code,
1009                               u64 tag)
1010 {
1011         struct srp_rsp *srp_rsp;
1012         struct sense_data *sense;
1013         int limit_delta;
1014         int sense_data_len = 0;
1015
1016         srp_rsp = ioctx->buf;
1017         memset(srp_rsp, 0, sizeof *srp_rsp);
1018
1019         limit_delta = atomic_read(&ch->req_lim_delta);
1020         atomic_sub(limit_delta, &ch->req_lim_delta);
1021
1022         srp_rsp->opcode = SRP_RSP;
1023         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1024         srp_rsp->tag = tag;
1025
1026         if (s_key != NO_SENSE) {
1027                 sense_data_len = sizeof *sense + (sizeof *sense % 4);
1028                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1029                 srp_rsp->status = SAM_STAT_CHECK_CONDITION;
1030                 srp_rsp->sense_data_len = cpu_to_be32(sense_data_len);
1031
1032                 sense = (struct sense_data *)(srp_rsp + 1);
1033                 sense->err_code = 0x70;
1034                 sense->key = s_key;
1035                 sense->asc_ascq = s_code;
1036         }
1037
1038         return sizeof(*srp_rsp) + sense_data_len;
1039 }
1040
1041 /**
1042  * Build a task management response, which is a specific SRP_RSP response.
1043  * @ch: RDMA channel through which the request has been received.
1044  * @ioctx: I/O context in which the SRP_RSP response will be built.
1045  * @rsp_code: RSP_CODE that will be stored in the response.
1046  * @tag: tag of the request for which this response is being generated.
1047  *
1048  * Returns the size in bytes of the SRP_RSP response.
1049  *
1050  * An SRP_RSP response contains a SCSI status or service response. See also
1051  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
1052  * response.
1053  */
1054 static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
1055                                   struct srpt_ioctx *ioctx, u8 rsp_code,
1056                                   u64 tag)
1057 {
1058         struct srp_rsp *srp_rsp;
1059         int limit_delta;
1060         int resp_data_len = 0;
1061
1062         dma_sync_single_for_cpu(ch->sport->sdev->device->dma_device, ioctx->dma,
1063                                 MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
1064
1065         srp_rsp = ioctx->buf;
1066         memset(srp_rsp, 0, sizeof *srp_rsp);
1067
1068         limit_delta = atomic_read(&ch->req_lim_delta);
1069         atomic_sub(limit_delta, &ch->req_lim_delta);
1070
1071         srp_rsp->opcode = SRP_RSP;
1072         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1073         srp_rsp->tag = tag;
1074
1075         if (rsp_code != SRP_TSK_MGMT_SUCCESS) {
1076                 resp_data_len = 4;
1077                 srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
1078                 srp_rsp->resp_data_len = cpu_to_be32(resp_data_len);
1079                 srp_rsp->data[3] = rsp_code;
1080         }
1081
1082         return sizeof(*srp_rsp) + resp_data_len;
1083 }
1084
1085 /*
1086  * Process SRP_CMD.
1087  */
1088 static int srpt_handle_cmd(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
1089 {
1090         struct scst_cmd *scmnd;
1091         struct srp_cmd *srp_cmd;
1092         struct srp_rsp *srp_rsp;
1093         scst_data_direction dir;
1094         int indirect_desc = 0;
1095         int ret;
1096         unsigned long flags;
1097
1098         srp_cmd = ioctx->buf;
1099         srp_rsp = ioctx->buf;
1100
1101         dir = SCST_DATA_NONE;
1102         if (srp_cmd->buf_fmt) {
1103                 ret = srpt_get_desc_tbl(ioctx, srp_cmd, &indirect_desc);
1104                 if (ret) {
1105                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1106                                            NO_ADD_SENSE, srp_cmd->tag);
1107                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1108                         goto err;
1109                 }
1110
1111                 if (indirect_desc) {
1112                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1113                                            NO_ADD_SENSE, srp_cmd->tag);
1114                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1115                         goto err;
1116                 }
1117
1118                 /*
1119                  * The lower four bits of the buffer format field contain the
1120                  * DATA-IN buffer descriptor format, and the highest four bits
1121                  * contain the DATA-OUT buffer descriptor format.
1122                  */
1123                 if (srp_cmd->buf_fmt & 0xf)
1124                         /* DATA-IN: transfer data from target to initiator. */
1125                         dir = SCST_DATA_READ;
1126                 else if (srp_cmd->buf_fmt >> 4)
1127                         /* DATA-OUT: transfer data from initiator to target. */
1128                         dir = SCST_DATA_WRITE;
1129         }
1130
1131         scmnd = scst_rx_cmd(ch->scst_sess, (u8 *) &srp_cmd->lun,
1132                             sizeof srp_cmd->lun, srp_cmd->cdb, 16,
1133                             thread ? SCST_NON_ATOMIC : SCST_ATOMIC);
1134         if (!scmnd) {
1135                 srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1136                                    NO_ADD_SENSE, srp_cmd->tag);
1137                 srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1138                 goto err;
1139         }
1140
1141         ioctx->scmnd = scmnd;
1142
1143         switch (srp_cmd->task_attr) {
1144         case SRP_CMD_HEAD_OF_Q:
1145                 scmnd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1146                 break;
1147         case SRP_CMD_ORDERED_Q:
1148                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1149                 break;
1150         case SRP_CMD_SIMPLE_Q:
1151                 scmnd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1152                 break;
1153         case SRP_CMD_ACA:
1154                 scmnd->queue_type = SCST_CMD_QUEUE_ACA;
1155                 break;
1156         default:
1157                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1158                 break;
1159         }
1160
1161         scst_cmd_set_tag(scmnd, srp_cmd->tag);
1162         scst_cmd_set_tgt_priv(scmnd, ioctx);
1163         scst_cmd_set_expected(scmnd, dir, ioctx->data_len);
1164
1165         spin_lock_irqsave(&ch->spinlock, flags);
1166         list_add_tail(&ioctx->scmnd_list, &ch->active_scmnd_list);
1167         ch->active_scmnd_cnt++;
1168         spin_unlock_irqrestore(&ch->spinlock, flags);
1169
1170         scst_cmd_init_done(scmnd, scst_estimate_context());
1171
1172         return 0;
1173
1174 err:
1175         WARN_ON(srp_rsp->opcode != SRP_RSP);
1176
1177         return -1;
1178 }
1179
1180 /*
1181  * Process an SRP_TSK_MGMT request.
1182  *
1183  * Returns 0 upon success and -1 upon failure.
1184  *
1185  * Each task management function is performed by calling one of the
1186  * scst_rx_mgmt_fn*() functions. These functions will either report failure
1187  * or process the task management function asynchronously. The function
1188  * srpt_tsk_mgmt_done() will be called by the SCST core upon completion of the
1189  * task management function. When srpt_handle_tsk_mgmt() reports failure
1190  * (i.e. returns -1) a response will have been built in ioctx->buf. This
1191  * information unit has to be sent back by the caller.
1192  *
1193  * For more information about SRP_TSK_MGMT information units, see also section
1194  * 6.7 in the T10 SRP r16a document.
1195  */
1196 static int srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
1197                                 struct srpt_ioctx *ioctx)
1198 {
1199         struct srp_tsk_mgmt *srp_tsk;
1200         struct srpt_mgmt_ioctx *mgmt_ioctx;
1201         int ret;
1202
1203         srp_tsk = ioctx->buf;
1204
1205         TRACE_DBG("recv_tsk_mgmt= %d for task_tag= %lld"
1206                   " using tag= %lld cm_id= %p sess= %p",
1207                   srp_tsk->tsk_mgmt_func,
1208                   (unsigned long long) srp_tsk->task_tag,
1209                   (unsigned long long) srp_tsk->tag,
1210                   ch->cm_id, ch->scst_sess);
1211
1212         mgmt_ioctx = kmalloc(sizeof *mgmt_ioctx, GFP_ATOMIC);
1213         if (!mgmt_ioctx) {
1214                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1215                                        srp_tsk->tag);
1216                 goto err;
1217         }
1218
1219         mgmt_ioctx->ioctx = ioctx;
1220         mgmt_ioctx->ch = ch;
1221         mgmt_ioctx->tag = srp_tsk->tag;
1222
1223         switch (srp_tsk->tsk_mgmt_func) {
1224         case SRP_TSK_ABORT_TASK:
1225                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK");
1226                 ret = scst_rx_mgmt_fn_tag(ch->scst_sess,
1227                                           SCST_ABORT_TASK,
1228                                           srp_tsk->task_tag,
1229                                           thread ?
1230                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1231                                           mgmt_ioctx);
1232                 break;
1233         case SRP_TSK_ABORT_TASK_SET:
1234                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK_SET");
1235                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1236                                           SCST_ABORT_TASK_SET,
1237                                           (u8 *) &srp_tsk->lun,
1238                                           sizeof srp_tsk->lun,
1239                                           thread ?
1240                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1241                                           mgmt_ioctx);
1242                 break;
1243         case SRP_TSK_CLEAR_TASK_SET:
1244                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_TASK_SET");
1245                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1246                                           SCST_CLEAR_TASK_SET,
1247                                           (u8 *) &srp_tsk->lun,
1248                                           sizeof srp_tsk->lun,
1249                                           thread ?
1250                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1251                                           mgmt_ioctx);
1252                 break;
1253         case SRP_TSK_LUN_RESET:
1254                 TRACE_DBG("%s", "Processing SRP_TSK_LUN_RESET");
1255                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1256                                           SCST_LUN_RESET,
1257                                           (u8 *) &srp_tsk->lun,
1258                                           sizeof srp_tsk->lun,
1259                                           thread ?
1260                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1261                                           mgmt_ioctx);
1262                 break;
1263         case SRP_TSK_CLEAR_ACA:
1264                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_ACA");
1265                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1266                                           SCST_CLEAR_ACA,
1267                                           (u8 *) &srp_tsk->lun,
1268                                           sizeof srp_tsk->lun,
1269                                           thread ?
1270                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1271                                           mgmt_ioctx);
1272                 break;
1273         default:
1274                 TRACE_DBG("%s", "Unsupported task management function.");
1275                 srpt_build_tskmgmt_rsp(ch, ioctx,
1276                                        SRP_TSK_MGMT_FUNC_NOT_SUPP,
1277                                        srp_tsk->tag);
1278                 goto err;
1279         }
1280
1281         if (ret) {
1282                 TRACE_DBG("%s", "Processing task management function failed.");
1283                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1284                                        srp_tsk->tag);
1285                 goto err;
1286         }
1287
1288         WARN_ON(srp_tsk->opcode == SRP_RSP);
1289
1290         return 0;
1291
1292 err:
1293         WARN_ON(srp_tsk->opcode != SRP_RSP);
1294
1295         kfree(mgmt_ioctx);
1296         return -1;
1297 }
1298
1299 /**
1300  * Process a receive completion event.
1301  * @ch: RDMA channel for which the completion event has been received.
1302  * @ioctx: SRPT I/O context for which the completion event has been received.
1303  */
1304 static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
1305                                struct srpt_ioctx *ioctx)
1306 {
1307         struct srp_cmd *srp_cmd;
1308         struct srp_rsp *srp_rsp;
1309         unsigned long flags;
1310         int len;
1311
1312         spin_lock_irqsave(&ch->spinlock, flags);
1313         if (ch->state != RDMA_CHANNEL_LIVE) {
1314                 if (ch->state == RDMA_CHANNEL_CONNECTING) {
1315                         list_add_tail(&ioctx->wait_list, &ch->cmd_wait_list);
1316                         spin_unlock_irqrestore(&ch->spinlock, flags);
1317                         return;
1318                 } else {
1319                         spin_unlock_irqrestore(&ch->spinlock, flags);
1320                         srpt_reset_ioctx(ch, ioctx);
1321                         return;
1322                 }
1323         }
1324         spin_unlock_irqrestore(&ch->spinlock, flags);
1325
1326         dma_sync_single_for_cpu(ch->sport->sdev->device->dma_device, ioctx->dma,
1327                                 MAX_MESSAGE_SIZE, DMA_FROM_DEVICE);
1328
1329         ioctx->data_len = 0;
1330         ioctx->n_rbuf = 0;
1331         ioctx->rbufs = NULL;
1332         ioctx->n_rdma = 0;
1333         ioctx->n_rdma_ius = 0;
1334         ioctx->rdma_ius = NULL;
1335         ioctx->scmnd = NULL;
1336         ioctx->state = SRPT_STATE_NEW;
1337
1338         srp_cmd = ioctx->buf;
1339         srp_rsp = ioctx->buf;
1340
1341         switch (srp_cmd->opcode) {
1342         case SRP_CMD:
1343                 if (srpt_handle_cmd(ch, ioctx) < 0)
1344                         goto err;
1345                 break;
1346
1347         case SRP_TSK_MGMT:
1348                 if (srpt_handle_tsk_mgmt(ch, ioctx) < 0)
1349                         goto err;
1350                 break;
1351
1352         case SRP_I_LOGOUT:
1353         case SRP_AER_REQ:
1354         default:
1355                 srpt_build_cmd_rsp(ch, ioctx, ILLEGAL_REQUEST, INVALID_CDB,
1356                                    srp_cmd->tag);
1357                 goto err;
1358         }
1359
1360         dma_sync_single_for_device(ch->sport->sdev->device->dma_device,
1361                                    ioctx->dma, MAX_MESSAGE_SIZE,
1362                                    DMA_FROM_DEVICE);
1363
1364         return;
1365
1366 err:
1367         WARN_ON(srp_rsp->opcode != SRP_RSP);
1368         len = (sizeof *srp_rsp) + be32_to_cpu(srp_rsp->sense_data_len);
1369
1370         if (ch->state != RDMA_CHANNEL_LIVE) {
1371                 /* Give up if another thread modified the channel state. */
1372                 PRINT_ERROR("%s: channel is in state %d", __func__, ch->state);
1373                 srpt_reset_ioctx(ch, ioctx);
1374         } else if (srpt_post_send(ch, ioctx, len)) {
1375                 PRINT_ERROR("%s: sending SRP_RSP response failed", __func__);
1376                 srpt_reset_ioctx(ch, ioctx);
1377         }
1378 }
1379
1380 /*
1381  * Returns true if the ioctx list is non-empty or if the ib_srpt kernel thread
1382  * should stop.
1383  * @pre thread != 0
1384  */
1385 static inline int srpt_test_ioctx_list(void)
1386 {
1387         int res = (!list_empty(&srpt_thread.thread_ioctx_list) ||
1388                    unlikely(kthread_should_stop()));
1389         return res;
1390 }
1391
1392 /*
1393  * Add 'ioctx' to the tail of the ioctx list and wake up the kernel thread.
1394  *
1395  * @pre thread != 0
1396  */
1397 static inline void srpt_schedule_thread(struct srpt_ioctx *ioctx)
1398 {
1399         unsigned long flags;
1400
1401         spin_lock_irqsave(&srpt_thread.thread_lock, flags);
1402         list_add_tail(&ioctx->comp_list, &srpt_thread.thread_ioctx_list);
1403         spin_unlock_irqrestore(&srpt_thread.thread_lock, flags);
1404         wake_up(&ioctx_list_waitQ);
1405 }
1406
1407 /**
1408  * InfiniBand completion queue callback function.
1409  * @cq: completion queue.
1410  * @ctx: completion queue context, which was passed as the fourth argument of
1411  *       the function ib_create_cq().
1412  */
1413 static void srpt_completion(struct ib_cq *cq, void *ctx)
1414 {
1415         struct srpt_rdma_ch *ch = ctx;
1416         struct srpt_device *sdev = ch->sport->sdev;
1417         struct ib_wc wc;
1418         struct srpt_ioctx *ioctx;
1419
1420         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1421         while (ib_poll_cq(ch->cq, 1, &wc) > 0) {
1422                 if (wc.status) {
1423                         PRINT_ERROR("failed %s status= %d\n",
1424                                wc.wr_id & SRPT_OP_RECV ? "receive" : "send",
1425                                wc.status);
1426                         srpt_handle_err_comp(ch, &wc);
1427                         break;
1428                 }
1429
1430                 if (wc.wr_id & SRPT_OP_RECV) {
1431                         ioctx = sdev->ioctx_ring[wc.wr_id & ~SRPT_OP_RECV];
1432                         if (thread) {
1433                                 ioctx->ch = ch;
1434                                 ioctx->op = IB_WC_RECV;
1435                                 srpt_schedule_thread(ioctx);
1436                         } else
1437                                 srpt_handle_new_iu(ch, ioctx);
1438                         continue;
1439                 } else
1440                         ioctx = sdev->ioctx_ring[wc.wr_id];
1441
1442                 if (thread) {
1443                         ioctx->ch = ch;
1444                         ioctx->op = wc.opcode;
1445                         srpt_schedule_thread(ioctx);
1446                 } else {
1447                         switch (wc.opcode) {
1448                         case IB_WC_SEND:
1449                                 srpt_handle_send_comp(ch, ioctx,
1450                                         scst_estimate_context());
1451                                 break;
1452                         case IB_WC_RDMA_WRITE:
1453                         case IB_WC_RDMA_READ:
1454                                 srpt_handle_rdma_comp(ch, ioctx);
1455                                 break;
1456                         default:
1457                                 break;
1458                         }
1459                 }
1460         }
1461 }
1462
1463 /*
1464  * Create a completion queue on the specified device.
1465  */
1466 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
1467 {
1468         struct ib_qp_init_attr *qp_init;
1469         struct srpt_device *sdev = ch->sport->sdev;
1470         int cqe;
1471         int ret;
1472
1473         qp_init = kzalloc(sizeof *qp_init, GFP_KERNEL);
1474         if (!qp_init)
1475                 return -ENOMEM;
1476
1477         /* Create a completion queue (CQ). */
1478
1479         cqe = SRPT_RQ_SIZE + SRPT_SQ_SIZE - 1;
1480 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(RHEL_RELEASE_CODE)
1481         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe);
1482 #else
1483         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe, 0);
1484 #endif
1485         if (IS_ERR(ch->cq)) {
1486                 ret = PTR_ERR(ch->cq);
1487                 PRINT_ERROR("failed to create_cq cqe= %d ret= %d\n",
1488                         cqe, ret);
1489                 goto out;
1490         }
1491
1492         /* Request completion notification. */
1493
1494         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1495
1496         /* Create a queue pair (QP). */
1497
1498         qp_init->qp_context = (void *)ch;
1499         qp_init->event_handler = srpt_qp_event;
1500         qp_init->send_cq = ch->cq;
1501         qp_init->recv_cq = ch->cq;
1502         qp_init->srq = sdev->srq;
1503         qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
1504         qp_init->qp_type = IB_QPT_RC;
1505         qp_init->cap.max_send_wr = SRPT_SQ_SIZE;
1506         qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE;
1507
1508         ch->qp = ib_create_qp(sdev->pd, qp_init);
1509         if (IS_ERR(ch->qp)) {
1510                 ret = PTR_ERR(ch->qp);
1511                 ib_destroy_cq(ch->cq);
1512                 PRINT_ERROR("failed to create_qp ret= %d\n", ret);
1513                 goto out;
1514         }
1515
1516         TRACE_DBG("%s: max_cqe= %d max_sge= %d cm_id= %p",
1517                __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
1518                ch->cm_id);
1519
1520         /* Modify the attributes and the state of queue pair ch->qp. */
1521
1522         ret = srpt_init_ch_qp(ch, ch->qp);
1523         if (ret) {
1524                 ib_destroy_qp(ch->qp);
1525                 ib_destroy_cq(ch->cq);
1526                 goto out;
1527         }
1528
1529         atomic_set(&ch->req_lim_delta, SRPT_RQ_SIZE);
1530 out:
1531         kfree(qp_init);
1532         return ret;
1533 }
1534
1535 /**
1536  * Look up the RDMA channel that corresponds to the specified cm_id.
1537  *
1538  * Return NULL if no matching RDMA channel has been found.
1539  */
1540 static struct srpt_rdma_ch *srpt_find_channel(struct ib_cm_id *cm_id, bool del)
1541 {
1542         struct srpt_device *sdev = cm_id->context;
1543         struct srpt_rdma_ch *ch;
1544
1545         spin_lock_irq(&sdev->spinlock);
1546         list_for_each_entry(ch, &sdev->rch_list, list) {
1547                 if (ch->cm_id == cm_id) {
1548                         if (del)
1549                                 list_del(&ch->list);
1550                         spin_unlock_irq(&sdev->spinlock);
1551                         return ch;
1552                 }
1553         }
1554
1555         spin_unlock_irq(&sdev->spinlock);
1556
1557         return NULL;
1558 }
1559
1560 /**
1561  * Release all resources associated with the specified RDMA channel.
1562  *
1563  * Note: the caller must have removed the channel from the channel list
1564  * before calling this function.
1565  */
1566 static void srpt_release_channel(struct srpt_rdma_ch *ch, int destroy_cmid)
1567 {
1568         TRACE_ENTRY();
1569
1570         WARN_ON(srpt_find_channel(ch->cm_id, false) == ch);
1571
1572         if (ch->cm_id && destroy_cmid) {
1573                 TRACE_DBG("%s: destroy cm_id= %p", __func__, ch->cm_id);
1574                 ib_destroy_cm_id(ch->cm_id);
1575                 ch->cm_id = NULL;
1576         }
1577
1578         ib_destroy_qp(ch->qp);
1579         ib_destroy_cq(ch->cq);
1580
1581         if (ch->scst_sess) {
1582                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1583
1584                 TRACE_DBG("%s: release sess= %p sess_name= %s active_cmd= %d",
1585                           __func__, ch->scst_sess, ch->sess_name,
1586                           ch->active_scmnd_cnt);
1587
1588                 spin_lock_irq(&ch->spinlock);
1589                 list_for_each_entry_safe(ioctx, ioctx_tmp,
1590                                          &ch->active_scmnd_list, scmnd_list) {
1591                         spin_unlock_irq(&ch->spinlock);
1592
1593                         if (ioctx->scmnd)
1594                                 srpt_abort_scst_cmd(ch->sport->sdev,
1595                                                     ioctx->scmnd, true);
1596
1597                         spin_lock_irq(&ch->spinlock);
1598                 }
1599                 WARN_ON(!list_empty(&ch->active_scmnd_list));
1600                 WARN_ON(ch->active_scmnd_cnt != 0);
1601                 spin_unlock_irq(&ch->spinlock);
1602
1603                 scst_unregister_session(ch->scst_sess, 0, NULL);
1604                 ch->scst_sess = NULL;
1605         }
1606
1607         kfree(ch);
1608
1609         TRACE_EXIT();
1610 }
1611
1612 static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
1613                             struct ib_cm_req_event_param *param,
1614                             void *private_data)
1615 {
1616         struct srpt_device *sdev = cm_id->context;
1617         struct srp_login_req *req;
1618         struct srp_login_rsp *rsp;
1619         struct srp_login_rej *rej;
1620         struct ib_cm_rep_param *rep_param;
1621         struct srpt_rdma_ch *ch, *tmp_ch;
1622         u32 it_iu_len;
1623         int ret = 0;
1624
1625 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
1626         WARN_ON(!sdev || !private_data);
1627         if (!sdev || !private_data)
1628                 return -EINVAL;
1629 #else
1630         if (WARN_ON(!sdev || !private_data))
1631                 return -EINVAL;
1632 #endif
1633
1634         req = (struct srp_login_req *)private_data;
1635
1636         it_iu_len = be32_to_cpu(req->req_it_iu_len);
1637
1638         PRINT_INFO("received SRP_LOGIN_REQ with"
1639             " i_port_id 0x%llx:0x%llx, t_port_id 0x%llx:0x%llx and length %d"
1640             " on port %d (guid=0x%llx:0x%llx)\n",
1641             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[0]),
1642             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[8]),
1643             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[0]),
1644             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[8]),
1645             it_iu_len,
1646             param->port,
1647             (unsigned long long)be64_to_cpu(*(u64 *)
1648                                 &sdev->port[param->port - 1].gid.raw[0]),
1649             (unsigned long long)be64_to_cpu(*(u64 *)
1650                                 &sdev->port[param->port - 1].gid.raw[8]));
1651
1652         rsp = kzalloc(sizeof *rsp, GFP_KERNEL);
1653         rej = kzalloc(sizeof *rej, GFP_KERNEL);
1654         rep_param = kzalloc(sizeof *rep_param, GFP_KERNEL);
1655
1656         if (!rsp || !rej || !rep_param) {
1657                 ret = -ENOMEM;
1658                 goto out;
1659         }
1660
1661         if (it_iu_len > MAX_MESSAGE_SIZE || it_iu_len < 64) {
1662                 rej->reason =
1663                     cpu_to_be32(SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
1664                 ret = -EINVAL;
1665                 PRINT_ERROR("rejected SRP_LOGIN_REQ because its"
1666                        " length (%d bytes) is invalid\n", it_iu_len);
1667                 goto reject;
1668         }
1669
1670         if ((req->req_flags & 0x3) == SRP_MULTICHAN_SINGLE) {
1671                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
1672
1673                 spin_lock_irq(&sdev->spinlock);
1674
1675                 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
1676                         if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
1677                             && !memcmp(ch->t_port_id, req->target_port_id, 16)
1678                             && param->port == ch->sport->port
1679                             && param->listen_id == ch->sport->sdev->cm_id
1680                             && ch->cm_id) {
1681                                 enum rdma_ch_state prev_state;
1682
1683                                 /* found an existing channel */
1684                                 TRACE_DBG("Found existing channel name= %s"
1685                                           " cm_id= %p state= %d",
1686                                           ch->sess_name, ch->cm_id, ch->state);
1687
1688                                 prev_state = ch->state;
1689                                 if (ch->state == RDMA_CHANNEL_LIVE)
1690                                         ch->state = RDMA_CHANNEL_DISCONNECTING;
1691                                 else if (ch->state == RDMA_CHANNEL_CONNECTING)
1692                                         list_del(&ch->list);
1693
1694                                 spin_unlock_irq(&sdev->spinlock);
1695
1696                                 rsp->rsp_flags =
1697                                         SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
1698
1699                                 if (prev_state == RDMA_CHANNEL_LIVE) {
1700                                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
1701                                         PRINT_ERROR("disconnected"
1702                                           " session %s because a new"
1703                                           " SRP_LOGIN_REQ has been received.\n",
1704                                           ch->sess_name);
1705                                 } else if (prev_state ==
1706                                          RDMA_CHANNEL_CONNECTING) {
1707                                         PRINT_ERROR("%s", "rejected"
1708                                           " SRP_LOGIN_REQ because another login"
1709                                           " request is being processed.\n");
1710                                         ib_send_cm_rej(ch->cm_id,
1711                                                        IB_CM_REJ_NO_RESOURCES,
1712                                                        NULL, 0, NULL, 0);
1713                                         srpt_release_channel(ch, 1);
1714                                 }
1715
1716                                 spin_lock_irq(&sdev->spinlock);
1717                         }
1718                 }
1719
1720                 spin_unlock_irq(&sdev->spinlock);
1721
1722         } else
1723                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
1724
1725         if (((u64) (*(u64 *) req->target_port_id) !=
1726              cpu_to_be64(srpt_service_guid)) ||
1727             ((u64) (*(u64 *) (req->target_port_id + 8)) !=
1728              cpu_to_be64(srpt_service_guid))) {
1729                 rej->reason =
1730                     cpu_to_be32(SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
1731                 ret = -ENOMEM;
1732                 PRINT_ERROR("%s", "rejected SRP_LOGIN_REQ because it"
1733                        " has an invalid target port identifier\n");
1734                 goto reject;
1735         }
1736
1737         ch = kzalloc(sizeof *ch, GFP_KERNEL);
1738         if (!ch) {
1739                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1740                 PRINT_ERROR("%s", "rejected SRP_LOGIN_REQ because out of memory\n");
1741                 ret = -ENOMEM;
1742                 goto reject;
1743         }
1744
1745         spin_lock_init(&ch->spinlock);
1746         memcpy(ch->i_port_id, req->initiator_port_id, 16);
1747         memcpy(ch->t_port_id, req->target_port_id, 16);
1748         ch->sport = &sdev->port[param->port - 1];
1749         ch->cm_id = cm_id;
1750         ch->state = RDMA_CHANNEL_CONNECTING;
1751         INIT_LIST_HEAD(&ch->cmd_wait_list);
1752         INIT_LIST_HEAD(&ch->active_scmnd_list);
1753
1754         ret = srpt_create_ch_ib(ch);
1755         if (ret) {
1756                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1757                 PRINT_ERROR("%s", "rejected SRP_LOGIN_REQ because creating"
1758                        " a new RDMA channel failed\n");
1759                 goto free_ch;
1760         }
1761
1762         ret = srpt_ch_qp_rtr(ch, ch->qp);
1763         if (ret) {
1764                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1765                 PRINT_ERROR("rejected SRP_LOGIN_REQ because enabling"
1766                        " RTR failed (error code = %d)\n", ret);
1767                 goto destroy_ib;
1768         }
1769
1770         if (use_port_guid_in_session_name) {
1771                 /*
1772                  * If the kernel module parameter use_port_guid_in_session_name
1773                  * has been specified, use a combination of the target port
1774                  * GUID and the initiator port ID as the session name. This
1775                  * was the original behavior of the SRP target implementation
1776                  * (i.e. before the SRPT was included in OFED 1.3).
1777                  */
1778                 snprintf(ch->sess_name, sizeof(ch->sess_name),
1779                          "0x%016llx%016llx",
1780                          (unsigned long long)be64_to_cpu(*(u64 *)
1781                                 &sdev->port[param->port - 1].gid.raw[8]),
1782                          (unsigned long long)be64_to_cpu(*(u64 *)
1783                                 (ch->i_port_id + 8)));
1784         } else {
1785                 /*
1786                  * Default behavior: use the initator port identifier as the
1787                  * session name.
1788                  */
1789                 snprintf(ch->sess_name, sizeof(ch->sess_name),
1790                          "0x%016llx%016llx",
1791                          (unsigned long long)be64_to_cpu(*(u64 *)ch->i_port_id),
1792                          (unsigned long long)be64_to_cpu(*(u64 *)
1793                                  (ch->i_port_id + 8)));
1794         }
1795
1796         TRACE_DBG("registering session %s", ch->sess_name);
1797
1798         BUG_ON(!sdev->scst_tgt);
1799         ch->scst_sess = scst_register_session(sdev->scst_tgt, 0, ch->sess_name,
1800                                               NULL, NULL);
1801         if (!ch->scst_sess) {
1802                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1803                 TRACE_DBG("%s", "Failed to create scst sess");
1804                 goto destroy_ib;
1805         }
1806
1807         TRACE_DBG("Establish connection sess=%p name=%s cm_id=%p",
1808                   ch->scst_sess, ch->sess_name, ch->cm_id);
1809
1810         scst_sess_set_tgt_priv(ch->scst_sess, ch);
1811
1812         /* create srp_login_response */
1813         rsp->opcode = SRP_LOGIN_RSP;
1814         rsp->tag = req->tag;
1815         rsp->max_it_iu_len = req->req_it_iu_len;
1816         rsp->max_ti_iu_len = req->req_it_iu_len;
1817         rsp->buf_fmt =
1818             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1819         rsp->req_lim_delta = cpu_to_be32(SRPT_RQ_SIZE);
1820         atomic_set(&ch->req_lim_delta, 0);
1821
1822         /* create cm reply */
1823         rep_param->qp_num = ch->qp->qp_num;
1824         rep_param->private_data = (void *)rsp;
1825         rep_param->private_data_len = sizeof *rsp;
1826         rep_param->rnr_retry_count = 7;
1827         rep_param->flow_control = 1;
1828         rep_param->failover_accepted = 0;
1829         rep_param->srq = 1;
1830         rep_param->responder_resources = 4;
1831         rep_param->initiator_depth = 4;
1832
1833         ret = ib_send_cm_rep(cm_id, rep_param);
1834         if (ret) {
1835                 PRINT_ERROR("sending SRP_LOGIN_REQ response failed"
1836                        " (error code = %d)\n", ret);
1837                 goto release_channel;
1838         }
1839
1840         spin_lock_irq(&sdev->spinlock);
1841         list_add_tail(&ch->list, &sdev->rch_list);
1842         spin_unlock_irq(&sdev->spinlock);
1843
1844         goto out;
1845
1846 release_channel:
1847         scst_unregister_session(ch->scst_sess, 0, NULL);
1848         ch->scst_sess = NULL;
1849
1850 destroy_ib:
1851         ib_destroy_qp(ch->qp);
1852         ib_destroy_cq(ch->cq);
1853
1854 free_ch:
1855         kfree(ch);
1856
1857 reject:
1858         rej->opcode = SRP_LOGIN_REJ;
1859         rej->tag = req->tag;
1860         rej->buf_fmt =
1861             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1862
1863         ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
1864                              (void *)rej, sizeof *rej);
1865
1866 out:
1867         kfree(rep_param);
1868         kfree(rsp);
1869         kfree(rej);
1870
1871         return ret;
1872 }
1873
1874 /**
1875  * Release the channel with the specified cm_id.
1876  *
1877  * Returns one to indicate that the caller of srpt_cm_handler() should destroy
1878  * the cm_id.
1879  */
1880 static void srpt_find_and_release_channel(struct ib_cm_id *cm_id)
1881 {
1882         struct srpt_rdma_ch *ch;
1883
1884         ch = srpt_find_channel(cm_id, true);
1885         if (ch)
1886                 srpt_release_channel(ch, 0);
1887 }
1888
1889 static void srpt_cm_rej_recv(struct ib_cm_id *cm_id)
1890 {
1891         TRACE_DBG("%s: cm_id=%p", __func__, cm_id);
1892         srpt_find_and_release_channel(cm_id);
1893 }
1894
1895 /**
1896  * Process an IB_CM_RTU_RECEIVED or IB_CM_USER_ESTABLISHED event.
1897  *
1898  * An IB_CM_RTU_RECEIVED message indicates that the connection is established
1899  * and that the recipient may begin transmitting (RTU = ready to use).
1900  */
1901 static int srpt_cm_rtu_recv(struct ib_cm_id *cm_id)
1902 {
1903         struct srpt_rdma_ch *ch;
1904         int ret;
1905
1906         ch = srpt_find_channel(cm_id, false);
1907         if (!ch)
1908                 return -EINVAL;
1909
1910         if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_CONNECTING,
1911                                             RDMA_CHANNEL_LIVE)) {
1912                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1913
1914                 ret = srpt_ch_qp_rts(ch, ch->qp);
1915
1916                 list_for_each_entry_safe(ioctx, ioctx_tmp, &ch->cmd_wait_list,
1917                                          wait_list) {
1918                         list_del(&ioctx->wait_list);
1919                         srpt_handle_new_iu(ch, ioctx);
1920                 }
1921                 if (ret && srpt_test_and_set_channel_state(ch,
1922                                         RDMA_CHANNEL_LIVE,
1923                                         RDMA_CHANNEL_DISCONNECTING)) {
1924                         TRACE_DBG("cm_id=%p sess_name=%s state=%d",
1925                                   cm_id, ch->sess_name, ch->state);
1926                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
1927                 }
1928         } else if (ch->state == RDMA_CHANNEL_DISCONNECTING) {
1929                 TRACE_DBG("cm_id=%p sess_name=%s state=%d",
1930                           cm_id, ch->sess_name, ch->state);
1931                 ib_send_cm_dreq(ch->cm_id, NULL, 0);
1932                 ret = -EAGAIN;
1933         } else
1934                 ret = 0;
1935
1936         return ret;
1937 }
1938
1939 static void srpt_cm_timewait_exit(struct ib_cm_id *cm_id)
1940 {
1941         TRACE_DBG("%s: cm_id=%p", __func__, cm_id);
1942         srpt_find_and_release_channel(cm_id);
1943 }
1944
1945 static void srpt_cm_rep_error(struct ib_cm_id *cm_id)
1946 {
1947         TRACE_DBG("%s: cm_id=%p", __func__, cm_id);
1948         srpt_find_and_release_channel(cm_id);
1949 }
1950
1951 static int srpt_cm_dreq_recv(struct ib_cm_id *cm_id)
1952 {
1953         struct srpt_rdma_ch *ch;
1954
1955         ch = srpt_find_channel(cm_id, false);
1956         if (!ch)
1957                 return -EINVAL;
1958
1959         TRACE_DBG("%s: cm_id= %p ch->state= %d",
1960                  __func__, cm_id, ch->state);
1961
1962         switch (ch->state) {
1963         case RDMA_CHANNEL_LIVE:
1964         case RDMA_CHANNEL_CONNECTING:
1965                 ib_send_cm_drep(ch->cm_id, NULL, 0);
1966                 break;
1967         case RDMA_CHANNEL_DISCONNECTING:
1968         default:
1969                 break;
1970         }
1971
1972         return 0;
1973 }
1974
1975 static void srpt_cm_drep_recv(struct ib_cm_id *cm_id)
1976 {
1977         TRACE_DBG("%s: cm_id=%p", __func__, cm_id);
1978         srpt_find_and_release_channel(cm_id);
1979 }
1980
1981 /**
1982  * IB connection manager callback function.
1983  *
1984  * A non-zero return value will make the caller destroy the CM ID.
1985  *
1986  * Note: srpt_add_one passes a struct srpt_device* as the third argument to
1987  * the ib_create_cm_id() call.
1988  */
1989 static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
1990 {
1991         int ret = 0;
1992
1993         switch (event->event) {
1994         case IB_CM_REQ_RECEIVED:
1995                 ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd,
1996                                        event->private_data);
1997                 break;
1998         case IB_CM_REJ_RECEIVED:
1999                 srpt_cm_rej_recv(cm_id);
2000                 ret = -EINVAL;
2001                 break;
2002         case IB_CM_RTU_RECEIVED:
2003         case IB_CM_USER_ESTABLISHED:
2004                 ret = srpt_cm_rtu_recv(cm_id);
2005                 break;
2006         case IB_CM_DREQ_RECEIVED:
2007                 ret = srpt_cm_dreq_recv(cm_id);
2008                 break;
2009         case IB_CM_DREP_RECEIVED:
2010                 srpt_cm_drep_recv(cm_id);
2011                 ret = -EINVAL;
2012                 break;
2013         case IB_CM_TIMEWAIT_EXIT:
2014                 srpt_cm_timewait_exit(cm_id);
2015                 ret = -EINVAL;
2016                 break;
2017         case IB_CM_REP_ERROR:
2018                 srpt_cm_rep_error(cm_id);
2019                 ret = -EINVAL;
2020                 break;
2021         default:
2022                 break;
2023         }
2024
2025         return ret;
2026 }
2027
2028 static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
2029                                  struct srpt_ioctx *ioctx,
2030                                  struct scst_cmd *scmnd)
2031 {
2032         struct scatterlist *scat;
2033         scst_data_direction dir;
2034         struct rdma_iu *riu;
2035         struct srp_direct_buf *db;
2036         dma_addr_t dma_addr;
2037         struct ib_sge *sge;
2038         u64 raddr;
2039         u32 rsize;
2040         u32 tsize;
2041         u32 dma_len;
2042         int count, nrdma;
2043         int i, j, k;
2044
2045         scat = scst_cmd_get_sg(scmnd);
2046         dir = scst_cmd_get_data_direction(scmnd);
2047         count = dma_map_sg(ch->sport->sdev->device->dma_device, scat,
2048                            scst_cmd_get_sg_cnt(scmnd),
2049                            scst_to_tgt_dma_dir(dir));
2050         if (unlikely(!count))
2051                 return -EBUSY;
2052
2053         if (ioctx->rdma_ius && ioctx->n_rdma_ius)
2054                 nrdma = ioctx->n_rdma_ius;
2055         else {
2056                 nrdma = count / SRPT_DEF_SG_PER_WQE + ioctx->n_rbuf;
2057
2058                 ioctx->rdma_ius = kzalloc(nrdma * sizeof *riu,
2059                                           scst_cmd_atomic(scmnd)
2060                                           ? GFP_ATOMIC : GFP_KERNEL);
2061                 if (!ioctx->rdma_ius) {
2062                         dma_unmap_sg(ch->sport->sdev->device->dma_device,
2063                                      scat, scst_cmd_get_sg_cnt(scmnd),
2064                                      scst_to_tgt_dma_dir(dir));
2065                         return -ENOMEM;
2066                 }
2067
2068                 ioctx->n_rdma_ius = nrdma;
2069         }
2070
2071         db = ioctx->rbufs;
2072         tsize = (dir == SCST_DATA_READ) ?
2073                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2074         dma_len = sg_dma_len(&scat[0]);
2075         riu = ioctx->rdma_ius;
2076
2077         /*
2078          * For each remote desc - calculate the #ib_sge.
2079          * If #ib_sge < SRPT_DEF_SG_PER_WQE per rdma operation then
2080          *      each remote desc rdma_iu is required a rdma wr;
2081          * else
2082          *      we need to allocate extra rdma_iu to carry extra #ib_sge in
2083          *      another rdma wr
2084          */
2085         for (i = 0, j = 0;
2086              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2087                 rsize = be32_to_cpu(db->len);
2088                 raddr = be64_to_cpu(db->va);
2089                 riu->raddr = raddr;
2090                 riu->rkey = be32_to_cpu(db->key);
2091                 riu->sge_cnt = 0;
2092
2093                 /* calculate how many sge required for this remote_buf */
2094                 while (rsize > 0 && tsize > 0) {
2095
2096                         if (rsize >= dma_len) {
2097                                 tsize -= dma_len;
2098                                 rsize -= dma_len;
2099                                 raddr += dma_len;
2100
2101                                 if (tsize > 0) {
2102                                         ++j;
2103                                         if (j < count)
2104                                                 dma_len = sg_dma_len(&scat[j]);
2105                                 }
2106                         } else {
2107                                 tsize -= rsize;
2108                                 dma_len -= rsize;
2109                                 rsize = 0;
2110                         }
2111
2112                         ++riu->sge_cnt;
2113
2114                         if (rsize > 0 && riu->sge_cnt == SRPT_DEF_SG_PER_WQE) {
2115                                 riu->sge =
2116                                     kmalloc(riu->sge_cnt * sizeof *riu->sge,
2117                                             scst_cmd_atomic(scmnd)
2118                                             ? GFP_ATOMIC : GFP_KERNEL);
2119                                 if (!riu->sge)
2120                                         goto free_mem;
2121
2122                                 ++ioctx->n_rdma;
2123                                 ++riu;
2124                                 riu->sge_cnt = 0;
2125                                 riu->raddr = raddr;
2126                                 riu->rkey = be32_to_cpu(db->key);
2127                         }
2128                 }
2129
2130                 riu->sge = kmalloc(riu->sge_cnt * sizeof *riu->sge,
2131                                    scst_cmd_atomic(scmnd)
2132                                    ? GFP_ATOMIC : GFP_KERNEL);
2133
2134                 if (!riu->sge)
2135                         goto free_mem;
2136
2137                 ++ioctx->n_rdma;
2138         }
2139
2140         db = ioctx->rbufs;
2141         scat = scst_cmd_get_sg(scmnd);
2142         tsize = (dir == SCST_DATA_READ) ?
2143                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2144         riu = ioctx->rdma_ius;
2145         dma_len = sg_dma_len(&scat[0]);
2146         dma_addr = sg_dma_address(&scat[0]);
2147
2148         /* this second loop is really mapped sg_addres to rdma_iu->ib_sge */
2149         for (i = 0, j = 0;
2150              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2151                 rsize = be32_to_cpu(db->len);
2152                 sge = riu->sge;
2153                 k = 0;
2154
2155                 while (rsize > 0 && tsize > 0) {
2156                         sge->addr = dma_addr;
2157                         sge->lkey = ch->sport->sdev->mr->lkey;
2158
2159                         if (rsize >= dma_len) {
2160                                 sge->length =
2161                                         (tsize < dma_len) ? tsize : dma_len;
2162                                 tsize -= dma_len;
2163                                 rsize -= dma_len;
2164
2165                                 if (tsize > 0) {
2166                                         ++j;
2167                                         if (j < count) {
2168                                                 dma_len = sg_dma_len(&scat[j]);
2169                                                 dma_addr =
2170                                                     sg_dma_address(&scat[j]);
2171                                         }
2172                                 }
2173                         } else {
2174                                 sge->length = (tsize < rsize) ? tsize : rsize;
2175                                 tsize -= rsize;
2176                                 dma_len -= rsize;
2177                                 dma_addr += rsize;
2178                                 rsize = 0;
2179                         }
2180
2181                         ++k;
2182                         if (k == riu->sge_cnt && rsize > 0) {
2183                                 ++riu;
2184                                 sge = riu->sge;
2185                                 k = 0;
2186                         } else if (rsize > 0)
2187                                 ++sge;
2188                 }
2189         }
2190
2191         return 0;
2192
2193 free_mem:
2194         while (ioctx->n_rdma)
2195                 kfree(ioctx->rdma_ius[ioctx->n_rdma--].sge);
2196
2197         kfree(ioctx->rdma_ius);
2198
2199         dma_unmap_sg(ch->sport->sdev->device->dma_device,
2200                      scat, scst_cmd_get_sg_cnt(scmnd),
2201                      scst_to_tgt_dma_dir(dir));
2202
2203         return -ENOMEM;
2204 }
2205
2206 static int srpt_perform_rdmas(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2207                               scst_data_direction dir)
2208 {
2209         struct ib_send_wr wr;
2210         struct ib_send_wr *bad_wr;
2211         struct rdma_iu *riu;
2212         int i;
2213         int ret = 0;
2214
2215         riu = ioctx->rdma_ius;
2216         memset(&wr, 0, sizeof wr);
2217
2218         for (i = 0; i < ioctx->n_rdma; ++i, ++riu) {
2219                 wr.opcode = (dir == SCST_DATA_READ) ?
2220                     IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;
2221                 wr.next = NULL;
2222                 wr.wr_id = ioctx->index;
2223                 wr.wr.rdma.remote_addr = riu->raddr;
2224                 wr.wr.rdma.rkey = riu->rkey;
2225                 wr.num_sge = riu->sge_cnt;
2226                 wr.sg_list = riu->sge;
2227
2228                 /* only get completion event for the last rdma wr */
2229                 if (i == (ioctx->n_rdma - 1) && dir == SCST_DATA_WRITE)
2230                         wr.send_flags = IB_SEND_SIGNALED;
2231
2232                 ret = ib_post_send(ch->qp, &wr, &bad_wr);
2233                 if (ret)
2234                         break;
2235         }
2236
2237         return ret;
2238 }
2239
2240 /*
2241  * Start data transfer between initiator and target. Must not block.
2242  */
2243 static int srpt_xfer_data(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2244                           struct scst_cmd *scmnd)
2245 {
2246         int ret;
2247
2248         ret = srpt_map_sg_to_ib_sge(ch, ioctx, scmnd);
2249         if (ret) {
2250                 PRINT_ERROR("%s[%d] ret=%d\n", __func__, __LINE__, ret);
2251                 ret = SCST_TGT_RES_QUEUE_FULL;
2252                 goto out;
2253         }
2254
2255         ret = srpt_perform_rdmas(ch, ioctx, scst_cmd_get_data_direction(scmnd));
2256         if (ret) {
2257                 PRINT_ERROR("%s[%d] ret=%d\n", __func__, __LINE__, ret);
2258                 if (ret == -EAGAIN || ret == -ENOMEM)
2259                         ret = SCST_TGT_RES_QUEUE_FULL;
2260                 else
2261                         ret = SCST_TGT_RES_FATAL_ERROR;
2262                 goto out;
2263         }
2264
2265         ret = SCST_TGT_RES_SUCCESS;
2266
2267 out:
2268         return ret;
2269 }
2270
2271 /*
2272  * Called by the SCST core to inform ib_srpt that data reception from the
2273  * initiator should start (SCST_DATA_WRITE). Must not block.
2274  */
2275 static int srpt_rdy_to_xfer(struct scst_cmd *scmnd)
2276 {
2277         struct srpt_rdma_ch *ch;
2278         struct srpt_ioctx *ioctx;
2279
2280         ioctx = scst_cmd_get_tgt_priv(scmnd);
2281         BUG_ON(!ioctx);
2282
2283         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2284         BUG_ON(!ch);
2285
2286         if (ch->state == RDMA_CHANNEL_DISCONNECTING)
2287                 return SCST_TGT_RES_FATAL_ERROR;
2288         else if (ch->state == RDMA_CHANNEL_CONNECTING)
2289                 return SCST_TGT_RES_QUEUE_FULL;
2290
2291         ioctx->state = SRPT_STATE_NEED_DATA;
2292
2293         return srpt_xfer_data(ch, ioctx, scmnd);
2294 }
2295
2296 /*
2297  * Called by the SCST core. Transmits the response buffer and status held in
2298  * 'scmnd'. Must not block.
2299  */
2300 static int srpt_xmit_response(struct scst_cmd *scmnd)
2301 {
2302         struct srpt_rdma_ch *ch;
2303         struct srpt_ioctx *ioctx;
2304         struct srp_rsp *srp_rsp;
2305         u64 tag;
2306         int ret = SCST_TGT_RES_SUCCESS;
2307         int dir;
2308         int status;
2309
2310         ioctx = scst_cmd_get_tgt_priv(scmnd);
2311         BUG_ON(!ioctx);
2312
2313         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2314         BUG_ON(!ch);
2315
2316         tag = scst_cmd_get_tag(scmnd);
2317
2318         ioctx->state = SRPT_STATE_PROCESSED;
2319
2320         if (ch->state != RDMA_CHANNEL_LIVE) {
2321                 PRINT_ERROR("%s: tag= %lld channel in bad state %d\n",
2322                        __func__, (unsigned long long)tag, ch->state);
2323
2324                 if (ch->state == RDMA_CHANNEL_DISCONNECTING)
2325                         ret = SCST_TGT_RES_FATAL_ERROR;
2326                 else if (ch->state == RDMA_CHANNEL_CONNECTING)
2327                         ret = SCST_TGT_RES_QUEUE_FULL;
2328
2329                 if (unlikely(scst_cmd_aborted(scmnd)))
2330                         goto out_aborted;
2331
2332                 goto out;
2333         }
2334
2335         dma_sync_single_for_cpu(ch->sport->sdev->device->dma_device, ioctx->dma,
2336                                 MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
2337
2338         srp_rsp = ioctx->buf;
2339
2340         if (unlikely(scst_cmd_aborted(scmnd))) {
2341                 PRINT_ERROR("%s: tag= %lld already get aborted\n",
2342                        __func__, (unsigned long long)tag);
2343                 goto out_aborted;
2344         }
2345
2346         dir = scst_cmd_get_data_direction(scmnd);
2347         status = scst_cmd_get_status(scmnd) & 0xff;
2348
2349         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE, NO_ADD_SENSE, tag);
2350
2351         if (SCST_SENSE_VALID(scst_cmd_get_sense_buffer(scmnd))) {
2352                 srp_rsp->sense_data_len = scst_cmd_get_sense_buffer_len(scmnd);
2353                 if (srp_rsp->sense_data_len >
2354                     (MAX_MESSAGE_SIZE - sizeof *srp_rsp))
2355                         srp_rsp->sense_data_len =
2356                             MAX_MESSAGE_SIZE - sizeof *srp_rsp;
2357
2358                 memcpy((u8 *) (srp_rsp + 1), scst_cmd_get_sense_buffer(scmnd),
2359                        srp_rsp->sense_data_len);
2360
2361                 srp_rsp->sense_data_len = cpu_to_be32(srp_rsp->sense_data_len);
2362                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
2363
2364                 if (!status)
2365                         status = SAM_STAT_CHECK_CONDITION;
2366         }
2367
2368         srp_rsp->status = status;
2369
2370         /* For read commands, transfer the data to the initiator. */
2371         if (dir == SCST_DATA_READ && scst_cmd_get_resp_data_len(scmnd)) {
2372                 ret = srpt_xfer_data(ch, ioctx, scmnd);
2373                 if (ret != SCST_TGT_RES_SUCCESS) {
2374                         PRINT_ERROR("%s: tag= %lld xfer_data failed\n",
2375                                __func__, (unsigned long long)tag);
2376                         goto out;
2377                 }
2378         }
2379
2380         if (srpt_post_send(ch, ioctx,
2381                            sizeof *srp_rsp +
2382                            be32_to_cpu(srp_rsp->sense_data_len))) {
2383                 PRINT_ERROR("%s: ch->state= %d tag= %lld\n",
2384                        __func__, ch->state,
2385                        (unsigned long long)tag);
2386                 ret = SCST_TGT_RES_FATAL_ERROR;
2387         }
2388
2389 out:
2390         return ret;
2391
2392 out_aborted:
2393         ret = SCST_TGT_RES_SUCCESS;
2394         scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_ABORTED);
2395         ioctx->state = SRPT_STATE_ABORTED;
2396         scst_tgt_cmd_done(scmnd, SCST_CONTEXT_SAME);
2397         goto out;
2398 }
2399
2400 /*
2401  * Called by the SCST core to inform ib_srpt that a received task management
2402  * function has been completed. Must not block.
2403  */
2404 static void srpt_tsk_mgmt_done(struct scst_mgmt_cmd *mcmnd)
2405 {
2406         struct srpt_rdma_ch *ch;
2407         struct srpt_mgmt_ioctx *mgmt_ioctx;
2408         struct srpt_ioctx *ioctx;
2409         int rsp_len;
2410
2411         mgmt_ioctx = scst_mgmt_cmd_get_tgt_priv(mcmnd);
2412         BUG_ON(!mgmt_ioctx);
2413
2414         ch = mgmt_ioctx->ch;
2415         BUG_ON(!ch);
2416
2417         ioctx = mgmt_ioctx->ioctx;
2418         BUG_ON(!ioctx);
2419
2420         TRACE_DBG("%s: tsk_mgmt_done for tag= %lld status=%d\n",
2421                   __func__, (unsigned long long)mgmt_ioctx->tag,
2422                   scst_mgmt_cmd_get_status(mcmnd));
2423
2424         rsp_len = srpt_build_tskmgmt_rsp(ch, ioctx,
2425                                          (scst_mgmt_cmd_get_status(mcmnd) ==
2426                                           SCST_MGMT_STATUS_SUCCESS) ?
2427                                          SRP_TSK_MGMT_SUCCESS :
2428                                          SRP_TSK_MGMT_FAILED,
2429                                          mgmt_ioctx->tag);
2430         srpt_post_send(ch, ioctx, rsp_len);
2431
2432         scst_mgmt_cmd_set_tgt_priv(mcmnd, NULL);
2433
2434         kfree(mgmt_ioctx);
2435 }
2436
2437 /*
2438  * Called by the SCST core to inform ib_srpt that the command 'scmnd' is about
2439  * to be freed. May be called in IRQ context.
2440  */
2441 static void srpt_on_free_cmd(struct scst_cmd *scmnd)
2442 {
2443         struct srpt_rdma_ch *ch;
2444         struct srpt_ioctx *ioctx;
2445
2446         ioctx = scst_cmd_get_tgt_priv(scmnd);
2447         BUG_ON(!ioctx);
2448
2449         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2450         BUG_ON(!ch);
2451
2452         spin_lock_irq(&ch->spinlock);
2453         list_del(&ioctx->scmnd_list);
2454         ch->active_scmnd_cnt--;
2455         spin_unlock_irq(&ch->spinlock);
2456
2457         srpt_reset_ioctx(ch, ioctx);
2458         scst_cmd_set_tgt_priv(scmnd, NULL);
2459 }
2460
2461 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2462 /* A vanilla 2.6.19 or older kernel without backported OFED kernel headers. */
2463 static void srpt_refresh_port_work(void *ctx)
2464 #else
2465 static void srpt_refresh_port_work(struct work_struct *work)
2466 #endif
2467 {
2468 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2469         struct srpt_port *sport = (struct srpt_port *)ctx;
2470 #else
2471         struct srpt_port *sport = container_of(work, struct srpt_port, work);
2472 #endif
2473
2474         srpt_refresh_port(sport);
2475 }
2476
2477 /*
2478  * Called by the SCST core to detect target adapters. Returns the number of
2479  * detected target adapters.
2480  */
2481 static int srpt_detect(struct scst_tgt_template *tp)
2482 {
2483         int device_count;
2484
2485         TRACE_ENTRY();
2486
2487         device_count = atomic_read(&srpt_device_count);
2488
2489         TRACE_EXIT_RES(device_count);
2490
2491         return device_count;
2492 }
2493
2494 /*
2495  * Callback function called by the SCST core from scst_unregister() to free up
2496  * the resources associated with device scst_tgt.
2497  */
2498 static int srpt_release(struct scst_tgt *scst_tgt)
2499 {
2500         struct srpt_device *sdev = scst_tgt_get_tgt_priv(scst_tgt);
2501         struct srpt_rdma_ch *ch, *tmp_ch;
2502
2503         TRACE_ENTRY();
2504
2505         BUG_ON(!scst_tgt);
2506 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
2507         WARN_ON(!sdev);
2508         if (!sdev)
2509                 return -ENODEV;
2510 #else
2511         if (WARN_ON(!sdev))
2512                 return -ENODEV;
2513 #endif
2514
2515         srpt_unregister_procfs_entry(scst_tgt->tgtt);
2516
2517         spin_lock_irq(&sdev->spinlock);
2518         list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
2519                 list_del(&ch->list);
2520                 spin_unlock_irq(&sdev->spinlock);
2521                 srpt_release_channel(ch, 1);
2522                 spin_lock_irq(&sdev->spinlock);
2523         }
2524         spin_unlock_irq(&sdev->spinlock);
2525
2526         srpt_unregister_mad_agent(sdev);
2527
2528         scst_tgt_set_tgt_priv(scst_tgt, NULL);
2529
2530         TRACE_EXIT();
2531
2532         return 0;
2533 }
2534
2535 /*
2536  * Entry point for ib_srpt's kernel thread. This kernel thread is only created
2537  * when the module parameter 'thread' is not zero (the default is zero).
2538  * This thread processes the ioctx list srpt_thread.thread_ioctx_list.
2539  *
2540  * @pre thread != 0
2541  */
2542 static int srpt_ioctx_thread(void *arg)
2543 {
2544         struct srpt_ioctx *ioctx;
2545
2546         /* Hibernation / freezing of the SRPT kernel thread is not supported. */
2547         current->flags |= PF_NOFREEZE;
2548
2549         spin_lock_irq(&srpt_thread.thread_lock);
2550         while (!kthread_should_stop()) {
2551                 wait_queue_t wait;
2552                 init_waitqueue_entry(&wait, current);
2553
2554                 if (!srpt_test_ioctx_list()) {
2555                         add_wait_queue_exclusive(&ioctx_list_waitQ, &wait);
2556
2557                         for (;;) {
2558                                 set_current_state(TASK_INTERRUPTIBLE);
2559                                 if (srpt_test_ioctx_list())
2560                                         break;
2561                                 spin_unlock_irq(&srpt_thread.thread_lock);
2562                                 schedule();
2563                                 spin_lock_irq(&srpt_thread.thread_lock);
2564                         }
2565                         set_current_state(TASK_RUNNING);
2566                         remove_wait_queue(&ioctx_list_waitQ, &wait);
2567                 }
2568
2569                 while (!list_empty(&srpt_thread.thread_ioctx_list)) {
2570                         ioctx = list_entry(srpt_thread.thread_ioctx_list.next,
2571                                            struct srpt_ioctx, comp_list);
2572
2573                         list_del(&ioctx->comp_list);
2574
2575                         spin_unlock_irq(&srpt_thread.thread_lock);
2576                         switch (ioctx->op) {
2577                         case IB_WC_SEND:
2578                                 srpt_handle_send_comp(ioctx->ch, ioctx,
2579                                         SCST_CONTEXT_DIRECT);
2580                                 break;
2581                         case IB_WC_RDMA_WRITE:
2582                         case IB_WC_RDMA_READ:
2583                                 srpt_handle_rdma_comp(ioctx->ch, ioctx);
2584                                 break;
2585                         case IB_WC_RECV:
2586                                 srpt_handle_new_iu(ioctx->ch, ioctx);
2587                                 break;
2588                         default:
2589                                 break;
2590                         }
2591                         spin_lock_irq(&srpt_thread.thread_lock);
2592                 }
2593         }
2594         spin_unlock_irq(&srpt_thread.thread_lock);
2595
2596         return 0;
2597 }
2598
2599 /* SCST target template for the SRP target implementation. */
2600 static struct scst_tgt_template srpt_template = {
2601         .name = DRV_NAME,
2602         .sg_tablesize = SRPT_DEF_SG_TABLESIZE,
2603         .xmit_response_atomic = 1,
2604         .rdy_to_xfer_atomic = 1,
2605         .no_proc_entry = 0,
2606         .detect = srpt_detect,
2607         .release = srpt_release,
2608         .xmit_response = srpt_xmit_response,
2609         .rdy_to_xfer = srpt_rdy_to_xfer,
2610         .on_free_cmd = srpt_on_free_cmd,
2611         .task_mgmt_fn_done = srpt_tsk_mgmt_done
2612 };
2613
2614 /*
2615  * The callback function srpt_release_class_dev() is called whenever a
2616  * device is removed from the /sys/class/infiniband_srpt device class.
2617  */
2618 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2619 static void srpt_release_class_dev(struct class_device *class_dev)
2620 #else
2621 static void srpt_release_class_dev(struct device *dev)
2622 #endif
2623 {
2624 }
2625
2626 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2627 static int srpt_trace_level_show(struct seq_file *seq, void *v)
2628 {
2629         return scst_proc_log_entry_read(seq, trace_flag, NULL);
2630 }
2631
2632 static ssize_t srpt_proc_trace_level_write(struct file *file,
2633         const char __user *buf, size_t length, loff_t *off)
2634 {
2635         return scst_proc_log_entry_write(file, buf, length, &trace_flag,
2636                 DEFAULT_SRPT_TRACE_FLAGS, NULL);
2637 }
2638
2639 static struct scst_proc_data srpt_log_proc_data = {
2640         SCST_DEF_RW_SEQ_OP(srpt_proc_trace_level_write)
2641         .show = srpt_trace_level_show,
2642 };
2643 #endif
2644
2645 static struct class_attribute srpt_class_attrs[] = {
2646         __ATTR_NULL,
2647 };
2648
2649 static struct class srpt_class = {
2650         .name = "infiniband_srpt",
2651 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2652         .release = srpt_release_class_dev,
2653 #else
2654         .dev_release = srpt_release_class_dev,
2655 #endif
2656         .class_attrs = srpt_class_attrs,
2657 };
2658
2659 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2660 static ssize_t show_login_info(struct class_device *class_dev, char *buf)
2661 #else
2662 static ssize_t show_login_info(struct device *dev,
2663                                struct device_attribute *attr, char *buf)
2664 #endif
2665 {
2666         struct srpt_device *sdev =
2667 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2668                 container_of(class_dev, struct srpt_device, class_dev);
2669 #else
2670                 container_of(dev, struct srpt_device, dev);
2671 #endif
2672         struct srpt_port *sport;
2673         int i;
2674         int len = 0;
2675
2676         for (i = 0; i < sdev->device->phys_port_cnt; i++) {
2677                 sport = &sdev->port[i];
2678
2679                 len += sprintf(buf + len,
2680                                "tid_ext=%016llx,ioc_guid=%016llx,pkey=ffff,"
2681                                "dgid=%04x%04x%04x%04x%04x%04x%04x%04x,"
2682                                "service_id=%016llx\n",
2683                                (unsigned long long) srpt_service_guid,
2684                                (unsigned long long) srpt_service_guid,
2685                                be16_to_cpu(((__be16 *) sport->gid.raw)[0]),
2686                                be16_to_cpu(((__be16 *) sport->gid.raw)[1]),
2687                                be16_to_cpu(((__be16 *) sport->gid.raw)[2]),
2688                                be16_to_cpu(((__be16 *) sport->gid.raw)[3]),
2689                                be16_to_cpu(((__be16 *) sport->gid.raw)[4]),
2690                                be16_to_cpu(((__be16 *) sport->gid.raw)[5]),
2691                                be16_to_cpu(((__be16 *) sport->gid.raw)[6]),
2692                                be16_to_cpu(((__be16 *) sport->gid.raw)[7]),
2693                                (unsigned long long) srpt_service_guid);
2694         }
2695
2696         return len;
2697 }
2698
2699 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2700 static CLASS_DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2701 #else
2702 static DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2703 #endif
2704
2705 /*
2706  * Callback function called by the InfiniBand core when either an InfiniBand
2707  * device has been added or during the ib_register_client() call for each
2708  * registered InfiniBand device.
2709  */
2710 static void srpt_add_one(struct ib_device *device)
2711 {
2712         struct srpt_device *sdev;
2713         struct srpt_port *sport;
2714         struct ib_srq_init_attr srq_attr;
2715         int i;
2716
2717         TRACE_ENTRY();
2718
2719         sdev = kzalloc(sizeof *sdev, GFP_KERNEL);
2720         if (!sdev)
2721                 return;
2722
2723         sdev->device = device;
2724
2725 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2726         sdev->class_dev.class = &srpt_class;
2727         sdev->class_dev.dev = device->dma_device;
2728         snprintf(sdev->class_dev.class_id, BUS_ID_SIZE,
2729                  "srpt-%s", device->name);
2730 #else
2731         sdev->dev.class = &srpt_class;
2732         sdev->dev.parent = device->dma_device;
2733 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
2734         snprintf(sdev->dev.bus_id, BUS_ID_SIZE, "srpt-%s", device->name);
2735 #else
2736         snprintf(sdev->init_name, sizeof(sdev->init_name),
2737                  "srpt-%s", device->name);
2738         sdev->dev.init_name = sdev->init_name;
2739 #endif
2740 #endif
2741
2742 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2743         if (class_device_register(&sdev->class_dev))
2744                 goto free_dev;
2745         if (class_device_create_file(&sdev->class_dev,
2746                                      &class_device_attr_login_info))
2747                 goto err_dev;
2748 #else
2749         if (device_register(&sdev->dev))
2750                 goto free_dev;
2751         if (device_create_file(&sdev->dev, &dev_attr_login_info))
2752                 goto err_dev;
2753 #endif
2754
2755         if (ib_query_device(device, &sdev->dev_attr))
2756                 goto err_dev;
2757
2758         sdev->pd = ib_alloc_pd(device);
2759         if (IS_ERR(sdev->pd))
2760                 goto err_dev;
2761
2762         sdev->mr = ib_get_dma_mr(sdev->pd, IB_ACCESS_LOCAL_WRITE);
2763         if (IS_ERR(sdev->mr))
2764                 goto err_pd;
2765
2766         srq_attr.event_handler = srpt_srq_event;
2767         srq_attr.srq_context = (void *)sdev;
2768         srq_attr.attr.max_wr = min(SRPT_SRQ_SIZE, sdev->dev_attr.max_srq_wr);
2769         srq_attr.attr.max_sge = 1;
2770         srq_attr.attr.srq_limit = 0;
2771
2772         sdev->srq = ib_create_srq(sdev->pd, &srq_attr);
2773         if (IS_ERR(sdev->srq))
2774                 goto err_mr;
2775
2776         TRACE_DBG("%s: create SRQ #wr= %d max_allow=%d dev= %s",
2777                __func__, srq_attr.attr.max_wr,
2778               sdev->dev_attr.max_srq_wr, device->name);
2779
2780         if (!srpt_service_guid)
2781                 srpt_service_guid = be64_to_cpu(device->node_guid);
2782
2783         sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);
2784         if (IS_ERR(sdev->cm_id))
2785                 goto err_srq;
2786
2787         /* print out target login information */
2788         TRACE_DBG("Target login info: id_ext=%016llx,"
2789                   "ioc_guid=%016llx,pkey=ffff,service_id=%016llx",
2790                   (unsigned long long) srpt_service_guid,
2791                   (unsigned long long) srpt_service_guid,
2792                   (unsigned long long) srpt_service_guid);
2793
2794         /*
2795          * We do not have a consistent service_id (ie. also id_ext of target_id)
2796          * to identify this target. We currently use the guid of the first HCA
2797          * in the system as service_id; therefore, the target_id will change
2798          * if this HCA is gone bad and replaced by different HCA
2799          */
2800         if (ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0, NULL))
2801                 goto err_cm;
2802
2803         INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
2804                               srpt_event_handler);
2805         if (ib_register_event_handler(&sdev->event_handler))
2806                 goto err_cm;
2807
2808         if (srpt_alloc_ioctx_ring(sdev))
2809                 goto err_event;
2810
2811         INIT_LIST_HEAD(&sdev->rch_list);
2812         spin_lock_init(&sdev->spinlock);
2813
2814         for (i = 0; i < SRPT_SRQ_SIZE; ++i)
2815                 srpt_post_recv(sdev, sdev->ioctx_ring[i]);
2816
2817         ib_set_client_data(device, &srpt_client, sdev);
2818
2819         sdev->scst_tgt = scst_register(&srpt_template, NULL);
2820         if (!sdev->scst_tgt) {
2821                 PRINT_ERROR("SCST registration failed for %s.\n",
2822                         sdev->device->name);
2823                 goto err_ring;
2824         }
2825
2826         scst_tgt_set_tgt_priv(sdev->scst_tgt, sdev);
2827
2828         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
2829                 sport = &sdev->port[i - 1];
2830                 sport->sdev = sdev;
2831                 sport->port = i;
2832 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2833                 /*
2834                  * A vanilla 2.6.19 or older kernel without backported OFED
2835                  * kernel headers.
2836                  */
2837                 INIT_WORK(&sport->work, srpt_refresh_port_work, sport);
2838 #else
2839                 INIT_WORK(&sport->work, srpt_refresh_port_work);
2840 #endif
2841                 if (srpt_refresh_port(sport)) {
2842                         PRINT_ERROR("MAD registration failed for %s-%d.\n",
2843                                     sdev->device->name, i);
2844                         goto err_refresh_port;
2845                 }
2846         }
2847
2848         atomic_inc(&srpt_device_count);
2849
2850         TRACE_EXIT();
2851
2852         return;
2853
2854 err_refresh_port:
2855         scst_unregister(sdev->scst_tgt);
2856 err_ring:
2857         ib_set_client_data(device, &srpt_client, NULL);
2858         srpt_free_ioctx_ring(sdev);
2859 err_event:
2860         ib_unregister_event_handler(&sdev->event_handler);
2861 err_cm:
2862         ib_destroy_cm_id(sdev->cm_id);
2863 err_srq:
2864         ib_destroy_srq(sdev->srq);
2865 err_mr:
2866         ib_dereg_mr(sdev->mr);
2867 err_pd:
2868         ib_dealloc_pd(sdev->pd);
2869 err_dev:
2870 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2871         class_device_unregister(&sdev->class_dev);
2872 #else
2873         device_unregister(&sdev->dev);
2874 #endif
2875 free_dev:
2876         kfree(sdev);
2877
2878         TRACE_EXIT();
2879 }
2880
2881 /*
2882  * Callback function called by the InfiniBand core when either an InfiniBand
2883  * device has been removed or during the ib_unregister_client() call for each
2884  * registered InfiniBand device.
2885  */
2886 static void srpt_remove_one(struct ib_device *device)
2887 {
2888         int i;
2889         struct srpt_device *sdev;
2890
2891         TRACE_ENTRY();
2892
2893         sdev = ib_get_client_data(device, &srpt_client);
2894 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
2895         WARN_ON(!sdev);
2896         if (!sdev)
2897                 return;
2898 #else
2899         if (WARN_ON(!sdev))
2900                 return;
2901 #endif
2902
2903         /*
2904          * Cancel the work if it is queued. Wait until srpt_refresh_port_work()
2905          * finished if it is running.
2906          */
2907         for (i = 0; i < sdev->device->phys_port_cnt; i++)
2908 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
2909                 cancel_work_sync(&sdev->port[i].work);
2910 #else
2911                 /*
2912                  * cancel_work_sync() was introduced in kernel 2.6.22. Older
2913                  * kernels do not have a facility to cancel scheduled work.
2914                  */
2915                 PRINT_ERROR("%s", 
2916                        "your kernel does not provide cancel_work_sync().\n");
2917 #endif
2918
2919         scst_unregister(sdev->scst_tgt);
2920         sdev->scst_tgt = NULL;
2921
2922         ib_unregister_event_handler(&sdev->event_handler);
2923         ib_destroy_cm_id(sdev->cm_id);
2924         ib_destroy_srq(sdev->srq);
2925         ib_dereg_mr(sdev->mr);
2926         ib_dealloc_pd(sdev->pd);
2927 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2928         class_device_unregister(&sdev->class_dev);
2929 #else
2930         device_unregister(&sdev->dev);
2931 #endif
2932
2933         srpt_free_ioctx_ring(sdev);
2934         kfree(sdev);
2935
2936         TRACE_EXIT();
2937 }
2938
2939 /**
2940  * Create procfs entries for srpt. Currently the only procfs entry created
2941  * by this function is the "trace_level" entry.
2942  */
2943 static int srpt_register_procfs_entry(struct scst_tgt_template *tgt)
2944 {
2945         int res = 0;
2946 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2947         struct proc_dir_entry *p, *root;
2948
2949         root = scst_proc_get_tgt_root(tgt);
2950         WARN_ON(!root);
2951         if (root) {
2952                 /*
2953                  * Fill in the scst_proc_data::data pointer, which is used in
2954                  * a printk(KERN_INFO ...) statement in
2955                  * scst_proc_log_entry_write() in scst_proc.c.
2956                  */
2957                 srpt_log_proc_data.data = (char *)tgt->name;
2958                 p = scst_create_proc_entry(root, SRPT_PROC_TRACE_LEVEL_NAME,
2959                                            &srpt_log_proc_data);
2960                 if (!p)
2961                         res = -ENOMEM;
2962         } else
2963                 res = -ENOMEM;
2964
2965 #endif
2966         return res;
2967 }
2968
2969 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt)
2970 {
2971 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2972         struct proc_dir_entry *root;
2973
2974         root = scst_proc_get_tgt_root(tgt);
2975         WARN_ON(!root);
2976         if (root)
2977                 remove_proc_entry(SRPT_PROC_TRACE_LEVEL_NAME, root);
2978 #endif
2979 }
2980
2981 /*
2982  * Module initialization.
2983  *
2984  * Note: since ib_register_client() registers callback functions, and since at
2985  * least one of these callback functions (srpt_add_one()) calls SCST functions,
2986  * the SCST target template must be registered before ib_register_client() is
2987  * called.
2988  */
2989 static int __init srpt_init_module(void)
2990 {
2991         int ret;
2992
2993         ret = class_register(&srpt_class);
2994         if (ret) {
2995                 PRINT_ERROR("%s", "couldn't register class ib_srpt\n");
2996                 goto out;
2997         }
2998
2999         ret = scst_register_target_template(&srpt_template);
3000         if (ret < 0) {
3001                 PRINT_ERROR("%s", "couldn't register with scst\n");
3002                 ret = -ENODEV;
3003                 goto out_unregister_class;
3004         }
3005
3006         ret = srpt_register_procfs_entry(&srpt_template);
3007         if (ret) {
3008                 PRINT_ERROR("%s", "couldn't register procfs entry\n");
3009                 goto out_unregister_target;
3010         }
3011
3012         ret = ib_register_client(&srpt_client);
3013         if (ret) {
3014                 PRINT_ERROR("%s", "couldn't register IB client\n");
3015                 goto out_unregister_target;
3016         }
3017
3018         if (thread) {
3019                 spin_lock_init(&srpt_thread.thread_lock);
3020                 INIT_LIST_HEAD(&srpt_thread.thread_ioctx_list);
3021                 srpt_thread.thread = kthread_run(srpt_ioctx_thread,
3022                                                  NULL, "srpt_thread");
3023                 if (IS_ERR(srpt_thread.thread)) {
3024                         srpt_thread.thread = NULL;
3025                         thread = 0;
3026                 }
3027         }
3028
3029         return 0;
3030
3031 out_unregister_target:
3032         /*
3033          * Note: the procfs entry is unregistered in srpt_release(), which is
3034          * called by scst_unregister_target_template().
3035          */
3036         scst_unregister_target_template(&srpt_template);
3037 out_unregister_class:
3038         class_unregister(&srpt_class);
3039 out:
3040         return ret;
3041 }
3042
3043 static void __exit srpt_cleanup_module(void)
3044 {
3045         TRACE_ENTRY();
3046
3047         if (srpt_thread.thread)
3048                 kthread_stop(srpt_thread.thread);
3049         ib_unregister_client(&srpt_client);
3050         scst_unregister_target_template(&srpt_template);
3051         class_unregister(&srpt_class);
3052
3053         TRACE_EXIT();
3054 }
3055
3056 module_init(srpt_init_module);
3057 module_exit(srpt_cleanup_module);