53a4aafa56f2c7d347a2ed93b5ea389d4030a49d
[mirror/scst/.git] / srpt / src / ib_srpt.c
1 /*
2  * Copyright (c) 2006 - 2009 Mellanox Technology Inc.  All rights reserved.
3  * Copyright (C) 2008 Vladislav Bolkhovitin <vst@vlnb.net>
4  * Copyright (C) 2008 - 2009 Bart Van Assche <bart.vanassche@gmail.com>
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  */
35
36 #include <linux/module.h>
37 #include <linux/init.h>
38 #include <linux/slab.h>
39 #include <linux/err.h>
40 #include <linux/ctype.h>
41 #include <linux/string.h>
42 #include <linux/kthread.h>
43 #include <asm/atomic.h>
44 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
45 #include <linux/proc_fs.h>
46 #include <linux/seq_file.h>
47 #endif
48 #include "ib_srpt.h"
49 #include "scst_debug.h"
50
51 /* Name of this kernel module. */
52 #define DRV_NAME                "ib_srpt"
53 /* Prefix for printk() kernel messages. */
54 #define LOG_PFX                 DRV_NAME ": "
55 #define DRV_VERSION             "1.0.1"
56 #define DRV_RELDATE             "July 10, 2008"
57 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
58 /* Flags to be used in SCST debug tracing statements. */
59 #define DEFAULT_SRPT_TRACE_FLAGS (TRACE_OUT_OF_MEM | TRACE_MINOR \
60                                   | TRACE_MGMT | TRACE_SPECIAL)
61 /* Name of the entry that will be created under /proc/scsi_tgt/ib_srpt. */
62 #define SRPT_PROC_TRACE_LEVEL_NAME      "trace_level"
63 #endif
64
65 #define MELLANOX_SRPT_ID_STRING "SCST SRP target"
66
67 MODULE_AUTHOR("Vu Pham");
68 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
69                    "v" DRV_VERSION " (" DRV_RELDATE ")");
70 MODULE_LICENSE("Dual BSD/GPL");
71
72 struct srpt_thread {
73         /* Protects thread_ioctx_list. */
74         spinlock_t thread_lock;
75         /* I/O contexts to be processed by the kernel thread. */
76         struct list_head thread_ioctx_list;
77         /* SRPT kernel thread. */
78         struct task_struct *thread;
79 };
80
81 /*
82  * Global Variables
83  */
84
85 static u64 srpt_service_guid;
86 /* List of srpt_device structures. */
87 static atomic_t srpt_device_count;
88 static int use_port_guid_in_session_name;
89 static int thread;
90 static struct srpt_thread srpt_thread;
91 static DECLARE_WAIT_QUEUE_HEAD(ioctx_list_waitQ);
92 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
93 static unsigned long trace_flag = DEFAULT_SRPT_TRACE_FLAGS;
94 module_param(trace_flag, long, 0644);
95 MODULE_PARM_DESC(trace_flag,
96                  "Trace flags for the ib_srpt kernel module.");
97 #endif
98 #if defined(CONFIG_SCST_DEBUG)
99 static unsigned long interrupt_processing_delay_in_us;
100 module_param(interrupt_processing_delay_in_us, long, 0744);
101 MODULE_PARM_DESC(interrupt_processing_delay_in_us,
102                  "CQ completion handler interrupt delay in microseconds.");
103 static unsigned long thread_processing_delay_in_us;
104 module_param(thread_processing_delay_in_us, long, 0744);
105 MODULE_PARM_DESC(thread_processing_delay_in_us,
106                  "SRP thread processing delay in microseconds.");
107 #endif
108
109 module_param(thread, int, 0444);
110 MODULE_PARM_DESC(thread,
111                  "Executing ioctx in thread context. Default 0, i.e. soft IRQ, "
112                  "where possible");
113
114 module_param(use_port_guid_in_session_name, bool, 0444);
115 MODULE_PARM_DESC(use_port_guid_in_session_name,
116                  "Use target port ID in the SCST session name such that"
117                  " redundant paths between multiport systems can be masked.");
118
119 static void srpt_add_one(struct ib_device *device);
120 static void srpt_remove_one(struct ib_device *device);
121 static void srpt_unregister_mad_agent(struct srpt_device *sdev);
122 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt);
123
124 static struct ib_client srpt_client = {
125         .name = DRV_NAME,
126         .add = srpt_add_one,
127         .remove = srpt_remove_one
128 };
129
130 /**
131  * Atomically test and set the channel state.
132  * @ch: RDMA channel.
133  * @old: channel state to compare with.
134  * @new: state to change the channel state to if the current state matches the
135  *       argument 'old'.
136  *
137  * Returns true if the channel state matched old upon entry of this function,
138  * and false otherwise.
139  */
140 static bool srpt_test_and_set_channel_state(struct srpt_rdma_ch *ch,
141                                             enum rdma_ch_state old,
142                                             enum rdma_ch_state new)
143 {
144         unsigned long flags;
145         enum rdma_ch_state cur;
146
147         spin_lock_irqsave(&ch->spinlock, flags);
148         cur = ch->state;
149         if (cur == old)
150                 ch->state = new;
151         spin_unlock_irqrestore(&ch->spinlock, flags);
152
153         return cur == old;
154 }
155
156 /*
157  * Callback function called by the InfiniBand core when an asynchronous IB
158  * event occurs. This callback may occur in interrupt context. See also
159  * section 11.5.2, Set Asynchronous Event Handler in the InfiniBand
160  * Architecture Specification.
161  */
162 static void srpt_event_handler(struct ib_event_handler *handler,
163                                struct ib_event *event)
164 {
165         struct srpt_device *sdev;
166         struct srpt_port *sport;
167
168         sdev = ib_get_client_data(event->device, &srpt_client);
169         if (!sdev || sdev->device != event->device)
170                 return;
171
172         TRACE_DBG("ASYNC event= %d on device= %s",
173                   event->event, sdev->device->name);
174
175         switch (event->event) {
176         case IB_EVENT_PORT_ERR:
177                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
178                         sport = &sdev->port[event->element.port_num - 1];
179                         sport->lid = 0;
180                         sport->sm_lid = 0;
181                 }
182                 break;
183         case IB_EVENT_PORT_ACTIVE:
184         case IB_EVENT_LID_CHANGE:
185         case IB_EVENT_PKEY_CHANGE:
186         case IB_EVENT_SM_CHANGE:
187         case IB_EVENT_CLIENT_REREGISTER:
188                 /*
189                  * Refresh port data asynchronously. Note: it is safe to call
190                  * schedule_work() even if &sport->work is already on the
191                  * global workqueue because schedule_work() tests for the
192                  * work_pending() condition before adding &sport->work to the
193                  * global work queue.
194                  */
195                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
196                         sport = &sdev->port[event->element.port_num - 1];
197                         if (!sport->lid && !sport->sm_lid)
198                                 schedule_work(&sport->work);
199                 }
200                 break;
201         default:
202                 break;
203         }
204
205 }
206
207 /*
208  * Callback function called by the InfiniBand core for SRQ (shared receive
209  * queue) events.
210  */
211 static void srpt_srq_event(struct ib_event *event, void *ctx)
212 {
213         TRACE_DBG("SRQ event %d", event->event);
214 }
215
216 /*
217  * Callback function called by the InfiniBand core for QP (queue pair) events.
218  */
219 static void srpt_qp_event(struct ib_event *event, void *ctx)
220 {
221         struct srpt_rdma_ch *ch = ctx;
222
223         TRACE_DBG("QP event %d on cm_id=%p sess_name=%s state=%d",
224                   event->event, ch->cm_id, ch->sess_name, ch->state);
225
226         switch (event->event) {
227         case IB_EVENT_COMM_EST:
228 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20) || defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
229                 ib_cm_notify(ch->cm_id, event->event);
230 #else
231                 /* Vanilla 2.6.19 kernel (or before) without OFED. */
232                 PRINT_ERROR("%s", "how to perform ib_cm_notify() on a"
233                             " vanilla 2.6.18 kernel ???");
234 #endif
235                 break;
236         case IB_EVENT_QP_LAST_WQE_REACHED:
237                 if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_LIVE,
238                                         RDMA_CHANNEL_DISCONNECTING)) {
239                         PRINT_INFO("disconnected session %s.", ch->sess_name);
240                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
241                 }
242                 break;
243         default:
244                 break;
245         }
246 }
247
248 /*
249  * Helper function for filling in an InfiniBand IOUnitInfo structure. Copies
250  * the lowest four bits of value in element slot of the array of four bit
251  * elements called c_list (controller list). The index slot is one-based.
252  *
253  * @pre 1 <= slot && 0 <= value && value < 16
254  */
255 static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
256 {
257         u16 id;
258         u8 tmp;
259
260         id = (slot - 1) / 2;
261         if (slot & 0x1) {
262                 tmp = c_list[id] & 0xf;
263                 c_list[id] = (value << 4) | tmp;
264         } else {
265                 tmp = c_list[id] & 0xf0;
266                 c_list[id] = (value & 0xf) | tmp;
267         }
268 }
269
270 /*
271  * Write InfiniBand ClassPortInfo to mad. See also section 16.3.3.1
272  * ClassPortInfo in the InfiniBand Architecture Specification.
273  */
274 static void srpt_get_class_port_info(struct ib_dm_mad *mad)
275 {
276         struct ib_class_port_info *cif;
277
278         cif = (struct ib_class_port_info *)mad->data;
279         memset(cif, 0, sizeof *cif);
280         cif->base_version = 1;
281         cif->class_version = 1;
282         cif->resp_time_value = 20;
283
284         mad->mad_hdr.status = 0;
285 }
286
287 /*
288  * Write IOUnitInfo to mad. See also section 16.3.3.3 IOUnitInfo in the
289  * InfiniBand Architecture Specification. See also section B.7,
290  * table B.6 in the T10 SRP r16a document.
291  */
292 static void srpt_get_iou(struct ib_dm_mad *mad)
293 {
294         struct ib_dm_iou_info *ioui;
295         u8 slot;
296         int i;
297
298         ioui = (struct ib_dm_iou_info *)mad->data;
299         ioui->change_id = 1;
300         ioui->max_controllers = 16;
301
302         /* set present for slot 1 and empty for the rest */
303         srpt_set_ioc(ioui->controller_list, 1, 1);
304         for (i = 1, slot = 2; i < 16; i++, slot++)
305                 srpt_set_ioc(ioui->controller_list, slot, 0);
306
307         mad->mad_hdr.status = 0;
308 }
309
310 /*
311  * Write IOControllerprofile to mad for I/O controller (sdev, slot). See also
312  * section 16.3.3.4 IOControllerProfile in the InfiniBand Architecture
313  * Specification. See also section B.7, table B.7 in the T10 SRP r16a
314  * document.
315  */
316 static void srpt_get_ioc(struct srpt_device *sdev, u32 slot,
317                          struct ib_dm_mad *mad)
318 {
319         struct ib_dm_ioc_profile *iocp;
320
321         iocp = (struct ib_dm_ioc_profile *)mad->data;
322
323         if (!slot || slot > 16) {
324                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
325                 return;
326         }
327
328         if (slot > 2) {
329                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
330                 return;
331         }
332
333         memset(iocp, 0, sizeof *iocp);
334         strcpy(iocp->id_string, MELLANOX_SRPT_ID_STRING);
335         iocp->guid = cpu_to_be64(srpt_service_guid);
336         iocp->vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
337         iocp->device_id = cpu_to_be32(sdev->dev_attr.vendor_part_id);
338         iocp->device_version = cpu_to_be16(sdev->dev_attr.hw_ver);
339         iocp->subsys_vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
340         iocp->subsys_device_id = 0x0;
341         iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
342         iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
343         iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
344         iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
345         iocp->send_queue_depth = cpu_to_be16(SRPT_SRQ_SIZE);
346         iocp->rdma_read_depth = 4;
347         iocp->send_size = cpu_to_be32(MAX_MESSAGE_SIZE);
348         iocp->rdma_size = cpu_to_be32(MAX_RDMA_SIZE);
349         iocp->num_svc_entries = 1;
350         iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
351                 SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
352
353         mad->mad_hdr.status = 0;
354 }
355
356 /*
357  * Device management: write ServiceEntries to mad for the given slot. See also
358  * section 16.3.3.5 ServiceEntries in the InfiniBand Architecture
359  * Specification. See also section B.7, table B.8 in the T10 SRP r16a document.
360  */
361 static void srpt_get_svc_entries(u64 ioc_guid,
362                                  u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
363 {
364         struct ib_dm_svc_entries *svc_entries;
365
366         WARN_ON(!ioc_guid);
367
368         if (!slot || slot > 16) {
369                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
370                 return;
371         }
372
373         if (slot > 2 || lo > hi || hi > 1) {
374                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
375                 return;
376         }
377
378         svc_entries = (struct ib_dm_svc_entries *)mad->data;
379         memset(svc_entries, 0, sizeof *svc_entries);
380         svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid);
381         snprintf(svc_entries->service_entries[0].name,
382                  sizeof(svc_entries->service_entries[0].name),
383                  "%s%016llx",
384                  SRP_SERVICE_NAME_PREFIX,
385                  (unsigned long long)ioc_guid);
386
387         mad->mad_hdr.status = 0;
388 }
389
390 /*
391  * Actual processing of a received MAD *rq_mad received through source port *sp
392  * (MAD = InfiniBand management datagram). The response to be sent back is
393  * written to *rsp_mad.
394  */
395 static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
396                                  struct ib_dm_mad *rsp_mad)
397 {
398         u16 attr_id;
399         u32 slot;
400         u8 hi, lo;
401
402         attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
403         switch (attr_id) {
404         case DM_ATTR_CLASS_PORT_INFO:
405                 srpt_get_class_port_info(rsp_mad);
406                 break;
407         case DM_ATTR_IOU_INFO:
408                 srpt_get_iou(rsp_mad);
409                 break;
410         case DM_ATTR_IOC_PROFILE:
411                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
412                 srpt_get_ioc(sp->sdev, slot, rsp_mad);
413                 break;
414         case DM_ATTR_SVC_ENTRIES:
415                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
416                 hi = (u8) ((slot >> 8) & 0xff);
417                 lo = (u8) (slot & 0xff);
418                 slot = (u16) ((slot >> 16) & 0xffff);
419                 srpt_get_svc_entries(srpt_service_guid,
420                                      slot, hi, lo, rsp_mad);
421                 break;
422         default:
423                 rsp_mad->mad_hdr.status =
424                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
425                 break;
426         }
427 }
428
429 /*
430  * Callback function that is called by the InfiniBand core after transmission of
431  * a MAD. (MAD = management datagram; AH = address handle.)
432  */
433 static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
434                                   struct ib_mad_send_wc *mad_wc)
435 {
436         ib_destroy_ah(mad_wc->send_buf->ah);
437         ib_free_send_mad(mad_wc->send_buf);
438 }
439
440 /*
441  * Callback function that is called by the InfiniBand core after reception of
442  * a MAD (management datagram).
443  */
444 static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
445                                   struct ib_mad_recv_wc *mad_wc)
446 {
447         struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
448         struct ib_ah *ah;
449         struct ib_mad_send_buf *rsp;
450         struct ib_dm_mad *dm_mad;
451
452         if (!mad_wc || !mad_wc->recv_buf.mad)
453                 return;
454
455         ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
456                                   mad_wc->recv_buf.grh, mad_agent->port_num);
457         if (IS_ERR(ah))
458                 goto err;
459
460         BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR);
461
462         rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
463                                  mad_wc->wc->pkey_index, 0,
464                                  IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA,
465                                  GFP_KERNEL);
466         if (IS_ERR(rsp))
467                 goto err_rsp;
468
469         rsp->ah = ah;
470
471         dm_mad = rsp->mad;
472         memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof *dm_mad);
473         dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
474         dm_mad->mad_hdr.status = 0;
475
476         switch (mad_wc->recv_buf.mad->mad_hdr.method) {
477         case IB_MGMT_METHOD_GET:
478                 srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
479                 break;
480         case IB_MGMT_METHOD_SET:
481                 dm_mad->mad_hdr.status =
482                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
483                 break;
484         default:
485                 dm_mad->mad_hdr.status =
486                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
487                 break;
488         }
489
490         if (!ib_post_send_mad(rsp, NULL)) {
491                 ib_free_recv_mad(mad_wc);
492                 /* will destroy_ah & free_send_mad in send completion */
493                 return;
494         }
495
496         ib_free_send_mad(rsp);
497
498 err_rsp:
499         ib_destroy_ah(ah);
500 err:
501         ib_free_recv_mad(mad_wc);
502 }
503
504 /*
505  * Enable InfiniBand management datagram processing, update the cached sm_lid,
506  * lid and gid values, and register a callback function for processing MADs
507  * on the specified port. It is safe to call this function more than once for
508  * the same port.
509  */
510 static int srpt_refresh_port(struct srpt_port *sport)
511 {
512         struct ib_mad_reg_req reg_req;
513         struct ib_port_modify port_modify;
514         struct ib_port_attr port_attr;
515         int ret;
516
517         memset(&port_modify, 0, sizeof port_modify);
518         port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
519         port_modify.clr_port_cap_mask = 0;
520
521         ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
522         if (ret)
523                 goto err_mod_port;
524
525         ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
526         if (ret)
527                 goto err_query_port;
528
529         sport->sm_lid = port_attr.sm_lid;
530         sport->lid = port_attr.lid;
531
532         ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid);
533         if (ret)
534                 goto err_query_port;
535
536         if (!sport->mad_agent) {
537                 memset(&reg_req, 0, sizeof reg_req);
538                 reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
539                 reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
540                 set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
541                 set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
542
543                 sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
544                                                          sport->port,
545                                                          IB_QPT_GSI,
546                                                          &reg_req, 0,
547                                                          srpt_mad_send_handler,
548                                                          srpt_mad_recv_handler,
549                                                          sport);
550                 if (IS_ERR(sport->mad_agent)) {
551                         ret = PTR_ERR(sport->mad_agent);
552                         sport->mad_agent = NULL;
553                         goto err_query_port;
554                 }
555         }
556
557         return 0;
558
559 err_query_port:
560
561         port_modify.set_port_cap_mask = 0;
562         port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
563         ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
564
565 err_mod_port:
566
567         return ret;
568 }
569
570 /*
571  * Unregister the callback function for processing MADs and disable MAD
572  * processing for all ports of the specified device. It is safe to call this
573  * function more than once for the same device.
574  */
575 static void srpt_unregister_mad_agent(struct srpt_device *sdev)
576 {
577         struct ib_port_modify port_modify = {
578                 .clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP,
579         };
580         struct srpt_port *sport;
581         int i;
582
583         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
584                 sport = &sdev->port[i - 1];
585                 WARN_ON(sport->port != i);
586                 if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
587                         PRINT_ERROR("%s", "disabling MAD processing failed.");
588                 if (sport->mad_agent) {
589                         ib_unregister_mad_agent(sport->mad_agent);
590                         sport->mad_agent = NULL;
591                 }
592         }
593 }
594
595 /*
596  * Allocate and initialize an SRPT I/O context structure.
597  */
598 static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev)
599 {
600         struct srpt_ioctx *ioctx;
601
602         ioctx = kmalloc(sizeof *ioctx, GFP_KERNEL);
603         if (!ioctx)
604                 goto out;
605
606         ioctx->buf = kzalloc(MAX_MESSAGE_SIZE, GFP_KERNEL);
607         if (!ioctx->buf)
608                 goto out_free_ioctx;
609
610         ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf,
611                                        MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
612         if (ib_dma_mapping_error(sdev->device, ioctx->dma))
613                 goto out_free_buf;
614
615         return ioctx;
616
617 out_free_buf:
618         kfree(ioctx->buf);
619 out_free_ioctx:
620         kfree(ioctx);
621 out:
622         return NULL;
623 }
624
625 /*
626  * Deallocate an SRPT I/O context structure.
627  */
628 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
629 {
630         if (!ioctx)
631                 return;
632
633         ib_dma_unmap_single(sdev->device, ioctx->dma,
634                             MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
635         kfree(ioctx->buf);
636         kfree(ioctx);
637 }
638
639 /*
640  * Associate a ring of SRPT I/O context structures with the specified device.
641  */
642 static int srpt_alloc_ioctx_ring(struct srpt_device *sdev)
643 {
644         int i;
645
646         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
647                 sdev->ioctx_ring[i] = srpt_alloc_ioctx(sdev);
648
649                 if (!sdev->ioctx_ring[i])
650                         goto err;
651
652                 sdev->ioctx_ring[i]->index = i;
653         }
654
655         return 0;
656
657 err:
658         while (--i > 0) {
659                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
660                 sdev->ioctx_ring[i] = NULL;
661         }
662         return -ENOMEM;
663 }
664
665 /* Free the ring of SRPT I/O context structures. */
666 static void srpt_free_ioctx_ring(struct srpt_device *sdev)
667 {
668         int i;
669
670         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
671                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
672                 sdev->ioctx_ring[i] = NULL;
673         }
674 }
675
676 /*
677  * Post a receive request on the work queue of InfiniBand device 'sdev'.
678  */
679 static int srpt_post_recv(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
680 {
681         struct ib_sge list;
682         struct ib_recv_wr wr, *bad_wr;
683
684         wr.wr_id = ioctx->index | SRPT_OP_RECV;
685
686         list.addr = ioctx->dma;
687         list.length = MAX_MESSAGE_SIZE;
688         list.lkey = sdev->mr->lkey;
689
690         wr.next = NULL;
691         wr.sg_list = &list;
692         wr.num_sge = 1;
693
694         return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);
695 }
696
697 /*
698  * Post an IB send request.
699  * @ch: RDMA channel to post the send request on.
700  * @ioctx: I/O context of the send request.
701  * @len: length of the request to be sent in bytes.
702  *
703  * Returns zero upon success and a non-zero value upon failure.
704  */
705 static int srpt_post_send(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
706                           int len)
707 {
708         struct ib_sge list;
709         struct ib_send_wr wr, *bad_wr;
710         struct srpt_device *sdev = ch->sport->sdev;
711
712         ib_dma_sync_single_for_device(sdev->device, ioctx->dma,
713                                       MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
714
715         list.addr = ioctx->dma;
716         list.length = len;
717         list.lkey = sdev->mr->lkey;
718
719         wr.next = NULL;
720         wr.wr_id = ioctx->index;
721         wr.sg_list = &list;
722         wr.num_sge = 1;
723         wr.opcode = IB_WR_SEND;
724         wr.send_flags = IB_SEND_SIGNALED;
725
726         return ib_post_send(ch->qp, &wr, &bad_wr);
727 }
728
729 static int srpt_get_desc_tbl(struct srpt_ioctx *ioctx, struct srp_cmd *srp_cmd,
730                              int *ind)
731 {
732         struct srp_indirect_buf *idb;
733         struct srp_direct_buf *db;
734
735         *ind = 0;
736         if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
737             ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
738                 ioctx->n_rbuf = 1;
739                 ioctx->rbufs = &ioctx->single_rbuf;
740
741                 db = (void *)srp_cmd->add_data;
742                 memcpy(ioctx->rbufs, db, sizeof *db);
743                 ioctx->data_len = be32_to_cpu(db->len);
744         } else {
745                 idb = (void *)srp_cmd->add_data;
746
747                 ioctx->n_rbuf = be32_to_cpu(idb->table_desc.len) / sizeof *db;
748
749                 if (ioctx->n_rbuf >
750                     (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
751                         *ind = 1;
752                         ioctx->n_rbuf = 0;
753                         goto out;
754                 }
755
756                 if (ioctx->n_rbuf == 1)
757                         ioctx->rbufs = &ioctx->single_rbuf;
758                 else
759                         ioctx->rbufs =
760                                 kmalloc(ioctx->n_rbuf * sizeof *db, GFP_ATOMIC);
761                 if (!ioctx->rbufs) {
762                         ioctx->n_rbuf = 0;
763                         return -ENOMEM;
764                 }
765
766                 db = idb->desc_list;
767                 memcpy(ioctx->rbufs, db, ioctx->n_rbuf * sizeof *db);
768                 ioctx->data_len = be32_to_cpu(idb->len);
769         }
770 out:
771         return 0;
772 }
773
774 /*
775  * Modify the attributes of queue pair 'qp': allow local write, remote read,
776  * and remote write. Also transition 'qp' to state IB_QPS_INIT.
777  */
778 static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
779 {
780         struct ib_qp_attr *attr;
781         int ret;
782
783         attr = kzalloc(sizeof *attr, GFP_KERNEL);
784         if (!attr)
785                 return -ENOMEM;
786
787         attr->qp_state = IB_QPS_INIT;
788         attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ |
789             IB_ACCESS_REMOTE_WRITE;
790         attr->port_num = ch->sport->port;
791         attr->pkey_index = 0;
792
793         ret = ib_modify_qp(qp, attr,
794                            IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
795                            IB_QP_PKEY_INDEX);
796
797         kfree(attr);
798         return ret;
799 }
800
801 /**
802  * Change the state of a channel to 'ready to receive' (RTR).
803  * @ch: channel of the queue pair.
804  * @qp: queue pair to change the state of.
805  *
806  * Returns zero upon success and a negative value upon failure.
807  *
808  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
809  * If this structure ever becomes larger, it might be necessary to allocate
810  * it dynamically instead of on the stack.
811  */
812 static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
813 {
814         struct ib_qp_attr qp_attr;
815         int attr_mask;
816         int ret;
817
818         qp_attr.qp_state = IB_QPS_RTR;
819         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
820         if (ret)
821                 goto out;
822
823         qp_attr.max_dest_rd_atomic = 4;
824
825         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
826
827 out:
828         return ret;
829 }
830
831 /**
832  * Change the state of a channel to 'ready to send' (RTS).
833  * @ch: channel of the queue pair.
834  * @qp: queue pair to change the state of.
835  *
836  * Returns zero upon success and a negative value upon failure.
837  *
838  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
839  * If this structure ever becomes larger, it might be necessary to allocate
840  * it dynamically instead of on the stack.
841  */
842 static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
843 {
844         struct ib_qp_attr qp_attr;
845         int attr_mask;
846         int ret;
847
848         qp_attr.qp_state = IB_QPS_RTS;
849         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
850         if (ret)
851                 goto out;
852
853         qp_attr.max_rd_atomic = 4;
854
855         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
856
857 out:
858         return ret;
859 }
860
861 static void srpt_reset_ioctx(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
862 {
863         int i;
864
865         if (ioctx->n_rdma_ius > 0 && ioctx->rdma_ius) {
866                 struct rdma_iu *riu = ioctx->rdma_ius;
867
868                 for (i = 0; i < ioctx->n_rdma_ius; ++i, ++riu)
869                         kfree(riu->sge);
870                 kfree(ioctx->rdma_ius);
871         }
872
873         if (ioctx->n_rbuf > 1)
874                 kfree(ioctx->rbufs);
875
876         /* If ch == NULL this means that the command has been aborted. */
877         if (!ch)
878                 return;
879
880         if (srpt_post_recv(ch->sport->sdev, ioctx))
881                 PRINT_ERROR("%s", "SRQ post_recv failed - this is serious.");
882                 /* we should queue it back to free_ioctx queue */
883         else
884                 atomic_inc(&ch->req_lim_delta);
885 }
886
887 static void srpt_abort_scst_cmd(struct srpt_device *sdev,
888                                 struct scst_cmd *scmnd,
889                                 bool tell_initiator)
890 {
891         struct srpt_ioctx *ioctx;
892         scst_data_direction dir;
893         struct srpt_rdma_ch *ch;
894
895         ioctx = scst_cmd_get_tgt_priv(scmnd);
896         BUG_ON(!ioctx);
897         dir = scst_cmd_get_data_direction(scmnd);
898         if (dir != SCST_DATA_NONE)
899                 ib_dma_unmap_sg(sdev->device,
900                                 scst_cmd_get_sg(scmnd),
901                                 scst_cmd_get_sg_cnt(scmnd),
902                                 scst_to_tgt_dma_dir(dir));
903
904         if (ioctx->state == SRPT_STATE_NEW) {
905                 /*
906                  * Do not try to abort the SCST command here but wait until
907                  * the SCST core has called srpt_rdy_to_xfer() or
908                  * srpt_xmit_response(). Since srpt_release_channel() will
909                  * finish before srpt_on_free_cmd() is called, set the channel
910                  * pointer inside the SCST command to NULL such that
911                  * srpt_on_free_cmd() will not dereference a dangling pointer.
912                  */
913                 ch = ioctx->ch;
914                 ioctx->ch = NULL;
915                 BUG_ON(!ch);
916                 spin_lock_irq(&ch->spinlock);
917                 list_del(&ioctx->scmnd_list);
918                 ch->active_scmnd_cnt--;
919                 spin_unlock_irq(&ch->spinlock);
920         } else if (ioctx->state == SRPT_STATE_NEED_DATA) {
921                 WARN_ON(scst_cmd_get_data_direction(ioctx->scmnd)
922                         == SCST_DATA_READ);
923                 scst_rx_data(scmnd,
924                              tell_initiator ? SCST_RX_STATUS_ERROR
925                              : SCST_RX_STATUS_ERROR_FATAL,
926                              SCST_CONTEXT_THREAD);
927         } else if (ioctx->state == SRPT_STATE_PROCESSED) {
928                 scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_FAILED);
929                 WARN_ON(scmnd->state != SCST_CMD_STATE_XMIT_WAIT);
930                 scst_tgt_cmd_done(scmnd, scst_estimate_context());
931         } else {
932                 TRACE_DBG("Aborting cmd with state %d", ioctx->state);
933                 WARN_ON("ERROR: unexpected command state");
934         }
935
936         ioctx->state = SRPT_STATE_ABORTED;
937 }
938
939 static void srpt_handle_err_comp(struct srpt_rdma_ch *ch, struct ib_wc *wc)
940 {
941         struct srpt_ioctx *ioctx;
942         struct srpt_device *sdev = ch->sport->sdev;
943
944         if (wc->wr_id & SRPT_OP_RECV) {
945                 ioctx = sdev->ioctx_ring[wc->wr_id & ~SRPT_OP_RECV];
946                 PRINT_ERROR("%s", "This is serious - SRQ is in bad state.");
947         } else {
948                 ioctx = sdev->ioctx_ring[wc->wr_id];
949
950                 if (ioctx->scmnd)
951                         srpt_abort_scst_cmd(sdev, ioctx->scmnd, true);
952                 else
953                         srpt_reset_ioctx(ch, ioctx);
954         }
955 }
956
957 static void srpt_handle_send_comp(struct srpt_rdma_ch *ch,
958                                   struct srpt_ioctx *ioctx,
959                                   enum scst_exec_context context)
960 {
961         if (ioctx->scmnd) {
962                 scst_data_direction dir =
963                         scst_cmd_get_data_direction(ioctx->scmnd);
964
965                 if (dir != SCST_DATA_NONE)
966                         ib_dma_unmap_sg(ch->sport->sdev->device,
967                                         scst_cmd_get_sg(ioctx->scmnd),
968                                         scst_cmd_get_sg_cnt(ioctx->scmnd),
969                                         scst_to_tgt_dma_dir(dir));
970
971                 WARN_ON(ioctx->scmnd->state != SCST_CMD_STATE_XMIT_WAIT);
972                 scst_tgt_cmd_done(ioctx->scmnd, context);
973         } else
974                 srpt_reset_ioctx(ch, ioctx);
975 }
976
977 /** Process an RDMA completion notification. */
978 static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch,
979                                   struct srpt_ioctx *ioctx)
980 {
981         if (!ioctx->scmnd) {
982                 WARN_ON("ERROR: ioctx->scmnd == NULL");
983                 srpt_reset_ioctx(ch, ioctx);
984                 return;
985         }
986
987         /*
988          * If an RDMA completion notification has been received for a write
989          * command, tell SCST that processing can continue by calling
990          * scst_rx_data().
991          */
992         if (ioctx->state == SRPT_STATE_NEED_DATA) {
993                 WARN_ON(scst_cmd_get_data_direction(ioctx->scmnd)
994                         == SCST_DATA_READ);
995                 ioctx->state = SRPT_STATE_DATA_IN;
996                 scst_rx_data(ioctx->scmnd, SCST_RX_STATUS_SUCCESS,
997                              scst_estimate_context());
998         }
999 }
1000
1001 /**
1002  * Build an SRP_RSP response.
1003  * @ch: RDMA channel through which the request has been received.
1004  * @ioctx: I/O context in which the SRP_RSP response will be built.
1005  * @s_key: sense key that will be stored in the response.
1006  * @s_code: value that will be stored in the asc_ascq field of the sense data.
1007  * @tag: tag of the request for which this response is being generated.
1008  *
1009  * Returns the size in bytes of the SRP_RSP response.
1010  *
1011  * An SRP_RSP response contains a SCSI status or service response. See also
1012  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
1013  * response. See also SPC-2 for more information about sense data.
1014  */
1015 static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
1016                               struct srpt_ioctx *ioctx, u8 s_key, u8 s_code,
1017                               u64 tag)
1018 {
1019         struct srp_rsp *srp_rsp;
1020         struct sense_data *sense;
1021         int limit_delta;
1022         int sense_data_len = 0;
1023
1024         srp_rsp = ioctx->buf;
1025         memset(srp_rsp, 0, sizeof *srp_rsp);
1026
1027         limit_delta = atomic_read(&ch->req_lim_delta);
1028         atomic_sub(limit_delta, &ch->req_lim_delta);
1029
1030         srp_rsp->opcode = SRP_RSP;
1031         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1032         srp_rsp->tag = tag;
1033
1034         if (s_key != NO_SENSE) {
1035                 sense_data_len = sizeof *sense + (sizeof *sense % 4);
1036                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1037                 srp_rsp->status = SAM_STAT_CHECK_CONDITION;
1038                 srp_rsp->sense_data_len = cpu_to_be32(sense_data_len);
1039
1040                 sense = (struct sense_data *)(srp_rsp + 1);
1041                 sense->err_code = 0x70;
1042                 sense->key = s_key;
1043                 sense->asc_ascq = s_code;
1044         }
1045
1046         return sizeof(*srp_rsp) + sense_data_len;
1047 }
1048
1049 /**
1050  * Build a task management response, which is a specific SRP_RSP response.
1051  * @ch: RDMA channel through which the request has been received.
1052  * @ioctx: I/O context in which the SRP_RSP response will be built.
1053  * @rsp_code: RSP_CODE that will be stored in the response.
1054  * @tag: tag of the request for which this response is being generated.
1055  *
1056  * Returns the size in bytes of the SRP_RSP response.
1057  *
1058  * An SRP_RSP response contains a SCSI status or service response. See also
1059  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
1060  * response.
1061  */
1062 static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
1063                                   struct srpt_ioctx *ioctx, u8 rsp_code,
1064                                   u64 tag)
1065 {
1066         struct srp_rsp *srp_rsp;
1067         int limit_delta;
1068         int resp_data_len = 0;
1069
1070         ib_dma_sync_single_for_cpu(ch->sport->sdev->device, ioctx->dma,
1071                                    MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
1072
1073         srp_rsp = ioctx->buf;
1074         memset(srp_rsp, 0, sizeof *srp_rsp);
1075
1076         limit_delta = atomic_read(&ch->req_lim_delta);
1077         atomic_sub(limit_delta, &ch->req_lim_delta);
1078
1079         srp_rsp->opcode = SRP_RSP;
1080         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1081         srp_rsp->tag = tag;
1082
1083         if (rsp_code != SRP_TSK_MGMT_SUCCESS) {
1084                 resp_data_len = 4;
1085                 srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
1086                 srp_rsp->resp_data_len = cpu_to_be32(resp_data_len);
1087                 srp_rsp->data[3] = rsp_code;
1088         }
1089
1090         return sizeof(*srp_rsp) + resp_data_len;
1091 }
1092
1093 /*
1094  * Process SRP_CMD.
1095  */
1096 static int srpt_handle_cmd(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
1097 {
1098         struct scst_cmd *scmnd;
1099         struct srp_cmd *srp_cmd;
1100         struct srp_rsp *srp_rsp;
1101         scst_data_direction dir;
1102         int indirect_desc = 0;
1103         int ret;
1104         unsigned long flags;
1105
1106         srp_cmd = ioctx->buf;
1107         srp_rsp = ioctx->buf;
1108
1109         dir = SCST_DATA_NONE;
1110         if (srp_cmd->buf_fmt) {
1111                 ret = srpt_get_desc_tbl(ioctx, srp_cmd, &indirect_desc);
1112                 if (ret) {
1113                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1114                                            NO_ADD_SENSE, srp_cmd->tag);
1115                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1116                         goto err;
1117                 }
1118
1119                 if (indirect_desc) {
1120                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1121                                            NO_ADD_SENSE, srp_cmd->tag);
1122                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1123                         goto err;
1124                 }
1125
1126                 /*
1127                  * The lower four bits of the buffer format field contain the
1128                  * DATA-IN buffer descriptor format, and the highest four bits
1129                  * contain the DATA-OUT buffer descriptor format.
1130                  */
1131                 if (srp_cmd->buf_fmt & 0xf)
1132                         /* DATA-IN: transfer data from target to initiator. */
1133                         dir = SCST_DATA_READ;
1134                 else if (srp_cmd->buf_fmt >> 4)
1135                         /* DATA-OUT: transfer data from initiator to target. */
1136                         dir = SCST_DATA_WRITE;
1137         }
1138
1139         scmnd = scst_rx_cmd(ch->scst_sess, (u8 *) &srp_cmd->lun,
1140                             sizeof srp_cmd->lun, srp_cmd->cdb, 16,
1141                             thread ? SCST_NON_ATOMIC : SCST_ATOMIC);
1142         if (!scmnd) {
1143                 srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1144                                    NO_ADD_SENSE, srp_cmd->tag);
1145                 srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1146                 goto err;
1147         }
1148
1149         ioctx->scmnd = scmnd;
1150
1151         switch (srp_cmd->task_attr) {
1152         case SRP_CMD_HEAD_OF_Q:
1153                 scmnd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1154                 break;
1155         case SRP_CMD_ORDERED_Q:
1156                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1157                 break;
1158         case SRP_CMD_SIMPLE_Q:
1159                 scmnd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1160                 break;
1161         case SRP_CMD_ACA:
1162                 scmnd->queue_type = SCST_CMD_QUEUE_ACA;
1163                 break;
1164         default:
1165                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1166                 break;
1167         }
1168
1169         scst_cmd_set_tag(scmnd, srp_cmd->tag);
1170         scst_cmd_set_tgt_priv(scmnd, ioctx);
1171         scst_cmd_set_expected(scmnd, dir, ioctx->data_len);
1172
1173         spin_lock_irqsave(&ch->spinlock, flags);
1174         list_add_tail(&ioctx->scmnd_list, &ch->active_scmnd_list);
1175         ch->active_scmnd_cnt++;
1176         spin_unlock_irqrestore(&ch->spinlock, flags);
1177
1178         scst_cmd_init_done(scmnd, scst_estimate_context());
1179
1180         return 0;
1181
1182 err:
1183         WARN_ON(srp_rsp->opcode != SRP_RSP);
1184
1185         return -1;
1186 }
1187
1188 /*
1189  * Process an SRP_TSK_MGMT request.
1190  *
1191  * Returns 0 upon success and -1 upon failure.
1192  *
1193  * Each task management function is performed by calling one of the
1194  * scst_rx_mgmt_fn*() functions. These functions will either report failure
1195  * or process the task management function asynchronously. The function
1196  * srpt_tsk_mgmt_done() will be called by the SCST core upon completion of the
1197  * task management function. When srpt_handle_tsk_mgmt() reports failure
1198  * (i.e. returns -1) a response will have been built in ioctx->buf. This
1199  * information unit has to be sent back by the caller.
1200  *
1201  * For more information about SRP_TSK_MGMT information units, see also section
1202  * 6.7 in the T10 SRP r16a document.
1203  */
1204 static int srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
1205                                 struct srpt_ioctx *ioctx)
1206 {
1207         struct srp_tsk_mgmt *srp_tsk;
1208         struct srpt_mgmt_ioctx *mgmt_ioctx;
1209         int ret;
1210
1211         srp_tsk = ioctx->buf;
1212
1213         TRACE_DBG("recv_tsk_mgmt= %d for task_tag= %lld"
1214                   " using tag= %lld cm_id= %p sess= %p",
1215                   srp_tsk->tsk_mgmt_func,
1216                   (unsigned long long) srp_tsk->task_tag,
1217                   (unsigned long long) srp_tsk->tag,
1218                   ch->cm_id, ch->scst_sess);
1219
1220         mgmt_ioctx = kmalloc(sizeof *mgmt_ioctx, GFP_ATOMIC);
1221         if (!mgmt_ioctx) {
1222                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1223                                        srp_tsk->tag);
1224                 goto err;
1225         }
1226
1227         mgmt_ioctx->ioctx = ioctx;
1228         mgmt_ioctx->ch = ch;
1229         mgmt_ioctx->tag = srp_tsk->tag;
1230
1231         switch (srp_tsk->tsk_mgmt_func) {
1232         case SRP_TSK_ABORT_TASK:
1233                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK");
1234                 ret = scst_rx_mgmt_fn_tag(ch->scst_sess,
1235                                           SCST_ABORT_TASK,
1236                                           srp_tsk->task_tag,
1237                                           thread ?
1238                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1239                                           mgmt_ioctx);
1240                 break;
1241         case SRP_TSK_ABORT_TASK_SET:
1242                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK_SET");
1243                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1244                                           SCST_ABORT_TASK_SET,
1245                                           (u8 *) &srp_tsk->lun,
1246                                           sizeof srp_tsk->lun,
1247                                           thread ?
1248                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1249                                           mgmt_ioctx);
1250                 break;
1251         case SRP_TSK_CLEAR_TASK_SET:
1252                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_TASK_SET");
1253                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1254                                           SCST_CLEAR_TASK_SET,
1255                                           (u8 *) &srp_tsk->lun,
1256                                           sizeof srp_tsk->lun,
1257                                           thread ?
1258                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1259                                           mgmt_ioctx);
1260                 break;
1261         case SRP_TSK_LUN_RESET:
1262                 TRACE_DBG("%s", "Processing SRP_TSK_LUN_RESET");
1263                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1264                                           SCST_LUN_RESET,
1265                                           (u8 *) &srp_tsk->lun,
1266                                           sizeof srp_tsk->lun,
1267                                           thread ?
1268                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1269                                           mgmt_ioctx);
1270                 break;
1271         case SRP_TSK_CLEAR_ACA:
1272                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_ACA");
1273                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1274                                           SCST_CLEAR_ACA,
1275                                           (u8 *) &srp_tsk->lun,
1276                                           sizeof srp_tsk->lun,
1277                                           thread ?
1278                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1279                                           mgmt_ioctx);
1280                 break;
1281         default:
1282                 TRACE_DBG("%s", "Unsupported task management function.");
1283                 srpt_build_tskmgmt_rsp(ch, ioctx,
1284                                        SRP_TSK_MGMT_FUNC_NOT_SUPP,
1285                                        srp_tsk->tag);
1286                 goto err;
1287         }
1288
1289         if (ret) {
1290                 TRACE_DBG("%s", "Processing task management function failed.");
1291                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1292                                        srp_tsk->tag);
1293                 goto err;
1294         }
1295
1296         WARN_ON(srp_tsk->opcode == SRP_RSP);
1297
1298         return 0;
1299
1300 err:
1301         WARN_ON(srp_tsk->opcode != SRP_RSP);
1302
1303         kfree(mgmt_ioctx);
1304         return -1;
1305 }
1306
1307 /**
1308  * Process a receive completion event.
1309  * @ch: RDMA channel for which the completion event has been received.
1310  * @ioctx: SRPT I/O context for which the completion event has been received.
1311  */
1312 static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
1313                                struct srpt_ioctx *ioctx)
1314 {
1315         struct srp_cmd *srp_cmd;
1316         struct srp_rsp *srp_rsp;
1317         unsigned long flags;
1318         int len;
1319
1320         spin_lock_irqsave(&ch->spinlock, flags);
1321         if (ch->state != RDMA_CHANNEL_LIVE) {
1322                 if (ch->state == RDMA_CHANNEL_CONNECTING) {
1323                         list_add_tail(&ioctx->wait_list, &ch->cmd_wait_list);
1324                         spin_unlock_irqrestore(&ch->spinlock, flags);
1325                         return;
1326                 } else {
1327                         spin_unlock_irqrestore(&ch->spinlock, flags);
1328                         srpt_reset_ioctx(ch, ioctx);
1329                         return;
1330                 }
1331         }
1332         spin_unlock_irqrestore(&ch->spinlock, flags);
1333
1334         ib_dma_sync_single_for_cpu(ch->sport->sdev->device, ioctx->dma,
1335                                    MAX_MESSAGE_SIZE, DMA_FROM_DEVICE);
1336
1337         ioctx->data_len = 0;
1338         ioctx->n_rbuf = 0;
1339         ioctx->rbufs = NULL;
1340         ioctx->n_rdma = 0;
1341         ioctx->n_rdma_ius = 0;
1342         ioctx->rdma_ius = NULL;
1343         ioctx->scmnd = NULL;
1344         ioctx->ch = ch;
1345         ioctx->state = SRPT_STATE_NEW;
1346
1347         srp_cmd = ioctx->buf;
1348         srp_rsp = ioctx->buf;
1349
1350         switch (srp_cmd->opcode) {
1351         case SRP_CMD:
1352                 if (srpt_handle_cmd(ch, ioctx) < 0)
1353                         goto err;
1354                 break;
1355
1356         case SRP_TSK_MGMT:
1357                 if (srpt_handle_tsk_mgmt(ch, ioctx) < 0)
1358                         goto err;
1359                 break;
1360
1361         case SRP_I_LOGOUT:
1362         case SRP_AER_REQ:
1363         default:
1364                 srpt_build_cmd_rsp(ch, ioctx, ILLEGAL_REQUEST, INVALID_CDB,
1365                                    srp_cmd->tag);
1366                 goto err;
1367         }
1368
1369         ib_dma_sync_single_for_device(ch->sport->sdev->device,
1370                                    ioctx->dma, MAX_MESSAGE_SIZE,
1371                                    DMA_FROM_DEVICE);
1372
1373         return;
1374
1375 err:
1376         WARN_ON(srp_rsp->opcode != SRP_RSP);
1377         len = (sizeof *srp_rsp) + be32_to_cpu(srp_rsp->sense_data_len);
1378
1379         if (ch->state != RDMA_CHANNEL_LIVE) {
1380                 /* Give up if another thread modified the channel state. */
1381                 PRINT_ERROR("%s: channel is in state %d", __func__, ch->state);
1382                 srpt_reset_ioctx(ch, ioctx);
1383         } else if (srpt_post_send(ch, ioctx, len)) {
1384                 PRINT_ERROR("%s: sending SRP_RSP response failed", __func__);
1385                 srpt_reset_ioctx(ch, ioctx);
1386         }
1387 }
1388
1389 /*
1390  * Returns true if the ioctx list is non-empty or if the ib_srpt kernel thread
1391  * should stop.
1392  * @pre thread != 0
1393  */
1394 static inline int srpt_test_ioctx_list(void)
1395 {
1396         int res = (!list_empty(&srpt_thread.thread_ioctx_list) ||
1397                    unlikely(kthread_should_stop()));
1398         return res;
1399 }
1400
1401 /*
1402  * Add 'ioctx' to the tail of the ioctx list and wake up the kernel thread.
1403  *
1404  * @pre thread != 0
1405  */
1406 static inline void srpt_schedule_thread(struct srpt_ioctx *ioctx)
1407 {
1408         unsigned long flags;
1409
1410         spin_lock_irqsave(&srpt_thread.thread_lock, flags);
1411         list_add_tail(&ioctx->comp_list, &srpt_thread.thread_ioctx_list);
1412         spin_unlock_irqrestore(&srpt_thread.thread_lock, flags);
1413         wake_up(&ioctx_list_waitQ);
1414 }
1415
1416 /**
1417  * InfiniBand completion queue callback function.
1418  * @cq: completion queue.
1419  * @ctx: completion queue context, which was passed as the fourth argument of
1420  *       the function ib_create_cq().
1421  */
1422 static void srpt_completion(struct ib_cq *cq, void *ctx)
1423 {
1424         struct srpt_rdma_ch *ch = ctx;
1425         struct srpt_device *sdev = ch->sport->sdev;
1426         struct ib_wc wc;
1427         struct srpt_ioctx *ioctx;
1428
1429         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1430         while (ib_poll_cq(ch->cq, 1, &wc) > 0) {
1431                 if (wc.status) {
1432                         PRINT_ERROR("failed %s status= %d",
1433                                wc.wr_id & SRPT_OP_RECV ? "receive" : "send",
1434                                wc.status);
1435                         srpt_handle_err_comp(ch, &wc);
1436                         break;
1437                 }
1438
1439                 if (wc.wr_id & SRPT_OP_RECV) {
1440                         ioctx = sdev->ioctx_ring[wc.wr_id & ~SRPT_OP_RECV];
1441                         if (thread) {
1442                                 ioctx->ch = ch;
1443                                 ioctx->op = IB_WC_RECV;
1444                                 srpt_schedule_thread(ioctx);
1445                         } else
1446                                 srpt_handle_new_iu(ch, ioctx);
1447                         continue;
1448                 } else
1449                         ioctx = sdev->ioctx_ring[wc.wr_id];
1450
1451                 if (thread) {
1452                         ioctx->ch = ch;
1453                         ioctx->op = wc.opcode;
1454                         srpt_schedule_thread(ioctx);
1455                 } else {
1456                         switch (wc.opcode) {
1457                         case IB_WC_SEND:
1458                                 srpt_handle_send_comp(ch, ioctx,
1459                                         scst_estimate_context());
1460                                 break;
1461                         case IB_WC_RDMA_WRITE:
1462                         case IB_WC_RDMA_READ:
1463                                 srpt_handle_rdma_comp(ch, ioctx);
1464                                 break;
1465                         default:
1466                                 break;
1467                         }
1468                 }
1469
1470 #if defined(CONFIG_SCST_DEBUG)
1471                 if (interrupt_processing_delay_in_us <= MAX_UDELAY_MS * 1000)
1472                         udelay(interrupt_processing_delay_in_us);
1473 #endif
1474         }
1475 }
1476
1477 /*
1478  * Create a completion queue on the specified device.
1479  */
1480 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
1481 {
1482         struct ib_qp_init_attr *qp_init;
1483         struct srpt_device *sdev = ch->sport->sdev;
1484         int cqe;
1485         int ret;
1486
1487         qp_init = kzalloc(sizeof *qp_init, GFP_KERNEL);
1488         if (!qp_init)
1489                 return -ENOMEM;
1490
1491         /* Create a completion queue (CQ). */
1492
1493         cqe = SRPT_RQ_SIZE + SRPT_SQ_SIZE - 1;
1494 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(RHEL_RELEASE_CODE)
1495         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe);
1496 #else
1497         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe, 0);
1498 #endif
1499         if (IS_ERR(ch->cq)) {
1500                 ret = PTR_ERR(ch->cq);
1501                 PRINT_ERROR("failed to create_cq cqe= %d ret= %d", cqe, ret);
1502                 goto out;
1503         }
1504
1505         /* Request completion notification. */
1506
1507         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1508
1509         /* Create a queue pair (QP). */
1510
1511         qp_init->qp_context = (void *)ch;
1512         qp_init->event_handler = srpt_qp_event;
1513         qp_init->send_cq = ch->cq;
1514         qp_init->recv_cq = ch->cq;
1515         qp_init->srq = sdev->srq;
1516         qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
1517         qp_init->qp_type = IB_QPT_RC;
1518         qp_init->cap.max_send_wr = SRPT_SQ_SIZE;
1519         qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE;
1520
1521         ch->qp = ib_create_qp(sdev->pd, qp_init);
1522         if (IS_ERR(ch->qp)) {
1523                 ret = PTR_ERR(ch->qp);
1524                 ib_destroy_cq(ch->cq);
1525                 PRINT_ERROR("failed to create_qp ret= %d", ret);
1526                 goto out;
1527         }
1528
1529         TRACE_DBG("%s: max_cqe= %d max_sge= %d cm_id= %p",
1530                __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
1531                ch->cm_id);
1532
1533         /* Modify the attributes and the state of queue pair ch->qp. */
1534
1535         ret = srpt_init_ch_qp(ch, ch->qp);
1536         if (ret) {
1537                 ib_destroy_qp(ch->qp);
1538                 ib_destroy_cq(ch->cq);
1539                 goto out;
1540         }
1541
1542         atomic_set(&ch->req_lim_delta, SRPT_RQ_SIZE);
1543 out:
1544         kfree(qp_init);
1545         return ret;
1546 }
1547
1548 /**
1549  * Look up the RDMA channel that corresponds to the specified cm_id.
1550  *
1551  * Return NULL if no matching RDMA channel has been found.
1552  */
1553 static struct srpt_rdma_ch *srpt_find_channel(struct ib_cm_id *cm_id, bool del)
1554 {
1555         struct srpt_device *sdev = cm_id->context;
1556         struct srpt_rdma_ch *ch;
1557
1558         spin_lock_irq(&sdev->spinlock);
1559         list_for_each_entry(ch, &sdev->rch_list, list) {
1560                 if (ch->cm_id == cm_id) {
1561                         if (del)
1562                                 list_del(&ch->list);
1563                         spin_unlock_irq(&sdev->spinlock);
1564                         return ch;
1565                 }
1566         }
1567
1568         spin_unlock_irq(&sdev->spinlock);
1569
1570         return NULL;
1571 }
1572
1573 /**
1574  * Release all resources associated with the specified RDMA channel.
1575  *
1576  * Note: the caller must have removed the channel from the channel list
1577  * before calling this function.
1578  */
1579 static void srpt_release_channel(struct srpt_rdma_ch *ch, int destroy_cmid)
1580 {
1581         TRACE_ENTRY();
1582
1583         WARN_ON(srpt_find_channel(ch->cm_id, false) == ch);
1584
1585         if (ch->cm_id && destroy_cmid) {
1586                 TRACE_DBG("%s: destroy cm_id= %p", __func__, ch->cm_id);
1587                 ib_destroy_cm_id(ch->cm_id);
1588                 ch->cm_id = NULL;
1589         }
1590
1591         ib_destroy_qp(ch->qp);
1592         ib_destroy_cq(ch->cq);
1593
1594         if (ch->scst_sess) {
1595                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1596
1597                 if (ch->active_scmnd_cnt)
1598                         PRINT_INFO("Releasing session %s which still has %d"
1599                                    " active commands",
1600                                    ch->sess_name, ch->active_scmnd_cnt);
1601                 else
1602                         PRINT_INFO("Releasing session %s", ch->sess_name);
1603
1604                 spin_lock_irq(&ch->spinlock);
1605                 list_for_each_entry_safe(ioctx, ioctx_tmp,
1606                                          &ch->active_scmnd_list, scmnd_list) {
1607                         spin_unlock_irq(&ch->spinlock);
1608
1609                         if (ioctx->scmnd)
1610                                 srpt_abort_scst_cmd(ch->sport->sdev,
1611                                                     ioctx->scmnd, true);
1612
1613                         spin_lock_irq(&ch->spinlock);
1614                 }
1615                 WARN_ON(!list_empty(&ch->active_scmnd_list));
1616                 WARN_ON(ch->active_scmnd_cnt != 0);
1617                 spin_unlock_irq(&ch->spinlock);
1618
1619                 scst_unregister_session(ch->scst_sess, 0, NULL);
1620                 ch->scst_sess = NULL;
1621         }
1622
1623         kfree(ch);
1624
1625         TRACE_EXIT();
1626 }
1627
1628 static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
1629                             struct ib_cm_req_event_param *param,
1630                             void *private_data)
1631 {
1632         struct srpt_device *sdev = cm_id->context;
1633         struct srp_login_req *req;
1634         struct srp_login_rsp *rsp;
1635         struct srp_login_rej *rej;
1636         struct ib_cm_rep_param *rep_param;
1637         struct srpt_rdma_ch *ch, *tmp_ch;
1638         u32 it_iu_len;
1639         int ret = 0;
1640
1641 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
1642         WARN_ON(!sdev || !private_data);
1643         if (!sdev || !private_data)
1644                 return -EINVAL;
1645 #else
1646         if (WARN_ON(!sdev || !private_data))
1647                 return -EINVAL;
1648 #endif
1649
1650         req = (struct srp_login_req *)private_data;
1651
1652         it_iu_len = be32_to_cpu(req->req_it_iu_len);
1653
1654         PRINT_INFO("Received SRP_LOGIN_REQ with"
1655             " i_port_id 0x%llx:0x%llx, t_port_id 0x%llx:0x%llx and length %d"
1656             " on port %d (guid=0x%llx:0x%llx)",
1657             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[0]),
1658             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[8]),
1659             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[0]),
1660             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[8]),
1661             it_iu_len,
1662             param->port,
1663             (unsigned long long)be64_to_cpu(*(u64 *)
1664                                 &sdev->port[param->port - 1].gid.raw[0]),
1665             (unsigned long long)be64_to_cpu(*(u64 *)
1666                                 &sdev->port[param->port - 1].gid.raw[8]));
1667
1668         rsp = kzalloc(sizeof *rsp, GFP_KERNEL);
1669         rej = kzalloc(sizeof *rej, GFP_KERNEL);
1670         rep_param = kzalloc(sizeof *rep_param, GFP_KERNEL);
1671
1672         if (!rsp || !rej || !rep_param) {
1673                 ret = -ENOMEM;
1674                 goto out;
1675         }
1676
1677         if (it_iu_len > MAX_MESSAGE_SIZE || it_iu_len < 64) {
1678                 rej->reason =
1679                     cpu_to_be32(SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
1680                 ret = -EINVAL;
1681                 PRINT_ERROR("rejected SRP_LOGIN_REQ because its"
1682                             " length (%d bytes) is invalid", it_iu_len);
1683                 goto reject;
1684         }
1685
1686         if ((req->req_flags & 0x3) == SRP_MULTICHAN_SINGLE) {
1687                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
1688
1689                 spin_lock_irq(&sdev->spinlock);
1690
1691                 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
1692                         if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
1693                             && !memcmp(ch->t_port_id, req->target_port_id, 16)
1694                             && param->port == ch->sport->port
1695                             && param->listen_id == ch->sport->sdev->cm_id
1696                             && ch->cm_id) {
1697                                 enum rdma_ch_state prev_state;
1698
1699                                 /* found an existing channel */
1700                                 TRACE_DBG("Found existing channel name= %s"
1701                                           " cm_id= %p state= %d",
1702                                           ch->sess_name, ch->cm_id, ch->state);
1703
1704                                 prev_state = ch->state;
1705                                 if (ch->state == RDMA_CHANNEL_LIVE)
1706                                         ch->state = RDMA_CHANNEL_DISCONNECTING;
1707                                 else if (ch->state == RDMA_CHANNEL_CONNECTING)
1708                                         list_del(&ch->list);
1709
1710                                 spin_unlock_irq(&sdev->spinlock);
1711
1712                                 rsp->rsp_flags =
1713                                         SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
1714
1715                                 if (prev_state == RDMA_CHANNEL_LIVE) {
1716                                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
1717                                         PRINT_INFO("disconnected"
1718                                           " session %s because a new"
1719                                           " SRP_LOGIN_REQ has been received.",
1720                                           ch->sess_name);
1721                                 } else if (prev_state ==
1722                                          RDMA_CHANNEL_CONNECTING) {
1723                                         PRINT_ERROR("%s", "rejected"
1724                                           " SRP_LOGIN_REQ because another login"
1725                                           " request is being processed.");
1726                                         ib_send_cm_rej(ch->cm_id,
1727                                                        IB_CM_REJ_NO_RESOURCES,
1728                                                        NULL, 0, NULL, 0);
1729                                         srpt_release_channel(ch, 1);
1730                                 }
1731
1732                                 spin_lock_irq(&sdev->spinlock);
1733                         }
1734                 }
1735
1736                 spin_unlock_irq(&sdev->spinlock);
1737
1738         } else
1739                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
1740
1741         if (((u64) (*(u64 *) req->target_port_id) !=
1742              cpu_to_be64(srpt_service_guid)) ||
1743             ((u64) (*(u64 *) (req->target_port_id + 8)) !=
1744              cpu_to_be64(srpt_service_guid))) {
1745                 rej->reason =
1746                     cpu_to_be32(SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
1747                 ret = -ENOMEM;
1748                 PRINT_ERROR("%s", "rejected SRP_LOGIN_REQ because it"
1749                        " has an invalid target port identifier.");
1750                 goto reject;
1751         }
1752
1753         ch = kzalloc(sizeof *ch, GFP_KERNEL);
1754         if (!ch) {
1755                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1756                 PRINT_ERROR("%s",
1757                             "rejected SRP_LOGIN_REQ because out of memory.");
1758                 ret = -ENOMEM;
1759                 goto reject;
1760         }
1761
1762         spin_lock_init(&ch->spinlock);
1763         memcpy(ch->i_port_id, req->initiator_port_id, 16);
1764         memcpy(ch->t_port_id, req->target_port_id, 16);
1765         ch->sport = &sdev->port[param->port - 1];
1766         ch->cm_id = cm_id;
1767         ch->state = RDMA_CHANNEL_CONNECTING;
1768         INIT_LIST_HEAD(&ch->cmd_wait_list);
1769         INIT_LIST_HEAD(&ch->active_scmnd_list);
1770
1771         ret = srpt_create_ch_ib(ch);
1772         if (ret) {
1773                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1774                 PRINT_ERROR("%s", "rejected SRP_LOGIN_REQ because creating"
1775                             " a new RDMA channel failed.");
1776                 goto free_ch;
1777         }
1778
1779         ret = srpt_ch_qp_rtr(ch, ch->qp);
1780         if (ret) {
1781                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1782                 PRINT_ERROR("rejected SRP_LOGIN_REQ because enabling"
1783                        " RTR failed (error code = %d)", ret);
1784                 goto destroy_ib;
1785         }
1786
1787         if (use_port_guid_in_session_name) {
1788                 /*
1789                  * If the kernel module parameter use_port_guid_in_session_name
1790                  * has been specified, use a combination of the target port
1791                  * GUID and the initiator port ID as the session name. This
1792                  * was the original behavior of the SRP target implementation
1793                  * (i.e. before the SRPT was included in OFED 1.3).
1794                  */
1795                 snprintf(ch->sess_name, sizeof(ch->sess_name),
1796                          "0x%016llx%016llx",
1797                          (unsigned long long)be64_to_cpu(*(u64 *)
1798                                 &sdev->port[param->port - 1].gid.raw[8]),
1799                          (unsigned long long)be64_to_cpu(*(u64 *)
1800                                 (ch->i_port_id + 8)));
1801         } else {
1802                 /*
1803                  * Default behavior: use the initator port identifier as the
1804                  * session name.
1805                  */
1806                 snprintf(ch->sess_name, sizeof(ch->sess_name),
1807                          "0x%016llx%016llx",
1808                          (unsigned long long)be64_to_cpu(*(u64 *)ch->i_port_id),
1809                          (unsigned long long)be64_to_cpu(*(u64 *)
1810                                  (ch->i_port_id + 8)));
1811         }
1812
1813         TRACE_DBG("registering session %s", ch->sess_name);
1814
1815         BUG_ON(!sdev->scst_tgt);
1816         ch->scst_sess = scst_register_session(sdev->scst_tgt, 0, ch->sess_name,
1817                                               NULL, NULL);
1818         if (!ch->scst_sess) {
1819                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1820                 TRACE_DBG("%s", "Failed to create scst sess");
1821                 goto destroy_ib;
1822         }
1823
1824         TRACE_DBG("Establish connection sess=%p name=%s cm_id=%p",
1825                   ch->scst_sess, ch->sess_name, ch->cm_id);
1826
1827         scst_sess_set_tgt_priv(ch->scst_sess, ch);
1828
1829         /* create srp_login_response */
1830         rsp->opcode = SRP_LOGIN_RSP;
1831         rsp->tag = req->tag;
1832         rsp->max_it_iu_len = req->req_it_iu_len;
1833         rsp->max_ti_iu_len = req->req_it_iu_len;
1834         rsp->buf_fmt =
1835             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1836         rsp->req_lim_delta = cpu_to_be32(SRPT_RQ_SIZE);
1837         atomic_set(&ch->req_lim_delta, 0);
1838
1839         /* create cm reply */
1840         rep_param->qp_num = ch->qp->qp_num;
1841         rep_param->private_data = (void *)rsp;
1842         rep_param->private_data_len = sizeof *rsp;
1843         rep_param->rnr_retry_count = 7;
1844         rep_param->flow_control = 1;
1845         rep_param->failover_accepted = 0;
1846         rep_param->srq = 1;
1847         rep_param->responder_resources = 4;
1848         rep_param->initiator_depth = 4;
1849
1850         ret = ib_send_cm_rep(cm_id, rep_param);
1851         if (ret) {
1852                 PRINT_ERROR("sending SRP_LOGIN_REQ response failed"
1853                             " (error code = %d)", ret);
1854                 goto release_channel;
1855         }
1856
1857         spin_lock_irq(&sdev->spinlock);
1858         list_add_tail(&ch->list, &sdev->rch_list);
1859         spin_unlock_irq(&sdev->spinlock);
1860
1861         goto out;
1862
1863 release_channel:
1864         scst_unregister_session(ch->scst_sess, 0, NULL);
1865         ch->scst_sess = NULL;
1866
1867 destroy_ib:
1868         ib_destroy_qp(ch->qp);
1869         ib_destroy_cq(ch->cq);
1870
1871 free_ch:
1872         kfree(ch);
1873
1874 reject:
1875         rej->opcode = SRP_LOGIN_REJ;
1876         rej->tag = req->tag;
1877         rej->buf_fmt =
1878             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1879
1880         ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
1881                              (void *)rej, sizeof *rej);
1882
1883 out:
1884         kfree(rep_param);
1885         kfree(rsp);
1886         kfree(rej);
1887
1888         return ret;
1889 }
1890
1891 /**
1892  * Release the channel with the specified cm_id.
1893  *
1894  * Returns one to indicate that the caller of srpt_cm_handler() should destroy
1895  * the cm_id.
1896  */
1897 static void srpt_find_and_release_channel(struct ib_cm_id *cm_id)
1898 {
1899         struct srpt_rdma_ch *ch;
1900
1901         ch = srpt_find_channel(cm_id, true);
1902         if (ch)
1903                 srpt_release_channel(ch, 0);
1904 }
1905
1906 static void srpt_cm_rej_recv(struct ib_cm_id *cm_id)
1907 {
1908         PRINT_INFO("%s", "Received InfiniBand REJ packet.");
1909         srpt_find_and_release_channel(cm_id);
1910 }
1911
1912 /**
1913  * Process an IB_CM_RTU_RECEIVED or IB_CM_USER_ESTABLISHED event.
1914  *
1915  * An IB_CM_RTU_RECEIVED message indicates that the connection is established
1916  * and that the recipient may begin transmitting (RTU = ready to use).
1917  */
1918 static int srpt_cm_rtu_recv(struct ib_cm_id *cm_id)
1919 {
1920         struct srpt_rdma_ch *ch;
1921         int ret;
1922
1923         ch = srpt_find_channel(cm_id, false);
1924         if (!ch)
1925                 return -EINVAL;
1926
1927         if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_CONNECTING,
1928                                             RDMA_CHANNEL_LIVE)) {
1929                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1930
1931                 ret = srpt_ch_qp_rts(ch, ch->qp);
1932
1933                 list_for_each_entry_safe(ioctx, ioctx_tmp, &ch->cmd_wait_list,
1934                                          wait_list) {
1935                         list_del(&ioctx->wait_list);
1936                         srpt_handle_new_iu(ch, ioctx);
1937                 }
1938                 if (ret && srpt_test_and_set_channel_state(ch,
1939                                         RDMA_CHANNEL_LIVE,
1940                                         RDMA_CHANNEL_DISCONNECTING)) {
1941                         TRACE_DBG("cm_id=%p sess_name=%s state=%d",
1942                                   cm_id, ch->sess_name, ch->state);
1943                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
1944                 }
1945         } else if (ch->state == RDMA_CHANNEL_DISCONNECTING) {
1946                 TRACE_DBG("cm_id=%p sess_name=%s state=%d",
1947                           cm_id, ch->sess_name, ch->state);
1948                 ib_send_cm_dreq(ch->cm_id, NULL, 0);
1949                 ret = -EAGAIN;
1950         } else
1951                 ret = 0;
1952
1953         return ret;
1954 }
1955
1956 static void srpt_cm_timewait_exit(struct ib_cm_id *cm_id)
1957 {
1958         PRINT_INFO("%s", "Received InfiniBand TimeWait exit.");
1959         srpt_find_and_release_channel(cm_id);
1960 }
1961
1962 static void srpt_cm_rep_error(struct ib_cm_id *cm_id)
1963 {
1964         PRINT_INFO("%s", "Received InfiniBand REP error.");
1965         srpt_find_and_release_channel(cm_id);
1966 }
1967
1968 static int srpt_cm_dreq_recv(struct ib_cm_id *cm_id)
1969 {
1970         struct srpt_rdma_ch *ch;
1971
1972         ch = srpt_find_channel(cm_id, false);
1973         if (!ch)
1974                 return -EINVAL;
1975
1976         TRACE_DBG("%s: cm_id= %p ch->state= %d",
1977                  __func__, cm_id, ch->state);
1978
1979         switch (ch->state) {
1980         case RDMA_CHANNEL_LIVE:
1981         case RDMA_CHANNEL_CONNECTING:
1982                 ib_send_cm_drep(ch->cm_id, NULL, 0);
1983                 PRINT_INFO("Received DREQ and sent DREP for session %s.",
1984                            ch->sess_name);
1985                 break;
1986         case RDMA_CHANNEL_DISCONNECTING:
1987         default:
1988                 break;
1989         }
1990
1991         return 0;
1992 }
1993
1994 static void srpt_cm_drep_recv(struct ib_cm_id *cm_id)
1995 {
1996         PRINT_INFO("%s", "Received InfiniBand DREP message.");
1997         srpt_find_and_release_channel(cm_id);
1998 }
1999
2000 /**
2001  * IB connection manager callback function.
2002  *
2003  * A non-zero return value will make the caller destroy the CM ID.
2004  *
2005  * Note: srpt_add_one passes a struct srpt_device* as the third argument to
2006  * the ib_create_cm_id() call.
2007  */
2008 static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2009 {
2010         int ret = 0;
2011
2012         switch (event->event) {
2013         case IB_CM_REQ_RECEIVED:
2014                 ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd,
2015                                        event->private_data);
2016                 break;
2017         case IB_CM_REJ_RECEIVED:
2018                 srpt_cm_rej_recv(cm_id);
2019                 ret = -EINVAL;
2020                 break;
2021         case IB_CM_RTU_RECEIVED:
2022         case IB_CM_USER_ESTABLISHED:
2023                 ret = srpt_cm_rtu_recv(cm_id);
2024                 break;
2025         case IB_CM_DREQ_RECEIVED:
2026                 ret = srpt_cm_dreq_recv(cm_id);
2027                 break;
2028         case IB_CM_DREP_RECEIVED:
2029                 srpt_cm_drep_recv(cm_id);
2030                 ret = -EINVAL;
2031                 break;
2032         case IB_CM_TIMEWAIT_EXIT:
2033                 srpt_cm_timewait_exit(cm_id);
2034                 ret = -EINVAL;
2035                 break;
2036         case IB_CM_REP_ERROR:
2037                 srpt_cm_rep_error(cm_id);
2038                 ret = -EINVAL;
2039                 break;
2040         default:
2041                 break;
2042         }
2043
2044         return ret;
2045 }
2046
2047 static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
2048                                  struct srpt_ioctx *ioctx,
2049                                  struct scst_cmd *scmnd)
2050 {
2051         struct scatterlist *scat;
2052         scst_data_direction dir;
2053         struct rdma_iu *riu;
2054         struct srp_direct_buf *db;
2055         dma_addr_t dma_addr;
2056         struct ib_sge *sge;
2057         u64 raddr;
2058         u32 rsize;
2059         u32 tsize;
2060         u32 dma_len;
2061         int count, nrdma;
2062         int i, j, k;
2063
2064         scat = scst_cmd_get_sg(scmnd);
2065         dir = scst_cmd_get_data_direction(scmnd);
2066         count = ib_dma_map_sg(ch->sport->sdev->device, scat,
2067                               scst_cmd_get_sg_cnt(scmnd),
2068                               scst_to_tgt_dma_dir(dir));
2069         if (unlikely(!count))
2070                 return -EBUSY;
2071
2072         if (ioctx->rdma_ius && ioctx->n_rdma_ius)
2073                 nrdma = ioctx->n_rdma_ius;
2074         else {
2075                 nrdma = count / SRPT_DEF_SG_PER_WQE + ioctx->n_rbuf;
2076
2077                 ioctx->rdma_ius = kzalloc(nrdma * sizeof *riu,
2078                                           scst_cmd_atomic(scmnd)
2079                                           ? GFP_ATOMIC : GFP_KERNEL);
2080                 if (!ioctx->rdma_ius) {
2081                         ib_dma_unmap_sg(ch->sport->sdev->device,
2082                                         scat, scst_cmd_get_sg_cnt(scmnd),
2083                                         scst_to_tgt_dma_dir(dir));
2084                         return -ENOMEM;
2085                 }
2086
2087                 ioctx->n_rdma_ius = nrdma;
2088         }
2089
2090         db = ioctx->rbufs;
2091         tsize = (dir == SCST_DATA_READ) ?
2092                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2093         dma_len = sg_dma_len(&scat[0]);
2094         riu = ioctx->rdma_ius;
2095
2096         /*
2097          * For each remote desc - calculate the #ib_sge.
2098          * If #ib_sge < SRPT_DEF_SG_PER_WQE per rdma operation then
2099          *      each remote desc rdma_iu is required a rdma wr;
2100          * else
2101          *      we need to allocate extra rdma_iu to carry extra #ib_sge in
2102          *      another rdma wr
2103          */
2104         for (i = 0, j = 0;
2105              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2106                 rsize = be32_to_cpu(db->len);
2107                 raddr = be64_to_cpu(db->va);
2108                 riu->raddr = raddr;
2109                 riu->rkey = be32_to_cpu(db->key);
2110                 riu->sge_cnt = 0;
2111
2112                 /* calculate how many sge required for this remote_buf */
2113                 while (rsize > 0 && tsize > 0) {
2114
2115                         if (rsize >= dma_len) {
2116                                 tsize -= dma_len;
2117                                 rsize -= dma_len;
2118                                 raddr += dma_len;
2119
2120                                 if (tsize > 0) {
2121                                         ++j;
2122                                         if (j < count)
2123                                                 dma_len = sg_dma_len(&scat[j]);
2124                                 }
2125                         } else {
2126                                 tsize -= rsize;
2127                                 dma_len -= rsize;
2128                                 rsize = 0;
2129                         }
2130
2131                         ++riu->sge_cnt;
2132
2133                         if (rsize > 0 && riu->sge_cnt == SRPT_DEF_SG_PER_WQE) {
2134                                 riu->sge =
2135                                     kmalloc(riu->sge_cnt * sizeof *riu->sge,
2136                                             scst_cmd_atomic(scmnd)
2137                                             ? GFP_ATOMIC : GFP_KERNEL);
2138                                 if (!riu->sge)
2139                                         goto free_mem;
2140
2141                                 ++ioctx->n_rdma;
2142                                 ++riu;
2143                                 riu->sge_cnt = 0;
2144                                 riu->raddr = raddr;
2145                                 riu->rkey = be32_to_cpu(db->key);
2146                         }
2147                 }
2148
2149                 riu->sge = kmalloc(riu->sge_cnt * sizeof *riu->sge,
2150                                    scst_cmd_atomic(scmnd)
2151                                    ? GFP_ATOMIC : GFP_KERNEL);
2152
2153                 if (!riu->sge)
2154                         goto free_mem;
2155
2156                 ++ioctx->n_rdma;
2157         }
2158
2159         db = ioctx->rbufs;
2160         scat = scst_cmd_get_sg(scmnd);
2161         tsize = (dir == SCST_DATA_READ) ?
2162                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2163         riu = ioctx->rdma_ius;
2164         dma_len = sg_dma_len(&scat[0]);
2165         dma_addr = sg_dma_address(&scat[0]);
2166
2167         /* this second loop is really mapped sg_addres to rdma_iu->ib_sge */
2168         for (i = 0, j = 0;
2169              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2170                 rsize = be32_to_cpu(db->len);
2171                 sge = riu->sge;
2172                 k = 0;
2173
2174                 while (rsize > 0 && tsize > 0) {
2175                         sge->addr = dma_addr;
2176                         sge->lkey = ch->sport->sdev->mr->lkey;
2177
2178                         if (rsize >= dma_len) {
2179                                 sge->length =
2180                                         (tsize < dma_len) ? tsize : dma_len;
2181                                 tsize -= dma_len;
2182                                 rsize -= dma_len;
2183
2184                                 if (tsize > 0) {
2185                                         ++j;
2186                                         if (j < count) {
2187                                                 dma_len = sg_dma_len(&scat[j]);
2188                                                 dma_addr =
2189                                                     sg_dma_address(&scat[j]);
2190                                         }
2191                                 }
2192                         } else {
2193                                 sge->length = (tsize < rsize) ? tsize : rsize;
2194                                 tsize -= rsize;
2195                                 dma_len -= rsize;
2196                                 dma_addr += rsize;
2197                                 rsize = 0;
2198                         }
2199
2200                         ++k;
2201                         if (k == riu->sge_cnt && rsize > 0) {
2202                                 ++riu;
2203                                 sge = riu->sge;
2204                                 k = 0;
2205                         } else if (rsize > 0)
2206                                 ++sge;
2207                 }
2208         }
2209
2210         return 0;
2211
2212 free_mem:
2213         while (ioctx->n_rdma)
2214                 kfree(ioctx->rdma_ius[ioctx->n_rdma--].sge);
2215
2216         kfree(ioctx->rdma_ius);
2217
2218         ib_dma_unmap_sg(ch->sport->sdev->device,
2219                         scat, scst_cmd_get_sg_cnt(scmnd),
2220                         scst_to_tgt_dma_dir(dir));
2221
2222         return -ENOMEM;
2223 }
2224
2225 static int srpt_perform_rdmas(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2226                               scst_data_direction dir)
2227 {
2228         struct ib_send_wr wr;
2229         struct ib_send_wr *bad_wr;
2230         struct rdma_iu *riu;
2231         int i;
2232         int ret = 0;
2233
2234         riu = ioctx->rdma_ius;
2235         memset(&wr, 0, sizeof wr);
2236
2237         for (i = 0; i < ioctx->n_rdma; ++i, ++riu) {
2238                 wr.opcode = (dir == SCST_DATA_READ) ?
2239                     IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;
2240                 wr.next = NULL;
2241                 wr.wr_id = ioctx->index;
2242                 wr.wr.rdma.remote_addr = riu->raddr;
2243                 wr.wr.rdma.rkey = riu->rkey;
2244                 wr.num_sge = riu->sge_cnt;
2245                 wr.sg_list = riu->sge;
2246
2247                 /* only get completion event for the last rdma wr */
2248                 if (i == (ioctx->n_rdma - 1) && dir == SCST_DATA_WRITE)
2249                         wr.send_flags = IB_SEND_SIGNALED;
2250
2251                 ret = ib_post_send(ch->qp, &wr, &bad_wr);
2252                 if (ret)
2253                         break;
2254         }
2255
2256         return ret;
2257 }
2258
2259 /*
2260  * Start data transfer between initiator and target. Must not block.
2261  */
2262 static int srpt_xfer_data(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2263                           struct scst_cmd *scmnd)
2264 {
2265         int ret;
2266
2267         ret = srpt_map_sg_to_ib_sge(ch, ioctx, scmnd);
2268         if (ret) {
2269                 PRINT_ERROR("%s[%d] ret=%d", __func__, __LINE__, ret);
2270                 ret = SCST_TGT_RES_QUEUE_FULL;
2271                 goto out;
2272         }
2273
2274         ret = srpt_perform_rdmas(ch, ioctx, scst_cmd_get_data_direction(scmnd));
2275         if (ret) {
2276                 PRINT_ERROR("%s[%d] ret=%d", __func__, __LINE__, ret);
2277                 if (ret == -EAGAIN || ret == -ENOMEM)
2278                         ret = SCST_TGT_RES_QUEUE_FULL;
2279                 else
2280                         ret = SCST_TGT_RES_FATAL_ERROR;
2281                 goto out;
2282         }
2283
2284         ret = SCST_TGT_RES_SUCCESS;
2285
2286 out:
2287         return ret;
2288 }
2289
2290 /*
2291  * Called by the SCST core to inform ib_srpt that data reception from the
2292  * initiator should start (SCST_DATA_WRITE). Must not block.
2293  */
2294 static int srpt_rdy_to_xfer(struct scst_cmd *scmnd)
2295 {
2296         struct srpt_rdma_ch *ch;
2297         struct srpt_ioctx *ioctx;
2298
2299         ioctx = scst_cmd_get_tgt_priv(scmnd);
2300         BUG_ON(!ioctx);
2301
2302         if (ioctx->state == SRPT_STATE_ABORTED)
2303                 return SCST_TGT_RES_FATAL_ERROR;
2304
2305         ch = ioctx->ch;
2306         WARN_ON(ch != scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd)));
2307         BUG_ON(!ch);
2308
2309         if (ch->state == RDMA_CHANNEL_DISCONNECTING)
2310                 return SCST_TGT_RES_FATAL_ERROR;
2311         else if (ch->state == RDMA_CHANNEL_CONNECTING)
2312                 return SCST_TGT_RES_QUEUE_FULL;
2313
2314         ioctx->state = SRPT_STATE_NEED_DATA;
2315
2316         return srpt_xfer_data(ch, ioctx, scmnd);
2317 }
2318
2319 /*
2320  * Called by the SCST core. Transmits the response buffer and status held in
2321  * 'scmnd'. Must not block.
2322  */
2323 static int srpt_xmit_response(struct scst_cmd *scmnd)
2324 {
2325         struct srpt_rdma_ch *ch;
2326         struct srpt_ioctx *ioctx;
2327         struct srp_rsp *srp_rsp;
2328         u64 tag;
2329         int ret = SCST_TGT_RES_SUCCESS;
2330         int dir;
2331         int status;
2332
2333         ioctx = scst_cmd_get_tgt_priv(scmnd);
2334         BUG_ON(!ioctx);
2335
2336         if (ioctx->state == SRPT_STATE_ABORTED) {
2337                 ret = SCST_TGT_RES_FATAL_ERROR;
2338                 goto out;
2339         }
2340
2341         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2342         BUG_ON(!ch);
2343
2344         tag = scst_cmd_get_tag(scmnd);
2345
2346         ioctx->state = SRPT_STATE_PROCESSED;
2347
2348         if (ch->state != RDMA_CHANNEL_LIVE) {
2349                 PRINT_ERROR("%s: tag= %lld channel in bad state %d",
2350                        __func__, (unsigned long long)tag, ch->state);
2351
2352                 if (ch->state == RDMA_CHANNEL_DISCONNECTING)
2353                         ret = SCST_TGT_RES_FATAL_ERROR;
2354                 else if (ch->state == RDMA_CHANNEL_CONNECTING)
2355                         ret = SCST_TGT_RES_QUEUE_FULL;
2356
2357                 if (unlikely(scst_cmd_aborted(scmnd)))
2358                         goto out_aborted;
2359
2360                 goto out;
2361         }
2362
2363         ib_dma_sync_single_for_cpu(ch->sport->sdev->device, ioctx->dma,
2364                                    MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
2365
2366         srp_rsp = ioctx->buf;
2367
2368         if (unlikely(scst_cmd_aborted(scmnd))) {
2369                 TRACE_MGMT_DBG("%s: tag= %lld already got aborted",
2370                                __func__, (unsigned long long)tag);
2371                 goto out_aborted;
2372         }
2373
2374         dir = scst_cmd_get_data_direction(scmnd);
2375         status = scst_cmd_get_status(scmnd) & 0xff;
2376
2377         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE, NO_ADD_SENSE, tag);
2378
2379         if (SCST_SENSE_VALID(scst_cmd_get_sense_buffer(scmnd))) {
2380                 srp_rsp->sense_data_len = scst_cmd_get_sense_buffer_len(scmnd);
2381                 if (srp_rsp->sense_data_len >
2382                     (MAX_MESSAGE_SIZE - sizeof *srp_rsp))
2383                         srp_rsp->sense_data_len =
2384                             MAX_MESSAGE_SIZE - sizeof *srp_rsp;
2385
2386                 memcpy((u8 *) (srp_rsp + 1), scst_cmd_get_sense_buffer(scmnd),
2387                        srp_rsp->sense_data_len);
2388
2389                 srp_rsp->sense_data_len = cpu_to_be32(srp_rsp->sense_data_len);
2390                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
2391
2392                 if (!status)
2393                         status = SAM_STAT_CHECK_CONDITION;
2394         }
2395
2396         srp_rsp->status = status;
2397
2398         /* For read commands, transfer the data to the initiator. */
2399         if (dir == SCST_DATA_READ && scst_cmd_get_resp_data_len(scmnd)) {
2400                 ret = srpt_xfer_data(ch, ioctx, scmnd);
2401                 if (ret != SCST_TGT_RES_SUCCESS) {
2402                         PRINT_ERROR("%s: tag= %lld xfer_data failed",
2403                                     __func__, (unsigned long long)tag);
2404                         goto out;
2405                 }
2406         }
2407
2408         if (srpt_post_send(ch, ioctx,
2409                            sizeof *srp_rsp +
2410                            be32_to_cpu(srp_rsp->sense_data_len))) {
2411                 PRINT_ERROR("%s: ch->state= %d tag= %lld",
2412                             __func__, ch->state,
2413                             (unsigned long long)tag);
2414                 ret = SCST_TGT_RES_FATAL_ERROR;
2415         }
2416
2417 out:
2418         return ret;
2419
2420 out_aborted:
2421         ret = SCST_TGT_RES_SUCCESS;
2422         scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_ABORTED);
2423         ioctx->state = SRPT_STATE_ABORTED;
2424         WARN_ON(scmnd->state != SCST_CMD_STATE_XMIT_WAIT);
2425         scst_tgt_cmd_done(scmnd, SCST_CONTEXT_SAME);
2426         goto out;
2427 }
2428
2429 /*
2430  * Called by the SCST core to inform ib_srpt that a received task management
2431  * function has been completed. Must not block.
2432  */
2433 static void srpt_tsk_mgmt_done(struct scst_mgmt_cmd *mcmnd)
2434 {
2435         struct srpt_rdma_ch *ch;
2436         struct srpt_mgmt_ioctx *mgmt_ioctx;
2437         struct srpt_ioctx *ioctx;
2438         int rsp_len;
2439
2440         mgmt_ioctx = scst_mgmt_cmd_get_tgt_priv(mcmnd);
2441         BUG_ON(!mgmt_ioctx);
2442
2443         ch = mgmt_ioctx->ch;
2444         BUG_ON(!ch);
2445
2446         ioctx = mgmt_ioctx->ioctx;
2447         BUG_ON(!ioctx);
2448
2449         TRACE_DBG("%s: tsk_mgmt_done for tag= %lld status=%d",
2450                   __func__, (unsigned long long)mgmt_ioctx->tag,
2451                   scst_mgmt_cmd_get_status(mcmnd));
2452
2453         ioctx->state = SRPT_STATE_PROCESSED;
2454
2455         rsp_len = srpt_build_tskmgmt_rsp(ch, ioctx,
2456                                          (scst_mgmt_cmd_get_status(mcmnd) ==
2457                                           SCST_MGMT_STATUS_SUCCESS) ?
2458                                          SRP_TSK_MGMT_SUCCESS :
2459                                          SRP_TSK_MGMT_FAILED,
2460                                          mgmt_ioctx->tag);
2461         srpt_post_send(ch, ioctx, rsp_len);
2462
2463         scst_mgmt_cmd_set_tgt_priv(mcmnd, NULL);
2464
2465         kfree(mgmt_ioctx);
2466 }
2467
2468 /*
2469  * Called by the SCST core to inform ib_srpt that the command 'scmnd' is about
2470  * to be freed. May be called in IRQ context.
2471  */
2472 static void srpt_on_free_cmd(struct scst_cmd *scmnd)
2473 {
2474         struct srpt_rdma_ch *ch;
2475         struct srpt_ioctx *ioctx;
2476
2477         ioctx = scst_cmd_get_tgt_priv(scmnd);
2478         BUG_ON(!ioctx);
2479
2480         ch = ioctx->ch;
2481         if (ch) {
2482                 spin_lock_irq(&ch->spinlock);
2483                 list_del(&ioctx->scmnd_list);
2484                 ch->active_scmnd_cnt--;
2485                 spin_unlock_irq(&ch->spinlock);
2486                 ioctx->ch = NULL;
2487         }
2488
2489         srpt_reset_ioctx(ch, ioctx);
2490         scst_cmd_set_tgt_priv(scmnd, NULL);
2491 }
2492
2493 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2494 /* A vanilla 2.6.19 or older kernel without backported OFED kernel headers. */
2495 static void srpt_refresh_port_work(void *ctx)
2496 #else
2497 static void srpt_refresh_port_work(struct work_struct *work)
2498 #endif
2499 {
2500 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2501         struct srpt_port *sport = (struct srpt_port *)ctx;
2502 #else
2503         struct srpt_port *sport = container_of(work, struct srpt_port, work);
2504 #endif
2505
2506         srpt_refresh_port(sport);
2507 }
2508
2509 /*
2510  * Called by the SCST core to detect target adapters. Returns the number of
2511  * detected target adapters.
2512  */
2513 static int srpt_detect(struct scst_tgt_template *tp)
2514 {
2515         int device_count;
2516
2517         TRACE_ENTRY();
2518
2519         device_count = atomic_read(&srpt_device_count);
2520
2521         TRACE_EXIT_RES(device_count);
2522
2523         return device_count;
2524 }
2525
2526 /*
2527  * Callback function called by the SCST core from scst_unregister() to free up
2528  * the resources associated with device scst_tgt.
2529  */
2530 static int srpt_release(struct scst_tgt *scst_tgt)
2531 {
2532         struct srpt_device *sdev = scst_tgt_get_tgt_priv(scst_tgt);
2533         struct srpt_rdma_ch *ch, *tmp_ch;
2534
2535         TRACE_ENTRY();
2536
2537         BUG_ON(!scst_tgt);
2538 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
2539         WARN_ON(!sdev);
2540         if (!sdev)
2541                 return -ENODEV;
2542 #else
2543         if (WARN_ON(!sdev))
2544                 return -ENODEV;
2545 #endif
2546
2547         srpt_unregister_procfs_entry(scst_tgt->tgtt);
2548
2549         spin_lock_irq(&sdev->spinlock);
2550         list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
2551                 list_del(&ch->list);
2552                 spin_unlock_irq(&sdev->spinlock);
2553                 srpt_release_channel(ch, 1);
2554                 spin_lock_irq(&sdev->spinlock);
2555         }
2556         spin_unlock_irq(&sdev->spinlock);
2557
2558         srpt_unregister_mad_agent(sdev);
2559
2560         scst_tgt_set_tgt_priv(scst_tgt, NULL);
2561
2562         TRACE_EXIT();
2563
2564         return 0;
2565 }
2566
2567 /*
2568  * Entry point for ib_srpt's kernel thread. This kernel thread is only created
2569  * when the module parameter 'thread' is not zero (the default is zero).
2570  * This thread processes the ioctx list srpt_thread.thread_ioctx_list.
2571  *
2572  * @pre thread != 0
2573  */
2574 static int srpt_ioctx_thread(void *arg)
2575 {
2576         struct srpt_ioctx *ioctx;
2577
2578         /* Hibernation / freezing of the SRPT kernel thread is not supported. */
2579         current->flags |= PF_NOFREEZE;
2580
2581         spin_lock_irq(&srpt_thread.thread_lock);
2582         while (!kthread_should_stop()) {
2583                 wait_queue_t wait;
2584                 init_waitqueue_entry(&wait, current);
2585
2586                 if (!srpt_test_ioctx_list()) {
2587                         add_wait_queue_exclusive(&ioctx_list_waitQ, &wait);
2588
2589                         for (;;) {
2590                                 set_current_state(TASK_INTERRUPTIBLE);
2591                                 if (srpt_test_ioctx_list())
2592                                         break;
2593                                 spin_unlock_irq(&srpt_thread.thread_lock);
2594                                 schedule();
2595                                 spin_lock_irq(&srpt_thread.thread_lock);
2596                         }
2597                         set_current_state(TASK_RUNNING);
2598                         remove_wait_queue(&ioctx_list_waitQ, &wait);
2599                 }
2600
2601                 while (!list_empty(&srpt_thread.thread_ioctx_list)) {
2602                         ioctx = list_entry(srpt_thread.thread_ioctx_list.next,
2603                                            struct srpt_ioctx, comp_list);
2604
2605                         list_del(&ioctx->comp_list);
2606
2607                         spin_unlock_irq(&srpt_thread.thread_lock);
2608                         switch (ioctx->op) {
2609                         case IB_WC_SEND:
2610                                 srpt_handle_send_comp(ioctx->ch, ioctx,
2611                                         SCST_CONTEXT_DIRECT);
2612                                 break;
2613                         case IB_WC_RDMA_WRITE:
2614                         case IB_WC_RDMA_READ:
2615                                 srpt_handle_rdma_comp(ioctx->ch, ioctx);
2616                                 break;
2617                         case IB_WC_RECV:
2618                                 srpt_handle_new_iu(ioctx->ch, ioctx);
2619                                 break;
2620                         default:
2621                                 break;
2622                         }
2623 #if defined(CONFIG_SCST_DEBUG)
2624                         if (thread_processing_delay_in_us
2625                             <= MAX_UDELAY_MS * 1000)
2626                                 udelay(thread_processing_delay_in_us);
2627 #endif
2628                         spin_lock_irq(&srpt_thread.thread_lock);
2629                 }
2630         }
2631         spin_unlock_irq(&srpt_thread.thread_lock);
2632
2633         return 0;
2634 }
2635
2636 /* SCST target template for the SRP target implementation. */
2637 static struct scst_tgt_template srpt_template = {
2638         .name = DRV_NAME,
2639         .sg_tablesize = SRPT_DEF_SG_TABLESIZE,
2640         .xmit_response_atomic = 1,
2641         .rdy_to_xfer_atomic = 1,
2642         .no_proc_entry = 0,
2643         .detect = srpt_detect,
2644         .release = srpt_release,
2645         .xmit_response = srpt_xmit_response,
2646         .rdy_to_xfer = srpt_rdy_to_xfer,
2647         .on_free_cmd = srpt_on_free_cmd,
2648         .task_mgmt_fn_done = srpt_tsk_mgmt_done
2649 };
2650
2651 /*
2652  * The callback function srpt_release_class_dev() is called whenever a
2653  * device is removed from the /sys/class/infiniband_srpt device class.
2654  */
2655 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2656 static void srpt_release_class_dev(struct class_device *class_dev)
2657 #else
2658 static void srpt_release_class_dev(struct device *dev)
2659 #endif
2660 {
2661 }
2662
2663 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2664 static int srpt_trace_level_show(struct seq_file *seq, void *v)
2665 {
2666         return scst_proc_log_entry_read(seq, trace_flag, NULL);
2667 }
2668
2669 static ssize_t srpt_proc_trace_level_write(struct file *file,
2670         const char __user *buf, size_t length, loff_t *off)
2671 {
2672         return scst_proc_log_entry_write(file, buf, length, &trace_flag,
2673                 DEFAULT_SRPT_TRACE_FLAGS, NULL);
2674 }
2675
2676 static struct scst_proc_data srpt_log_proc_data = {
2677         SCST_DEF_RW_SEQ_OP(srpt_proc_trace_level_write)
2678         .show = srpt_trace_level_show,
2679 };
2680 #endif
2681
2682 static struct class_attribute srpt_class_attrs[] = {
2683         __ATTR_NULL,
2684 };
2685
2686 static struct class srpt_class = {
2687         .name = "infiniband_srpt",
2688 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2689         .release = srpt_release_class_dev,
2690 #else
2691         .dev_release = srpt_release_class_dev,
2692 #endif
2693         .class_attrs = srpt_class_attrs,
2694 };
2695
2696 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2697 static ssize_t show_login_info(struct class_device *class_dev, char *buf)
2698 #else
2699 static ssize_t show_login_info(struct device *dev,
2700                                struct device_attribute *attr, char *buf)
2701 #endif
2702 {
2703         struct srpt_device *sdev =
2704 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2705                 container_of(class_dev, struct srpt_device, class_dev);
2706 #else
2707                 container_of(dev, struct srpt_device, dev);
2708 #endif
2709         struct srpt_port *sport;
2710         int i;
2711         int len = 0;
2712
2713         for (i = 0; i < sdev->device->phys_port_cnt; i++) {
2714                 sport = &sdev->port[i];
2715
2716                 len += sprintf(buf + len,
2717                                "tid_ext=%016llx,ioc_guid=%016llx,pkey=ffff,"
2718                                "dgid=%04x%04x%04x%04x%04x%04x%04x%04x,"
2719                                "service_id=%016llx\n",
2720                                (unsigned long long) srpt_service_guid,
2721                                (unsigned long long) srpt_service_guid,
2722                                be16_to_cpu(((__be16 *) sport->gid.raw)[0]),
2723                                be16_to_cpu(((__be16 *) sport->gid.raw)[1]),
2724                                be16_to_cpu(((__be16 *) sport->gid.raw)[2]),
2725                                be16_to_cpu(((__be16 *) sport->gid.raw)[3]),
2726                                be16_to_cpu(((__be16 *) sport->gid.raw)[4]),
2727                                be16_to_cpu(((__be16 *) sport->gid.raw)[5]),
2728                                be16_to_cpu(((__be16 *) sport->gid.raw)[6]),
2729                                be16_to_cpu(((__be16 *) sport->gid.raw)[7]),
2730                                (unsigned long long) srpt_service_guid);
2731         }
2732
2733         return len;
2734 }
2735
2736 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2737 static CLASS_DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2738 #else
2739 static DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2740 #endif
2741
2742 /*
2743  * Callback function called by the InfiniBand core when either an InfiniBand
2744  * device has been added or during the ib_register_client() call for each
2745  * registered InfiniBand device.
2746  */
2747 static void srpt_add_one(struct ib_device *device)
2748 {
2749         struct srpt_device *sdev;
2750         struct srpt_port *sport;
2751         struct ib_srq_init_attr srq_attr;
2752         int i;
2753
2754         TRACE_ENTRY();
2755
2756         sdev = kzalloc(sizeof *sdev, GFP_KERNEL);
2757         if (!sdev)
2758                 return;
2759
2760         sdev->device = device;
2761
2762 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2763         sdev->class_dev.class = &srpt_class;
2764         sdev->class_dev.dev = device->dma_device;
2765         snprintf(sdev->class_dev.class_id, BUS_ID_SIZE,
2766                  "srpt-%s", device->name);
2767 #else
2768         sdev->dev.class = &srpt_class;
2769         sdev->dev.parent = device->dma_device;
2770 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
2771         snprintf(sdev->dev.bus_id, BUS_ID_SIZE, "srpt-%s", device->name);
2772 #else
2773         dev_set_name(&sdev->dev, "srpt-%s", device->name);
2774 #endif
2775 #endif
2776
2777 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2778         if (class_device_register(&sdev->class_dev))
2779                 goto free_dev;
2780         if (class_device_create_file(&sdev->class_dev,
2781                                      &class_device_attr_login_info))
2782                 goto err_dev;
2783 #else
2784         if (device_register(&sdev->dev))
2785                 goto free_dev;
2786         if (device_create_file(&sdev->dev, &dev_attr_login_info))
2787                 goto err_dev;
2788 #endif
2789
2790         if (ib_query_device(device, &sdev->dev_attr))
2791                 goto err_dev;
2792
2793         sdev->pd = ib_alloc_pd(device);
2794         if (IS_ERR(sdev->pd))
2795                 goto err_dev;
2796
2797         sdev->mr = ib_get_dma_mr(sdev->pd, IB_ACCESS_LOCAL_WRITE);
2798         if (IS_ERR(sdev->mr))
2799                 goto err_pd;
2800
2801         srq_attr.event_handler = srpt_srq_event;
2802         srq_attr.srq_context = (void *)sdev;
2803         srq_attr.attr.max_wr = min(SRPT_SRQ_SIZE, sdev->dev_attr.max_srq_wr);
2804         srq_attr.attr.max_sge = 1;
2805         srq_attr.attr.srq_limit = 0;
2806
2807         sdev->srq = ib_create_srq(sdev->pd, &srq_attr);
2808         if (IS_ERR(sdev->srq))
2809                 goto err_mr;
2810
2811         TRACE_DBG("%s: create SRQ #wr= %d max_allow=%d dev= %s",
2812                __func__, srq_attr.attr.max_wr,
2813               sdev->dev_attr.max_srq_wr, device->name);
2814
2815         if (!srpt_service_guid)
2816                 srpt_service_guid = be64_to_cpu(device->node_guid);
2817
2818         sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);
2819         if (IS_ERR(sdev->cm_id))
2820                 goto err_srq;
2821
2822         /* print out target login information */
2823         TRACE_DBG("Target login info: id_ext=%016llx,"
2824                   "ioc_guid=%016llx,pkey=ffff,service_id=%016llx",
2825                   (unsigned long long) srpt_service_guid,
2826                   (unsigned long long) srpt_service_guid,
2827                   (unsigned long long) srpt_service_guid);
2828
2829         /*
2830          * We do not have a consistent service_id (ie. also id_ext of target_id)
2831          * to identify this target. We currently use the guid of the first HCA
2832          * in the system as service_id; therefore, the target_id will change
2833          * if this HCA is gone bad and replaced by different HCA
2834          */
2835         if (ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0, NULL))
2836                 goto err_cm;
2837
2838         INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
2839                               srpt_event_handler);
2840         if (ib_register_event_handler(&sdev->event_handler))
2841                 goto err_cm;
2842
2843         if (srpt_alloc_ioctx_ring(sdev))
2844                 goto err_event;
2845
2846         INIT_LIST_HEAD(&sdev->rch_list);
2847         spin_lock_init(&sdev->spinlock);
2848
2849         for (i = 0; i < SRPT_SRQ_SIZE; ++i)
2850                 srpt_post_recv(sdev, sdev->ioctx_ring[i]);
2851
2852         ib_set_client_data(device, &srpt_client, sdev);
2853
2854         sdev->scst_tgt = scst_register(&srpt_template, NULL);
2855         if (!sdev->scst_tgt) {
2856                 PRINT_ERROR("SCST registration failed for %s.",
2857                             sdev->device->name);
2858                 goto err_ring;
2859         }
2860
2861         scst_tgt_set_tgt_priv(sdev->scst_tgt, sdev);
2862
2863         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
2864                 sport = &sdev->port[i - 1];
2865                 sport->sdev = sdev;
2866                 sport->port = i;
2867 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2868                 /*
2869                  * A vanilla 2.6.19 or older kernel without backported OFED
2870                  * kernel headers.
2871                  */
2872                 INIT_WORK(&sport->work, srpt_refresh_port_work, sport);
2873 #else
2874                 INIT_WORK(&sport->work, srpt_refresh_port_work);
2875 #endif
2876                 if (srpt_refresh_port(sport)) {
2877                         PRINT_ERROR("MAD registration failed for %s-%d.",
2878                                     sdev->device->name, i);
2879                         goto err_refresh_port;
2880                 }
2881         }
2882
2883         atomic_inc(&srpt_device_count);
2884
2885         TRACE_EXIT();
2886
2887         return;
2888
2889 err_refresh_port:
2890         scst_unregister(sdev->scst_tgt);
2891 err_ring:
2892         ib_set_client_data(device, &srpt_client, NULL);
2893         srpt_free_ioctx_ring(sdev);
2894 err_event:
2895         ib_unregister_event_handler(&sdev->event_handler);
2896 err_cm:
2897         ib_destroy_cm_id(sdev->cm_id);
2898 err_srq:
2899         ib_destroy_srq(sdev->srq);
2900 err_mr:
2901         ib_dereg_mr(sdev->mr);
2902 err_pd:
2903         ib_dealloc_pd(sdev->pd);
2904 err_dev:
2905 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2906         class_device_unregister(&sdev->class_dev);
2907 #else
2908         device_unregister(&sdev->dev);
2909 #endif
2910 free_dev:
2911         kfree(sdev);
2912
2913         TRACE_EXIT();
2914 }
2915
2916 /*
2917  * Callback function called by the InfiniBand core when either an InfiniBand
2918  * device has been removed or during the ib_unregister_client() call for each
2919  * registered InfiniBand device.
2920  */
2921 static void srpt_remove_one(struct ib_device *device)
2922 {
2923         int i;
2924         struct srpt_device *sdev;
2925
2926         TRACE_ENTRY();
2927
2928         sdev = ib_get_client_data(device, &srpt_client);
2929 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
2930         WARN_ON(!sdev);
2931         if (!sdev)
2932                 return;
2933 #else
2934         if (WARN_ON(!sdev))
2935                 return;
2936 #endif
2937
2938         /*
2939          * Cancel the work if it is queued. Wait until srpt_refresh_port_work()
2940          * finished if it is running.
2941          */
2942         for (i = 0; i < sdev->device->phys_port_cnt; i++)
2943 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
2944                 cancel_work_sync(&sdev->port[i].work);
2945 #else
2946                 /*
2947                  * cancel_work_sync() was introduced in kernel 2.6.22. Older
2948                  * kernels do not have a facility to cancel scheduled work.
2949                  */
2950                 PRINT_ERROR("%s", 
2951                        "your kernel does not provide cancel_work_sync().");
2952 #endif
2953
2954         scst_unregister(sdev->scst_tgt);
2955         sdev->scst_tgt = NULL;
2956
2957         ib_unregister_event_handler(&sdev->event_handler);
2958         ib_destroy_cm_id(sdev->cm_id);
2959         ib_destroy_srq(sdev->srq);
2960         ib_dereg_mr(sdev->mr);
2961         ib_dealloc_pd(sdev->pd);
2962 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2963         class_device_unregister(&sdev->class_dev);
2964 #else
2965         device_unregister(&sdev->dev);
2966 #endif
2967
2968         srpt_free_ioctx_ring(sdev);
2969         kfree(sdev);
2970
2971         TRACE_EXIT();
2972 }
2973
2974 /**
2975  * Create procfs entries for srpt. Currently the only procfs entry created
2976  * by this function is the "trace_level" entry.
2977  */
2978 static int srpt_register_procfs_entry(struct scst_tgt_template *tgt)
2979 {
2980         int res = 0;
2981 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2982         struct proc_dir_entry *p, *root;
2983
2984         root = scst_proc_get_tgt_root(tgt);
2985         WARN_ON(!root);
2986         if (root) {
2987                 /*
2988                  * Fill in the scst_proc_data::data pointer, which is used in
2989                  * a printk(KERN_INFO ...) statement in
2990                  * scst_proc_log_entry_write() in scst_proc.c.
2991                  */
2992                 srpt_log_proc_data.data = (char *)tgt->name;
2993                 p = scst_create_proc_entry(root, SRPT_PROC_TRACE_LEVEL_NAME,
2994                                            &srpt_log_proc_data);
2995                 if (!p)
2996                         res = -ENOMEM;
2997         } else
2998                 res = -ENOMEM;
2999
3000 #endif
3001         return res;
3002 }
3003
3004 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt)
3005 {
3006 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
3007         struct proc_dir_entry *root;
3008
3009         root = scst_proc_get_tgt_root(tgt);
3010         WARN_ON(!root);
3011         if (root)
3012                 remove_proc_entry(SRPT_PROC_TRACE_LEVEL_NAME, root);
3013 #endif
3014 }
3015
3016 /*
3017  * Module initialization.
3018  *
3019  * Note: since ib_register_client() registers callback functions, and since at
3020  * least one of these callback functions (srpt_add_one()) calls SCST functions,
3021  * the SCST target template must be registered before ib_register_client() is
3022  * called.
3023  */
3024 static int __init srpt_init_module(void)
3025 {
3026         int ret;
3027
3028         ret = class_register(&srpt_class);
3029         if (ret) {
3030                 PRINT_ERROR("%s", "couldn't register class ib_srpt");
3031                 goto out;
3032         }
3033
3034         ret = scst_register_target_template(&srpt_template);
3035         if (ret < 0) {
3036                 PRINT_ERROR("%s", "couldn't register with scst");
3037                 ret = -ENODEV;
3038                 goto out_unregister_class;
3039         }
3040
3041         ret = srpt_register_procfs_entry(&srpt_template);
3042         if (ret) {
3043                 PRINT_ERROR("%s", "couldn't register procfs entry");
3044                 goto out_unregister_target;
3045         }
3046
3047         ret = ib_register_client(&srpt_client);
3048         if (ret) {
3049                 PRINT_ERROR("%s", "couldn't register IB client");
3050                 goto out_unregister_target;
3051         }
3052
3053         if (thread) {
3054                 spin_lock_init(&srpt_thread.thread_lock);
3055                 INIT_LIST_HEAD(&srpt_thread.thread_ioctx_list);
3056                 srpt_thread.thread = kthread_run(srpt_ioctx_thread,
3057                                                  NULL, "srpt_thread");
3058                 if (IS_ERR(srpt_thread.thread)) {
3059                         srpt_thread.thread = NULL;
3060                         thread = 0;
3061                 }
3062         }
3063
3064         return 0;
3065
3066 out_unregister_target:
3067         /*
3068          * Note: the procfs entry is unregistered in srpt_release(), which is
3069          * called by scst_unregister_target_template().
3070          */
3071         scst_unregister_target_template(&srpt_template);
3072 out_unregister_class:
3073         class_unregister(&srpt_class);
3074 out:
3075         return ret;
3076 }
3077
3078 static void __exit srpt_cleanup_module(void)
3079 {
3080         TRACE_ENTRY();
3081
3082         if (srpt_thread.thread)
3083                 kthread_stop(srpt_thread.thread);
3084         ib_unregister_client(&srpt_client);
3085         scst_unregister_target_template(&srpt_template);
3086         class_unregister(&srpt_class);
3087
3088         TRACE_EXIT();
3089 }
3090
3091 module_init(srpt_init_module);
3092 module_exit(srpt_cleanup_module);