Compiles again on RHEL 5 / CentOS 5 with CONFIG_SCST_DEBUG enabled.
[mirror/scst/.git] / srpt / src / ib_srpt.c
1 /*
2  * Copyright (c) 2006 - 2009 Mellanox Technology Inc.  All rights reserved.
3  * Copyright (C) 2008 Vladislav Bolkhovitin <vst@vlnb.net>
4  * Copyright (C) 2008 - 2009 Bart Van Assche <bart.vanassche@gmail.com>
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  */
35
36 #include <linux/module.h>
37 #include <linux/init.h>
38 #include <linux/slab.h>
39 #include <linux/err.h>
40 #include <linux/ctype.h>
41 #include <linux/string.h>
42 #include <linux/kthread.h>
43 #include <linux/delay.h>
44 #include <asm/atomic.h>
45 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
46 #include <linux/proc_fs.h>
47 #include <linux/seq_file.h>
48 #endif
49 #include "ib_srpt.h"
50 #include "scst_debug.h"
51
52 /* Name of this kernel module. */
53 #define DRV_NAME                "ib_srpt"
54 /* Prefix for printk() kernel messages. */
55 #define LOG_PFX                 DRV_NAME ": "
56 #define DRV_VERSION             "1.0.1"
57 #define DRV_RELDATE             "July 10, 2008"
58 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
59 /* Flags to be used in SCST debug tracing statements. */
60 #define DEFAULT_SRPT_TRACE_FLAGS (TRACE_OUT_OF_MEM | TRACE_MINOR \
61                                   | TRACE_MGMT | TRACE_SPECIAL)
62 /* Name of the entry that will be created under /proc/scsi_tgt/ib_srpt. */
63 #define SRPT_PROC_TRACE_LEVEL_NAME      "trace_level"
64 #endif
65
66 #define MELLANOX_SRPT_ID_STRING "SCST SRP target"
67
68 MODULE_AUTHOR("Vu Pham");
69 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
70                    "v" DRV_VERSION " (" DRV_RELDATE ")");
71 MODULE_LICENSE("Dual BSD/GPL");
72
73 struct srpt_thread {
74         /* Protects thread_ioctx_list. */
75         spinlock_t thread_lock;
76         /* I/O contexts to be processed by the kernel thread. */
77         struct list_head thread_ioctx_list;
78         /* SRPT kernel thread. */
79         struct task_struct *thread;
80 };
81
82 /*
83  * Global Variables
84  */
85
86 static u64 srpt_service_guid;
87 /* List of srpt_device structures. */
88 static atomic_t srpt_device_count;
89 static int use_port_guid_in_session_name;
90 static int thread;
91 static struct srpt_thread srpt_thread;
92 static DECLARE_WAIT_QUEUE_HEAD(ioctx_list_waitQ);
93 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
94 static unsigned long trace_flag = DEFAULT_SRPT_TRACE_FLAGS;
95 module_param(trace_flag, long, 0644);
96 MODULE_PARM_DESC(trace_flag,
97                  "Trace flags for the ib_srpt kernel module.");
98 #endif
99 #if defined(CONFIG_SCST_DEBUG)
100 static unsigned long interrupt_processing_delay_in_us;
101 module_param(interrupt_processing_delay_in_us, long, 0744);
102 MODULE_PARM_DESC(interrupt_processing_delay_in_us,
103                  "CQ completion handler interrupt delay in microseconds.");
104 static unsigned long thread_processing_delay_in_us;
105 module_param(thread_processing_delay_in_us, long, 0744);
106 MODULE_PARM_DESC(thread_processing_delay_in_us,
107                  "SRP thread processing delay in microseconds.");
108 #endif
109
110 module_param(thread, int, 0444);
111 MODULE_PARM_DESC(thread,
112                  "Executing ioctx in thread context. Default 0, i.e. soft IRQ, "
113                  "where possible");
114
115 module_param(use_port_guid_in_session_name, bool, 0444);
116 MODULE_PARM_DESC(use_port_guid_in_session_name,
117                  "Use target port ID in the SCST session name such that"
118                  " redundant paths between multiport systems can be masked.");
119
120 static void srpt_add_one(struct ib_device *device);
121 static void srpt_remove_one(struct ib_device *device);
122 static void srpt_unregister_mad_agent(struct srpt_device *sdev);
123 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt);
124
125 static struct ib_client srpt_client = {
126         .name = DRV_NAME,
127         .add = srpt_add_one,
128         .remove = srpt_remove_one
129 };
130
131 /**
132  * Atomically test and set the channel state.
133  * @ch: RDMA channel.
134  * @old: channel state to compare with.
135  * @new: state to change the channel state to if the current state matches the
136  *       argument 'old'.
137  *
138  * Returns true if the channel state matched old upon entry of this function,
139  * and false otherwise.
140  */
141 static bool srpt_test_and_set_channel_state(struct srpt_rdma_ch *ch,
142                                             enum rdma_ch_state old,
143                                             enum rdma_ch_state new)
144 {
145         unsigned long flags;
146         enum rdma_ch_state cur;
147
148         spin_lock_irqsave(&ch->spinlock, flags);
149         cur = ch->state;
150         if (cur == old)
151                 ch->state = new;
152         spin_unlock_irqrestore(&ch->spinlock, flags);
153
154         return cur == old;
155 }
156
157 /*
158  * Callback function called by the InfiniBand core when an asynchronous IB
159  * event occurs. This callback may occur in interrupt context. See also
160  * section 11.5.2, Set Asynchronous Event Handler in the InfiniBand
161  * Architecture Specification.
162  */
163 static void srpt_event_handler(struct ib_event_handler *handler,
164                                struct ib_event *event)
165 {
166         struct srpt_device *sdev;
167         struct srpt_port *sport;
168
169         sdev = ib_get_client_data(event->device, &srpt_client);
170         if (!sdev || sdev->device != event->device)
171                 return;
172
173         TRACE_DBG("ASYNC event= %d on device= %s",
174                   event->event, sdev->device->name);
175
176         switch (event->event) {
177         case IB_EVENT_PORT_ERR:
178                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
179                         sport = &sdev->port[event->element.port_num - 1];
180                         sport->lid = 0;
181                         sport->sm_lid = 0;
182                 }
183                 break;
184         case IB_EVENT_PORT_ACTIVE:
185         case IB_EVENT_LID_CHANGE:
186         case IB_EVENT_PKEY_CHANGE:
187         case IB_EVENT_SM_CHANGE:
188         case IB_EVENT_CLIENT_REREGISTER:
189                 /*
190                  * Refresh port data asynchronously. Note: it is safe to call
191                  * schedule_work() even if &sport->work is already on the
192                  * global workqueue because schedule_work() tests for the
193                  * work_pending() condition before adding &sport->work to the
194                  * global work queue.
195                  */
196                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
197                         sport = &sdev->port[event->element.port_num - 1];
198                         if (!sport->lid && !sport->sm_lid)
199                                 schedule_work(&sport->work);
200                 }
201                 break;
202         default:
203                 break;
204         }
205
206 }
207
208 /*
209  * Callback function called by the InfiniBand core for SRQ (shared receive
210  * queue) events.
211  */
212 static void srpt_srq_event(struct ib_event *event, void *ctx)
213 {
214         TRACE_DBG("SRQ event %d", event->event);
215 }
216
217 /*
218  * Callback function called by the InfiniBand core for QP (queue pair) events.
219  */
220 static void srpt_qp_event(struct ib_event *event, void *ctx)
221 {
222         struct srpt_rdma_ch *ch = ctx;
223
224         TRACE_DBG("QP event %d on cm_id=%p sess_name=%s state=%d",
225                   event->event, ch->cm_id, ch->sess_name, ch->state);
226
227         switch (event->event) {
228         case IB_EVENT_COMM_EST:
229 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20) || defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
230                 ib_cm_notify(ch->cm_id, event->event);
231 #else
232                 /* Vanilla 2.6.19 kernel (or before) without OFED. */
233                 PRINT_ERROR("%s", "how to perform ib_cm_notify() on a"
234                             " vanilla 2.6.18 kernel ???");
235 #endif
236                 break;
237         case IB_EVENT_QP_LAST_WQE_REACHED:
238                 if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_LIVE,
239                                         RDMA_CHANNEL_DISCONNECTING)) {
240                         PRINT_INFO("disconnected session %s.", ch->sess_name);
241                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
242                 }
243                 break;
244         default:
245                 break;
246         }
247 }
248
249 /*
250  * Helper function for filling in an InfiniBand IOUnitInfo structure. Copies
251  * the lowest four bits of value in element slot of the array of four bit
252  * elements called c_list (controller list). The index slot is one-based.
253  *
254  * @pre 1 <= slot && 0 <= value && value < 16
255  */
256 static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
257 {
258         u16 id;
259         u8 tmp;
260
261         id = (slot - 1) / 2;
262         if (slot & 0x1) {
263                 tmp = c_list[id] & 0xf;
264                 c_list[id] = (value << 4) | tmp;
265         } else {
266                 tmp = c_list[id] & 0xf0;
267                 c_list[id] = (value & 0xf) | tmp;
268         }
269 }
270
271 /*
272  * Write InfiniBand ClassPortInfo to mad. See also section 16.3.3.1
273  * ClassPortInfo in the InfiniBand Architecture Specification.
274  */
275 static void srpt_get_class_port_info(struct ib_dm_mad *mad)
276 {
277         struct ib_class_port_info *cif;
278
279         cif = (struct ib_class_port_info *)mad->data;
280         memset(cif, 0, sizeof *cif);
281         cif->base_version = 1;
282         cif->class_version = 1;
283         cif->resp_time_value = 20;
284
285         mad->mad_hdr.status = 0;
286 }
287
288 /*
289  * Write IOUnitInfo to mad. See also section 16.3.3.3 IOUnitInfo in the
290  * InfiniBand Architecture Specification. See also section B.7,
291  * table B.6 in the T10 SRP r16a document.
292  */
293 static void srpt_get_iou(struct ib_dm_mad *mad)
294 {
295         struct ib_dm_iou_info *ioui;
296         u8 slot;
297         int i;
298
299         ioui = (struct ib_dm_iou_info *)mad->data;
300         ioui->change_id = 1;
301         ioui->max_controllers = 16;
302
303         /* set present for slot 1 and empty for the rest */
304         srpt_set_ioc(ioui->controller_list, 1, 1);
305         for (i = 1, slot = 2; i < 16; i++, slot++)
306                 srpt_set_ioc(ioui->controller_list, slot, 0);
307
308         mad->mad_hdr.status = 0;
309 }
310
311 /*
312  * Write IOControllerprofile to mad for I/O controller (sdev, slot). See also
313  * section 16.3.3.4 IOControllerProfile in the InfiniBand Architecture
314  * Specification. See also section B.7, table B.7 in the T10 SRP r16a
315  * document.
316  */
317 static void srpt_get_ioc(struct srpt_device *sdev, u32 slot,
318                          struct ib_dm_mad *mad)
319 {
320         struct ib_dm_ioc_profile *iocp;
321
322         iocp = (struct ib_dm_ioc_profile *)mad->data;
323
324         if (!slot || slot > 16) {
325                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
326                 return;
327         }
328
329         if (slot > 2) {
330                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
331                 return;
332         }
333
334         memset(iocp, 0, sizeof *iocp);
335         strcpy(iocp->id_string, MELLANOX_SRPT_ID_STRING);
336         iocp->guid = cpu_to_be64(srpt_service_guid);
337         iocp->vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
338         iocp->device_id = cpu_to_be32(sdev->dev_attr.vendor_part_id);
339         iocp->device_version = cpu_to_be16(sdev->dev_attr.hw_ver);
340         iocp->subsys_vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
341         iocp->subsys_device_id = 0x0;
342         iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
343         iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
344         iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
345         iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
346         iocp->send_queue_depth = cpu_to_be16(SRPT_SRQ_SIZE);
347         iocp->rdma_read_depth = 4;
348         iocp->send_size = cpu_to_be32(MAX_MESSAGE_SIZE);
349         iocp->rdma_size = cpu_to_be32(MAX_RDMA_SIZE);
350         iocp->num_svc_entries = 1;
351         iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
352                 SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
353
354         mad->mad_hdr.status = 0;
355 }
356
357 /*
358  * Device management: write ServiceEntries to mad for the given slot. See also
359  * section 16.3.3.5 ServiceEntries in the InfiniBand Architecture
360  * Specification. See also section B.7, table B.8 in the T10 SRP r16a document.
361  */
362 static void srpt_get_svc_entries(u64 ioc_guid,
363                                  u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
364 {
365         struct ib_dm_svc_entries *svc_entries;
366
367         WARN_ON(!ioc_guid);
368
369         if (!slot || slot > 16) {
370                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
371                 return;
372         }
373
374         if (slot > 2 || lo > hi || hi > 1) {
375                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
376                 return;
377         }
378
379         svc_entries = (struct ib_dm_svc_entries *)mad->data;
380         memset(svc_entries, 0, sizeof *svc_entries);
381         svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid);
382         snprintf(svc_entries->service_entries[0].name,
383                  sizeof(svc_entries->service_entries[0].name),
384                  "%s%016llx",
385                  SRP_SERVICE_NAME_PREFIX,
386                  (unsigned long long)ioc_guid);
387
388         mad->mad_hdr.status = 0;
389 }
390
391 /*
392  * Actual processing of a received MAD *rq_mad received through source port *sp
393  * (MAD = InfiniBand management datagram). The response to be sent back is
394  * written to *rsp_mad.
395  */
396 static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
397                                  struct ib_dm_mad *rsp_mad)
398 {
399         u16 attr_id;
400         u32 slot;
401         u8 hi, lo;
402
403         attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
404         switch (attr_id) {
405         case DM_ATTR_CLASS_PORT_INFO:
406                 srpt_get_class_port_info(rsp_mad);
407                 break;
408         case DM_ATTR_IOU_INFO:
409                 srpt_get_iou(rsp_mad);
410                 break;
411         case DM_ATTR_IOC_PROFILE:
412                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
413                 srpt_get_ioc(sp->sdev, slot, rsp_mad);
414                 break;
415         case DM_ATTR_SVC_ENTRIES:
416                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
417                 hi = (u8) ((slot >> 8) & 0xff);
418                 lo = (u8) (slot & 0xff);
419                 slot = (u16) ((slot >> 16) & 0xffff);
420                 srpt_get_svc_entries(srpt_service_guid,
421                                      slot, hi, lo, rsp_mad);
422                 break;
423         default:
424                 rsp_mad->mad_hdr.status =
425                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
426                 break;
427         }
428 }
429
430 /*
431  * Callback function that is called by the InfiniBand core after transmission of
432  * a MAD. (MAD = management datagram; AH = address handle.)
433  */
434 static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
435                                   struct ib_mad_send_wc *mad_wc)
436 {
437         ib_destroy_ah(mad_wc->send_buf->ah);
438         ib_free_send_mad(mad_wc->send_buf);
439 }
440
441 /*
442  * Callback function that is called by the InfiniBand core after reception of
443  * a MAD (management datagram).
444  */
445 static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
446                                   struct ib_mad_recv_wc *mad_wc)
447 {
448         struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
449         struct ib_ah *ah;
450         struct ib_mad_send_buf *rsp;
451         struct ib_dm_mad *dm_mad;
452
453         if (!mad_wc || !mad_wc->recv_buf.mad)
454                 return;
455
456         ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
457                                   mad_wc->recv_buf.grh, mad_agent->port_num);
458         if (IS_ERR(ah))
459                 goto err;
460
461         BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR);
462
463         rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
464                                  mad_wc->wc->pkey_index, 0,
465                                  IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA,
466                                  GFP_KERNEL);
467         if (IS_ERR(rsp))
468                 goto err_rsp;
469
470         rsp->ah = ah;
471
472         dm_mad = rsp->mad;
473         memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof *dm_mad);
474         dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
475         dm_mad->mad_hdr.status = 0;
476
477         switch (mad_wc->recv_buf.mad->mad_hdr.method) {
478         case IB_MGMT_METHOD_GET:
479                 srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
480                 break;
481         case IB_MGMT_METHOD_SET:
482                 dm_mad->mad_hdr.status =
483                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
484                 break;
485         default:
486                 dm_mad->mad_hdr.status =
487                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
488                 break;
489         }
490
491         if (!ib_post_send_mad(rsp, NULL)) {
492                 ib_free_recv_mad(mad_wc);
493                 /* will destroy_ah & free_send_mad in send completion */
494                 return;
495         }
496
497         ib_free_send_mad(rsp);
498
499 err_rsp:
500         ib_destroy_ah(ah);
501 err:
502         ib_free_recv_mad(mad_wc);
503 }
504
505 /*
506  * Enable InfiniBand management datagram processing, update the cached sm_lid,
507  * lid and gid values, and register a callback function for processing MADs
508  * on the specified port. It is safe to call this function more than once for
509  * the same port.
510  */
511 static int srpt_refresh_port(struct srpt_port *sport)
512 {
513         struct ib_mad_reg_req reg_req;
514         struct ib_port_modify port_modify;
515         struct ib_port_attr port_attr;
516         int ret;
517
518         memset(&port_modify, 0, sizeof port_modify);
519         port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
520         port_modify.clr_port_cap_mask = 0;
521
522         ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
523         if (ret)
524                 goto err_mod_port;
525
526         ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
527         if (ret)
528                 goto err_query_port;
529
530         sport->sm_lid = port_attr.sm_lid;
531         sport->lid = port_attr.lid;
532
533         ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid);
534         if (ret)
535                 goto err_query_port;
536
537         if (!sport->mad_agent) {
538                 memset(&reg_req, 0, sizeof reg_req);
539                 reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
540                 reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
541                 set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
542                 set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
543
544                 sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
545                                                          sport->port,
546                                                          IB_QPT_GSI,
547                                                          &reg_req, 0,
548                                                          srpt_mad_send_handler,
549                                                          srpt_mad_recv_handler,
550                                                          sport);
551                 if (IS_ERR(sport->mad_agent)) {
552                         ret = PTR_ERR(sport->mad_agent);
553                         sport->mad_agent = NULL;
554                         goto err_query_port;
555                 }
556         }
557
558         return 0;
559
560 err_query_port:
561
562         port_modify.set_port_cap_mask = 0;
563         port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
564         ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
565
566 err_mod_port:
567
568         return ret;
569 }
570
571 /*
572  * Unregister the callback function for processing MADs and disable MAD
573  * processing for all ports of the specified device. It is safe to call this
574  * function more than once for the same device.
575  */
576 static void srpt_unregister_mad_agent(struct srpt_device *sdev)
577 {
578         struct ib_port_modify port_modify = {
579                 .clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP,
580         };
581         struct srpt_port *sport;
582         int i;
583
584         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
585                 sport = &sdev->port[i - 1];
586                 WARN_ON(sport->port != i);
587                 if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
588                         PRINT_ERROR("%s", "disabling MAD processing failed.");
589                 if (sport->mad_agent) {
590                         ib_unregister_mad_agent(sport->mad_agent);
591                         sport->mad_agent = NULL;
592                 }
593         }
594 }
595
596 /*
597  * Allocate and initialize an SRPT I/O context structure.
598  */
599 static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev)
600 {
601         struct srpt_ioctx *ioctx;
602
603         ioctx = kmalloc(sizeof *ioctx, GFP_KERNEL);
604         if (!ioctx)
605                 goto out;
606
607         ioctx->buf = kzalloc(MAX_MESSAGE_SIZE, GFP_KERNEL);
608         if (!ioctx->buf)
609                 goto out_free_ioctx;
610
611         ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf,
612                                        MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
613         if (ib_dma_mapping_error(sdev->device, ioctx->dma))
614                 goto out_free_buf;
615
616         return ioctx;
617
618 out_free_buf:
619         kfree(ioctx->buf);
620 out_free_ioctx:
621         kfree(ioctx);
622 out:
623         return NULL;
624 }
625
626 /*
627  * Deallocate an SRPT I/O context structure.
628  */
629 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
630 {
631         if (!ioctx)
632                 return;
633
634         ib_dma_unmap_single(sdev->device, ioctx->dma,
635                             MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
636         kfree(ioctx->buf);
637         kfree(ioctx);
638 }
639
640 /*
641  * Associate a ring of SRPT I/O context structures with the specified device.
642  */
643 static int srpt_alloc_ioctx_ring(struct srpt_device *sdev)
644 {
645         int i;
646
647         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
648                 sdev->ioctx_ring[i] = srpt_alloc_ioctx(sdev);
649
650                 if (!sdev->ioctx_ring[i])
651                         goto err;
652
653                 sdev->ioctx_ring[i]->index = i;
654         }
655
656         return 0;
657
658 err:
659         while (--i > 0) {
660                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
661                 sdev->ioctx_ring[i] = NULL;
662         }
663         return -ENOMEM;
664 }
665
666 /* Free the ring of SRPT I/O context structures. */
667 static void srpt_free_ioctx_ring(struct srpt_device *sdev)
668 {
669         int i;
670
671         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
672                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
673                 sdev->ioctx_ring[i] = NULL;
674         }
675 }
676
677 /*
678  * Post a receive request on the work queue of InfiniBand device 'sdev'.
679  */
680 static int srpt_post_recv(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
681 {
682         struct ib_sge list;
683         struct ib_recv_wr wr, *bad_wr;
684
685         wr.wr_id = ioctx->index | SRPT_OP_RECV;
686
687         list.addr = ioctx->dma;
688         list.length = MAX_MESSAGE_SIZE;
689         list.lkey = sdev->mr->lkey;
690
691         wr.next = NULL;
692         wr.sg_list = &list;
693         wr.num_sge = 1;
694
695         return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);
696 }
697
698 /*
699  * Post an IB send request.
700  * @ch: RDMA channel to post the send request on.
701  * @ioctx: I/O context of the send request.
702  * @len: length of the request to be sent in bytes.
703  *
704  * Returns zero upon success and a non-zero value upon failure.
705  */
706 static int srpt_post_send(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
707                           int len)
708 {
709         struct ib_sge list;
710         struct ib_send_wr wr, *bad_wr;
711         struct srpt_device *sdev = ch->sport->sdev;
712
713         ib_dma_sync_single_for_device(sdev->device, ioctx->dma,
714                                       MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
715
716         list.addr = ioctx->dma;
717         list.length = len;
718         list.lkey = sdev->mr->lkey;
719
720         wr.next = NULL;
721         wr.wr_id = ioctx->index;
722         wr.sg_list = &list;
723         wr.num_sge = 1;
724         wr.opcode = IB_WR_SEND;
725         wr.send_flags = IB_SEND_SIGNALED;
726
727         return ib_post_send(ch->qp, &wr, &bad_wr);
728 }
729
730 static int srpt_get_desc_tbl(struct srpt_ioctx *ioctx, struct srp_cmd *srp_cmd,
731                              int *ind)
732 {
733         struct srp_indirect_buf *idb;
734         struct srp_direct_buf *db;
735
736         *ind = 0;
737         if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
738             ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
739                 ioctx->n_rbuf = 1;
740                 ioctx->rbufs = &ioctx->single_rbuf;
741
742                 db = (void *)srp_cmd->add_data;
743                 memcpy(ioctx->rbufs, db, sizeof *db);
744                 ioctx->data_len = be32_to_cpu(db->len);
745         } else {
746                 idb = (void *)srp_cmd->add_data;
747
748                 ioctx->n_rbuf = be32_to_cpu(idb->table_desc.len) / sizeof *db;
749
750                 if (ioctx->n_rbuf >
751                     (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
752                         *ind = 1;
753                         ioctx->n_rbuf = 0;
754                         goto out;
755                 }
756
757                 if (ioctx->n_rbuf == 1)
758                         ioctx->rbufs = &ioctx->single_rbuf;
759                 else
760                         ioctx->rbufs =
761                                 kmalloc(ioctx->n_rbuf * sizeof *db, GFP_ATOMIC);
762                 if (!ioctx->rbufs) {
763                         ioctx->n_rbuf = 0;
764                         return -ENOMEM;
765                 }
766
767                 db = idb->desc_list;
768                 memcpy(ioctx->rbufs, db, ioctx->n_rbuf * sizeof *db);
769                 ioctx->data_len = be32_to_cpu(idb->len);
770         }
771 out:
772         return 0;
773 }
774
775 /*
776  * Modify the attributes of queue pair 'qp': allow local write, remote read,
777  * and remote write. Also transition 'qp' to state IB_QPS_INIT.
778  */
779 static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
780 {
781         struct ib_qp_attr *attr;
782         int ret;
783
784         attr = kzalloc(sizeof *attr, GFP_KERNEL);
785         if (!attr)
786                 return -ENOMEM;
787
788         attr->qp_state = IB_QPS_INIT;
789         attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ |
790             IB_ACCESS_REMOTE_WRITE;
791         attr->port_num = ch->sport->port;
792         attr->pkey_index = 0;
793
794         ret = ib_modify_qp(qp, attr,
795                            IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
796                            IB_QP_PKEY_INDEX);
797
798         kfree(attr);
799         return ret;
800 }
801
802 /**
803  * Change the state of a channel to 'ready to receive' (RTR).
804  * @ch: channel of the queue pair.
805  * @qp: queue pair to change the state of.
806  *
807  * Returns zero upon success and a negative value upon failure.
808  *
809  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
810  * If this structure ever becomes larger, it might be necessary to allocate
811  * it dynamically instead of on the stack.
812  */
813 static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
814 {
815         struct ib_qp_attr qp_attr;
816         int attr_mask;
817         int ret;
818
819         qp_attr.qp_state = IB_QPS_RTR;
820         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
821         if (ret)
822                 goto out;
823
824         qp_attr.max_dest_rd_atomic = 4;
825
826         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
827
828 out:
829         return ret;
830 }
831
832 /**
833  * Change the state of a channel to 'ready to send' (RTS).
834  * @ch: channel of the queue pair.
835  * @qp: queue pair to change the state of.
836  *
837  * Returns zero upon success and a negative value upon failure.
838  *
839  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
840  * If this structure ever becomes larger, it might be necessary to allocate
841  * it dynamically instead of on the stack.
842  */
843 static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
844 {
845         struct ib_qp_attr qp_attr;
846         int attr_mask;
847         int ret;
848
849         qp_attr.qp_state = IB_QPS_RTS;
850         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
851         if (ret)
852                 goto out;
853
854         qp_attr.max_rd_atomic = 4;
855
856         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
857
858 out:
859         return ret;
860 }
861
862 static void srpt_reset_ioctx(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
863 {
864         int i;
865
866         if (ioctx->n_rdma_ius > 0 && ioctx->rdma_ius) {
867                 struct rdma_iu *riu = ioctx->rdma_ius;
868
869                 for (i = 0; i < ioctx->n_rdma_ius; ++i, ++riu)
870                         kfree(riu->sge);
871                 kfree(ioctx->rdma_ius);
872         }
873
874         if (ioctx->n_rbuf > 1)
875                 kfree(ioctx->rbufs);
876
877         /* If ch == NULL this means that the command has been aborted. */
878         if (!ch)
879                 return;
880
881         if (srpt_post_recv(ch->sport->sdev, ioctx))
882                 PRINT_ERROR("%s", "SRQ post_recv failed - this is serious.");
883                 /* we should queue it back to free_ioctx queue */
884         else
885                 atomic_inc(&ch->req_lim_delta);
886 }
887
888 static void srpt_abort_scst_cmd(struct srpt_device *sdev,
889                                 struct scst_cmd *scmnd,
890                                 bool tell_initiator)
891 {
892         struct srpt_ioctx *ioctx;
893         scst_data_direction dir;
894         struct srpt_rdma_ch *ch;
895
896         ioctx = scst_cmd_get_tgt_priv(scmnd);
897         BUG_ON(!ioctx);
898         dir = scst_cmd_get_data_direction(scmnd);
899         if (dir != SCST_DATA_NONE)
900                 ib_dma_unmap_sg(sdev->device,
901                                 scst_cmd_get_sg(scmnd),
902                                 scst_cmd_get_sg_cnt(scmnd),
903                                 scst_to_tgt_dma_dir(dir));
904
905         if (ioctx->state == SRPT_STATE_NEW) {
906                 /*
907                  * Do not try to abort the SCST command here but wait until
908                  * the SCST core has called srpt_rdy_to_xfer() or
909                  * srpt_xmit_response(). Since srpt_release_channel() will
910                  * finish before srpt_on_free_cmd() is called, set the channel
911                  * pointer inside the SCST command to NULL such that
912                  * srpt_on_free_cmd() will not dereference a dangling pointer.
913                  */
914                 ch = ioctx->ch;
915                 ioctx->ch = NULL;
916                 BUG_ON(!ch);
917                 spin_lock_irq(&ch->spinlock);
918                 list_del(&ioctx->scmnd_list);
919                 ch->active_scmnd_cnt--;
920                 spin_unlock_irq(&ch->spinlock);
921         } else if (ioctx->state == SRPT_STATE_NEED_DATA) {
922                 WARN_ON(scst_cmd_get_data_direction(ioctx->scmnd)
923                         == SCST_DATA_READ);
924                 scst_rx_data(scmnd,
925                              tell_initiator ? SCST_RX_STATUS_ERROR
926                              : SCST_RX_STATUS_ERROR_FATAL,
927                              SCST_CONTEXT_THREAD);
928         } else if (ioctx->state == SRPT_STATE_PROCESSED) {
929                 scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_FAILED);
930                 WARN_ON(scmnd->state != SCST_CMD_STATE_XMIT_WAIT);
931                 scst_tgt_cmd_done(scmnd, scst_estimate_context());
932         } else {
933                 TRACE_DBG("Aborting cmd with state %d", ioctx->state);
934                 WARN_ON("ERROR: unexpected command state");
935         }
936
937         ioctx->state = SRPT_STATE_ABORTED;
938 }
939
940 static void srpt_handle_err_comp(struct srpt_rdma_ch *ch, struct ib_wc *wc)
941 {
942         struct srpt_ioctx *ioctx;
943         struct srpt_device *sdev = ch->sport->sdev;
944
945         if (wc->wr_id & SRPT_OP_RECV) {
946                 ioctx = sdev->ioctx_ring[wc->wr_id & ~SRPT_OP_RECV];
947                 PRINT_ERROR("%s", "This is serious - SRQ is in bad state.");
948         } else {
949                 ioctx = sdev->ioctx_ring[wc->wr_id];
950
951                 if (ioctx->scmnd)
952                         srpt_abort_scst_cmd(sdev, ioctx->scmnd, true);
953                 else
954                         srpt_reset_ioctx(ch, ioctx);
955         }
956 }
957
958 static void srpt_handle_send_comp(struct srpt_rdma_ch *ch,
959                                   struct srpt_ioctx *ioctx,
960                                   enum scst_exec_context context)
961 {
962         if (ioctx->scmnd) {
963                 scst_data_direction dir =
964                         scst_cmd_get_data_direction(ioctx->scmnd);
965
966                 if (dir != SCST_DATA_NONE)
967                         ib_dma_unmap_sg(ch->sport->sdev->device,
968                                         scst_cmd_get_sg(ioctx->scmnd),
969                                         scst_cmd_get_sg_cnt(ioctx->scmnd),
970                                         scst_to_tgt_dma_dir(dir));
971
972                 WARN_ON(ioctx->scmnd->state != SCST_CMD_STATE_XMIT_WAIT);
973                 scst_tgt_cmd_done(ioctx->scmnd, context);
974         } else
975                 srpt_reset_ioctx(ch, ioctx);
976 }
977
978 /** Process an RDMA completion notification. */
979 static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch,
980                                   struct srpt_ioctx *ioctx)
981 {
982         if (!ioctx->scmnd) {
983                 WARN_ON("ERROR: ioctx->scmnd == NULL");
984                 srpt_reset_ioctx(ch, ioctx);
985                 return;
986         }
987
988         /*
989          * If an RDMA completion notification has been received for a write
990          * command, tell SCST that processing can continue by calling
991          * scst_rx_data().
992          */
993         if (ioctx->state == SRPT_STATE_NEED_DATA) {
994                 WARN_ON(scst_cmd_get_data_direction(ioctx->scmnd)
995                         == SCST_DATA_READ);
996                 ioctx->state = SRPT_STATE_DATA_IN;
997                 scst_rx_data(ioctx->scmnd, SCST_RX_STATUS_SUCCESS,
998                              scst_estimate_context());
999         }
1000 }
1001
1002 /**
1003  * Build an SRP_RSP response.
1004  * @ch: RDMA channel through which the request has been received.
1005  * @ioctx: I/O context in which the SRP_RSP response will be built.
1006  * @s_key: sense key that will be stored in the response.
1007  * @s_code: value that will be stored in the asc_ascq field of the sense data.
1008  * @tag: tag of the request for which this response is being generated.
1009  *
1010  * Returns the size in bytes of the SRP_RSP response.
1011  *
1012  * An SRP_RSP response contains a SCSI status or service response. See also
1013  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
1014  * response. See also SPC-2 for more information about sense data.
1015  */
1016 static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
1017                               struct srpt_ioctx *ioctx, u8 s_key, u8 s_code,
1018                               u64 tag)
1019 {
1020         struct srp_rsp *srp_rsp;
1021         struct sense_data *sense;
1022         int limit_delta;
1023         int sense_data_len = 0;
1024
1025         srp_rsp = ioctx->buf;
1026         memset(srp_rsp, 0, sizeof *srp_rsp);
1027
1028         limit_delta = atomic_read(&ch->req_lim_delta);
1029         atomic_sub(limit_delta, &ch->req_lim_delta);
1030
1031         srp_rsp->opcode = SRP_RSP;
1032         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1033         srp_rsp->tag = tag;
1034
1035         if (s_key != NO_SENSE) {
1036                 sense_data_len = sizeof *sense + (sizeof *sense % 4);
1037                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1038                 srp_rsp->status = SAM_STAT_CHECK_CONDITION;
1039                 srp_rsp->sense_data_len = cpu_to_be32(sense_data_len);
1040
1041                 sense = (struct sense_data *)(srp_rsp + 1);
1042                 sense->err_code = 0x70;
1043                 sense->key = s_key;
1044                 sense->asc_ascq = s_code;
1045         }
1046
1047         return sizeof(*srp_rsp) + sense_data_len;
1048 }
1049
1050 /**
1051  * Build a task management response, which is a specific SRP_RSP response.
1052  * @ch: RDMA channel through which the request has been received.
1053  * @ioctx: I/O context in which the SRP_RSP response will be built.
1054  * @rsp_code: RSP_CODE that will be stored in the response.
1055  * @tag: tag of the request for which this response is being generated.
1056  *
1057  * Returns the size in bytes of the SRP_RSP response.
1058  *
1059  * An SRP_RSP response contains a SCSI status or service response. See also
1060  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
1061  * response.
1062  */
1063 static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
1064                                   struct srpt_ioctx *ioctx, u8 rsp_code,
1065                                   u64 tag)
1066 {
1067         struct srp_rsp *srp_rsp;
1068         int limit_delta;
1069         int resp_data_len = 0;
1070
1071         ib_dma_sync_single_for_cpu(ch->sport->sdev->device, ioctx->dma,
1072                                    MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
1073
1074         srp_rsp = ioctx->buf;
1075         memset(srp_rsp, 0, sizeof *srp_rsp);
1076
1077         limit_delta = atomic_read(&ch->req_lim_delta);
1078         atomic_sub(limit_delta, &ch->req_lim_delta);
1079
1080         srp_rsp->opcode = SRP_RSP;
1081         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1082         srp_rsp->tag = tag;
1083
1084         if (rsp_code != SRP_TSK_MGMT_SUCCESS) {
1085                 resp_data_len = 4;
1086                 srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
1087                 srp_rsp->resp_data_len = cpu_to_be32(resp_data_len);
1088                 srp_rsp->data[3] = rsp_code;
1089         }
1090
1091         return sizeof(*srp_rsp) + resp_data_len;
1092 }
1093
1094 /*
1095  * Process SRP_CMD.
1096  */
1097 static int srpt_handle_cmd(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
1098 {
1099         struct scst_cmd *scmnd;
1100         struct srp_cmd *srp_cmd;
1101         struct srp_rsp *srp_rsp;
1102         scst_data_direction dir;
1103         int indirect_desc = 0;
1104         int ret;
1105         unsigned long flags;
1106
1107         srp_cmd = ioctx->buf;
1108         srp_rsp = ioctx->buf;
1109
1110         dir = SCST_DATA_NONE;
1111         if (srp_cmd->buf_fmt) {
1112                 ret = srpt_get_desc_tbl(ioctx, srp_cmd, &indirect_desc);
1113                 if (ret) {
1114                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1115                                            NO_ADD_SENSE, srp_cmd->tag);
1116                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1117                         goto err;
1118                 }
1119
1120                 if (indirect_desc) {
1121                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1122                                            NO_ADD_SENSE, srp_cmd->tag);
1123                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1124                         goto err;
1125                 }
1126
1127                 /*
1128                  * The lower four bits of the buffer format field contain the
1129                  * DATA-IN buffer descriptor format, and the highest four bits
1130                  * contain the DATA-OUT buffer descriptor format.
1131                  */
1132                 if (srp_cmd->buf_fmt & 0xf)
1133                         /* DATA-IN: transfer data from target to initiator. */
1134                         dir = SCST_DATA_READ;
1135                 else if (srp_cmd->buf_fmt >> 4)
1136                         /* DATA-OUT: transfer data from initiator to target. */
1137                         dir = SCST_DATA_WRITE;
1138         }
1139
1140         scmnd = scst_rx_cmd(ch->scst_sess, (u8 *) &srp_cmd->lun,
1141                             sizeof srp_cmd->lun, srp_cmd->cdb, 16,
1142                             thread ? SCST_NON_ATOMIC : SCST_ATOMIC);
1143         if (!scmnd) {
1144                 srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1145                                    NO_ADD_SENSE, srp_cmd->tag);
1146                 srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1147                 goto err;
1148         }
1149
1150         ioctx->scmnd = scmnd;
1151
1152         switch (srp_cmd->task_attr) {
1153         case SRP_CMD_HEAD_OF_Q:
1154                 scmnd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1155                 break;
1156         case SRP_CMD_ORDERED_Q:
1157                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1158                 break;
1159         case SRP_CMD_SIMPLE_Q:
1160                 scmnd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1161                 break;
1162         case SRP_CMD_ACA:
1163                 scmnd->queue_type = SCST_CMD_QUEUE_ACA;
1164                 break;
1165         default:
1166                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1167                 break;
1168         }
1169
1170         scst_cmd_set_tag(scmnd, srp_cmd->tag);
1171         scst_cmd_set_tgt_priv(scmnd, ioctx);
1172         scst_cmd_set_expected(scmnd, dir, ioctx->data_len);
1173
1174         spin_lock_irqsave(&ch->spinlock, flags);
1175         list_add_tail(&ioctx->scmnd_list, &ch->active_scmnd_list);
1176         ch->active_scmnd_cnt++;
1177         spin_unlock_irqrestore(&ch->spinlock, flags);
1178
1179         scst_cmd_init_done(scmnd, scst_estimate_context());
1180
1181         return 0;
1182
1183 err:
1184         WARN_ON(srp_rsp->opcode != SRP_RSP);
1185
1186         return -1;
1187 }
1188
1189 /*
1190  * Process an SRP_TSK_MGMT request.
1191  *
1192  * Returns 0 upon success and -1 upon failure.
1193  *
1194  * Each task management function is performed by calling one of the
1195  * scst_rx_mgmt_fn*() functions. These functions will either report failure
1196  * or process the task management function asynchronously. The function
1197  * srpt_tsk_mgmt_done() will be called by the SCST core upon completion of the
1198  * task management function. When srpt_handle_tsk_mgmt() reports failure
1199  * (i.e. returns -1) a response will have been built in ioctx->buf. This
1200  * information unit has to be sent back by the caller.
1201  *
1202  * For more information about SRP_TSK_MGMT information units, see also section
1203  * 6.7 in the T10 SRP r16a document.
1204  */
1205 static int srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
1206                                 struct srpt_ioctx *ioctx)
1207 {
1208         struct srp_tsk_mgmt *srp_tsk;
1209         struct srpt_mgmt_ioctx *mgmt_ioctx;
1210         int ret;
1211
1212         srp_tsk = ioctx->buf;
1213
1214         TRACE_DBG("recv_tsk_mgmt= %d for task_tag= %lld"
1215                   " using tag= %lld cm_id= %p sess= %p",
1216                   srp_tsk->tsk_mgmt_func,
1217                   (unsigned long long) srp_tsk->task_tag,
1218                   (unsigned long long) srp_tsk->tag,
1219                   ch->cm_id, ch->scst_sess);
1220
1221         mgmt_ioctx = kmalloc(sizeof *mgmt_ioctx, GFP_ATOMIC);
1222         if (!mgmt_ioctx) {
1223                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1224                                        srp_tsk->tag);
1225                 goto err;
1226         }
1227
1228         mgmt_ioctx->ioctx = ioctx;
1229         mgmt_ioctx->ch = ch;
1230         mgmt_ioctx->tag = srp_tsk->tag;
1231
1232         switch (srp_tsk->tsk_mgmt_func) {
1233         case SRP_TSK_ABORT_TASK:
1234                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK");
1235                 ret = scst_rx_mgmt_fn_tag(ch->scst_sess,
1236                                           SCST_ABORT_TASK,
1237                                           srp_tsk->task_tag,
1238                                           thread ?
1239                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1240                                           mgmt_ioctx);
1241                 break;
1242         case SRP_TSK_ABORT_TASK_SET:
1243                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK_SET");
1244                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1245                                           SCST_ABORT_TASK_SET,
1246                                           (u8 *) &srp_tsk->lun,
1247                                           sizeof srp_tsk->lun,
1248                                           thread ?
1249                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1250                                           mgmt_ioctx);
1251                 break;
1252         case SRP_TSK_CLEAR_TASK_SET:
1253                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_TASK_SET");
1254                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1255                                           SCST_CLEAR_TASK_SET,
1256                                           (u8 *) &srp_tsk->lun,
1257                                           sizeof srp_tsk->lun,
1258                                           thread ?
1259                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1260                                           mgmt_ioctx);
1261                 break;
1262         case SRP_TSK_LUN_RESET:
1263                 TRACE_DBG("%s", "Processing SRP_TSK_LUN_RESET");
1264                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1265                                           SCST_LUN_RESET,
1266                                           (u8 *) &srp_tsk->lun,
1267                                           sizeof srp_tsk->lun,
1268                                           thread ?
1269                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1270                                           mgmt_ioctx);
1271                 break;
1272         case SRP_TSK_CLEAR_ACA:
1273                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_ACA");
1274                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1275                                           SCST_CLEAR_ACA,
1276                                           (u8 *) &srp_tsk->lun,
1277                                           sizeof srp_tsk->lun,
1278                                           thread ?
1279                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1280                                           mgmt_ioctx);
1281                 break;
1282         default:
1283                 TRACE_DBG("%s", "Unsupported task management function.");
1284                 srpt_build_tskmgmt_rsp(ch, ioctx,
1285                                        SRP_TSK_MGMT_FUNC_NOT_SUPP,
1286                                        srp_tsk->tag);
1287                 goto err;
1288         }
1289
1290         if (ret) {
1291                 TRACE_DBG("%s", "Processing task management function failed.");
1292                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1293                                        srp_tsk->tag);
1294                 goto err;
1295         }
1296
1297         WARN_ON(srp_tsk->opcode == SRP_RSP);
1298
1299         return 0;
1300
1301 err:
1302         WARN_ON(srp_tsk->opcode != SRP_RSP);
1303
1304         kfree(mgmt_ioctx);
1305         return -1;
1306 }
1307
1308 /**
1309  * Process a receive completion event.
1310  * @ch: RDMA channel for which the completion event has been received.
1311  * @ioctx: SRPT I/O context for which the completion event has been received.
1312  */
1313 static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
1314                                struct srpt_ioctx *ioctx)
1315 {
1316         struct srp_cmd *srp_cmd;
1317         struct srp_rsp *srp_rsp;
1318         unsigned long flags;
1319         int len;
1320
1321         spin_lock_irqsave(&ch->spinlock, flags);
1322         if (ch->state != RDMA_CHANNEL_LIVE) {
1323                 if (ch->state == RDMA_CHANNEL_CONNECTING) {
1324                         list_add_tail(&ioctx->wait_list, &ch->cmd_wait_list);
1325                         spin_unlock_irqrestore(&ch->spinlock, flags);
1326                         return;
1327                 } else {
1328                         spin_unlock_irqrestore(&ch->spinlock, flags);
1329                         srpt_reset_ioctx(ch, ioctx);
1330                         return;
1331                 }
1332         }
1333         spin_unlock_irqrestore(&ch->spinlock, flags);
1334
1335         ib_dma_sync_single_for_cpu(ch->sport->sdev->device, ioctx->dma,
1336                                    MAX_MESSAGE_SIZE, DMA_FROM_DEVICE);
1337
1338         ioctx->data_len = 0;
1339         ioctx->n_rbuf = 0;
1340         ioctx->rbufs = NULL;
1341         ioctx->n_rdma = 0;
1342         ioctx->n_rdma_ius = 0;
1343         ioctx->rdma_ius = NULL;
1344         ioctx->scmnd = NULL;
1345         ioctx->ch = ch;
1346         ioctx->state = SRPT_STATE_NEW;
1347
1348         srp_cmd = ioctx->buf;
1349         srp_rsp = ioctx->buf;
1350
1351         switch (srp_cmd->opcode) {
1352         case SRP_CMD:
1353                 if (srpt_handle_cmd(ch, ioctx) < 0)
1354                         goto err;
1355                 break;
1356
1357         case SRP_TSK_MGMT:
1358                 if (srpt_handle_tsk_mgmt(ch, ioctx) < 0)
1359                         goto err;
1360                 break;
1361
1362         case SRP_I_LOGOUT:
1363         case SRP_AER_REQ:
1364         default:
1365                 srpt_build_cmd_rsp(ch, ioctx, ILLEGAL_REQUEST, INVALID_CDB,
1366                                    srp_cmd->tag);
1367                 goto err;
1368         }
1369
1370         ib_dma_sync_single_for_device(ch->sport->sdev->device,
1371                                    ioctx->dma, MAX_MESSAGE_SIZE,
1372                                    DMA_FROM_DEVICE);
1373
1374         return;
1375
1376 err:
1377         WARN_ON(srp_rsp->opcode != SRP_RSP);
1378         len = (sizeof *srp_rsp) + be32_to_cpu(srp_rsp->sense_data_len);
1379
1380         if (ch->state != RDMA_CHANNEL_LIVE) {
1381                 /* Give up if another thread modified the channel state. */
1382                 PRINT_ERROR("%s: channel is in state %d", __func__, ch->state);
1383                 srpt_reset_ioctx(ch, ioctx);
1384         } else if (srpt_post_send(ch, ioctx, len)) {
1385                 PRINT_ERROR("%s: sending SRP_RSP response failed", __func__);
1386                 srpt_reset_ioctx(ch, ioctx);
1387         }
1388 }
1389
1390 /*
1391  * Returns true if the ioctx list is non-empty or if the ib_srpt kernel thread
1392  * should stop.
1393  * @pre thread != 0
1394  */
1395 static inline int srpt_test_ioctx_list(void)
1396 {
1397         int res = (!list_empty(&srpt_thread.thread_ioctx_list) ||
1398                    unlikely(kthread_should_stop()));
1399         return res;
1400 }
1401
1402 /*
1403  * Add 'ioctx' to the tail of the ioctx list and wake up the kernel thread.
1404  *
1405  * @pre thread != 0
1406  */
1407 static inline void srpt_schedule_thread(struct srpt_ioctx *ioctx)
1408 {
1409         unsigned long flags;
1410
1411         spin_lock_irqsave(&srpt_thread.thread_lock, flags);
1412         list_add_tail(&ioctx->comp_list, &srpt_thread.thread_ioctx_list);
1413         spin_unlock_irqrestore(&srpt_thread.thread_lock, flags);
1414         wake_up(&ioctx_list_waitQ);
1415 }
1416
1417 /**
1418  * InfiniBand completion queue callback function.
1419  * @cq: completion queue.
1420  * @ctx: completion queue context, which was passed as the fourth argument of
1421  *       the function ib_create_cq().
1422  */
1423 static void srpt_completion(struct ib_cq *cq, void *ctx)
1424 {
1425         struct srpt_rdma_ch *ch = ctx;
1426         struct srpt_device *sdev = ch->sport->sdev;
1427         struct ib_wc wc;
1428         struct srpt_ioctx *ioctx;
1429
1430         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1431         while (ib_poll_cq(ch->cq, 1, &wc) > 0) {
1432                 if (wc.status) {
1433                         PRINT_ERROR("failed %s status= %d",
1434                                wc.wr_id & SRPT_OP_RECV ? "receive" : "send",
1435                                wc.status);
1436                         srpt_handle_err_comp(ch, &wc);
1437                         break;
1438                 }
1439
1440                 if (wc.wr_id & SRPT_OP_RECV) {
1441                         ioctx = sdev->ioctx_ring[wc.wr_id & ~SRPT_OP_RECV];
1442                         if (thread) {
1443                                 ioctx->ch = ch;
1444                                 ioctx->op = IB_WC_RECV;
1445                                 srpt_schedule_thread(ioctx);
1446                         } else
1447                                 srpt_handle_new_iu(ch, ioctx);
1448                         continue;
1449                 } else
1450                         ioctx = sdev->ioctx_ring[wc.wr_id];
1451
1452                 if (thread) {
1453                         ioctx->ch = ch;
1454                         ioctx->op = wc.opcode;
1455                         srpt_schedule_thread(ioctx);
1456                 } else {
1457                         switch (wc.opcode) {
1458                         case IB_WC_SEND:
1459                                 srpt_handle_send_comp(ch, ioctx,
1460                                         scst_estimate_context());
1461                                 break;
1462                         case IB_WC_RDMA_WRITE:
1463                         case IB_WC_RDMA_READ:
1464                                 srpt_handle_rdma_comp(ch, ioctx);
1465                                 break;
1466                         default:
1467                                 break;
1468                         }
1469                 }
1470
1471 #if defined(CONFIG_SCST_DEBUG)
1472                 if (interrupt_processing_delay_in_us <= MAX_UDELAY_MS * 1000)
1473                         udelay(interrupt_processing_delay_in_us);
1474 #endif
1475         }
1476 }
1477
1478 /*
1479  * Create a completion queue on the specified device.
1480  */
1481 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
1482 {
1483         struct ib_qp_init_attr *qp_init;
1484         struct srpt_device *sdev = ch->sport->sdev;
1485         int cqe;
1486         int ret;
1487
1488         qp_init = kzalloc(sizeof *qp_init, GFP_KERNEL);
1489         if (!qp_init)
1490                 return -ENOMEM;
1491
1492         /* Create a completion queue (CQ). */
1493
1494         cqe = SRPT_RQ_SIZE + SRPT_SQ_SIZE - 1;
1495 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(RHEL_RELEASE_CODE)
1496         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe);
1497 #else
1498         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe, 0);
1499 #endif
1500         if (IS_ERR(ch->cq)) {
1501                 ret = PTR_ERR(ch->cq);
1502                 PRINT_ERROR("failed to create_cq cqe= %d ret= %d", cqe, ret);
1503                 goto out;
1504         }
1505
1506         /* Request completion notification. */
1507
1508         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1509
1510         /* Create a queue pair (QP). */
1511
1512         qp_init->qp_context = (void *)ch;
1513         qp_init->event_handler = srpt_qp_event;
1514         qp_init->send_cq = ch->cq;
1515         qp_init->recv_cq = ch->cq;
1516         qp_init->srq = sdev->srq;
1517         qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
1518         qp_init->qp_type = IB_QPT_RC;
1519         qp_init->cap.max_send_wr = SRPT_SQ_SIZE;
1520         qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE;
1521
1522         ch->qp = ib_create_qp(sdev->pd, qp_init);
1523         if (IS_ERR(ch->qp)) {
1524                 ret = PTR_ERR(ch->qp);
1525                 ib_destroy_cq(ch->cq);
1526                 PRINT_ERROR("failed to create_qp ret= %d", ret);
1527                 goto out;
1528         }
1529
1530         TRACE_DBG("%s: max_cqe= %d max_sge= %d cm_id= %p",
1531                __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
1532                ch->cm_id);
1533
1534         /* Modify the attributes and the state of queue pair ch->qp. */
1535
1536         ret = srpt_init_ch_qp(ch, ch->qp);
1537         if (ret) {
1538                 ib_destroy_qp(ch->qp);
1539                 ib_destroy_cq(ch->cq);
1540                 goto out;
1541         }
1542
1543         atomic_set(&ch->req_lim_delta, SRPT_RQ_SIZE);
1544 out:
1545         kfree(qp_init);
1546         return ret;
1547 }
1548
1549 /**
1550  * Look up the RDMA channel that corresponds to the specified cm_id.
1551  *
1552  * Return NULL if no matching RDMA channel has been found.
1553  */
1554 static struct srpt_rdma_ch *srpt_find_channel(struct ib_cm_id *cm_id, bool del)
1555 {
1556         struct srpt_device *sdev = cm_id->context;
1557         struct srpt_rdma_ch *ch;
1558
1559         spin_lock_irq(&sdev->spinlock);
1560         list_for_each_entry(ch, &sdev->rch_list, list) {
1561                 if (ch->cm_id == cm_id) {
1562                         if (del)
1563                                 list_del(&ch->list);
1564                         spin_unlock_irq(&sdev->spinlock);
1565                         return ch;
1566                 }
1567         }
1568
1569         spin_unlock_irq(&sdev->spinlock);
1570
1571         return NULL;
1572 }
1573
1574 /**
1575  * Release all resources associated with the specified RDMA channel.
1576  *
1577  * Note: the caller must have removed the channel from the channel list
1578  * before calling this function.
1579  */
1580 static void srpt_release_channel(struct srpt_rdma_ch *ch, int destroy_cmid)
1581 {
1582         TRACE_ENTRY();
1583
1584         WARN_ON(srpt_find_channel(ch->cm_id, false) == ch);
1585
1586         if (ch->cm_id && destroy_cmid) {
1587                 TRACE_DBG("%s: destroy cm_id= %p", __func__, ch->cm_id);
1588                 ib_destroy_cm_id(ch->cm_id);
1589                 ch->cm_id = NULL;
1590         }
1591
1592         ib_destroy_qp(ch->qp);
1593         ib_destroy_cq(ch->cq);
1594
1595         if (ch->scst_sess) {
1596                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1597
1598                 if (ch->active_scmnd_cnt)
1599                         PRINT_INFO("Releasing session %s which still has %d"
1600                                    " active commands",
1601                                    ch->sess_name, ch->active_scmnd_cnt);
1602                 else
1603                         PRINT_INFO("Releasing session %s", ch->sess_name);
1604
1605                 spin_lock_irq(&ch->spinlock);
1606                 list_for_each_entry_safe(ioctx, ioctx_tmp,
1607                                          &ch->active_scmnd_list, scmnd_list) {
1608                         spin_unlock_irq(&ch->spinlock);
1609
1610                         if (ioctx->scmnd)
1611                                 srpt_abort_scst_cmd(ch->sport->sdev,
1612                                                     ioctx->scmnd, true);
1613
1614                         spin_lock_irq(&ch->spinlock);
1615                 }
1616                 WARN_ON(!list_empty(&ch->active_scmnd_list));
1617                 WARN_ON(ch->active_scmnd_cnt != 0);
1618                 spin_unlock_irq(&ch->spinlock);
1619
1620                 scst_unregister_session(ch->scst_sess, 0, NULL);
1621                 ch->scst_sess = NULL;
1622         }
1623
1624         kfree(ch);
1625
1626         TRACE_EXIT();
1627 }
1628
1629 static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
1630                             struct ib_cm_req_event_param *param,
1631                             void *private_data)
1632 {
1633         struct srpt_device *sdev = cm_id->context;
1634         struct srp_login_req *req;
1635         struct srp_login_rsp *rsp;
1636         struct srp_login_rej *rej;
1637         struct ib_cm_rep_param *rep_param;
1638         struct srpt_rdma_ch *ch, *tmp_ch;
1639         u32 it_iu_len;
1640         int ret = 0;
1641
1642 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
1643         WARN_ON(!sdev || !private_data);
1644         if (!sdev || !private_data)
1645                 return -EINVAL;
1646 #else
1647         if (WARN_ON(!sdev || !private_data))
1648                 return -EINVAL;
1649 #endif
1650
1651         req = (struct srp_login_req *)private_data;
1652
1653         it_iu_len = be32_to_cpu(req->req_it_iu_len);
1654
1655         PRINT_INFO("Received SRP_LOGIN_REQ with"
1656             " i_port_id 0x%llx:0x%llx, t_port_id 0x%llx:0x%llx and length %d"
1657             " on port %d (guid=0x%llx:0x%llx)",
1658             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[0]),
1659             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[8]),
1660             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[0]),
1661             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[8]),
1662             it_iu_len,
1663             param->port,
1664             (unsigned long long)be64_to_cpu(*(u64 *)
1665                                 &sdev->port[param->port - 1].gid.raw[0]),
1666             (unsigned long long)be64_to_cpu(*(u64 *)
1667                                 &sdev->port[param->port - 1].gid.raw[8]));
1668
1669         rsp = kzalloc(sizeof *rsp, GFP_KERNEL);
1670         rej = kzalloc(sizeof *rej, GFP_KERNEL);
1671         rep_param = kzalloc(sizeof *rep_param, GFP_KERNEL);
1672
1673         if (!rsp || !rej || !rep_param) {
1674                 ret = -ENOMEM;
1675                 goto out;
1676         }
1677
1678         if (it_iu_len > MAX_MESSAGE_SIZE || it_iu_len < 64) {
1679                 rej->reason =
1680                     cpu_to_be32(SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
1681                 ret = -EINVAL;
1682                 PRINT_ERROR("rejected SRP_LOGIN_REQ because its"
1683                             " length (%d bytes) is invalid", it_iu_len);
1684                 goto reject;
1685         }
1686
1687         if ((req->req_flags & 0x3) == SRP_MULTICHAN_SINGLE) {
1688                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
1689
1690                 spin_lock_irq(&sdev->spinlock);
1691
1692                 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
1693                         if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
1694                             && !memcmp(ch->t_port_id, req->target_port_id, 16)
1695                             && param->port == ch->sport->port
1696                             && param->listen_id == ch->sport->sdev->cm_id
1697                             && ch->cm_id) {
1698                                 enum rdma_ch_state prev_state;
1699
1700                                 /* found an existing channel */
1701                                 TRACE_DBG("Found existing channel name= %s"
1702                                           " cm_id= %p state= %d",
1703                                           ch->sess_name, ch->cm_id, ch->state);
1704
1705                                 prev_state = ch->state;
1706                                 if (ch->state == RDMA_CHANNEL_LIVE)
1707                                         ch->state = RDMA_CHANNEL_DISCONNECTING;
1708                                 else if (ch->state == RDMA_CHANNEL_CONNECTING)
1709                                         list_del(&ch->list);
1710
1711                                 spin_unlock_irq(&sdev->spinlock);
1712
1713                                 rsp->rsp_flags =
1714                                         SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
1715
1716                                 if (prev_state == RDMA_CHANNEL_LIVE) {
1717                                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
1718                                         PRINT_INFO("disconnected"
1719                                           " session %s because a new"
1720                                           " SRP_LOGIN_REQ has been received.",
1721                                           ch->sess_name);
1722                                 } else if (prev_state ==
1723                                          RDMA_CHANNEL_CONNECTING) {
1724                                         PRINT_ERROR("%s", "rejected"
1725                                           " SRP_LOGIN_REQ because another login"
1726                                           " request is being processed.");
1727                                         ib_send_cm_rej(ch->cm_id,
1728                                                        IB_CM_REJ_NO_RESOURCES,
1729                                                        NULL, 0, NULL, 0);
1730                                         srpt_release_channel(ch, 1);
1731                                 }
1732
1733                                 spin_lock_irq(&sdev->spinlock);
1734                         }
1735                 }
1736
1737                 spin_unlock_irq(&sdev->spinlock);
1738
1739         } else
1740                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
1741
1742         if (((u64) (*(u64 *) req->target_port_id) !=
1743              cpu_to_be64(srpt_service_guid)) ||
1744             ((u64) (*(u64 *) (req->target_port_id + 8)) !=
1745              cpu_to_be64(srpt_service_guid))) {
1746                 rej->reason =
1747                     cpu_to_be32(SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
1748                 ret = -ENOMEM;
1749                 PRINT_ERROR("%s", "rejected SRP_LOGIN_REQ because it"
1750                        " has an invalid target port identifier.");
1751                 goto reject;
1752         }
1753
1754         ch = kzalloc(sizeof *ch, GFP_KERNEL);
1755         if (!ch) {
1756                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1757                 PRINT_ERROR("%s",
1758                             "rejected SRP_LOGIN_REQ because out of memory.");
1759                 ret = -ENOMEM;
1760                 goto reject;
1761         }
1762
1763         spin_lock_init(&ch->spinlock);
1764         memcpy(ch->i_port_id, req->initiator_port_id, 16);
1765         memcpy(ch->t_port_id, req->target_port_id, 16);
1766         ch->sport = &sdev->port[param->port - 1];
1767         ch->cm_id = cm_id;
1768         ch->state = RDMA_CHANNEL_CONNECTING;
1769         INIT_LIST_HEAD(&ch->cmd_wait_list);
1770         INIT_LIST_HEAD(&ch->active_scmnd_list);
1771
1772         ret = srpt_create_ch_ib(ch);
1773         if (ret) {
1774                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1775                 PRINT_ERROR("%s", "rejected SRP_LOGIN_REQ because creating"
1776                             " a new RDMA channel failed.");
1777                 goto free_ch;
1778         }
1779
1780         ret = srpt_ch_qp_rtr(ch, ch->qp);
1781         if (ret) {
1782                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1783                 PRINT_ERROR("rejected SRP_LOGIN_REQ because enabling"
1784                        " RTR failed (error code = %d)", ret);
1785                 goto destroy_ib;
1786         }
1787
1788         if (use_port_guid_in_session_name) {
1789                 /*
1790                  * If the kernel module parameter use_port_guid_in_session_name
1791                  * has been specified, use a combination of the target port
1792                  * GUID and the initiator port ID as the session name. This
1793                  * was the original behavior of the SRP target implementation
1794                  * (i.e. before the SRPT was included in OFED 1.3).
1795                  */
1796                 snprintf(ch->sess_name, sizeof(ch->sess_name),
1797                          "0x%016llx%016llx",
1798                          (unsigned long long)be64_to_cpu(*(u64 *)
1799                                 &sdev->port[param->port - 1].gid.raw[8]),
1800                          (unsigned long long)be64_to_cpu(*(u64 *)
1801                                 (ch->i_port_id + 8)));
1802         } else {
1803                 /*
1804                  * Default behavior: use the initator port identifier as the
1805                  * session name.
1806                  */
1807                 snprintf(ch->sess_name, sizeof(ch->sess_name),
1808                          "0x%016llx%016llx",
1809                          (unsigned long long)be64_to_cpu(*(u64 *)ch->i_port_id),
1810                          (unsigned long long)be64_to_cpu(*(u64 *)
1811                                  (ch->i_port_id + 8)));
1812         }
1813
1814         TRACE_DBG("registering session %s", ch->sess_name);
1815
1816         BUG_ON(!sdev->scst_tgt);
1817         ch->scst_sess = scst_register_session(sdev->scst_tgt, 0, ch->sess_name,
1818                                               NULL, NULL);
1819         if (!ch->scst_sess) {
1820                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1821                 TRACE_DBG("%s", "Failed to create scst sess");
1822                 goto destroy_ib;
1823         }
1824
1825         TRACE_DBG("Establish connection sess=%p name=%s cm_id=%p",
1826                   ch->scst_sess, ch->sess_name, ch->cm_id);
1827
1828         scst_sess_set_tgt_priv(ch->scst_sess, ch);
1829
1830         /* create srp_login_response */
1831         rsp->opcode = SRP_LOGIN_RSP;
1832         rsp->tag = req->tag;
1833         rsp->max_it_iu_len = req->req_it_iu_len;
1834         rsp->max_ti_iu_len = req->req_it_iu_len;
1835         rsp->buf_fmt =
1836             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1837         rsp->req_lim_delta = cpu_to_be32(SRPT_RQ_SIZE);
1838         atomic_set(&ch->req_lim_delta, 0);
1839
1840         /* create cm reply */
1841         rep_param->qp_num = ch->qp->qp_num;
1842         rep_param->private_data = (void *)rsp;
1843         rep_param->private_data_len = sizeof *rsp;
1844         rep_param->rnr_retry_count = 7;
1845         rep_param->flow_control = 1;
1846         rep_param->failover_accepted = 0;
1847         rep_param->srq = 1;
1848         rep_param->responder_resources = 4;
1849         rep_param->initiator_depth = 4;
1850
1851         ret = ib_send_cm_rep(cm_id, rep_param);
1852         if (ret) {
1853                 PRINT_ERROR("sending SRP_LOGIN_REQ response failed"
1854                             " (error code = %d)", ret);
1855                 goto release_channel;
1856         }
1857
1858         spin_lock_irq(&sdev->spinlock);
1859         list_add_tail(&ch->list, &sdev->rch_list);
1860         spin_unlock_irq(&sdev->spinlock);
1861
1862         goto out;
1863
1864 release_channel:
1865         scst_unregister_session(ch->scst_sess, 0, NULL);
1866         ch->scst_sess = NULL;
1867
1868 destroy_ib:
1869         ib_destroy_qp(ch->qp);
1870         ib_destroy_cq(ch->cq);
1871
1872 free_ch:
1873         kfree(ch);
1874
1875 reject:
1876         rej->opcode = SRP_LOGIN_REJ;
1877         rej->tag = req->tag;
1878         rej->buf_fmt =
1879             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1880
1881         ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
1882                              (void *)rej, sizeof *rej);
1883
1884 out:
1885         kfree(rep_param);
1886         kfree(rsp);
1887         kfree(rej);
1888
1889         return ret;
1890 }
1891
1892 /**
1893  * Release the channel with the specified cm_id.
1894  *
1895  * Returns one to indicate that the caller of srpt_cm_handler() should destroy
1896  * the cm_id.
1897  */
1898 static void srpt_find_and_release_channel(struct ib_cm_id *cm_id)
1899 {
1900         struct srpt_rdma_ch *ch;
1901
1902         ch = srpt_find_channel(cm_id, true);
1903         if (ch)
1904                 srpt_release_channel(ch, 0);
1905 }
1906
1907 static void srpt_cm_rej_recv(struct ib_cm_id *cm_id)
1908 {
1909         PRINT_INFO("%s", "Received InfiniBand REJ packet.");
1910         srpt_find_and_release_channel(cm_id);
1911 }
1912
1913 /**
1914  * Process an IB_CM_RTU_RECEIVED or IB_CM_USER_ESTABLISHED event.
1915  *
1916  * An IB_CM_RTU_RECEIVED message indicates that the connection is established
1917  * and that the recipient may begin transmitting (RTU = ready to use).
1918  */
1919 static int srpt_cm_rtu_recv(struct ib_cm_id *cm_id)
1920 {
1921         struct srpt_rdma_ch *ch;
1922         int ret;
1923
1924         ch = srpt_find_channel(cm_id, false);
1925         if (!ch)
1926                 return -EINVAL;
1927
1928         if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_CONNECTING,
1929                                             RDMA_CHANNEL_LIVE)) {
1930                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1931
1932                 ret = srpt_ch_qp_rts(ch, ch->qp);
1933
1934                 list_for_each_entry_safe(ioctx, ioctx_tmp, &ch->cmd_wait_list,
1935                                          wait_list) {
1936                         list_del(&ioctx->wait_list);
1937                         srpt_handle_new_iu(ch, ioctx);
1938                 }
1939                 if (ret && srpt_test_and_set_channel_state(ch,
1940                                         RDMA_CHANNEL_LIVE,
1941                                         RDMA_CHANNEL_DISCONNECTING)) {
1942                         TRACE_DBG("cm_id=%p sess_name=%s state=%d",
1943                                   cm_id, ch->sess_name, ch->state);
1944                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
1945                 }
1946         } else if (ch->state == RDMA_CHANNEL_DISCONNECTING) {
1947                 TRACE_DBG("cm_id=%p sess_name=%s state=%d",
1948                           cm_id, ch->sess_name, ch->state);
1949                 ib_send_cm_dreq(ch->cm_id, NULL, 0);
1950                 ret = -EAGAIN;
1951         } else
1952                 ret = 0;
1953
1954         return ret;
1955 }
1956
1957 static void srpt_cm_timewait_exit(struct ib_cm_id *cm_id)
1958 {
1959         PRINT_INFO("%s", "Received InfiniBand TimeWait exit.");
1960         srpt_find_and_release_channel(cm_id);
1961 }
1962
1963 static void srpt_cm_rep_error(struct ib_cm_id *cm_id)
1964 {
1965         PRINT_INFO("%s", "Received InfiniBand REP error.");
1966         srpt_find_and_release_channel(cm_id);
1967 }
1968
1969 static int srpt_cm_dreq_recv(struct ib_cm_id *cm_id)
1970 {
1971         struct srpt_rdma_ch *ch;
1972
1973         ch = srpt_find_channel(cm_id, false);
1974         if (!ch)
1975                 return -EINVAL;
1976
1977         TRACE_DBG("%s: cm_id= %p ch->state= %d",
1978                  __func__, cm_id, ch->state);
1979
1980         switch (ch->state) {
1981         case RDMA_CHANNEL_LIVE:
1982         case RDMA_CHANNEL_CONNECTING:
1983                 ib_send_cm_drep(ch->cm_id, NULL, 0);
1984                 PRINT_INFO("Received DREQ and sent DREP for session %s.",
1985                            ch->sess_name);
1986                 break;
1987         case RDMA_CHANNEL_DISCONNECTING:
1988         default:
1989                 break;
1990         }
1991
1992         return 0;
1993 }
1994
1995 static void srpt_cm_drep_recv(struct ib_cm_id *cm_id)
1996 {
1997         PRINT_INFO("%s", "Received InfiniBand DREP message.");
1998         srpt_find_and_release_channel(cm_id);
1999 }
2000
2001 /**
2002  * IB connection manager callback function.
2003  *
2004  * A non-zero return value will make the caller destroy the CM ID.
2005  *
2006  * Note: srpt_add_one passes a struct srpt_device* as the third argument to
2007  * the ib_create_cm_id() call.
2008  */
2009 static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2010 {
2011         int ret = 0;
2012
2013         switch (event->event) {
2014         case IB_CM_REQ_RECEIVED:
2015                 ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd,
2016                                        event->private_data);
2017                 break;
2018         case IB_CM_REJ_RECEIVED:
2019                 srpt_cm_rej_recv(cm_id);
2020                 ret = -EINVAL;
2021                 break;
2022         case IB_CM_RTU_RECEIVED:
2023         case IB_CM_USER_ESTABLISHED:
2024                 ret = srpt_cm_rtu_recv(cm_id);
2025                 break;
2026         case IB_CM_DREQ_RECEIVED:
2027                 ret = srpt_cm_dreq_recv(cm_id);
2028                 break;
2029         case IB_CM_DREP_RECEIVED:
2030                 srpt_cm_drep_recv(cm_id);
2031                 ret = -EINVAL;
2032                 break;
2033         case IB_CM_TIMEWAIT_EXIT:
2034                 srpt_cm_timewait_exit(cm_id);
2035                 ret = -EINVAL;
2036                 break;
2037         case IB_CM_REP_ERROR:
2038                 srpt_cm_rep_error(cm_id);
2039                 ret = -EINVAL;
2040                 break;
2041         default:
2042                 break;
2043         }
2044
2045         return ret;
2046 }
2047
2048 static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
2049                                  struct srpt_ioctx *ioctx,
2050                                  struct scst_cmd *scmnd)
2051 {
2052         struct scatterlist *scat;
2053         scst_data_direction dir;
2054         struct rdma_iu *riu;
2055         struct srp_direct_buf *db;
2056         dma_addr_t dma_addr;
2057         struct ib_sge *sge;
2058         u64 raddr;
2059         u32 rsize;
2060         u32 tsize;
2061         u32 dma_len;
2062         int count, nrdma;
2063         int i, j, k;
2064
2065         scat = scst_cmd_get_sg(scmnd);
2066         dir = scst_cmd_get_data_direction(scmnd);
2067         count = ib_dma_map_sg(ch->sport->sdev->device, scat,
2068                               scst_cmd_get_sg_cnt(scmnd),
2069                               scst_to_tgt_dma_dir(dir));
2070         if (unlikely(!count))
2071                 return -EBUSY;
2072
2073         if (ioctx->rdma_ius && ioctx->n_rdma_ius)
2074                 nrdma = ioctx->n_rdma_ius;
2075         else {
2076                 nrdma = count / SRPT_DEF_SG_PER_WQE + ioctx->n_rbuf;
2077
2078                 ioctx->rdma_ius = kzalloc(nrdma * sizeof *riu,
2079                                           scst_cmd_atomic(scmnd)
2080                                           ? GFP_ATOMIC : GFP_KERNEL);
2081                 if (!ioctx->rdma_ius) {
2082                         ib_dma_unmap_sg(ch->sport->sdev->device,
2083                                         scat, scst_cmd_get_sg_cnt(scmnd),
2084                                         scst_to_tgt_dma_dir(dir));
2085                         return -ENOMEM;
2086                 }
2087
2088                 ioctx->n_rdma_ius = nrdma;
2089         }
2090
2091         db = ioctx->rbufs;
2092         tsize = (dir == SCST_DATA_READ) ?
2093                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2094         dma_len = sg_dma_len(&scat[0]);
2095         riu = ioctx->rdma_ius;
2096
2097         /*
2098          * For each remote desc - calculate the #ib_sge.
2099          * If #ib_sge < SRPT_DEF_SG_PER_WQE per rdma operation then
2100          *      each remote desc rdma_iu is required a rdma wr;
2101          * else
2102          *      we need to allocate extra rdma_iu to carry extra #ib_sge in
2103          *      another rdma wr
2104          */
2105         for (i = 0, j = 0;
2106              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2107                 rsize = be32_to_cpu(db->len);
2108                 raddr = be64_to_cpu(db->va);
2109                 riu->raddr = raddr;
2110                 riu->rkey = be32_to_cpu(db->key);
2111                 riu->sge_cnt = 0;
2112
2113                 /* calculate how many sge required for this remote_buf */
2114                 while (rsize > 0 && tsize > 0) {
2115
2116                         if (rsize >= dma_len) {
2117                                 tsize -= dma_len;
2118                                 rsize -= dma_len;
2119                                 raddr += dma_len;
2120
2121                                 if (tsize > 0) {
2122                                         ++j;
2123                                         if (j < count)
2124                                                 dma_len = sg_dma_len(&scat[j]);
2125                                 }
2126                         } else {
2127                                 tsize -= rsize;
2128                                 dma_len -= rsize;
2129                                 rsize = 0;
2130                         }
2131
2132                         ++riu->sge_cnt;
2133
2134                         if (rsize > 0 && riu->sge_cnt == SRPT_DEF_SG_PER_WQE) {
2135                                 riu->sge =
2136                                     kmalloc(riu->sge_cnt * sizeof *riu->sge,
2137                                             scst_cmd_atomic(scmnd)
2138                                             ? GFP_ATOMIC : GFP_KERNEL);
2139                                 if (!riu->sge)
2140                                         goto free_mem;
2141
2142                                 ++ioctx->n_rdma;
2143                                 ++riu;
2144                                 riu->sge_cnt = 0;
2145                                 riu->raddr = raddr;
2146                                 riu->rkey = be32_to_cpu(db->key);
2147                         }
2148                 }
2149
2150                 riu->sge = kmalloc(riu->sge_cnt * sizeof *riu->sge,
2151                                    scst_cmd_atomic(scmnd)
2152                                    ? GFP_ATOMIC : GFP_KERNEL);
2153
2154                 if (!riu->sge)
2155                         goto free_mem;
2156
2157                 ++ioctx->n_rdma;
2158         }
2159
2160         db = ioctx->rbufs;
2161         scat = scst_cmd_get_sg(scmnd);
2162         tsize = (dir == SCST_DATA_READ) ?
2163                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2164         riu = ioctx->rdma_ius;
2165         dma_len = sg_dma_len(&scat[0]);
2166         dma_addr = sg_dma_address(&scat[0]);
2167
2168         /* this second loop is really mapped sg_addres to rdma_iu->ib_sge */
2169         for (i = 0, j = 0;
2170              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2171                 rsize = be32_to_cpu(db->len);
2172                 sge = riu->sge;
2173                 k = 0;
2174
2175                 while (rsize > 0 && tsize > 0) {
2176                         sge->addr = dma_addr;
2177                         sge->lkey = ch->sport->sdev->mr->lkey;
2178
2179                         if (rsize >= dma_len) {
2180                                 sge->length =
2181                                         (tsize < dma_len) ? tsize : dma_len;
2182                                 tsize -= dma_len;
2183                                 rsize -= dma_len;
2184
2185                                 if (tsize > 0) {
2186                                         ++j;
2187                                         if (j < count) {
2188                                                 dma_len = sg_dma_len(&scat[j]);
2189                                                 dma_addr =
2190                                                     sg_dma_address(&scat[j]);
2191                                         }
2192                                 }
2193                         } else {
2194                                 sge->length = (tsize < rsize) ? tsize : rsize;
2195                                 tsize -= rsize;
2196                                 dma_len -= rsize;
2197                                 dma_addr += rsize;
2198                                 rsize = 0;
2199                         }
2200
2201                         ++k;
2202                         if (k == riu->sge_cnt && rsize > 0) {
2203                                 ++riu;
2204                                 sge = riu->sge;
2205                                 k = 0;
2206                         } else if (rsize > 0)
2207                                 ++sge;
2208                 }
2209         }
2210
2211         return 0;
2212
2213 free_mem:
2214         while (ioctx->n_rdma)
2215                 kfree(ioctx->rdma_ius[ioctx->n_rdma--].sge);
2216
2217         kfree(ioctx->rdma_ius);
2218
2219         ib_dma_unmap_sg(ch->sport->sdev->device,
2220                         scat, scst_cmd_get_sg_cnt(scmnd),
2221                         scst_to_tgt_dma_dir(dir));
2222
2223         return -ENOMEM;
2224 }
2225
2226 static int srpt_perform_rdmas(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2227                               scst_data_direction dir)
2228 {
2229         struct ib_send_wr wr;
2230         struct ib_send_wr *bad_wr;
2231         struct rdma_iu *riu;
2232         int i;
2233         int ret = 0;
2234
2235         riu = ioctx->rdma_ius;
2236         memset(&wr, 0, sizeof wr);
2237
2238         for (i = 0; i < ioctx->n_rdma; ++i, ++riu) {
2239                 wr.opcode = (dir == SCST_DATA_READ) ?
2240                     IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;
2241                 wr.next = NULL;
2242                 wr.wr_id = ioctx->index;
2243                 wr.wr.rdma.remote_addr = riu->raddr;
2244                 wr.wr.rdma.rkey = riu->rkey;
2245                 wr.num_sge = riu->sge_cnt;
2246                 wr.sg_list = riu->sge;
2247
2248                 /* only get completion event for the last rdma wr */
2249                 if (i == (ioctx->n_rdma - 1) && dir == SCST_DATA_WRITE)
2250                         wr.send_flags = IB_SEND_SIGNALED;
2251
2252                 ret = ib_post_send(ch->qp, &wr, &bad_wr);
2253                 if (ret)
2254                         break;
2255         }
2256
2257         return ret;
2258 }
2259
2260 /*
2261  * Start data transfer between initiator and target. Must not block.
2262  */
2263 static int srpt_xfer_data(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2264                           struct scst_cmd *scmnd)
2265 {
2266         int ret;
2267
2268         ret = srpt_map_sg_to_ib_sge(ch, ioctx, scmnd);
2269         if (ret) {
2270                 PRINT_ERROR("%s[%d] ret=%d", __func__, __LINE__, ret);
2271                 ret = SCST_TGT_RES_QUEUE_FULL;
2272                 goto out;
2273         }
2274
2275         ret = srpt_perform_rdmas(ch, ioctx, scst_cmd_get_data_direction(scmnd));
2276         if (ret) {
2277                 PRINT_ERROR("%s[%d] ret=%d", __func__, __LINE__, ret);
2278                 if (ret == -EAGAIN || ret == -ENOMEM)
2279                         ret = SCST_TGT_RES_QUEUE_FULL;
2280                 else
2281                         ret = SCST_TGT_RES_FATAL_ERROR;
2282                 goto out;
2283         }
2284
2285         ret = SCST_TGT_RES_SUCCESS;
2286
2287 out:
2288         return ret;
2289 }
2290
2291 /*
2292  * Called by the SCST core to inform ib_srpt that data reception from the
2293  * initiator should start (SCST_DATA_WRITE). Must not block.
2294  */
2295 static int srpt_rdy_to_xfer(struct scst_cmd *scmnd)
2296 {
2297         struct srpt_rdma_ch *ch;
2298         struct srpt_ioctx *ioctx;
2299
2300         ioctx = scst_cmd_get_tgt_priv(scmnd);
2301         BUG_ON(!ioctx);
2302
2303         if (ioctx->state == SRPT_STATE_ABORTED)
2304                 return SCST_TGT_RES_FATAL_ERROR;
2305
2306         ch = ioctx->ch;
2307         WARN_ON(ch != scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd)));
2308         BUG_ON(!ch);
2309
2310         if (ch->state == RDMA_CHANNEL_DISCONNECTING)
2311                 return SCST_TGT_RES_FATAL_ERROR;
2312         else if (ch->state == RDMA_CHANNEL_CONNECTING)
2313                 return SCST_TGT_RES_QUEUE_FULL;
2314
2315         ioctx->state = SRPT_STATE_NEED_DATA;
2316
2317         return srpt_xfer_data(ch, ioctx, scmnd);
2318 }
2319
2320 /*
2321  * Called by the SCST core. Transmits the response buffer and status held in
2322  * 'scmnd'. Must not block.
2323  */
2324 static int srpt_xmit_response(struct scst_cmd *scmnd)
2325 {
2326         struct srpt_rdma_ch *ch;
2327         struct srpt_ioctx *ioctx;
2328         struct srp_rsp *srp_rsp;
2329         u64 tag;
2330         int ret = SCST_TGT_RES_SUCCESS;
2331         int dir;
2332         int status;
2333
2334         ioctx = scst_cmd_get_tgt_priv(scmnd);
2335         BUG_ON(!ioctx);
2336
2337         if (ioctx->state == SRPT_STATE_ABORTED) {
2338                 ret = SCST_TGT_RES_FATAL_ERROR;
2339                 goto out;
2340         }
2341
2342         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2343         BUG_ON(!ch);
2344
2345         tag = scst_cmd_get_tag(scmnd);
2346
2347         ioctx->state = SRPT_STATE_PROCESSED;
2348
2349         if (ch->state != RDMA_CHANNEL_LIVE) {
2350                 PRINT_ERROR("%s: tag= %lld channel in bad state %d",
2351                        __func__, (unsigned long long)tag, ch->state);
2352
2353                 if (ch->state == RDMA_CHANNEL_DISCONNECTING)
2354                         ret = SCST_TGT_RES_FATAL_ERROR;
2355                 else if (ch->state == RDMA_CHANNEL_CONNECTING)
2356                         ret = SCST_TGT_RES_QUEUE_FULL;
2357
2358                 if (unlikely(scst_cmd_aborted(scmnd)))
2359                         goto out_aborted;
2360
2361                 goto out;
2362         }
2363
2364         ib_dma_sync_single_for_cpu(ch->sport->sdev->device, ioctx->dma,
2365                                    MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
2366
2367         srp_rsp = ioctx->buf;
2368
2369         if (unlikely(scst_cmd_aborted(scmnd))) {
2370                 TRACE_MGMT_DBG("%s: tag= %lld already got aborted",
2371                                __func__, (unsigned long long)tag);
2372                 goto out_aborted;
2373         }
2374
2375         dir = scst_cmd_get_data_direction(scmnd);
2376         status = scst_cmd_get_status(scmnd) & 0xff;
2377
2378         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE, NO_ADD_SENSE, tag);
2379
2380         if (SCST_SENSE_VALID(scst_cmd_get_sense_buffer(scmnd))) {
2381                 srp_rsp->sense_data_len = scst_cmd_get_sense_buffer_len(scmnd);
2382                 if (srp_rsp->sense_data_len >
2383                     (MAX_MESSAGE_SIZE - sizeof *srp_rsp))
2384                         srp_rsp->sense_data_len =
2385                             MAX_MESSAGE_SIZE - sizeof *srp_rsp;
2386
2387                 memcpy((u8 *) (srp_rsp + 1), scst_cmd_get_sense_buffer(scmnd),
2388                        srp_rsp->sense_data_len);
2389
2390                 srp_rsp->sense_data_len = cpu_to_be32(srp_rsp->sense_data_len);
2391                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
2392
2393                 if (!status)
2394                         status = SAM_STAT_CHECK_CONDITION;
2395         }
2396
2397         srp_rsp->status = status;
2398
2399         /* For read commands, transfer the data to the initiator. */
2400         if (dir == SCST_DATA_READ && scst_cmd_get_resp_data_len(scmnd)) {
2401                 ret = srpt_xfer_data(ch, ioctx, scmnd);
2402                 if (ret != SCST_TGT_RES_SUCCESS) {
2403                         PRINT_ERROR("%s: tag= %lld xfer_data failed",
2404                                     __func__, (unsigned long long)tag);
2405                         goto out;
2406                 }
2407         }
2408
2409         if (srpt_post_send(ch, ioctx,
2410                            sizeof *srp_rsp +
2411                            be32_to_cpu(srp_rsp->sense_data_len))) {
2412                 PRINT_ERROR("%s: ch->state= %d tag= %lld",
2413                             __func__, ch->state,
2414                             (unsigned long long)tag);
2415                 ret = SCST_TGT_RES_FATAL_ERROR;
2416         }
2417
2418 out:
2419         return ret;
2420
2421 out_aborted:
2422         ret = SCST_TGT_RES_SUCCESS;
2423         scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_ABORTED);
2424         ioctx->state = SRPT_STATE_ABORTED;
2425         WARN_ON(scmnd->state != SCST_CMD_STATE_XMIT_WAIT);
2426         scst_tgt_cmd_done(scmnd, SCST_CONTEXT_SAME);
2427         goto out;
2428 }
2429
2430 /*
2431  * Called by the SCST core to inform ib_srpt that a received task management
2432  * function has been completed. Must not block.
2433  */
2434 static void srpt_tsk_mgmt_done(struct scst_mgmt_cmd *mcmnd)
2435 {
2436         struct srpt_rdma_ch *ch;
2437         struct srpt_mgmt_ioctx *mgmt_ioctx;
2438         struct srpt_ioctx *ioctx;
2439         int rsp_len;
2440
2441         mgmt_ioctx = scst_mgmt_cmd_get_tgt_priv(mcmnd);
2442         BUG_ON(!mgmt_ioctx);
2443
2444         ch = mgmt_ioctx->ch;
2445         BUG_ON(!ch);
2446
2447         ioctx = mgmt_ioctx->ioctx;
2448         BUG_ON(!ioctx);
2449
2450         TRACE_DBG("%s: tsk_mgmt_done for tag= %lld status=%d",
2451                   __func__, (unsigned long long)mgmt_ioctx->tag,
2452                   scst_mgmt_cmd_get_status(mcmnd));
2453
2454         ioctx->state = SRPT_STATE_PROCESSED;
2455
2456         rsp_len = srpt_build_tskmgmt_rsp(ch, ioctx,
2457                                          (scst_mgmt_cmd_get_status(mcmnd) ==
2458                                           SCST_MGMT_STATUS_SUCCESS) ?
2459                                          SRP_TSK_MGMT_SUCCESS :
2460                                          SRP_TSK_MGMT_FAILED,
2461                                          mgmt_ioctx->tag);
2462         srpt_post_send(ch, ioctx, rsp_len);
2463
2464         scst_mgmt_cmd_set_tgt_priv(mcmnd, NULL);
2465
2466         kfree(mgmt_ioctx);
2467 }
2468
2469 /*
2470  * Called by the SCST core to inform ib_srpt that the command 'scmnd' is about
2471  * to be freed. May be called in IRQ context.
2472  */
2473 static void srpt_on_free_cmd(struct scst_cmd *scmnd)
2474 {
2475         struct srpt_rdma_ch *ch;
2476         struct srpt_ioctx *ioctx;
2477
2478         ioctx = scst_cmd_get_tgt_priv(scmnd);
2479         BUG_ON(!ioctx);
2480
2481         ch = ioctx->ch;
2482         if (ch) {
2483                 spin_lock_irq(&ch->spinlock);
2484                 list_del(&ioctx->scmnd_list);
2485                 ch->active_scmnd_cnt--;
2486                 spin_unlock_irq(&ch->spinlock);
2487                 ioctx->ch = NULL;
2488         }
2489
2490         srpt_reset_ioctx(ch, ioctx);
2491         scst_cmd_set_tgt_priv(scmnd, NULL);
2492 }
2493
2494 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2495 /* A vanilla 2.6.19 or older kernel without backported OFED kernel headers. */
2496 static void srpt_refresh_port_work(void *ctx)
2497 #else
2498 static void srpt_refresh_port_work(struct work_struct *work)
2499 #endif
2500 {
2501 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2502         struct srpt_port *sport = (struct srpt_port *)ctx;
2503 #else
2504         struct srpt_port *sport = container_of(work, struct srpt_port, work);
2505 #endif
2506
2507         srpt_refresh_port(sport);
2508 }
2509
2510 /*
2511  * Called by the SCST core to detect target adapters. Returns the number of
2512  * detected target adapters.
2513  */
2514 static int srpt_detect(struct scst_tgt_template *tp)
2515 {
2516         int device_count;
2517
2518         TRACE_ENTRY();
2519
2520         device_count = atomic_read(&srpt_device_count);
2521
2522         TRACE_EXIT_RES(device_count);
2523
2524         return device_count;
2525 }
2526
2527 /*
2528  * Callback function called by the SCST core from scst_unregister() to free up
2529  * the resources associated with device scst_tgt.
2530  */
2531 static int srpt_release(struct scst_tgt *scst_tgt)
2532 {
2533         struct srpt_device *sdev = scst_tgt_get_tgt_priv(scst_tgt);
2534         struct srpt_rdma_ch *ch, *tmp_ch;
2535
2536         TRACE_ENTRY();
2537
2538         BUG_ON(!scst_tgt);
2539 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
2540         WARN_ON(!sdev);
2541         if (!sdev)
2542                 return -ENODEV;
2543 #else
2544         if (WARN_ON(!sdev))
2545                 return -ENODEV;
2546 #endif
2547
2548         srpt_unregister_procfs_entry(scst_tgt->tgtt);
2549
2550         spin_lock_irq(&sdev->spinlock);
2551         list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
2552                 list_del(&ch->list);
2553                 spin_unlock_irq(&sdev->spinlock);
2554                 srpt_release_channel(ch, 1);
2555                 spin_lock_irq(&sdev->spinlock);
2556         }
2557         spin_unlock_irq(&sdev->spinlock);
2558
2559         srpt_unregister_mad_agent(sdev);
2560
2561         scst_tgt_set_tgt_priv(scst_tgt, NULL);
2562
2563         TRACE_EXIT();
2564
2565         return 0;
2566 }
2567
2568 /*
2569  * Entry point for ib_srpt's kernel thread. This kernel thread is only created
2570  * when the module parameter 'thread' is not zero (the default is zero).
2571  * This thread processes the ioctx list srpt_thread.thread_ioctx_list.
2572  *
2573  * @pre thread != 0
2574  */
2575 static int srpt_ioctx_thread(void *arg)
2576 {
2577         struct srpt_ioctx *ioctx;
2578
2579         /* Hibernation / freezing of the SRPT kernel thread is not supported. */
2580         current->flags |= PF_NOFREEZE;
2581
2582         spin_lock_irq(&srpt_thread.thread_lock);
2583         while (!kthread_should_stop()) {
2584                 wait_queue_t wait;
2585                 init_waitqueue_entry(&wait, current);
2586
2587                 if (!srpt_test_ioctx_list()) {
2588                         add_wait_queue_exclusive(&ioctx_list_waitQ, &wait);
2589
2590                         for (;;) {
2591                                 set_current_state(TASK_INTERRUPTIBLE);
2592                                 if (srpt_test_ioctx_list())
2593                                         break;
2594                                 spin_unlock_irq(&srpt_thread.thread_lock);
2595                                 schedule();
2596                                 spin_lock_irq(&srpt_thread.thread_lock);
2597                         }
2598                         set_current_state(TASK_RUNNING);
2599                         remove_wait_queue(&ioctx_list_waitQ, &wait);
2600                 }
2601
2602                 while (!list_empty(&srpt_thread.thread_ioctx_list)) {
2603                         ioctx = list_entry(srpt_thread.thread_ioctx_list.next,
2604                                            struct srpt_ioctx, comp_list);
2605
2606                         list_del(&ioctx->comp_list);
2607
2608                         spin_unlock_irq(&srpt_thread.thread_lock);
2609                         switch (ioctx->op) {
2610                         case IB_WC_SEND:
2611                                 srpt_handle_send_comp(ioctx->ch, ioctx,
2612                                         SCST_CONTEXT_DIRECT);
2613                                 break;
2614                         case IB_WC_RDMA_WRITE:
2615                         case IB_WC_RDMA_READ:
2616                                 srpt_handle_rdma_comp(ioctx->ch, ioctx);
2617                                 break;
2618                         case IB_WC_RECV:
2619                                 srpt_handle_new_iu(ioctx->ch, ioctx);
2620                                 break;
2621                         default:
2622                                 break;
2623                         }
2624 #if defined(CONFIG_SCST_DEBUG)
2625                         if (thread_processing_delay_in_us
2626                             <= MAX_UDELAY_MS * 1000)
2627                                 udelay(thread_processing_delay_in_us);
2628 #endif
2629                         spin_lock_irq(&srpt_thread.thread_lock);
2630                 }
2631         }
2632         spin_unlock_irq(&srpt_thread.thread_lock);
2633
2634         return 0;
2635 }
2636
2637 /* SCST target template for the SRP target implementation. */
2638 static struct scst_tgt_template srpt_template = {
2639         .name = DRV_NAME,
2640         .sg_tablesize = SRPT_DEF_SG_TABLESIZE,
2641         .xmit_response_atomic = 1,
2642         .rdy_to_xfer_atomic = 1,
2643         .no_proc_entry = 0,
2644         .detect = srpt_detect,
2645         .release = srpt_release,
2646         .xmit_response = srpt_xmit_response,
2647         .rdy_to_xfer = srpt_rdy_to_xfer,
2648         .on_free_cmd = srpt_on_free_cmd,
2649         .task_mgmt_fn_done = srpt_tsk_mgmt_done
2650 };
2651
2652 /*
2653  * The callback function srpt_release_class_dev() is called whenever a
2654  * device is removed from the /sys/class/infiniband_srpt device class.
2655  */
2656 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2657 static void srpt_release_class_dev(struct class_device *class_dev)
2658 #else
2659 static void srpt_release_class_dev(struct device *dev)
2660 #endif
2661 {
2662 }
2663
2664 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2665 static int srpt_trace_level_show(struct seq_file *seq, void *v)
2666 {
2667         return scst_proc_log_entry_read(seq, trace_flag, NULL);
2668 }
2669
2670 static ssize_t srpt_proc_trace_level_write(struct file *file,
2671         const char __user *buf, size_t length, loff_t *off)
2672 {
2673         return scst_proc_log_entry_write(file, buf, length, &trace_flag,
2674                 DEFAULT_SRPT_TRACE_FLAGS, NULL);
2675 }
2676
2677 static struct scst_proc_data srpt_log_proc_data = {
2678         SCST_DEF_RW_SEQ_OP(srpt_proc_trace_level_write)
2679         .show = srpt_trace_level_show,
2680 };
2681 #endif
2682
2683 static struct class_attribute srpt_class_attrs[] = {
2684         __ATTR_NULL,
2685 };
2686
2687 static struct class srpt_class = {
2688         .name = "infiniband_srpt",
2689 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2690         .release = srpt_release_class_dev,
2691 #else
2692         .dev_release = srpt_release_class_dev,
2693 #endif
2694         .class_attrs = srpt_class_attrs,
2695 };
2696
2697 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2698 static ssize_t show_login_info(struct class_device *class_dev, char *buf)
2699 #else
2700 static ssize_t show_login_info(struct device *dev,
2701                                struct device_attribute *attr, char *buf)
2702 #endif
2703 {
2704         struct srpt_device *sdev =
2705 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2706                 container_of(class_dev, struct srpt_device, class_dev);
2707 #else
2708                 container_of(dev, struct srpt_device, dev);
2709 #endif
2710         struct srpt_port *sport;
2711         int i;
2712         int len = 0;
2713
2714         for (i = 0; i < sdev->device->phys_port_cnt; i++) {
2715                 sport = &sdev->port[i];
2716
2717                 len += sprintf(buf + len,
2718                                "tid_ext=%016llx,ioc_guid=%016llx,pkey=ffff,"
2719                                "dgid=%04x%04x%04x%04x%04x%04x%04x%04x,"
2720                                "service_id=%016llx\n",
2721                                (unsigned long long) srpt_service_guid,
2722                                (unsigned long long) srpt_service_guid,
2723                                be16_to_cpu(((__be16 *) sport->gid.raw)[0]),
2724                                be16_to_cpu(((__be16 *) sport->gid.raw)[1]),
2725                                be16_to_cpu(((__be16 *) sport->gid.raw)[2]),
2726                                be16_to_cpu(((__be16 *) sport->gid.raw)[3]),
2727                                be16_to_cpu(((__be16 *) sport->gid.raw)[4]),
2728                                be16_to_cpu(((__be16 *) sport->gid.raw)[5]),
2729                                be16_to_cpu(((__be16 *) sport->gid.raw)[6]),
2730                                be16_to_cpu(((__be16 *) sport->gid.raw)[7]),
2731                                (unsigned long long) srpt_service_guid);
2732         }
2733
2734         return len;
2735 }
2736
2737 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2738 static CLASS_DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2739 #else
2740 static DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2741 #endif
2742
2743 /*
2744  * Callback function called by the InfiniBand core when either an InfiniBand
2745  * device has been added or during the ib_register_client() call for each
2746  * registered InfiniBand device.
2747  */
2748 static void srpt_add_one(struct ib_device *device)
2749 {
2750         struct srpt_device *sdev;
2751         struct srpt_port *sport;
2752         struct ib_srq_init_attr srq_attr;
2753         int i;
2754
2755         TRACE_ENTRY();
2756
2757         sdev = kzalloc(sizeof *sdev, GFP_KERNEL);
2758         if (!sdev)
2759                 return;
2760
2761         sdev->device = device;
2762
2763 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2764         sdev->class_dev.class = &srpt_class;
2765         sdev->class_dev.dev = device->dma_device;
2766         snprintf(sdev->class_dev.class_id, BUS_ID_SIZE,
2767                  "srpt-%s", device->name);
2768 #else
2769         sdev->dev.class = &srpt_class;
2770         sdev->dev.parent = device->dma_device;
2771 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
2772         snprintf(sdev->dev.bus_id, BUS_ID_SIZE, "srpt-%s", device->name);
2773 #else
2774         dev_set_name(&sdev->dev, "srpt-%s", device->name);
2775 #endif
2776 #endif
2777
2778 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2779         if (class_device_register(&sdev->class_dev))
2780                 goto free_dev;
2781         if (class_device_create_file(&sdev->class_dev,
2782                                      &class_device_attr_login_info))
2783                 goto err_dev;
2784 #else
2785         if (device_register(&sdev->dev))
2786                 goto free_dev;
2787         if (device_create_file(&sdev->dev, &dev_attr_login_info))
2788                 goto err_dev;
2789 #endif
2790
2791         if (ib_query_device(device, &sdev->dev_attr))
2792                 goto err_dev;
2793
2794         sdev->pd = ib_alloc_pd(device);
2795         if (IS_ERR(sdev->pd))
2796                 goto err_dev;
2797
2798         sdev->mr = ib_get_dma_mr(sdev->pd, IB_ACCESS_LOCAL_WRITE);
2799         if (IS_ERR(sdev->mr))
2800                 goto err_pd;
2801
2802         srq_attr.event_handler = srpt_srq_event;
2803         srq_attr.srq_context = (void *)sdev;
2804         srq_attr.attr.max_wr = min(SRPT_SRQ_SIZE, sdev->dev_attr.max_srq_wr);
2805         srq_attr.attr.max_sge = 1;
2806         srq_attr.attr.srq_limit = 0;
2807
2808         sdev->srq = ib_create_srq(sdev->pd, &srq_attr);
2809         if (IS_ERR(sdev->srq))
2810                 goto err_mr;
2811
2812         TRACE_DBG("%s: create SRQ #wr= %d max_allow=%d dev= %s",
2813                __func__, srq_attr.attr.max_wr,
2814               sdev->dev_attr.max_srq_wr, device->name);
2815
2816         if (!srpt_service_guid)
2817                 srpt_service_guid = be64_to_cpu(device->node_guid);
2818
2819         sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);
2820         if (IS_ERR(sdev->cm_id))
2821                 goto err_srq;
2822
2823         /* print out target login information */
2824         TRACE_DBG("Target login info: id_ext=%016llx,"
2825                   "ioc_guid=%016llx,pkey=ffff,service_id=%016llx",
2826                   (unsigned long long) srpt_service_guid,
2827                   (unsigned long long) srpt_service_guid,
2828                   (unsigned long long) srpt_service_guid);
2829
2830         /*
2831          * We do not have a consistent service_id (ie. also id_ext of target_id)
2832          * to identify this target. We currently use the guid of the first HCA
2833          * in the system as service_id; therefore, the target_id will change
2834          * if this HCA is gone bad and replaced by different HCA
2835          */
2836         if (ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0, NULL))
2837                 goto err_cm;
2838
2839         INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
2840                               srpt_event_handler);
2841         if (ib_register_event_handler(&sdev->event_handler))
2842                 goto err_cm;
2843
2844         if (srpt_alloc_ioctx_ring(sdev))
2845                 goto err_event;
2846
2847         INIT_LIST_HEAD(&sdev->rch_list);
2848         spin_lock_init(&sdev->spinlock);
2849
2850         for (i = 0; i < SRPT_SRQ_SIZE; ++i)
2851                 srpt_post_recv(sdev, sdev->ioctx_ring[i]);
2852
2853         ib_set_client_data(device, &srpt_client, sdev);
2854
2855         sdev->scst_tgt = scst_register(&srpt_template, NULL);
2856         if (!sdev->scst_tgt) {
2857                 PRINT_ERROR("SCST registration failed for %s.",
2858                             sdev->device->name);
2859                 goto err_ring;
2860         }
2861
2862         scst_tgt_set_tgt_priv(sdev->scst_tgt, sdev);
2863
2864         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
2865                 sport = &sdev->port[i - 1];
2866                 sport->sdev = sdev;
2867                 sport->port = i;
2868 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2869                 /*
2870                  * A vanilla 2.6.19 or older kernel without backported OFED
2871                  * kernel headers.
2872                  */
2873                 INIT_WORK(&sport->work, srpt_refresh_port_work, sport);
2874 #else
2875                 INIT_WORK(&sport->work, srpt_refresh_port_work);
2876 #endif
2877                 if (srpt_refresh_port(sport)) {
2878                         PRINT_ERROR("MAD registration failed for %s-%d.",
2879                                     sdev->device->name, i);
2880                         goto err_refresh_port;
2881                 }
2882         }
2883
2884         atomic_inc(&srpt_device_count);
2885
2886         TRACE_EXIT();
2887
2888         return;
2889
2890 err_refresh_port:
2891         scst_unregister(sdev->scst_tgt);
2892 err_ring:
2893         ib_set_client_data(device, &srpt_client, NULL);
2894         srpt_free_ioctx_ring(sdev);
2895 err_event:
2896         ib_unregister_event_handler(&sdev->event_handler);
2897 err_cm:
2898         ib_destroy_cm_id(sdev->cm_id);
2899 err_srq:
2900         ib_destroy_srq(sdev->srq);
2901 err_mr:
2902         ib_dereg_mr(sdev->mr);
2903 err_pd:
2904         ib_dealloc_pd(sdev->pd);
2905 err_dev:
2906 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2907         class_device_unregister(&sdev->class_dev);
2908 #else
2909         device_unregister(&sdev->dev);
2910 #endif
2911 free_dev:
2912         kfree(sdev);
2913
2914         TRACE_EXIT();
2915 }
2916
2917 /*
2918  * Callback function called by the InfiniBand core when either an InfiniBand
2919  * device has been removed or during the ib_unregister_client() call for each
2920  * registered InfiniBand device.
2921  */
2922 static void srpt_remove_one(struct ib_device *device)
2923 {
2924         int i;
2925         struct srpt_device *sdev;
2926
2927         TRACE_ENTRY();
2928
2929         sdev = ib_get_client_data(device, &srpt_client);
2930 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
2931         WARN_ON(!sdev);
2932         if (!sdev)
2933                 return;
2934 #else
2935         if (WARN_ON(!sdev))
2936                 return;
2937 #endif
2938
2939         /*
2940          * Cancel the work if it is queued. Wait until srpt_refresh_port_work()
2941          * finished if it is running.
2942          */
2943         for (i = 0; i < sdev->device->phys_port_cnt; i++)
2944 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
2945                 cancel_work_sync(&sdev->port[i].work);
2946 #else
2947                 /*
2948                  * cancel_work_sync() was introduced in kernel 2.6.22. Older
2949                  * kernels do not have a facility to cancel scheduled work.
2950                  */
2951                 PRINT_ERROR("%s", 
2952                        "your kernel does not provide cancel_work_sync().");
2953 #endif
2954
2955         scst_unregister(sdev->scst_tgt);
2956         sdev->scst_tgt = NULL;
2957
2958         ib_unregister_event_handler(&sdev->event_handler);
2959         ib_destroy_cm_id(sdev->cm_id);
2960         ib_destroy_srq(sdev->srq);
2961         ib_dereg_mr(sdev->mr);
2962         ib_dealloc_pd(sdev->pd);
2963 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2964         class_device_unregister(&sdev->class_dev);
2965 #else
2966         device_unregister(&sdev->dev);
2967 #endif
2968
2969         srpt_free_ioctx_ring(sdev);
2970         kfree(sdev);
2971
2972         TRACE_EXIT();
2973 }
2974
2975 /**
2976  * Create procfs entries for srpt. Currently the only procfs entry created
2977  * by this function is the "trace_level" entry.
2978  */
2979 static int srpt_register_procfs_entry(struct scst_tgt_template *tgt)
2980 {
2981         int res = 0;
2982 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2983         struct proc_dir_entry *p, *root;
2984
2985         root = scst_proc_get_tgt_root(tgt);
2986         WARN_ON(!root);
2987         if (root) {
2988                 /*
2989                  * Fill in the scst_proc_data::data pointer, which is used in
2990                  * a printk(KERN_INFO ...) statement in
2991                  * scst_proc_log_entry_write() in scst_proc.c.
2992                  */
2993                 srpt_log_proc_data.data = (char *)tgt->name;
2994                 p = scst_create_proc_entry(root, SRPT_PROC_TRACE_LEVEL_NAME,
2995                                            &srpt_log_proc_data);
2996                 if (!p)
2997                         res = -ENOMEM;
2998         } else
2999                 res = -ENOMEM;
3000
3001 #endif
3002         return res;
3003 }
3004
3005 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt)
3006 {
3007 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
3008         struct proc_dir_entry *root;
3009
3010         root = scst_proc_get_tgt_root(tgt);
3011         WARN_ON(!root);
3012         if (root)
3013                 remove_proc_entry(SRPT_PROC_TRACE_LEVEL_NAME, root);
3014 #endif
3015 }
3016
3017 /*
3018  * Module initialization.
3019  *
3020  * Note: since ib_register_client() registers callback functions, and since at
3021  * least one of these callback functions (srpt_add_one()) calls SCST functions,
3022  * the SCST target template must be registered before ib_register_client() is
3023  * called.
3024  */
3025 static int __init srpt_init_module(void)
3026 {
3027         int ret;
3028
3029         ret = class_register(&srpt_class);
3030         if (ret) {
3031                 PRINT_ERROR("%s", "couldn't register class ib_srpt");
3032                 goto out;
3033         }
3034
3035         ret = scst_register_target_template(&srpt_template);
3036         if (ret < 0) {
3037                 PRINT_ERROR("%s", "couldn't register with scst");
3038                 ret = -ENODEV;
3039                 goto out_unregister_class;
3040         }
3041
3042         ret = srpt_register_procfs_entry(&srpt_template);
3043         if (ret) {
3044                 PRINT_ERROR("%s", "couldn't register procfs entry");
3045                 goto out_unregister_target;
3046         }
3047
3048         ret = ib_register_client(&srpt_client);
3049         if (ret) {
3050                 PRINT_ERROR("%s", "couldn't register IB client");
3051                 goto out_unregister_target;
3052         }
3053
3054         if (thread) {
3055                 spin_lock_init(&srpt_thread.thread_lock);
3056                 INIT_LIST_HEAD(&srpt_thread.thread_ioctx_list);
3057                 srpt_thread.thread = kthread_run(srpt_ioctx_thread,
3058                                                  NULL, "srpt_thread");
3059                 if (IS_ERR(srpt_thread.thread)) {
3060                         srpt_thread.thread = NULL;
3061                         thread = 0;
3062                 }
3063         }
3064
3065         return 0;
3066
3067 out_unregister_target:
3068         /*
3069          * Note: the procfs entry is unregistered in srpt_release(), which is
3070          * called by scst_unregister_target_template().
3071          */
3072         scst_unregister_target_template(&srpt_template);
3073 out_unregister_class:
3074         class_unregister(&srpt_class);
3075 out:
3076         return ret;
3077 }
3078
3079 static void __exit srpt_cleanup_module(void)
3080 {
3081         TRACE_ENTRY();
3082
3083         if (srpt_thread.thread)
3084                 kthread_stop(srpt_thread.thread);
3085         ib_unregister_client(&srpt_client);
3086         scst_unregister_target_template(&srpt_template);
3087         class_unregister(&srpt_class);
3088
3089         TRACE_EXIT();
3090 }
3091
3092 module_init(srpt_init_module);
3093 module_exit(srpt_cleanup_module);