Added a note about the size of struct ib_qp_attr.
[mirror/scst/.git] / srpt / src / ib_srpt.c
1 /*
2  * Copyright (c) 2006 - 2009 Mellanox Technology Inc.  All rights reserved.
3  * Copyright (C) 2008 Vladislav Bolkhovitin <vst@vlnb.net>
4  * Copyright (C) 2008 - 2009 Bart Van Assche <bart.vanassche@gmail.com>
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  */
35
36 #include <linux/module.h>
37 #include <linux/init.h>
38 #include <linux/slab.h>
39 #include <linux/err.h>
40 #include <linux/ctype.h>
41 #include <linux/string.h>
42 #include <linux/kthread.h>
43 #include <asm/atomic.h>
44 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
45 #include <linux/proc_fs.h>
46 #include <linux/seq_file.h>
47 #endif
48 #include "ib_srpt.h"
49 #include "scst_debug.h"
50
51 /* Name of this kernel module. */
52 #define DRV_NAME                "ib_srpt"
53 /* Prefix for printk() kernel messages. */
54 #define PFX                     DRV_NAME ": "
55 #define DRV_VERSION             "1.0.1"
56 #define DRV_RELDATE             "July 10, 2008"
57 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
58 /* Flags to be used in SCST debug tracing statements. */
59 #define DEFAULT_SRPT_TRACE_FLAGS (TRACE_OUT_OF_MEM | TRACE_MINOR \
60                                   | TRACE_MGMT | TRACE_SPECIAL)
61 /* Name of the entry that will be created under /proc/scsi_tgt/ib_srpt. */
62 #define SRPT_PROC_TRACE_LEVEL_NAME      "trace_level"
63 #endif
64
65 #define MELLANOX_SRPT_ID_STRING "Mellanox OFED SRP target"
66
67 MODULE_AUTHOR("Vu Pham");
68 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
69                    "v" DRV_VERSION " (" DRV_RELDATE ")");
70 MODULE_LICENSE("Dual BSD/GPL");
71
72 struct srpt_thread {
73         /* Protects thread_ioctx_list. */
74         spinlock_t thread_lock;
75         /* I/O contexts to be processed by the kernel thread. */
76         struct list_head thread_ioctx_list;
77         /* SRPT kernel thread. */
78         struct task_struct *thread;
79 };
80
81 /*
82  * Global Variables
83  */
84
85 static u64 global_ioc_guid;
86 /* List of srpt_device structures. */
87 static atomic_t srpt_device_count;
88 static int thread;
89 static int one_guid_per_ioc;
90 static struct srpt_thread srpt_thread;
91 static DECLARE_WAIT_QUEUE_HEAD(ioctx_list_waitQ);
92 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
93 static unsigned long trace_flag = DEFAULT_SRPT_TRACE_FLAGS;
94 module_param(trace_flag, long, 0644);
95 MODULE_PARM_DESC(trace_flag,
96                  "Trace flags for the ib_srpt kernel module.");
97 #endif
98
99 module_param(thread, int, 0444);
100 MODULE_PARM_DESC(thread,
101                  "Executing ioctx in thread context. Default 0, i.e. soft IRQ, "
102                  "where possible");
103 module_param(one_guid_per_ioc, bool, 0444);
104 MODULE_PARM_DESC(one_guid_per_ioc,
105                  "Assign a unique GUID to each IOC instead of using one GUID "
106                  "for all IOCs.");
107
108 static void srpt_add_one(struct ib_device *device);
109 static void srpt_remove_one(struct ib_device *device);
110 static void srpt_unregister_mad_agent(struct srpt_device *sdev);
111 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt);
112
113 static struct ib_client srpt_client = {
114         .name = DRV_NAME,
115         .add = srpt_add_one,
116         .remove = srpt_remove_one
117 };
118
119 /**
120  * Atomically test and set the channel state.
121  * @ch: RDMA channel.
122  * @old: channel state to compare with.
123  * @new: state to change the channel state to if the current state matches the
124  *       argument 'old'.
125  *
126  * Returns true if the channel state matched old upon entry of this function,
127  * and false otherwise.
128  */
129 static bool srpt_test_and_set_channel_state(struct srpt_rdma_ch *ch,
130                                             enum rdma_ch_state old,
131                                             enum rdma_ch_state new)
132 {
133         unsigned long flags;
134         enum rdma_ch_state cur;
135
136         spin_lock_irqsave(&ch->spinlock, flags);
137         cur = ch->state;
138         if (cur == old)
139                 ch->state = new;
140         spin_unlock_irqrestore(&ch->spinlock, flags);
141
142         return cur == old;
143 }
144
145 /*
146  * Callback function called by the InfiniBand core when an asynchronous IB
147  * event occurs. This callback may occur in interrupt context. See also
148  * section 11.5.2, Set Asynchronous Event Handler in the InfiniBand
149  * Architecture Specification.
150  */
151 static void srpt_event_handler(struct ib_event_handler *handler,
152                                struct ib_event *event)
153 {
154         struct srpt_device *sdev;
155         struct srpt_port *sport;
156
157         sdev = ib_get_client_data(event->device, &srpt_client);
158         if (!sdev || sdev->device != event->device)
159                 return;
160
161         TRACE_DBG("ASYNC event= %d on device= %s",
162                   event->event, sdev->device->name);
163
164         switch (event->event) {
165         case IB_EVENT_PORT_ERR:
166                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
167                         sport = &sdev->port[event->element.port_num - 1];
168                         sport->lid = 0;
169                         sport->sm_lid = 0;
170                 }
171                 break;
172         case IB_EVENT_PORT_ACTIVE:
173         case IB_EVENT_LID_CHANGE:
174         case IB_EVENT_PKEY_CHANGE:
175         case IB_EVENT_SM_CHANGE:
176         case IB_EVENT_CLIENT_REREGISTER:
177                 /*
178                  * Refresh port data asynchronously. Note: it is safe to call
179                  * schedule_work() even if &sport->work is already on the
180                  * global workqueue because schedule_work() tests for the
181                  * work_pending() condition before adding &sport->work to the
182                  * global work queue.
183                  */
184                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
185                         sport = &sdev->port[event->element.port_num - 1];
186                         if (!sport->lid && !sport->sm_lid)
187                                 schedule_work(&sport->work);
188                 }
189                 break;
190         default:
191                 break;
192         }
193
194 }
195
196 /*
197  * Callback function called by the InfiniBand core for SRQ (shared receive
198  * queue) events.
199  */
200 static void srpt_srq_event(struct ib_event *event, void *ctx)
201 {
202         TRACE_DBG("SRQ event %d", event->event);
203 }
204
205 /*
206  * Callback function called by the InfiniBand core for QP (queue pair) events.
207  */
208 static void srpt_qp_event(struct ib_event *event, void *ctx)
209 {
210         struct srpt_rdma_ch *ch = ctx;
211
212         TRACE_DBG("QP event %d on cm_id=%p sess_name=%s state=%d",
213                   event->event, ch->cm_id, ch->sess_name, ch->state);
214
215         switch (event->event) {
216         case IB_EVENT_COMM_EST:
217 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20) || defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
218                 ib_cm_notify(ch->cm_id, event->event);
219 #else
220                 /* Vanilla 2.6.19 kernel (or before) without OFED. */
221                 printk(KERN_ERR PFX "how to perform ib_cm_notify() on a"
222                         " vanilla 2.6.18 kernel ???\n");
223 #endif
224                 break;
225         case IB_EVENT_QP_LAST_WQE_REACHED:
226                 if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_LIVE,
227                                         RDMA_CHANNEL_DISCONNECTING)) {
228                         TRACE_DBG("%s", "Disconnecting channel.");
229                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
230                 }
231                 break;
232         default:
233                 break;
234         }
235 }
236
237 /*
238  * Helper function for filling in an InfiniBand IOUnitInfo structure. Copies
239  * the lowest four bits of value in element slot of the array of four bit
240  * elements called c_list (controller list). The index slot is one-based.
241  *
242  * @pre 1 <= slot && 0 <= value && value < 16
243  */
244 static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
245 {
246         u16 id;
247         u8 tmp;
248
249         id = (slot - 1) / 2;
250         if (slot & 0x1) {
251                 tmp = c_list[id] & 0xf;
252                 c_list[id] = (value << 4) | tmp;
253         } else {
254                 tmp = c_list[id] & 0xf0;
255                 c_list[id] = (value & 0xf) | tmp;
256         }
257 }
258
259 /*
260  * Write InfiniBand ClassPortInfo to mad. See also section 16.3.3.1
261  * ClassPortInfo in the InfiniBand Architecture Specification.
262  */
263 static void srpt_get_class_port_info(struct ib_dm_mad *mad)
264 {
265         struct ib_class_port_info *cif;
266
267         cif = (struct ib_class_port_info *)mad->data;
268         memset(cif, 0, sizeof *cif);
269         cif->base_version = 1;
270         cif->class_version = 1;
271         cif->resp_time_value = 20;
272
273         mad->mad_hdr.status = 0;
274 }
275
276 /*
277  * Write IOUnitInfo to mad. See also section 16.3.3.3 IOUnitInfo in the
278  * InfiniBand Architecture Specification. See also section B.7,
279  * table B.6 in the T10 SRP r16a document.
280  */
281 static void srpt_get_iou(struct ib_dm_mad *mad)
282 {
283         struct ib_dm_iou_info *ioui;
284         u8 slot;
285         int i;
286
287         ioui = (struct ib_dm_iou_info *)mad->data;
288         ioui->change_id = 1;
289         ioui->max_controllers = 16;
290
291         /* set present for slot 1 and empty for the rest */
292         srpt_set_ioc(ioui->controller_list, 1, 1);
293         for (i = 1, slot = 2; i < 16; i++, slot++)
294                 srpt_set_ioc(ioui->controller_list, slot, 0);
295
296         mad->mad_hdr.status = 0;
297 }
298
299 /**
300  * Return the GUID that will be communicated to the initiator for identifying
301  * the SRPT target. Depending on the mode variable one_guid_per_ioc, either the
302  * GUID of the specified IOC is returned or the GUID of the first IOC.
303  */
304 static u64 srpt_get_ioc_guid(struct ib_device *device)
305 {
306         BUG_ON(!device);
307         WARN_ON(!global_ioc_guid);
308         WARN_ON(!device->node_guid);
309
310         return one_guid_per_ioc ? be64_to_cpu(device->node_guid)
311                 : global_ioc_guid;
312 }
313
314 /*
315  * Write IOControllerprofile to mad for I/O controller (sdev, slot). See also
316  * section 16.3.3.4 IOControllerProfile in the InfiniBand Architecture
317  * Specification. See also section B.7, table B.7 in the T10 SRP r16a
318  * document.
319  */
320 static void srpt_get_ioc(struct srpt_device *sdev, u32 slot,
321                          struct ib_dm_mad *mad)
322 {
323         struct ib_dm_ioc_profile *iocp;
324
325         iocp = (struct ib_dm_ioc_profile *)mad->data;
326
327         if (!slot || slot > 16) {
328                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
329                 return;
330         }
331
332         if (slot > 2) {
333                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
334                 return;
335         }
336
337         memset(iocp, 0, sizeof *iocp);
338         strcpy(iocp->id_string, MELLANOX_SRPT_ID_STRING);
339         iocp->guid = cpu_to_be64(srpt_get_ioc_guid(sdev->device));
340         iocp->vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
341         iocp->device_id = cpu_to_be32(sdev->dev_attr.vendor_part_id);
342         iocp->device_version = cpu_to_be16(sdev->dev_attr.hw_ver);
343         iocp->subsys_vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
344         iocp->subsys_device_id = 0x0;
345         iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
346         iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
347         iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
348         iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
349         iocp->send_queue_depth = cpu_to_be16(SRPT_SRQ_SIZE);
350         iocp->rdma_read_depth = 4;
351         iocp->send_size = cpu_to_be32(MAX_MESSAGE_SIZE);
352         iocp->rdma_size = cpu_to_be32(MAX_RDMA_SIZE);
353         iocp->num_svc_entries = 1;
354         iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
355                 SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
356
357         mad->mad_hdr.status = 0;
358 }
359
360 /*
361  * Device management: write ServiceEntries to mad for the given slot. See also
362  * section 16.3.3.5 ServiceEntries in the InfiniBand Architecture
363  * Specification. See also section B.7, table B.8 in the T10 SRP r16a document.
364  */
365 static void srpt_get_svc_entries(u64 ioc_guid,
366                                  u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
367 {
368         struct ib_dm_svc_entries *svc_entries;
369
370         WARN_ON(!ioc_guid);
371
372         if (!slot || slot > 16) {
373                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
374                 return;
375         }
376
377         if (slot > 2 || lo > hi || hi > 1) {
378                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
379                 return;
380         }
381
382         svc_entries = (struct ib_dm_svc_entries *)mad->data;
383         memset(svc_entries, 0, sizeof *svc_entries);
384         svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid);
385         snprintf(svc_entries->service_entries[0].name,
386                  sizeof(svc_entries->service_entries[0].name),
387                  "%s%016llx",
388                  SRP_SERVICE_NAME_PREFIX,
389                  (unsigned long long)ioc_guid);
390
391         mad->mad_hdr.status = 0;
392 }
393
394 /*
395  * Actual processing of a received MAD *rq_mad received through source port *sp
396  * (MAD = InfiniBand management datagram). The response to be sent back is
397  * written to *rsp_mad.
398  */
399 static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
400                                  struct ib_dm_mad *rsp_mad)
401 {
402         u16 attr_id;
403         u32 slot;
404         u8 hi, lo;
405
406         attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
407         switch (attr_id) {
408         case DM_ATTR_CLASS_PORT_INFO:
409                 srpt_get_class_port_info(rsp_mad);
410                 break;
411         case DM_ATTR_IOU_INFO:
412                 srpt_get_iou(rsp_mad);
413                 break;
414         case DM_ATTR_IOC_PROFILE:
415                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
416                 srpt_get_ioc(sp->sdev, slot, rsp_mad);
417                 break;
418         case DM_ATTR_SVC_ENTRIES:
419                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
420                 hi = (u8) ((slot >> 8) & 0xff);
421                 lo = (u8) (slot & 0xff);
422                 slot = (u16) ((slot >> 16) & 0xffff);
423                 srpt_get_svc_entries(srpt_get_ioc_guid(sp->sdev->device),
424                                      slot, hi, lo, rsp_mad);
425                 break;
426         default:
427                 rsp_mad->mad_hdr.status =
428                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
429                 break;
430         }
431 }
432
433 /*
434  * Callback function that is called by the InfiniBand core after transmission of
435  * a MAD. (MAD = management datagram; AH = address handle.)
436  */
437 static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
438                                   struct ib_mad_send_wc *mad_wc)
439 {
440         ib_destroy_ah(mad_wc->send_buf->ah);
441         ib_free_send_mad(mad_wc->send_buf);
442 }
443
444 /*
445  * Callback function that is called by the InfiniBand core after reception of
446  * a MAD (management datagram).
447  */
448 static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
449                                   struct ib_mad_recv_wc *mad_wc)
450 {
451         struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
452         struct ib_ah *ah;
453         struct ib_mad_send_buf *rsp;
454         struct ib_dm_mad *dm_mad;
455
456         if (!mad_wc || !mad_wc->recv_buf.mad)
457                 return;
458
459         ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
460                                   mad_wc->recv_buf.grh, mad_agent->port_num);
461         if (IS_ERR(ah))
462                 goto err;
463
464         BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR);
465
466         rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
467                                  mad_wc->wc->pkey_index, 0,
468                                  IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA,
469                                  GFP_KERNEL);
470         if (IS_ERR(rsp))
471                 goto err_rsp;
472
473         rsp->ah = ah;
474
475         dm_mad = rsp->mad;
476         memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof *dm_mad);
477         dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
478         dm_mad->mad_hdr.status = 0;
479
480         switch (mad_wc->recv_buf.mad->mad_hdr.method) {
481         case IB_MGMT_METHOD_GET:
482                 srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
483                 break;
484         case IB_MGMT_METHOD_SET:
485                 dm_mad->mad_hdr.status =
486                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
487                 break;
488         default:
489                 dm_mad->mad_hdr.status =
490                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
491                 break;
492         }
493
494         if (!ib_post_send_mad(rsp, NULL)) {
495                 ib_free_recv_mad(mad_wc);
496                 /* will destroy_ah & free_send_mad in send completion */
497                 return;
498         }
499
500         ib_free_send_mad(rsp);
501
502 err_rsp:
503         ib_destroy_ah(ah);
504 err:
505         ib_free_recv_mad(mad_wc);
506 }
507
508 /*
509  * Enable InfiniBand management datagram processing, update the cached sm_lid,
510  * lid and gid values, and register a callback function for processing MADs
511  * on the specified port. It is safe to call this function more than once for
512  * the same port.
513  */
514 static int srpt_refresh_port(struct srpt_port *sport)
515 {
516         struct ib_mad_reg_req reg_req;
517         struct ib_port_modify port_modify;
518         struct ib_port_attr port_attr;
519         int ret;
520
521         memset(&port_modify, 0, sizeof port_modify);
522         port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
523         port_modify.clr_port_cap_mask = 0;
524
525         ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
526         if (ret)
527                 goto err_mod_port;
528
529         ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
530         if (ret)
531                 goto err_query_port;
532
533         sport->sm_lid = port_attr.sm_lid;
534         sport->lid = port_attr.lid;
535
536         ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid);
537         if (ret)
538                 goto err_query_port;
539
540         if (!sport->mad_agent) {
541                 memset(&reg_req, 0, sizeof reg_req);
542                 reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
543                 reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
544                 set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
545                 set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
546
547                 sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
548                                                          sport->port,
549                                                          IB_QPT_GSI,
550                                                          &reg_req, 0,
551                                                          srpt_mad_send_handler,
552                                                          srpt_mad_recv_handler,
553                                                          sport);
554                 if (IS_ERR(sport->mad_agent)) {
555                         ret = PTR_ERR(sport->mad_agent);
556                         sport->mad_agent = NULL;
557                         goto err_query_port;
558                 }
559         }
560
561         return 0;
562
563 err_query_port:
564
565         port_modify.set_port_cap_mask = 0;
566         port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
567         ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
568
569 err_mod_port:
570
571         return ret;
572 }
573
574 /*
575  * Unregister the callback function for processing MADs and disable MAD
576  * processing for all ports of the specified device. It is safe to call this
577  * function more than once for the same device.
578  */
579 static void srpt_unregister_mad_agent(struct srpt_device *sdev)
580 {
581         struct ib_port_modify port_modify = {
582                 .clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP,
583         };
584         struct srpt_port *sport;
585         int i;
586
587         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
588                 sport = &sdev->port[i - 1];
589                 WARN_ON(sport->port != i);
590                 if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
591                         printk(KERN_ERR PFX "disabling MAD processing"
592                                " failed.\n");
593                 if (sport->mad_agent) {
594                         ib_unregister_mad_agent(sport->mad_agent);
595                         sport->mad_agent = NULL;
596                 }
597         }
598 }
599
600 /*
601  * Allocate and initialize an SRPT I/O context structure.
602  */
603 static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev)
604 {
605         struct srpt_ioctx *ioctx;
606
607         ioctx = kmalloc(sizeof *ioctx, GFP_KERNEL);
608         if (!ioctx)
609                 goto out;
610
611         ioctx->buf = kzalloc(MAX_MESSAGE_SIZE, GFP_KERNEL);
612         if (!ioctx->buf)
613                 goto out_free_ioctx;
614
615         ioctx->dma = dma_map_single(sdev->device->dma_device, ioctx->buf,
616                                     MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
617 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
618         if (dma_mapping_error(sdev->device->dma_device, ioctx->dma))
619 #else
620         if (dma_mapping_error(ioctx->dma))
621 #endif
622                 goto out_free_buf;
623
624         return ioctx;
625
626 out_free_buf:
627         kfree(ioctx->buf);
628 out_free_ioctx:
629         kfree(ioctx);
630 out:
631         return NULL;
632 }
633
634 /*
635  * Deallocate an SRPT I/O context structure.
636  */
637 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
638 {
639         if (!ioctx)
640                 return;
641
642         dma_unmap_single(sdev->device->dma_device, ioctx->dma,
643                          MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
644         kfree(ioctx->buf);
645         kfree(ioctx);
646 }
647
648 /*
649  * Associate a ring of SRPT I/O context structures with the specified device.
650  */
651 static int srpt_alloc_ioctx_ring(struct srpt_device *sdev)
652 {
653         int i;
654
655         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
656                 sdev->ioctx_ring[i] = srpt_alloc_ioctx(sdev);
657
658                 if (!sdev->ioctx_ring[i])
659                         goto err;
660
661                 sdev->ioctx_ring[i]->index = i;
662         }
663
664         return 0;
665
666 err:
667         while (--i > 0) {
668                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
669                 sdev->ioctx_ring[i] = NULL;
670         }
671         return -ENOMEM;
672 }
673
674 /* Free the ring of SRPT I/O context structures. */
675 static void srpt_free_ioctx_ring(struct srpt_device *sdev)
676 {
677         int i;
678
679         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
680                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
681                 sdev->ioctx_ring[i] = NULL;
682         }
683 }
684
685 /*
686  * Post a receive request on the work queue of InfiniBand device 'sdev'.
687  */
688 static int srpt_post_recv(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
689 {
690         struct ib_sge list;
691         struct ib_recv_wr wr, *bad_wr;
692
693         wr.wr_id = ioctx->index | SRPT_OP_RECV;
694
695         list.addr = ioctx->dma;
696         list.length = MAX_MESSAGE_SIZE;
697         list.lkey = sdev->mr->lkey;
698
699         wr.next = NULL;
700         wr.sg_list = &list;
701         wr.num_sge = 1;
702
703         return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);
704 }
705
706 /*
707  * Post an IB send request.
708  * @ch: RDMA channel to post the send request on.
709  * @ioctx: I/O context of the send request.
710  * @len: length of the request to be sent in bytes.
711  *
712  * Returns zero upon success and a non-zero value upon failure.
713  */
714 static int srpt_post_send(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
715                           int len)
716 {
717         struct ib_sge list;
718         struct ib_send_wr wr, *bad_wr;
719         struct srpt_device *sdev = ch->sport->sdev;
720
721         dma_sync_single_for_device(sdev->device->dma_device, ioctx->dma,
722                                    MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
723
724         list.addr = ioctx->dma;
725         list.length = len;
726         list.lkey = sdev->mr->lkey;
727
728         wr.next = NULL;
729         wr.wr_id = ioctx->index;
730         wr.sg_list = &list;
731         wr.num_sge = 1;
732         wr.opcode = IB_WR_SEND;
733         wr.send_flags = IB_SEND_SIGNALED;
734
735         return ib_post_send(ch->qp, &wr, &bad_wr);
736 }
737
738 static int srpt_get_desc_tbl(struct srpt_ioctx *ioctx, struct srp_cmd *srp_cmd,
739                              int *ind)
740 {
741         struct srp_indirect_buf *idb;
742         struct srp_direct_buf *db;
743
744         *ind = 0;
745         if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
746             ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
747                 ioctx->n_rbuf = 1;
748                 ioctx->rbufs = &ioctx->single_rbuf;
749
750                 db = (void *)srp_cmd->add_data;
751                 memcpy(ioctx->rbufs, db, sizeof *db);
752                 ioctx->data_len = be32_to_cpu(db->len);
753         } else {
754                 idb = (void *)srp_cmd->add_data;
755
756                 ioctx->n_rbuf = be32_to_cpu(idb->table_desc.len) / sizeof *db;
757
758                 if (ioctx->n_rbuf >
759                     (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
760                         *ind = 1;
761                         ioctx->n_rbuf = 0;
762                         goto out;
763                 }
764
765                 if (ioctx->n_rbuf == 1)
766                         ioctx->rbufs = &ioctx->single_rbuf;
767                 else
768                         ioctx->rbufs =
769                                 kmalloc(ioctx->n_rbuf * sizeof *db, GFP_ATOMIC);
770                 if (!ioctx->rbufs) {
771                         ioctx->n_rbuf = 0;
772                         return -ENOMEM;
773                 }
774
775                 db = idb->desc_list;
776                 memcpy(ioctx->rbufs, db, ioctx->n_rbuf * sizeof *db);
777                 ioctx->data_len = be32_to_cpu(idb->len);
778         }
779 out:
780         return 0;
781 }
782
783 /*
784  * Modify the attributes of queue pair 'qp': allow local write, remote read,
785  * and remote write. Also transition 'qp' to state IB_QPS_INIT.
786  */
787 static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
788 {
789         struct ib_qp_attr *attr;
790         int ret;
791
792         attr = kzalloc(sizeof *attr, GFP_KERNEL);
793         if (!attr)
794                 return -ENOMEM;
795
796         attr->qp_state = IB_QPS_INIT;
797         attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ |
798             IB_ACCESS_REMOTE_WRITE;
799         attr->port_num = ch->sport->port;
800         attr->pkey_index = 0;
801
802         ret = ib_modify_qp(qp, attr,
803                            IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
804                            IB_QP_PKEY_INDEX);
805
806         kfree(attr);
807         return ret;
808 }
809
810 /**
811  * Change the state of a channel to 'ready to receive' (RTR).
812  * @ch: channel of the queue pair.
813  * @qp: queue pair to change the state of.
814  *
815  * Returns zero upon success and a negative value upon failure.
816  *
817  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
818  * If this structure ever becomes larger, it might be necessary to allocate
819  * it dynamically instead of on the stack.
820  */
821 static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
822 {
823         struct ib_qp_attr qp_attr;
824         int attr_mask;
825         int ret;
826
827         qp_attr.qp_state = IB_QPS_RTR;
828         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
829         if (ret)
830                 goto out;
831
832         qp_attr.max_dest_rd_atomic = 4;
833
834         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
835
836 out:
837         return ret;
838 }
839
840 /**
841  * Change the state of a channel to 'ready to send' (RTS).
842  * @ch: channel of the queue pair.
843  * @qp: queue pair to change the state of.
844  *
845  * Returns zero upon success and a negative value upon failure.
846  *
847  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
848  * If this structure ever becomes larger, it might be necessary to allocate
849  * it dynamically instead of on the stack.
850  */
851 static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
852 {
853         struct ib_qp_attr qp_attr;
854         int attr_mask;
855         int ret;
856
857         qp_attr.qp_state = IB_QPS_RTS;
858         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
859         if (ret)
860                 goto out;
861
862         qp_attr.max_rd_atomic = 4;
863
864         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
865
866 out:
867         return ret;
868 }
869
870 static void srpt_reset_ioctx(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
871 {
872         int i;
873
874         if (ioctx->n_rdma_ius > 0 && ioctx->rdma_ius) {
875                 struct rdma_iu *riu = ioctx->rdma_ius;
876
877                 for (i = 0; i < ioctx->n_rdma_ius; ++i, ++riu)
878                         kfree(riu->sge);
879                 kfree(ioctx->rdma_ius);
880         }
881
882         if (ioctx->n_rbuf > 1)
883                 kfree(ioctx->rbufs);
884
885         if (srpt_post_recv(ch->sport->sdev, ioctx))
886                 printk(KERN_ERR PFX "SRQ post_recv failed - this is serious\n");
887                 /* we should queue it back to free_ioctx queue */
888         else
889                 atomic_inc(&ch->req_lim_delta);
890 }
891
892 static void srpt_abort_scst_cmd(struct srpt_device *sdev,
893                                 struct scst_cmd *scmnd,
894                                 bool tell_initiator)
895 {
896         struct srpt_ioctx *ioctx;
897         scst_data_direction dir;
898
899         ioctx = scst_cmd_get_tgt_priv(scmnd);
900         BUG_ON(!ioctx);
901         dir = scst_cmd_get_data_direction(scmnd);
902         if (dir != SCST_DATA_NONE) {
903                 dma_unmap_sg(sdev->device->dma_device,
904                              scst_cmd_get_sg(scmnd),
905                              scst_cmd_get_sg_cnt(scmnd),
906                              scst_to_tgt_dma_dir(dir));
907
908 #if 1
909                 switch (scmnd->state) {
910                 case SCST_CMD_STATE_DATA_WAIT:
911                         WARN_ON(ioctx->state != SRPT_STATE_NEED_DATA);
912                         break;
913                 case SCST_CMD_STATE_XMIT_WAIT:
914                         WARN_ON(ioctx->state != SRPT_STATE_PROCESSED);
915                         break;
916                 default:
917                         WARN_ON(ioctx->state == SRPT_STATE_NEED_DATA ||
918                                 ioctx->state == SRPT_STATE_PROCESSED);
919                 }
920 #endif
921
922                 if (ioctx->state == SRPT_STATE_NEED_DATA) {
923                         scst_rx_data(scmnd,
924                                      tell_initiator ? SCST_RX_STATUS_ERROR
925                                      : SCST_RX_STATUS_ERROR_FATAL,
926                                      SCST_CONTEXT_THREAD);
927                         goto out;
928                 } else if (ioctx->state == SRPT_STATE_PROCESSED)
929                         ;
930                 else {
931                         printk(KERN_ERR PFX
932                                "unexpected cmd state %d (SCST) %d (SRPT)\n",
933                                scmnd->state, ioctx->state);
934                         WARN_ON("unexpected cmd state");
935                 }
936         }
937
938         scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_FAILED);
939         scst_tgt_cmd_done(scmnd, scst_estimate_context());
940 out:
941         return;
942 }
943
944 static void srpt_handle_err_comp(struct srpt_rdma_ch *ch, struct ib_wc *wc)
945 {
946         struct srpt_ioctx *ioctx;
947         struct srpt_device *sdev = ch->sport->sdev;
948
949         if (wc->wr_id & SRPT_OP_RECV) {
950                 ioctx = sdev->ioctx_ring[wc->wr_id & ~SRPT_OP_RECV];
951                 printk(KERN_ERR PFX "This is serious - SRQ is in bad state\n");
952         } else {
953                 ioctx = sdev->ioctx_ring[wc->wr_id];
954
955                 if (ioctx->scmnd)
956                         srpt_abort_scst_cmd(sdev, ioctx->scmnd, true);
957                 else
958                         srpt_reset_ioctx(ch, ioctx);
959         }
960 }
961
962 static void srpt_handle_send_comp(struct srpt_rdma_ch *ch,
963                                   struct srpt_ioctx *ioctx,
964                                   enum scst_exec_context context)
965 {
966         if (ioctx->scmnd) {
967                 scst_data_direction dir =
968                         scst_cmd_get_data_direction(ioctx->scmnd);
969
970                 if (dir != SCST_DATA_NONE)
971                         dma_unmap_sg(ch->sport->sdev->device->dma_device,
972                                      scst_cmd_get_sg(ioctx->scmnd),
973                                      scst_cmd_get_sg_cnt(ioctx->scmnd),
974                                      scst_to_tgt_dma_dir(dir));
975
976                 scst_tgt_cmd_done(ioctx->scmnd, context);
977         } else
978                 srpt_reset_ioctx(ch, ioctx);
979 }
980
981 static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch,
982                                   struct srpt_ioctx *ioctx)
983 {
984         if (!ioctx->scmnd) {
985                 srpt_reset_ioctx(ch, ioctx);
986                 return;
987         }
988
989         if (scst_cmd_get_data_direction(ioctx->scmnd) == SCST_DATA_WRITE)
990                 scst_rx_data(ioctx->scmnd, SCST_RX_STATUS_SUCCESS,
991                         scst_estimate_context());
992 }
993
994 /**
995  * Build an SRP_RSP response PDU.
996  * @ch: RDMA channel through which the request has been received.
997  * @ioctx: I/O context in which the SRP_RSP PDU will be built.
998  * @s_key: sense key that will be stored in the response.
999  * @s_code: value that will be stored in the asc_ascq field of the sense data.
1000  * @tag: tag of the request for which this response is being generated.
1001  *
1002  * Returns the size in bytes of the SRP_RSP response PDU.
1003  *
1004  * An SRP_RSP PDU contains a SCSI status or service response. See also
1005  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP PDU.
1006  * See also SPC-2 for more information about sense data.
1007  */
1008 static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
1009                               struct srpt_ioctx *ioctx, u8 s_key, u8 s_code,
1010                               u64 tag)
1011 {
1012         struct srp_rsp *srp_rsp;
1013         struct sense_data *sense;
1014         int limit_delta;
1015         int sense_data_len = 0;
1016
1017         srp_rsp = ioctx->buf;
1018         memset(srp_rsp, 0, sizeof *srp_rsp);
1019
1020         limit_delta = atomic_read(&ch->req_lim_delta);
1021         atomic_sub(limit_delta, &ch->req_lim_delta);
1022
1023         srp_rsp->opcode = SRP_RSP;
1024         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1025         srp_rsp->tag = tag;
1026
1027         if (s_key != NO_SENSE) {
1028                 sense_data_len = sizeof *sense + (sizeof *sense % 4);
1029                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1030                 srp_rsp->status = SAM_STAT_CHECK_CONDITION;
1031                 srp_rsp->sense_data_len = cpu_to_be32(sense_data_len);
1032
1033                 sense = (struct sense_data *)(srp_rsp + 1);
1034                 sense->err_code = 0x70;
1035                 sense->key = s_key;
1036                 sense->asc_ascq = s_code;
1037         }
1038
1039         return sizeof(*srp_rsp) + sense_data_len;
1040 }
1041
1042 /**
1043  * Build a task management response, which is a specific SRP_RSP response PDU.
1044  * @ch: RDMA channel through which the request has been received.
1045  * @ioctx: I/O context in which the SRP_RSP PDU will be built.
1046  * @rsp_code: RSP_CODE that will be stored in the response.
1047  * @tag: tag of the request for which this response is being generated.
1048  *
1049  * Returns the size in bytes of the SRP_RSP response PDU.
1050  *
1051  * An SRP_RSP PDU contains a SCSI status or service response. See also
1052  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP PDU.
1053  */
1054 static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
1055                                   struct srpt_ioctx *ioctx, u8 rsp_code,
1056                                   u64 tag)
1057 {
1058         struct srp_rsp *srp_rsp;
1059         int limit_delta;
1060         int resp_data_len = 0;
1061
1062         dma_sync_single_for_cpu(ch->sport->sdev->device->dma_device, ioctx->dma,
1063                                 MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
1064
1065         srp_rsp = ioctx->buf;
1066         memset(srp_rsp, 0, sizeof *srp_rsp);
1067
1068         limit_delta = atomic_read(&ch->req_lim_delta);
1069         atomic_sub(limit_delta, &ch->req_lim_delta);
1070
1071         srp_rsp->opcode = SRP_RSP;
1072         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1073         srp_rsp->tag = tag;
1074
1075         if (rsp_code != SRP_TSK_MGMT_SUCCESS) {
1076                 resp_data_len = 4;
1077                 srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
1078                 srp_rsp->resp_data_len = cpu_to_be32(resp_data_len);
1079                 srp_rsp->data[3] = rsp_code;
1080         }
1081
1082         return sizeof(*srp_rsp) + resp_data_len;
1083 }
1084
1085 /*
1086  * Process SRP_CMD.
1087  */
1088 static int srpt_handle_cmd(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
1089 {
1090         struct scst_cmd *scmnd;
1091         struct srp_cmd *srp_cmd;
1092         struct srp_rsp *srp_rsp;
1093         scst_data_direction dir = SCST_DATA_NONE;
1094         int indirect_desc = 0;
1095         int ret;
1096         unsigned long flags;
1097
1098         srp_cmd = ioctx->buf;
1099         srp_rsp = ioctx->buf;
1100
1101         if (srp_cmd->buf_fmt) {
1102                 ret = srpt_get_desc_tbl(ioctx, srp_cmd, &indirect_desc);
1103                 if (ret) {
1104                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1105                                            NO_ADD_SENSE, srp_cmd->tag);
1106                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1107                         goto err;
1108                 }
1109
1110                 if (indirect_desc) {
1111                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1112                                            NO_ADD_SENSE, srp_cmd->tag);
1113                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1114                         goto err;
1115                 }
1116
1117                 if (srp_cmd->buf_fmt & 0xf)
1118                         dir = SCST_DATA_READ;
1119                 else if (srp_cmd->buf_fmt >> 4)
1120                         dir = SCST_DATA_WRITE;
1121                 else
1122                         dir = SCST_DATA_NONE;
1123         } else
1124                 dir = SCST_DATA_NONE;
1125
1126         scmnd = scst_rx_cmd(ch->scst_sess, (u8 *) &srp_cmd->lun,
1127                             sizeof srp_cmd->lun, srp_cmd->cdb, 16,
1128                             thread ? SCST_NON_ATOMIC : SCST_ATOMIC);
1129         if (!scmnd) {
1130                 srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1131                                    NO_ADD_SENSE, srp_cmd->tag);
1132                 srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1133                 goto err;
1134         }
1135
1136         ioctx->scmnd = scmnd;
1137
1138         switch (srp_cmd->task_attr) {
1139         case SRP_CMD_HEAD_OF_Q:
1140                 scmnd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1141                 break;
1142         case SRP_CMD_ORDERED_Q:
1143                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1144                 break;
1145         case SRP_CMD_SIMPLE_Q:
1146                 scmnd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1147                 break;
1148         case SRP_CMD_ACA:
1149                 scmnd->queue_type = SCST_CMD_QUEUE_ACA;
1150                 break;
1151         default:
1152                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1153                 break;
1154         }
1155
1156         scst_cmd_set_tag(scmnd, srp_cmd->tag);
1157         scst_cmd_set_tgt_priv(scmnd, ioctx);
1158         scst_cmd_set_expected(scmnd, dir, ioctx->data_len);
1159
1160         spin_lock_irqsave(&ch->spinlock, flags);
1161         list_add_tail(&ioctx->scmnd_list, &ch->active_scmnd_list);
1162         ch->active_scmnd_cnt++;
1163         spin_unlock_irqrestore(&ch->spinlock, flags);
1164
1165         scst_cmd_init_done(scmnd, scst_estimate_context());
1166
1167         return 0;
1168
1169 err:
1170         WARN_ON(srp_rsp->opcode != SRP_RSP);
1171
1172         return -1;
1173 }
1174
1175 /*
1176  * Process an SRP_TSK_MGMT request PDU.
1177  *
1178  * Returns 0 upon success and -1 upon failure.
1179  *
1180  * Each task management function is performed by calling one of the
1181  * scst_rx_mgmt_fn*() functions. These functions will either report failure
1182  * or process the task management function asynchronously. The function
1183  * srpt_tsk_mgmt_done() will be called by the SCST core upon completion of the
1184  * task management function. When srpt_handle_tsk_mgmt() reports failure
1185  * (i.e. returns -1) a response PDU will have been built in ioctx->buf. This
1186  * PDU has to be sent back by the caller.
1187  *
1188  * For more information about SRP_TSK_MGMT PDU's, see also section 6.7 in
1189  * the T10 SRP r16a document.
1190  */
1191 static int srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
1192                                 struct srpt_ioctx *ioctx)
1193 {
1194         struct srp_tsk_mgmt *srp_tsk;
1195         struct srpt_mgmt_ioctx *mgmt_ioctx;
1196         int ret;
1197
1198         srp_tsk = ioctx->buf;
1199
1200         TRACE_DBG("recv_tsk_mgmt= %d for task_tag= %lld"
1201                   " using tag= %lld cm_id= %p sess= %p",
1202                   srp_tsk->tsk_mgmt_func,
1203                   (unsigned long long) srp_tsk->task_tag,
1204                   (unsigned long long) srp_tsk->tag,
1205                   ch->cm_id, ch->scst_sess);
1206
1207         mgmt_ioctx = kmalloc(sizeof *mgmt_ioctx, GFP_ATOMIC);
1208         if (!mgmt_ioctx) {
1209                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1210                                        srp_tsk->tag);
1211                 goto err;
1212         }
1213
1214         mgmt_ioctx->ioctx = ioctx;
1215         mgmt_ioctx->ch = ch;
1216         mgmt_ioctx->tag = srp_tsk->tag;
1217
1218         switch (srp_tsk->tsk_mgmt_func) {
1219         case SRP_TSK_ABORT_TASK:
1220                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK");
1221                 ret = scst_rx_mgmt_fn_tag(ch->scst_sess,
1222                                           SCST_ABORT_TASK,
1223                                           srp_tsk->task_tag,
1224                                           thread ?
1225                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1226                                           mgmt_ioctx);
1227                 break;
1228         case SRP_TSK_ABORT_TASK_SET:
1229                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK_SET");
1230                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1231                                           SCST_ABORT_TASK_SET,
1232                                           (u8 *) &srp_tsk->lun,
1233                                           sizeof srp_tsk->lun,
1234                                           thread ?
1235                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1236                                           mgmt_ioctx);
1237                 break;
1238         case SRP_TSK_CLEAR_TASK_SET:
1239                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_TASK_SET");
1240                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1241                                           SCST_CLEAR_TASK_SET,
1242                                           (u8 *) &srp_tsk->lun,
1243                                           sizeof srp_tsk->lun,
1244                                           thread ?
1245                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1246                                           mgmt_ioctx);
1247                 break;
1248         case SRP_TSK_LUN_RESET:
1249                 TRACE_DBG("%s", "Processing SRP_TSK_LUN_RESET");
1250                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1251                                           SCST_LUN_RESET,
1252                                           (u8 *) &srp_tsk->lun,
1253                                           sizeof srp_tsk->lun,
1254                                           thread ?
1255                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1256                                           mgmt_ioctx);
1257                 break;
1258         case SRP_TSK_CLEAR_ACA:
1259                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_ACA");
1260                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1261                                           SCST_CLEAR_ACA,
1262                                           (u8 *) &srp_tsk->lun,
1263                                           sizeof srp_tsk->lun,
1264                                           thread ?
1265                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1266                                           mgmt_ioctx);
1267                 break;
1268         default:
1269                 TRACE_DBG("%s", "Unsupported task management function.");
1270                 srpt_build_tskmgmt_rsp(ch, ioctx,
1271                                        SRP_TSK_MGMT_FUNC_NOT_SUPP,
1272                                        srp_tsk->tag);
1273                 goto err;
1274         }
1275
1276         if (ret) {
1277                 TRACE_DBG("%s", "Processing task management function failed.");
1278                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1279                                        srp_tsk->tag);
1280                 goto err;
1281         }
1282
1283         WARN_ON(srp_tsk->opcode == SRP_RSP);
1284
1285         return 0;
1286
1287 err:
1288         WARN_ON(srp_tsk->opcode != SRP_RSP);
1289
1290         kfree(mgmt_ioctx);
1291         return -1;
1292 }
1293
1294 /**
1295  * Process a receive completion event.
1296  * @ch: RDMA channel for which the completion event has been received.
1297  * @ioctx: SRPT I/O context for which the completion event has been received.
1298  */
1299 static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
1300                                struct srpt_ioctx *ioctx)
1301 {
1302         struct srp_cmd *srp_cmd;
1303         struct srp_rsp *srp_rsp;
1304         unsigned long flags;
1305         int len;
1306
1307         spin_lock_irqsave(&ch->spinlock, flags);
1308         if (ch->state != RDMA_CHANNEL_LIVE) {
1309                 if (ch->state == RDMA_CHANNEL_CONNECTING) {
1310                         list_add_tail(&ioctx->wait_list, &ch->cmd_wait_list);
1311                         spin_unlock_irqrestore(&ch->spinlock, flags);
1312                         return;
1313                 } else {
1314                         spin_unlock_irqrestore(&ch->spinlock, flags);
1315                         srpt_reset_ioctx(ch, ioctx);
1316                         return;
1317                 }
1318         }
1319         spin_unlock_irqrestore(&ch->spinlock, flags);
1320
1321         dma_sync_single_for_cpu(ch->sport->sdev->device->dma_device, ioctx->dma,
1322                                 MAX_MESSAGE_SIZE, DMA_FROM_DEVICE);
1323
1324         ioctx->data_len = 0;
1325         ioctx->n_rbuf = 0;
1326         ioctx->rbufs = NULL;
1327         ioctx->n_rdma = 0;
1328         ioctx->n_rdma_ius = 0;
1329         ioctx->rdma_ius = NULL;
1330         ioctx->scmnd = NULL;
1331         ioctx->state = SRPT_STATE_NEW;
1332
1333         srp_cmd = ioctx->buf;
1334         srp_rsp = ioctx->buf;
1335
1336         switch (srp_cmd->opcode) {
1337         case SRP_CMD:
1338                 if (srpt_handle_cmd(ch, ioctx) < 0)
1339                         goto err;
1340                 break;
1341
1342         case SRP_TSK_MGMT:
1343                 if (srpt_handle_tsk_mgmt(ch, ioctx) < 0)
1344                         goto err;
1345                 break;
1346
1347         case SRP_I_LOGOUT:
1348         case SRP_AER_REQ:
1349         default:
1350                 srpt_build_cmd_rsp(ch, ioctx, ILLEGAL_REQUEST, INVALID_CDB,
1351                                    srp_cmd->tag);
1352                 goto err;
1353         }
1354
1355         dma_sync_single_for_device(ch->sport->sdev->device->dma_device,
1356                                    ioctx->dma, MAX_MESSAGE_SIZE,
1357                                    DMA_FROM_DEVICE);
1358
1359         return;
1360
1361 err:
1362         WARN_ON(srp_rsp->opcode != SRP_RSP);
1363         len = (sizeof *srp_rsp) + be32_to_cpu(srp_rsp->sense_data_len);
1364
1365         if (ch->state != RDMA_CHANNEL_LIVE) {
1366                 /* Give up if another thread modified the channel state. */
1367                 printk(KERN_ERR PFX "%s: channel is in state %d",
1368                        __func__, ch->state);
1369                 srpt_reset_ioctx(ch, ioctx);
1370         } else if (srpt_post_send(ch, ioctx, len)) {
1371                 printk(KERN_ERR PFX "%s: sending SRP_RSP PDU failed",
1372                        __func__);
1373                 srpt_reset_ioctx(ch, ioctx);
1374         }
1375 }
1376
1377 /*
1378  * Returns true if the ioctx list is non-empty or if the ib_srpt kernel thread
1379  * should stop.
1380  * @pre thread != 0
1381  */
1382 static inline int srpt_test_ioctx_list(void)
1383 {
1384         int res = (!list_empty(&srpt_thread.thread_ioctx_list) ||
1385                    unlikely(kthread_should_stop()));
1386         return res;
1387 }
1388
1389 /*
1390  * Add 'ioctx' to the tail of the ioctx list and wake up the kernel thread.
1391  *
1392  * @pre thread != 0
1393  */
1394 static inline void srpt_schedule_thread(struct srpt_ioctx *ioctx)
1395 {
1396         unsigned long flags;
1397
1398         spin_lock_irqsave(&srpt_thread.thread_lock, flags);
1399         list_add_tail(&ioctx->comp_list, &srpt_thread.thread_ioctx_list);
1400         spin_unlock_irqrestore(&srpt_thread.thread_lock, flags);
1401         wake_up(&ioctx_list_waitQ);
1402 }
1403
1404 /**
1405  * InfiniBand completion queue callback function.
1406  * @cq: completion queue.
1407  * @ctx: completion queue context, which was passed as the fourth argument of
1408  *       the function ib_create_cq().
1409  */
1410 static void srpt_completion(struct ib_cq *cq, void *ctx)
1411 {
1412         struct srpt_rdma_ch *ch = ctx;
1413         struct srpt_device *sdev = ch->sport->sdev;
1414         struct ib_wc wc;
1415         struct srpt_ioctx *ioctx;
1416
1417         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1418         while (ib_poll_cq(ch->cq, 1, &wc) > 0) {
1419                 if (wc.status) {
1420                         printk(KERN_ERR PFX "failed %s status= %d\n",
1421                                wc.wr_id & SRPT_OP_RECV ? "receive" : "send",
1422                                wc.status);
1423                         srpt_handle_err_comp(ch, &wc);
1424                         break;
1425                 }
1426
1427                 if (wc.wr_id & SRPT_OP_RECV) {
1428                         ioctx = sdev->ioctx_ring[wc.wr_id & ~SRPT_OP_RECV];
1429                         if (thread) {
1430                                 ioctx->ch = ch;
1431                                 ioctx->op = IB_WC_RECV;
1432                                 srpt_schedule_thread(ioctx);
1433                         } else
1434                                 srpt_handle_new_iu(ch, ioctx);
1435                         continue;
1436                 } else
1437                         ioctx = sdev->ioctx_ring[wc.wr_id];
1438
1439                 if (thread) {
1440                         ioctx->ch = ch;
1441                         ioctx->op = wc.opcode;
1442                         srpt_schedule_thread(ioctx);
1443                 } else {
1444                         switch (wc.opcode) {
1445                         case IB_WC_SEND:
1446                                 srpt_handle_send_comp(ch, ioctx,
1447                                         scst_estimate_context());
1448                                 break;
1449                         case IB_WC_RDMA_WRITE:
1450                         case IB_WC_RDMA_READ:
1451                                 srpt_handle_rdma_comp(ch, ioctx);
1452                                 break;
1453                         default:
1454                                 break;
1455                         }
1456                 }
1457         }
1458 }
1459
1460 /*
1461  * Create a completion queue on the specified device.
1462  */
1463 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
1464 {
1465         struct ib_qp_init_attr *qp_init;
1466         struct srpt_device *sdev = ch->sport->sdev;
1467         int cqe;
1468         int ret;
1469
1470         qp_init = kzalloc(sizeof *qp_init, GFP_KERNEL);
1471         if (!qp_init)
1472                 return -ENOMEM;
1473
1474         /* Create a completion queue (CQ). */
1475
1476         cqe = SRPT_RQ_SIZE + SRPT_SQ_SIZE - 1;
1477 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(RHEL_RELEASE_CODE)
1478         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe);
1479 #else
1480         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe, 0);
1481 #endif
1482         if (IS_ERR(ch->cq)) {
1483                 ret = PTR_ERR(ch->cq);
1484                 printk(KERN_ERR PFX "failed to create_cq cqe= %d ret= %d\n",
1485                         cqe, ret);
1486                 goto out;
1487         }
1488
1489         /* Request completion notification. */
1490
1491         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1492
1493         /* Create a queue pair (QP). */
1494
1495         qp_init->qp_context = (void *)ch;
1496         qp_init->event_handler = srpt_qp_event;
1497         qp_init->send_cq = ch->cq;
1498         qp_init->recv_cq = ch->cq;
1499         qp_init->srq = sdev->srq;
1500         qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
1501         qp_init->qp_type = IB_QPT_RC;
1502         qp_init->cap.max_send_wr = SRPT_SQ_SIZE;
1503         qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE;
1504
1505         ch->qp = ib_create_qp(sdev->pd, qp_init);
1506         if (IS_ERR(ch->qp)) {
1507                 ret = PTR_ERR(ch->qp);
1508                 ib_destroy_cq(ch->cq);
1509                 printk(KERN_ERR PFX "failed to create_qp ret= %d\n", ret);
1510                 goto out;
1511         }
1512
1513         TRACE_DBG("%s: max_cqe= %d max_sge= %d cm_id= %p",
1514                __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
1515                ch->cm_id);
1516
1517         /* Modify the attributes and the state of queue pair ch->qp. */
1518
1519         ret = srpt_init_ch_qp(ch, ch->qp);
1520         if (ret) {
1521                 ib_destroy_qp(ch->qp);
1522                 ib_destroy_cq(ch->cq);
1523                 goto out;
1524         }
1525
1526         atomic_set(&ch->req_lim_delta, SRPT_RQ_SIZE);
1527 out:
1528         kfree(qp_init);
1529         return ret;
1530 }
1531
1532 /**
1533  * Look up the RDMA channel that corresponds to the specified cm_id.
1534  *
1535  * Return NULL if no matching RDMA channel has been found.
1536  */
1537 static struct srpt_rdma_ch *srpt_find_channel(struct ib_cm_id *cm_id, bool del)
1538 {
1539         struct srpt_device *sdev = cm_id->context;
1540         struct srpt_rdma_ch *ch;
1541
1542         spin_lock_irq(&sdev->spinlock);
1543         list_for_each_entry(ch, &sdev->rch_list, list) {
1544                 if (ch->cm_id == cm_id) {
1545                         if (del)
1546                                 list_del(&ch->list);
1547                         spin_unlock_irq(&sdev->spinlock);
1548                         return ch;
1549                 }
1550         }
1551
1552         spin_unlock_irq(&sdev->spinlock);
1553
1554         return NULL;
1555 }
1556
1557 /**
1558  * Release all resources associated with the specified RDMA channel.
1559  *
1560  * Note: the caller must have removed the channel from the channel list
1561  * before calling this function.
1562  */
1563 static void srpt_release_channel(struct srpt_rdma_ch *ch, int destroy_cmid)
1564 {
1565         TRACE_ENTRY();
1566
1567         WARN_ON(srpt_find_channel(ch->cm_id, false) == ch);
1568
1569         if (ch->cm_id && destroy_cmid) {
1570                 TRACE_DBG("%s: destroy cm_id= %p", __func__, ch->cm_id);
1571                 ib_destroy_cm_id(ch->cm_id);
1572                 ch->cm_id = NULL;
1573         }
1574
1575         ib_destroy_qp(ch->qp);
1576         ib_destroy_cq(ch->cq);
1577
1578         if (ch->scst_sess) {
1579                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1580
1581                 TRACE_DBG("%s: release sess= %p sess_name= %s active_cmd= %d",
1582                           __func__, ch->scst_sess, ch->sess_name,
1583                           ch->active_scmnd_cnt);
1584
1585                 spin_lock_irq(&ch->spinlock);
1586                 list_for_each_entry_safe(ioctx, ioctx_tmp,
1587                                          &ch->active_scmnd_list, scmnd_list) {
1588                         spin_unlock_irq(&ch->spinlock);
1589
1590                         if (ioctx->scmnd)
1591                                 srpt_abort_scst_cmd(ch->sport->sdev,
1592                                                     ioctx->scmnd, true);
1593
1594                         spin_lock_irq(&ch->spinlock);
1595                 }
1596                 WARN_ON(!list_empty(&ch->active_scmnd_list));
1597                 WARN_ON(ch->active_scmnd_cnt != 0);
1598                 spin_unlock_irq(&ch->spinlock);
1599
1600                 scst_unregister_session(ch->scst_sess, 0, NULL);
1601                 ch->scst_sess = NULL;
1602         }
1603
1604         kfree(ch);
1605
1606         TRACE_EXIT();
1607 }
1608
1609 static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
1610                             struct ib_cm_req_event_param *param,
1611                             void *private_data)
1612 {
1613         struct srpt_device *sdev = cm_id->context;
1614         struct srp_login_req *req;
1615         struct srp_login_rsp *rsp;
1616         struct srp_login_rej *rej;
1617         struct ib_cm_rep_param *rep_param;
1618         struct srpt_rdma_ch *ch, *tmp_ch;
1619         u32 it_iu_len;
1620         u64 ioc_guid;
1621         int ret = 0;
1622
1623         if (!sdev || !private_data)
1624                 return -EINVAL;
1625
1626         rsp = kzalloc(sizeof *rsp, GFP_KERNEL);
1627         rej = kzalloc(sizeof *rej, GFP_KERNEL);
1628         rep_param = kzalloc(sizeof *rep_param, GFP_KERNEL);
1629
1630         if (!rsp || !rej || !rep_param) {
1631                 ret = -ENOMEM;
1632                 goto out;
1633         }
1634
1635         req = (struct srp_login_req *)private_data;
1636
1637         it_iu_len = be32_to_cpu(req->req_it_iu_len);
1638
1639         TRACE_DBG("Host login i_port_id=0x%llx:0x%llx t_port_id=0x%llx:0x%llx"
1640             " it_iu_len=%d",
1641             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[0]),
1642             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[8]),
1643             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[0]),
1644             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[8]),
1645             it_iu_len);
1646
1647         if (it_iu_len > MAX_MESSAGE_SIZE || it_iu_len < 64) {
1648                 rej->reason =
1649                     cpu_to_be32(SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
1650                 ret = -EINVAL;
1651                 TRACE_DBG("Reject invalid it_iu_len=%d", it_iu_len);
1652                 goto reject;
1653         }
1654
1655         if ((req->req_flags & 0x3) == SRP_MULTICHAN_SINGLE) {
1656                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
1657
1658                 spin_lock_irq(&sdev->spinlock);
1659
1660                 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
1661                         if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
1662                             && !memcmp(ch->t_port_id, req->target_port_id, 16)
1663                             && param->port == ch->sport->port
1664                             && param->listen_id == ch->sport->sdev->cm_id
1665                             && ch->cm_id) {
1666                                 enum rdma_ch_state prev_state;
1667
1668                                 /* found an existing channel */
1669                                 TRACE_DBG("Found existing channel name= %s"
1670                                           " cm_id= %p state= %d",
1671                                           ch->sess_name, ch->cm_id, ch->state);
1672
1673                                 prev_state = ch->state;
1674                                 if (ch->state == RDMA_CHANNEL_LIVE)
1675                                         ch->state = RDMA_CHANNEL_DISCONNECTING;
1676                                 else if (ch->state == RDMA_CHANNEL_CONNECTING)
1677                                         list_del(&ch->list);
1678
1679                                 spin_unlock_irq(&sdev->spinlock);
1680
1681                                 rsp->rsp_flags =
1682                                         SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
1683
1684                                 if (prev_state == RDMA_CHANNEL_LIVE)
1685                                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
1686                                 else if (prev_state ==
1687                                          RDMA_CHANNEL_CONNECTING) {
1688                                         ib_send_cm_rej(ch->cm_id,
1689                                                        IB_CM_REJ_NO_RESOURCES,
1690                                                        NULL, 0, NULL, 0);
1691                                         srpt_release_channel(ch, 1);
1692                                 }
1693
1694                                 spin_lock_irq(&sdev->spinlock);
1695                         }
1696                 }
1697
1698                 spin_unlock_irq(&sdev->spinlock);
1699
1700         } else
1701                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
1702
1703         ioc_guid = srpt_get_ioc_guid(sdev->device);
1704
1705         if (((u64) (*(u64 *) req->target_port_id) != cpu_to_be64(ioc_guid)) ||
1706             ((u64) (*(u64 *) (req->target_port_id + 8)) !=
1707              cpu_to_be64(ioc_guid))) {
1708                 rej->reason =
1709                     cpu_to_be32(SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
1710                 ret = -ENOMEM;
1711                 TRACE_DBG("%s", "Reject invalid target_port_id");
1712                 goto reject;
1713         }
1714
1715         ch = kzalloc(sizeof *ch, GFP_KERNEL);
1716         if (!ch) {
1717                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1718                 TRACE_DBG("%s", "Reject failed allocate rdma_ch");
1719                 ret = -ENOMEM;
1720                 goto reject;
1721         }
1722
1723         spin_lock_init(&ch->spinlock);
1724         memcpy(ch->i_port_id, req->initiator_port_id, 16);
1725         memcpy(ch->t_port_id, req->target_port_id, 16);
1726         ch->sport = &sdev->port[param->port - 1];
1727         ch->cm_id = cm_id;
1728         ch->state = RDMA_CHANNEL_CONNECTING;
1729         INIT_LIST_HEAD(&ch->cmd_wait_list);
1730         INIT_LIST_HEAD(&ch->active_scmnd_list);
1731
1732         ret = srpt_create_ch_ib(ch);
1733         if (ret) {
1734                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1735                 TRACE_DBG("%s", "Reject failed to create rdma_ch");
1736                 goto free_ch;
1737         }
1738
1739         ret = srpt_ch_qp_rtr(ch, ch->qp);
1740         if (ret) {
1741                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1742                 TRACE_DBG("Reject failed qp to rtr/rts ret=%d", ret);
1743                 goto destroy_ib;
1744         }
1745
1746         snprintf(ch->sess_name, sizeof(ch->sess_name),
1747                  "0x%016llx%016llx",
1748                  (unsigned long long)be64_to_cpu(*(u64 *)ch->i_port_id),
1749                  (unsigned long long)be64_to_cpu(*(u64 *)(ch->i_port_id + 8)));
1750
1751         TRACE_DBG("registering session %s", ch->sess_name);
1752
1753         BUG_ON(!sdev->scst_tgt);
1754         ch->scst_sess = scst_register_session(sdev->scst_tgt, 0, ch->sess_name,
1755                                   NULL, NULL);
1756         if (!ch->scst_sess) {
1757                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1758                 TRACE_DBG("%s", "Failed to create scst sess");
1759                 goto destroy_ib;
1760         }
1761
1762         TRACE_DBG("Establish connection sess=%p name=%s cm_id=%p",
1763                   ch->scst_sess, ch->sess_name, ch->cm_id);
1764
1765         scst_sess_set_tgt_priv(ch->scst_sess, ch);
1766
1767         /* create srp_login_response */
1768         rsp->opcode = SRP_LOGIN_RSP;
1769         rsp->tag = req->tag;
1770         rsp->max_it_iu_len = req->req_it_iu_len;
1771         rsp->max_ti_iu_len = req->req_it_iu_len;
1772         rsp->buf_fmt =
1773             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1774         rsp->req_lim_delta = cpu_to_be32(SRPT_RQ_SIZE);
1775         atomic_set(&ch->req_lim_delta, 0);
1776
1777         /* create cm reply */
1778         rep_param->qp_num = ch->qp->qp_num;
1779         rep_param->private_data = (void *)rsp;
1780         rep_param->private_data_len = sizeof *rsp;
1781         rep_param->rnr_retry_count = 7;
1782         rep_param->flow_control = 1;
1783         rep_param->failover_accepted = 0;
1784         rep_param->srq = 1;
1785         rep_param->responder_resources = 4;
1786         rep_param->initiator_depth = 4;
1787
1788         ret = ib_send_cm_rep(cm_id, rep_param);
1789         if (ret)
1790                 goto release_channel;
1791
1792         spin_lock_irq(&sdev->spinlock);
1793         list_add_tail(&ch->list, &sdev->rch_list);
1794         spin_unlock_irq(&sdev->spinlock);
1795
1796         goto out;
1797
1798 release_channel:
1799         scst_unregister_session(ch->scst_sess, 0, NULL);
1800         ch->scst_sess = NULL;
1801
1802 destroy_ib:
1803         ib_destroy_qp(ch->qp);
1804         ib_destroy_cq(ch->cq);
1805
1806 free_ch:
1807         kfree(ch);
1808
1809 reject:
1810         rej->opcode = SRP_LOGIN_REJ;
1811         rej->tag = req->tag;
1812         rej->buf_fmt =
1813             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1814
1815         ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
1816                              (void *)rej, sizeof *rej);
1817
1818 out:
1819         kfree(rep_param);
1820         kfree(rsp);
1821         kfree(rej);
1822
1823         return ret;
1824 }
1825
1826 /**
1827  * Release the channel with the specified cm_id.
1828  *
1829  * Returns one to indicate that the caller of srpt_cm_handler() should destroy
1830  * the cm_id.
1831  */
1832 static void srpt_find_and_release_channel(struct ib_cm_id *cm_id)
1833 {
1834         struct srpt_rdma_ch *ch;
1835
1836         ch = srpt_find_channel(cm_id, true);
1837         if (ch)
1838                 srpt_release_channel(ch, 0);
1839 }
1840
1841 static void srpt_cm_rej_recv(struct ib_cm_id *cm_id)
1842 {
1843         TRACE_DBG("%s: cm_id=%p", __func__, cm_id);
1844         srpt_find_and_release_channel(cm_id);
1845 }
1846
1847 /**
1848  * Process an IB_CM_RTU_RECEIVED or IB_CM_USER_ESTABLISHED event.
1849  *
1850  * An IB_CM_RTU_RECEIVED message indicates that the connection is established
1851  * and that the recipient may begin transmitting (RTU = ready to use).
1852  */
1853 static int srpt_cm_rtu_recv(struct ib_cm_id *cm_id)
1854 {
1855         struct srpt_rdma_ch *ch;
1856         int ret;
1857
1858         ch = srpt_find_channel(cm_id, false);
1859         if (!ch)
1860                 return -EINVAL;
1861
1862         if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_CONNECTING,
1863                                             RDMA_CHANNEL_LIVE)) {
1864                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1865
1866                 ret = srpt_ch_qp_rts(ch, ch->qp);
1867
1868                 list_for_each_entry_safe(ioctx, ioctx_tmp, &ch->cmd_wait_list,
1869                                          wait_list) {
1870                         list_del(&ioctx->wait_list);
1871                         srpt_handle_new_iu(ch, ioctx);
1872                 }
1873                 if (ret && srpt_test_and_set_channel_state(ch,
1874                                         RDMA_CHANNEL_LIVE,
1875                                         RDMA_CHANNEL_DISCONNECTING)) {
1876                         TRACE_DBG("cm_id=%p sess_name=%s state=%d",
1877                                   cm_id, ch->sess_name, ch->state);
1878                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
1879                 }
1880         } else if (ch->state == RDMA_CHANNEL_DISCONNECTING) {
1881                 TRACE_DBG("cm_id=%p sess_name=%s state=%d",
1882                           cm_id, ch->sess_name, ch->state);
1883                 ib_send_cm_dreq(ch->cm_id, NULL, 0);
1884                 ret = -EAGAIN;
1885         } else
1886                 ret = 0;
1887
1888         return ret;
1889 }
1890
1891 static void srpt_cm_timewait_exit(struct ib_cm_id *cm_id)
1892 {
1893         TRACE_DBG("%s: cm_id=%p", __func__, cm_id);
1894         srpt_find_and_release_channel(cm_id);
1895 }
1896
1897 static void srpt_cm_rep_error(struct ib_cm_id *cm_id)
1898 {
1899         TRACE_DBG("%s: cm_id=%p", __func__, cm_id);
1900         srpt_find_and_release_channel(cm_id);
1901 }
1902
1903 static int srpt_cm_dreq_recv(struct ib_cm_id *cm_id)
1904 {
1905         struct srpt_rdma_ch *ch;
1906
1907         ch = srpt_find_channel(cm_id, false);
1908         if (!ch)
1909                 return -EINVAL;
1910
1911         TRACE_DBG("%s: cm_id= %p ch->state= %d",
1912                  __func__, cm_id, ch->state);
1913
1914         switch (ch->state) {
1915         case RDMA_CHANNEL_LIVE:
1916         case RDMA_CHANNEL_CONNECTING:
1917                 ib_send_cm_drep(ch->cm_id, NULL, 0);
1918                 break;
1919         case RDMA_CHANNEL_DISCONNECTING:
1920         default:
1921                 break;
1922         }
1923
1924         return 0;
1925 }
1926
1927 static void srpt_cm_drep_recv(struct ib_cm_id *cm_id)
1928 {
1929         TRACE_DBG("%s: cm_id=%p", __func__, cm_id);
1930         srpt_find_and_release_channel(cm_id);
1931 }
1932
1933 /**
1934  * IB connection manager callback function.
1935  *
1936  * A non-zero return value will make the caller destroy the CM ID.
1937  *
1938  * Note: srpt_add_one passes a struct srpt_device* as the third argument to
1939  * the ib_create_cm_id() call.
1940  */
1941 static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
1942 {
1943         int ret = 0;
1944
1945         switch (event->event) {
1946         case IB_CM_REQ_RECEIVED:
1947                 ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd,
1948                                        event->private_data);
1949                 break;
1950         case IB_CM_REJ_RECEIVED:
1951                 srpt_cm_rej_recv(cm_id);
1952                 ret = -EINVAL;
1953                 break;
1954         case IB_CM_RTU_RECEIVED:
1955         case IB_CM_USER_ESTABLISHED:
1956                 ret = srpt_cm_rtu_recv(cm_id);
1957                 break;
1958         case IB_CM_DREQ_RECEIVED:
1959                 ret = srpt_cm_dreq_recv(cm_id);
1960                 break;
1961         case IB_CM_DREP_RECEIVED:
1962                 srpt_cm_drep_recv(cm_id);
1963                 ret = -EINVAL;
1964                 break;
1965         case IB_CM_TIMEWAIT_EXIT:
1966                 srpt_cm_timewait_exit(cm_id);
1967                 ret = -EINVAL;
1968                 break;
1969         case IB_CM_REP_ERROR:
1970                 srpt_cm_rep_error(cm_id);
1971                 ret = -EINVAL;
1972                 break;
1973         default:
1974                 break;
1975         }
1976
1977         return ret;
1978 }
1979
1980 static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
1981                                  struct srpt_ioctx *ioctx,
1982                                  struct scst_cmd *scmnd)
1983 {
1984         struct scatterlist *scat;
1985         scst_data_direction dir;
1986         struct rdma_iu *riu;
1987         struct srp_direct_buf *db;
1988         dma_addr_t dma_addr;
1989         struct ib_sge *sge;
1990         u64 raddr;
1991         u32 rsize;
1992         u32 tsize;
1993         u32 dma_len;
1994         int count, nrdma;
1995         int i, j, k;
1996
1997         scat = scst_cmd_get_sg(scmnd);
1998         dir = scst_cmd_get_data_direction(scmnd);
1999         count = dma_map_sg(ch->sport->sdev->device->dma_device, scat,
2000                            scst_cmd_get_sg_cnt(scmnd),
2001                            scst_to_tgt_dma_dir(dir));
2002         if (unlikely(!count))
2003                 return -EBUSY;
2004
2005         if (ioctx->rdma_ius && ioctx->n_rdma_ius)
2006                 nrdma = ioctx->n_rdma_ius;
2007         else {
2008                 nrdma = count / SRPT_DEF_SG_PER_WQE + ioctx->n_rbuf;
2009
2010                 ioctx->rdma_ius = kzalloc(nrdma * sizeof *riu,
2011                                           scst_cmd_atomic(scmnd)
2012                                           ? GFP_ATOMIC : GFP_KERNEL);
2013                 if (!ioctx->rdma_ius) {
2014                         dma_unmap_sg(ch->sport->sdev->device->dma_device,
2015                                      scat, scst_cmd_get_sg_cnt(scmnd),
2016                                      scst_to_tgt_dma_dir(dir));
2017                         return -ENOMEM;
2018                 }
2019
2020                 ioctx->n_rdma_ius = nrdma;
2021         }
2022
2023         db = ioctx->rbufs;
2024         tsize = (dir == SCST_DATA_READ) ?
2025                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2026         dma_len = sg_dma_len(&scat[0]);
2027         riu = ioctx->rdma_ius;
2028
2029         /*
2030          * For each remote desc - calculate the #ib_sge.
2031          * If #ib_sge < SRPT_DEF_SG_PER_WQE per rdma operation then
2032          *      each remote desc rdma_iu is required a rdma wr;
2033          * else
2034          *      we need to allocate extra rdma_iu to carry extra #ib_sge in
2035          *      another rdma wr
2036          */
2037         for (i = 0, j = 0;
2038              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2039                 rsize = be32_to_cpu(db->len);
2040                 raddr = be64_to_cpu(db->va);
2041                 riu->raddr = raddr;
2042                 riu->rkey = be32_to_cpu(db->key);
2043                 riu->sge_cnt = 0;
2044
2045                 /* calculate how many sge required for this remote_buf */
2046                 while (rsize > 0 && tsize > 0) {
2047
2048                         if (rsize >= dma_len) {
2049                                 tsize -= dma_len;
2050                                 rsize -= dma_len;
2051                                 raddr += dma_len;
2052
2053                                 if (tsize > 0) {
2054                                         ++j;
2055                                         if (j < count)
2056                                                 dma_len = sg_dma_len(&scat[j]);
2057                                 }
2058                         } else {
2059                                 tsize -= rsize;
2060                                 dma_len -= rsize;
2061                                 rsize = 0;
2062                         }
2063
2064                         ++riu->sge_cnt;
2065
2066                         if (rsize > 0 && riu->sge_cnt == SRPT_DEF_SG_PER_WQE) {
2067                                 riu->sge =
2068                                     kmalloc(riu->sge_cnt * sizeof *riu->sge,
2069                                             scst_cmd_atomic(scmnd)
2070                                             ? GFP_ATOMIC : GFP_KERNEL);
2071                                 if (!riu->sge)
2072                                         goto free_mem;
2073
2074                                 ++ioctx->n_rdma;
2075                                 ++riu;
2076                                 riu->sge_cnt = 0;
2077                                 riu->raddr = raddr;
2078                                 riu->rkey = be32_to_cpu(db->key);
2079                         }
2080                 }
2081
2082                 riu->sge = kmalloc(riu->sge_cnt * sizeof *riu->sge,
2083                                    scst_cmd_atomic(scmnd)
2084                                    ? GFP_ATOMIC : GFP_KERNEL);
2085
2086                 if (!riu->sge)
2087                         goto free_mem;
2088
2089                 ++ioctx->n_rdma;
2090         }
2091
2092         db = ioctx->rbufs;
2093         scat = scst_cmd_get_sg(scmnd);
2094         tsize = (dir == SCST_DATA_READ) ?
2095                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2096         riu = ioctx->rdma_ius;
2097         dma_len = sg_dma_len(&scat[0]);
2098         dma_addr = sg_dma_address(&scat[0]);
2099
2100         /* this second loop is really mapped sg_addres to rdma_iu->ib_sge */
2101         for (i = 0, j = 0;
2102              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2103                 rsize = be32_to_cpu(db->len);
2104                 sge = riu->sge;
2105                 k = 0;
2106
2107                 while (rsize > 0 && tsize > 0) {
2108                         sge->addr = dma_addr;
2109                         sge->lkey = ch->sport->sdev->mr->lkey;
2110
2111                         if (rsize >= dma_len) {
2112                                 sge->length =
2113                                         (tsize < dma_len) ? tsize : dma_len;
2114                                 tsize -= dma_len;
2115                                 rsize -= dma_len;
2116
2117                                 if (tsize > 0) {
2118                                         ++j;
2119                                         if (j < count) {
2120                                                 dma_len = sg_dma_len(&scat[j]);
2121                                                 dma_addr =
2122                                                     sg_dma_address(&scat[j]);
2123                                         }
2124                                 }
2125                         } else {
2126                                 sge->length = (tsize < rsize) ? tsize : rsize;
2127                                 tsize -= rsize;
2128                                 dma_len -= rsize;
2129                                 dma_addr += rsize;
2130                                 rsize = 0;
2131                         }
2132
2133                         ++k;
2134                         if (k == riu->sge_cnt && rsize > 0) {
2135                                 ++riu;
2136                                 sge = riu->sge;
2137                                 k = 0;
2138                         } else if (rsize > 0)
2139                                 ++sge;
2140                 }
2141         }
2142
2143         return 0;
2144
2145 free_mem:
2146         while (ioctx->n_rdma)
2147                 kfree(ioctx->rdma_ius[ioctx->n_rdma--].sge);
2148
2149         kfree(ioctx->rdma_ius);
2150
2151         dma_unmap_sg(ch->sport->sdev->device->dma_device,
2152                      scat, scst_cmd_get_sg_cnt(scmnd),
2153                      scst_to_tgt_dma_dir(dir));
2154
2155         return -ENOMEM;
2156 }
2157
2158 static int srpt_perform_rdmas(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2159                               scst_data_direction dir)
2160 {
2161         struct ib_send_wr wr;
2162         struct ib_send_wr *bad_wr;
2163         struct rdma_iu *riu;
2164         int i;
2165         int ret = 0;
2166
2167         riu = ioctx->rdma_ius;
2168         memset(&wr, 0, sizeof wr);
2169
2170         for (i = 0; i < ioctx->n_rdma; ++i, ++riu) {
2171                 wr.opcode = (dir == SCST_DATA_READ) ?
2172                     IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;
2173                 wr.next = NULL;
2174                 wr.wr_id = ioctx->index;
2175                 wr.wr.rdma.remote_addr = riu->raddr;
2176                 wr.wr.rdma.rkey = riu->rkey;
2177                 wr.num_sge = riu->sge_cnt;
2178                 wr.sg_list = riu->sge;
2179
2180                 /* only get completion event for the last rdma wr */
2181                 if (i == (ioctx->n_rdma - 1) && dir == SCST_DATA_WRITE)
2182                         wr.send_flags = IB_SEND_SIGNALED;
2183
2184                 ret = ib_post_send(ch->qp, &wr, &bad_wr);
2185                 if (ret)
2186                         break;
2187         }
2188
2189         return ret;
2190 }
2191
2192 /*
2193  * Start data reception. Must not block.
2194  */
2195 static int srpt_xfer_data(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2196                           struct scst_cmd *scmnd)
2197 {
2198         int ret;
2199
2200         ret = srpt_map_sg_to_ib_sge(ch, ioctx, scmnd);
2201         if (ret) {
2202                 printk(KERN_ERR PFX "%s[%d] ret=%d\n", __func__, __LINE__, ret);
2203                 ret = SCST_TGT_RES_QUEUE_FULL;
2204                 goto out;
2205         }
2206
2207         ret = srpt_perform_rdmas(ch, ioctx, scst_cmd_get_data_direction(scmnd));
2208         if (ret) {
2209                 printk(KERN_ERR PFX "%s[%d] ret=%d\n", __func__, __LINE__, ret);
2210                 if (ret == -EAGAIN || ret == -ENOMEM)
2211                         ret = SCST_TGT_RES_QUEUE_FULL;
2212                 else
2213                         ret = SCST_TGT_RES_FATAL_ERROR;
2214                 goto out;
2215         }
2216
2217         ret = SCST_TGT_RES_SUCCESS;
2218
2219 out:
2220         return ret;
2221 }
2222
2223 /*
2224  * Called by the SCST core to inform ib_srpt that data reception should start.
2225  * Must not block.
2226  */
2227 static int srpt_rdy_to_xfer(struct scst_cmd *scmnd)
2228 {
2229         struct srpt_rdma_ch *ch;
2230         struct srpt_ioctx *ioctx;
2231
2232         ioctx = scst_cmd_get_tgt_priv(scmnd);
2233         BUG_ON(!ioctx);
2234
2235         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2236         BUG_ON(!ch);
2237
2238         if (ch->state == RDMA_CHANNEL_DISCONNECTING)
2239                 return SCST_TGT_RES_FATAL_ERROR;
2240         else if (ch->state == RDMA_CHANNEL_CONNECTING)
2241                 return SCST_TGT_RES_QUEUE_FULL;
2242
2243         ioctx->state = SRPT_STATE_NEED_DATA;
2244
2245         return srpt_xfer_data(ch, ioctx, scmnd);
2246 }
2247
2248 /*
2249  * Called by the SCST core. Transmits the response buffer and status held in
2250  * 'scmnd'. Must not block.
2251  */
2252 static int srpt_xmit_response(struct scst_cmd *scmnd)
2253 {
2254         struct srpt_rdma_ch *ch;
2255         struct srpt_ioctx *ioctx;
2256         struct srp_rsp *srp_rsp;
2257         u64 tag;
2258         int ret = SCST_TGT_RES_SUCCESS;
2259         int dir;
2260         int status;
2261
2262         ioctx = scst_cmd_get_tgt_priv(scmnd);
2263         BUG_ON(!ioctx);
2264
2265         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2266         BUG_ON(!ch);
2267
2268         tag = scst_cmd_get_tag(scmnd);
2269
2270         if (ch->state != RDMA_CHANNEL_LIVE) {
2271                 printk(KERN_ERR PFX
2272                        "%s: tag= %lld channel in bad state %d\n",
2273                        __func__, (unsigned long long)tag, ch->state);
2274
2275                 if (ch->state == RDMA_CHANNEL_DISCONNECTING)
2276                         ret = SCST_TGT_RES_FATAL_ERROR;
2277                 else if (ch->state == RDMA_CHANNEL_CONNECTING)
2278                         ret = SCST_TGT_RES_QUEUE_FULL;
2279
2280                 if (unlikely(scst_cmd_aborted(scmnd)))
2281                         goto out_aborted;
2282
2283                 goto out;
2284         }
2285
2286         dma_sync_single_for_cpu(ch->sport->sdev->device->dma_device, ioctx->dma,
2287                                 MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
2288
2289         srp_rsp = ioctx->buf;
2290
2291         if (unlikely(scst_cmd_aborted(scmnd))) {
2292                 printk(KERN_ERR PFX
2293                        "%s: tag= %lld already get aborted\n",
2294                        __func__, (unsigned long long)tag);
2295                 goto out_aborted;
2296         }
2297
2298         dir = scst_cmd_get_data_direction(scmnd);
2299         status = scst_cmd_get_status(scmnd) & 0xff;
2300
2301         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE, NO_ADD_SENSE, tag);
2302
2303         if (SCST_SENSE_VALID(scst_cmd_get_sense_buffer(scmnd))) {
2304                 srp_rsp->sense_data_len = scst_cmd_get_sense_buffer_len(scmnd);
2305                 if (srp_rsp->sense_data_len >
2306                     (MAX_MESSAGE_SIZE - sizeof *srp_rsp))
2307                         srp_rsp->sense_data_len =
2308                             MAX_MESSAGE_SIZE - sizeof *srp_rsp;
2309
2310                 memcpy((u8 *) (srp_rsp + 1), scst_cmd_get_sense_buffer(scmnd),
2311                        srp_rsp->sense_data_len);
2312
2313                 srp_rsp->sense_data_len = cpu_to_be32(srp_rsp->sense_data_len);
2314                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
2315
2316                 if (!status)
2317                         status = SAM_STAT_CHECK_CONDITION;
2318         }
2319
2320         srp_rsp->status = status;
2321
2322         /* transfer read data if any */
2323         if (dir == SCST_DATA_READ && scst_cmd_get_resp_data_len(scmnd)) {
2324                 ret = srpt_xfer_data(ch, ioctx, scmnd);
2325                 if (ret != SCST_TGT_RES_SUCCESS) {
2326                         printk(KERN_ERR PFX
2327                                "%s: tag= %lld xfer_data failed\n",
2328                                __func__, (unsigned long long)tag);
2329                         goto out;
2330                 }
2331         }
2332
2333         ioctx->state = SRPT_STATE_PROCESSED;
2334
2335         if (srpt_post_send(ch, ioctx,
2336                            sizeof *srp_rsp +
2337                            be32_to_cpu(srp_rsp->sense_data_len))) {
2338                 printk(KERN_ERR PFX "%s: ch->state= %d tag= %lld\n",
2339                        __func__, ch->state,
2340                        (unsigned long long)tag);
2341                 ret = SCST_TGT_RES_FATAL_ERROR;
2342         }
2343
2344 out:
2345         return ret;
2346
2347 out_aborted:
2348         ret = SCST_TGT_RES_SUCCESS;
2349         scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_ABORTED);
2350         ioctx->state = SRPT_STATE_ABORTED;
2351         scst_tgt_cmd_done(scmnd, SCST_CONTEXT_SAME);
2352         goto out;
2353 }
2354
2355 /*
2356  * Called by the SCST core to inform ib_srpt that a received task management
2357  * function has been completed. Must not block.
2358  */
2359 static void srpt_tsk_mgmt_done(struct scst_mgmt_cmd *mcmnd)
2360 {
2361         struct srpt_rdma_ch *ch;
2362         struct srpt_mgmt_ioctx *mgmt_ioctx;
2363         struct srpt_ioctx *ioctx;
2364         int rsp_len;
2365
2366         mgmt_ioctx = scst_mgmt_cmd_get_tgt_priv(mcmnd);
2367         BUG_ON(!mgmt_ioctx);
2368
2369         ch = mgmt_ioctx->ch;
2370         BUG_ON(!ch);
2371
2372         ioctx = mgmt_ioctx->ioctx;
2373         BUG_ON(!ioctx);
2374
2375         TRACE_DBG("%s: tsk_mgmt_done for tag= %lld status=%d\n",
2376                   __func__, (unsigned long long)mgmt_ioctx->tag,
2377                   scst_mgmt_cmd_get_status(mcmnd));
2378
2379         rsp_len = srpt_build_tskmgmt_rsp(ch, ioctx,
2380                                          (scst_mgmt_cmd_get_status(mcmnd) ==
2381                                           SCST_MGMT_STATUS_SUCCESS) ?
2382                                          SRP_TSK_MGMT_SUCCESS :
2383                                          SRP_TSK_MGMT_FAILED,
2384                                          mgmt_ioctx->tag);
2385         srpt_post_send(ch, ioctx, rsp_len);
2386
2387         scst_mgmt_cmd_set_tgt_priv(mcmnd, NULL);
2388
2389         kfree(mgmt_ioctx);
2390 }
2391
2392 /*
2393  * Called by the SCST core to inform ib_srpt that the command 'scmnd' is about
2394  * to be freed. May be called in IRQ context.
2395  */
2396 static void srpt_on_free_cmd(struct scst_cmd *scmnd)
2397 {
2398         struct srpt_rdma_ch *ch;
2399         struct srpt_ioctx *ioctx;
2400
2401         ioctx = scst_cmd_get_tgt_priv(scmnd);
2402         BUG_ON(!ioctx);
2403
2404         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2405         BUG_ON(!ch);
2406
2407         spin_lock_irq(&ch->spinlock);
2408         list_del(&ioctx->scmnd_list);
2409         ch->active_scmnd_cnt--;
2410         spin_unlock_irq(&ch->spinlock);
2411
2412         srpt_reset_ioctx(ch, ioctx);
2413         scst_cmd_set_tgt_priv(scmnd, NULL);
2414 }
2415
2416 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2417 /* A vanilla 2.6.19 or older kernel without backported OFED kernel headers. */
2418 static void srpt_refresh_port_work(void *ctx)
2419 #else
2420 static void srpt_refresh_port_work(struct work_struct *work)
2421 #endif
2422 {
2423 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2424         struct srpt_port *sport = (struct srpt_port *)ctx;
2425 #else
2426         struct srpt_port *sport = container_of(work, struct srpt_port, work);
2427 #endif
2428
2429         srpt_refresh_port(sport);
2430 }
2431
2432 /*
2433  * Called by the SCST core to detect target adapters. Returns the number of
2434  * detected target adapters.
2435  */
2436 static int srpt_detect(struct scst_tgt_template *tp)
2437 {
2438         int device_count;
2439
2440         TRACE_ENTRY();
2441
2442         device_count = atomic_read(&srpt_device_count);
2443
2444         TRACE_EXIT_RES(device_count);
2445
2446         return device_count;
2447 }
2448
2449 /*
2450  * Callback function called by the SCST core from scst_unregister() to free up
2451  * the resources associated with device scst_tgt.
2452  */
2453 static int srpt_release(struct scst_tgt *scst_tgt)
2454 {
2455         struct srpt_device *sdev = scst_tgt_get_tgt_priv(scst_tgt);
2456         struct srpt_rdma_ch *ch, *tmp_ch;
2457
2458         TRACE_ENTRY();
2459
2460         BUG_ON(!scst_tgt);
2461 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
2462         WARN_ON(!sdev);
2463         if (!sdev)
2464                 return -ENODEV;
2465 #else
2466         if (WARN_ON(!sdev))
2467                 return -ENODEV;
2468 #endif
2469
2470         srpt_unregister_procfs_entry(scst_tgt->tgtt);
2471
2472         spin_lock_irq(&sdev->spinlock);
2473         list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
2474                 list_del(&ch->list);
2475                 spin_unlock_irq(&sdev->spinlock);
2476                 srpt_release_channel(ch, 1);
2477                 spin_lock_irq(&sdev->spinlock);
2478         }
2479         spin_unlock_irq(&sdev->spinlock);
2480
2481         srpt_unregister_mad_agent(sdev);
2482
2483         scst_tgt_set_tgt_priv(scst_tgt, NULL);
2484
2485         TRACE_EXIT();
2486
2487         return 0;
2488 }
2489
2490 /*
2491  * Entry point for ib_srpt's kernel thread. This kernel thread is only created
2492  * when the module parameter 'thread' is not zero (the default is zero).
2493  * This thread processes the ioctx list srpt_thread.thread_ioctx_list.
2494  *
2495  * @pre thread != 0
2496  */
2497 static int srpt_ioctx_thread(void *arg)
2498 {
2499         struct srpt_ioctx *ioctx;
2500
2501         /* Hibernation / freezing of the SRPT kernel thread is not supported. */
2502         current->flags |= PF_NOFREEZE;
2503
2504         spin_lock_irq(&srpt_thread.thread_lock);
2505         while (!kthread_should_stop()) {
2506                 wait_queue_t wait;
2507                 init_waitqueue_entry(&wait, current);
2508
2509                 if (!srpt_test_ioctx_list()) {
2510                         add_wait_queue_exclusive(&ioctx_list_waitQ, &wait);
2511
2512                         for (;;) {
2513                                 set_current_state(TASK_INTERRUPTIBLE);
2514                                 if (srpt_test_ioctx_list())
2515                                         break;
2516                                 spin_unlock_irq(&srpt_thread.thread_lock);
2517                                 schedule();
2518                                 spin_lock_irq(&srpt_thread.thread_lock);
2519                         }
2520                         set_current_state(TASK_RUNNING);
2521                         remove_wait_queue(&ioctx_list_waitQ, &wait);
2522                 }
2523
2524                 while (!list_empty(&srpt_thread.thread_ioctx_list)) {
2525                         ioctx = list_entry(srpt_thread.thread_ioctx_list.next,
2526                                            struct srpt_ioctx, comp_list);
2527
2528                         list_del(&ioctx->comp_list);
2529
2530                         spin_unlock_irq(&srpt_thread.thread_lock);
2531                         switch (ioctx->op) {
2532                         case IB_WC_SEND:
2533                                 srpt_handle_send_comp(ioctx->ch, ioctx,
2534                                         SCST_CONTEXT_DIRECT);
2535                                 break;
2536                         case IB_WC_RDMA_WRITE:
2537                         case IB_WC_RDMA_READ:
2538                                 srpt_handle_rdma_comp(ioctx->ch, ioctx);
2539                                 break;
2540                         case IB_WC_RECV:
2541                                 srpt_handle_new_iu(ioctx->ch, ioctx);
2542                                 break;
2543                         default:
2544                                 break;
2545                         }
2546                         spin_lock_irq(&srpt_thread.thread_lock);
2547                 }
2548         }
2549         spin_unlock_irq(&srpt_thread.thread_lock);
2550
2551         return 0;
2552 }
2553
2554 /* SCST target template for the SRP target implementation. */
2555 static struct scst_tgt_template srpt_template = {
2556         .name = DRV_NAME,
2557         .sg_tablesize = SRPT_DEF_SG_TABLESIZE,
2558         .xmit_response_atomic = 1,
2559         .rdy_to_xfer_atomic = 1,
2560         .no_proc_entry = 0,
2561         .detect = srpt_detect,
2562         .release = srpt_release,
2563         .xmit_response = srpt_xmit_response,
2564         .rdy_to_xfer = srpt_rdy_to_xfer,
2565         .on_free_cmd = srpt_on_free_cmd,
2566         .task_mgmt_fn_done = srpt_tsk_mgmt_done
2567 };
2568
2569 /*
2570  * The callback function srpt_release_class_dev() is called whenever a
2571  * device is removed from the /sys/class/infiniband_srpt device class.
2572  */
2573 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2574 static void srpt_release_class_dev(struct class_device *class_dev)
2575 #else
2576 static void srpt_release_class_dev(struct device *dev)
2577 #endif
2578 {
2579 }
2580
2581 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2582 static int srpt_trace_level_show(struct seq_file *seq, void *v)
2583 {
2584         return scst_proc_log_entry_read(seq, trace_flag, NULL);
2585 }
2586
2587 static ssize_t srpt_proc_trace_level_write(struct file *file,
2588         const char __user *buf, size_t length, loff_t *off)
2589 {
2590         return scst_proc_log_entry_write(file, buf, length, &trace_flag,
2591                 DEFAULT_SRPT_TRACE_FLAGS, NULL);
2592 }
2593
2594 static struct scst_proc_data srpt_log_proc_data = {
2595         SCST_DEF_RW_SEQ_OP(srpt_proc_trace_level_write)
2596         .show = srpt_trace_level_show,
2597 };
2598 #endif
2599
2600 static struct class_attribute srpt_class_attrs[] = {
2601         __ATTR_NULL,
2602 };
2603
2604 static struct class srpt_class = {
2605         .name = "infiniband_srpt",
2606 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2607         .release = srpt_release_class_dev,
2608 #else
2609         .dev_release = srpt_release_class_dev,
2610 #endif
2611         .class_attrs = srpt_class_attrs,
2612 };
2613
2614 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2615 static ssize_t show_login_info(struct class_device *class_dev, char *buf)
2616 #else
2617 static ssize_t show_login_info(struct device *dev,
2618                                struct device_attribute *attr, char *buf)
2619 #endif
2620 {
2621         struct srpt_device *sdev =
2622 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2623                 container_of(class_dev, struct srpt_device, class_dev);
2624 #else
2625                 container_of(dev, struct srpt_device, dev);
2626 #endif
2627         struct srpt_port *sport;
2628         u64 ioc_guid;
2629         int i;
2630         int len = 0;
2631
2632         for (i = 0; i < sdev->device->phys_port_cnt; i++) {
2633                 sport = &sdev->port[i];
2634
2635                 ioc_guid = srpt_get_ioc_guid(sdev->device);
2636                 len += sprintf(buf + len,
2637                                "tid_ext=%016llx,ioc_guid=%016llx,pkey=ffff,"
2638                                "dgid=%04x%04x%04x%04x%04x%04x%04x%04x,"
2639                                "service_id=%016llx\n",
2640                                (unsigned long long) ioc_guid,
2641                                (unsigned long long) ioc_guid,
2642                                be16_to_cpu(((__be16 *) sport->gid.raw)[0]),
2643                                be16_to_cpu(((__be16 *) sport->gid.raw)[1]),
2644                                be16_to_cpu(((__be16 *) sport->gid.raw)[2]),
2645                                be16_to_cpu(((__be16 *) sport->gid.raw)[3]),
2646                                be16_to_cpu(((__be16 *) sport->gid.raw)[4]),
2647                                be16_to_cpu(((__be16 *) sport->gid.raw)[5]),
2648                                be16_to_cpu(((__be16 *) sport->gid.raw)[6]),
2649                                be16_to_cpu(((__be16 *) sport->gid.raw)[7]),
2650                                (unsigned long long) ioc_guid);
2651         }
2652
2653         return len;
2654 }
2655
2656 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2657 static CLASS_DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2658 #else
2659 static DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2660 #endif
2661
2662 /*
2663  * Callback function called by the InfiniBand core when either an InfiniBand
2664  * device has been added or during the ib_register_client() call for each
2665  * registered InfiniBand device.
2666  */
2667 static void srpt_add_one(struct ib_device *device)
2668 {
2669         struct srpt_device *sdev;
2670         struct srpt_port *sport;
2671         struct ib_srq_init_attr srq_attr;
2672         int i;
2673
2674         TRACE_ENTRY();
2675
2676         sdev = kzalloc(sizeof *sdev, GFP_KERNEL);
2677         if (!sdev)
2678                 return;
2679
2680         sdev->device = device;
2681
2682 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2683         sdev->class_dev.class = &srpt_class;
2684         sdev->class_dev.dev = device->dma_device;
2685         snprintf(sdev->class_dev.class_id, BUS_ID_SIZE,
2686                  "srpt-%s", device->name);
2687 #else
2688         sdev->dev.class = &srpt_class;
2689         sdev->dev.parent = device->dma_device;
2690 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
2691         snprintf(sdev->dev.bus_id, BUS_ID_SIZE, "srpt-%s", device->name);
2692 #else
2693         snprintf(sdev->init_name, sizeof(sdev->init_name),
2694                  "srpt-%s", device->name);
2695         sdev->dev.init_name = sdev->init_name;
2696 #endif
2697 #endif
2698
2699 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2700         if (class_device_register(&sdev->class_dev))
2701                 goto free_dev;
2702         if (class_device_create_file(&sdev->class_dev,
2703                                      &class_device_attr_login_info))
2704                 goto err_dev;
2705 #else
2706         if (device_register(&sdev->dev))
2707                 goto free_dev;
2708         if (device_create_file(&sdev->dev, &dev_attr_login_info))
2709                 goto err_dev;
2710 #endif
2711
2712         if (ib_query_device(device, &sdev->dev_attr))
2713                 goto err_dev;
2714
2715         sdev->pd = ib_alloc_pd(device);
2716         if (IS_ERR(sdev->pd))
2717                 goto err_dev;
2718
2719         sdev->mr = ib_get_dma_mr(sdev->pd, IB_ACCESS_LOCAL_WRITE);
2720         if (IS_ERR(sdev->mr))
2721                 goto err_pd;
2722
2723         srq_attr.event_handler = srpt_srq_event;
2724         srq_attr.srq_context = (void *)sdev;
2725         srq_attr.attr.max_wr = min(SRPT_SRQ_SIZE, sdev->dev_attr.max_srq_wr);
2726         srq_attr.attr.max_sge = 1;
2727         srq_attr.attr.srq_limit = 0;
2728
2729         sdev->srq = ib_create_srq(sdev->pd, &srq_attr);
2730         if (IS_ERR(sdev->srq))
2731                 goto err_mr;
2732
2733         TRACE_DBG("%s: create SRQ #wr= %d max_allow=%d dev= %s",
2734                __func__, srq_attr.attr.max_wr,
2735               sdev->dev_attr.max_srq_wr, device->name);
2736
2737         if (!global_ioc_guid)
2738                 global_ioc_guid = be64_to_cpu(device->node_guid);
2739
2740         sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);
2741         if (IS_ERR(sdev->cm_id))
2742                 goto err_srq;
2743
2744         /* print out target login information */
2745         TRACE_DBG("Target login info: id_ext=%016llx,"
2746                   "ioc_guid=%016llx,pkey=ffff,service_id=%016llx",
2747                   (unsigned long long) srpt_get_ioc_guid(sdev->device),
2748                   (unsigned long long) srpt_get_ioc_guid(sdev->device),
2749                   (unsigned long long) srpt_get_ioc_guid(sdev->device));
2750
2751         /*
2752          * We do not have a consistent service_id (ie. also id_ext of target_id)
2753          * to identify this target. We currently use the guid of the first HCA
2754          * in the system as service_id; therefore, the target_id will change
2755          * if this HCA is gone bad and replaced by different HCA
2756          */
2757         if (ib_cm_listen(sdev->cm_id, cpu_to_be64(global_ioc_guid), 0, NULL))
2758                 goto err_cm;
2759
2760         INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
2761                               srpt_event_handler);
2762         if (ib_register_event_handler(&sdev->event_handler))
2763                 goto err_cm;
2764
2765         if (srpt_alloc_ioctx_ring(sdev))
2766                 goto err_event;
2767
2768         INIT_LIST_HEAD(&sdev->rch_list);
2769         spin_lock_init(&sdev->spinlock);
2770
2771         for (i = 0; i < SRPT_SRQ_SIZE; ++i)
2772                 srpt_post_recv(sdev, sdev->ioctx_ring[i]);
2773
2774         ib_set_client_data(device, &srpt_client, sdev);
2775
2776         sdev->scst_tgt = scst_register(&srpt_template, NULL);
2777         if (!sdev->scst_tgt) {
2778                 printk(KERN_ERR PFX "SCST registration failed for %s.\n",
2779                         sdev->device->name);
2780                 goto err_ring;
2781         }
2782
2783         scst_tgt_set_tgt_priv(sdev->scst_tgt, sdev);
2784
2785         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
2786                 sport = &sdev->port[i - 1];
2787                 sport->sdev = sdev;
2788                 sport->port = i;
2789 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2790                 /*
2791                  * A vanilla 2.6.19 or older kernel without backported OFED
2792                  * kernel headers.
2793                  */
2794                 INIT_WORK(&sport->work, srpt_refresh_port_work, sport);
2795 #else
2796                 INIT_WORK(&sport->work, srpt_refresh_port_work);
2797 #endif
2798                 if (srpt_refresh_port(sport)) {
2799                         printk(KERN_ERR PFX "MAD registration failed"
2800                                " for %s-%d.\n", sdev->device->name, i);
2801                         goto err_refresh_port;
2802                 }
2803         }
2804
2805         atomic_inc(&srpt_device_count);
2806
2807         TRACE_EXIT();
2808
2809         return;
2810
2811 err_refresh_port:
2812         scst_unregister(sdev->scst_tgt);
2813 err_ring:
2814         ib_set_client_data(device, &srpt_client, NULL);
2815         srpt_free_ioctx_ring(sdev);
2816 err_event:
2817         ib_unregister_event_handler(&sdev->event_handler);
2818 err_cm:
2819         ib_destroy_cm_id(sdev->cm_id);
2820 err_srq:
2821         ib_destroy_srq(sdev->srq);
2822 err_mr:
2823         ib_dereg_mr(sdev->mr);
2824 err_pd:
2825         ib_dealloc_pd(sdev->pd);
2826 err_dev:
2827 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2828         class_device_unregister(&sdev->class_dev);
2829 #else
2830         device_unregister(&sdev->dev);
2831 #endif
2832 free_dev:
2833         kfree(sdev);
2834
2835         TRACE_EXIT();
2836 }
2837
2838 /*
2839  * Callback function called by the InfiniBand core when either an InfiniBand
2840  * device has been removed or during the ib_unregister_client() call for each
2841  * registered InfiniBand device.
2842  */
2843 static void srpt_remove_one(struct ib_device *device)
2844 {
2845         int i;
2846         struct srpt_device *sdev;
2847
2848         TRACE_ENTRY();
2849
2850         sdev = ib_get_client_data(device, &srpt_client);
2851 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
2852         WARN_ON(!sdev);
2853         if (!sdev)
2854                 return;
2855 #else
2856         if (WARN_ON(!sdev))
2857                 return;
2858 #endif
2859
2860         /*
2861          * Cancel the work if it is queued. Wait until srpt_refresh_port_work()
2862          * finished if it is running.
2863          */
2864         for (i = 0; i < sdev->device->phys_port_cnt; i++)
2865 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
2866                 cancel_work_sync(&sdev->port[i].work);
2867 #else
2868                 /*
2869                  * cancel_work_sync() was introduced in kernel 2.6.22. Older
2870                  * kernels do not have a facility to cancel scheduled work.
2871                  */
2872                 printk(KERN_ERR PFX
2873                        "your kernel does not provide cancel_work_sync().\n");
2874 #endif
2875
2876         scst_unregister(sdev->scst_tgt);
2877         sdev->scst_tgt = NULL;
2878
2879         ib_unregister_event_handler(&sdev->event_handler);
2880         ib_destroy_cm_id(sdev->cm_id);
2881         ib_destroy_srq(sdev->srq);
2882         ib_dereg_mr(sdev->mr);
2883         ib_dealloc_pd(sdev->pd);
2884 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2885         class_device_unregister(&sdev->class_dev);
2886 #else
2887         device_unregister(&sdev->dev);
2888 #endif
2889
2890         srpt_free_ioctx_ring(sdev);
2891         kfree(sdev);
2892
2893         TRACE_EXIT();
2894 }
2895
2896 /**
2897  * Create procfs entries for srpt. Currently the only procfs entry created
2898  * by this function is the "trace_level" entry.
2899  */
2900 static int srpt_register_procfs_entry(struct scst_tgt_template *tgt)
2901 {
2902         int res = 0;
2903 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2904         struct proc_dir_entry *p, *root;
2905
2906         root = scst_proc_get_tgt_root(tgt);
2907         WARN_ON(!root);
2908         if (root) {
2909                 /*
2910                  * Fill in the scst_proc_data::data pointer, which is used in
2911                  * a printk(KERN_INFO ...) statement in
2912                  * scst_proc_log_entry_write() in scst_proc.c.
2913                  */
2914                 srpt_log_proc_data.data = (char *)tgt->name;
2915                 p = scst_create_proc_entry(root, SRPT_PROC_TRACE_LEVEL_NAME,
2916                                            &srpt_log_proc_data);
2917                 if (!p)
2918                         res = -ENOMEM;
2919         } else
2920                 res = -ENOMEM;
2921
2922 #endif
2923         return res;
2924 }
2925
2926 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt)
2927 {
2928 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2929         struct proc_dir_entry *root;
2930
2931         root = scst_proc_get_tgt_root(tgt);
2932         WARN_ON(!root);
2933         if (root)
2934                 remove_proc_entry(SRPT_PROC_TRACE_LEVEL_NAME, root);
2935 #endif
2936 }
2937
2938 /*
2939  * Module initialization.
2940  *
2941  * Note: since ib_register_client() registers callback functions, and since at
2942  * least one of these callback functions (srpt_add_one()) calls SCST functions,
2943  * the SCST target template must be registered before ib_register_client() is
2944  * called.
2945  */
2946 static int __init srpt_init_module(void)
2947 {
2948         int ret;
2949
2950         ret = class_register(&srpt_class);
2951         if (ret) {
2952                 printk(KERN_ERR PFX "couldn't register class ib_srpt\n");
2953                 goto out;
2954         }
2955
2956         ret = scst_register_target_template(&srpt_template);
2957         if (ret < 0) {
2958                 printk(KERN_ERR PFX "couldn't register with scst\n");
2959                 ret = -ENODEV;
2960                 goto out_unregister_class;
2961         }
2962
2963         ret = srpt_register_procfs_entry(&srpt_template);
2964         if (ret) {
2965                 printk(KERN_ERR PFX "couldn't register procfs entry\n");
2966                 goto out_unregister_target;
2967         }
2968
2969         ret = ib_register_client(&srpt_client);
2970         if (ret) {
2971                 printk(KERN_ERR PFX "couldn't register IB client\n");
2972                 goto out_unregister_target;
2973         }
2974
2975         if (thread) {
2976                 spin_lock_init(&srpt_thread.thread_lock);
2977                 INIT_LIST_HEAD(&srpt_thread.thread_ioctx_list);
2978                 srpt_thread.thread = kthread_run(srpt_ioctx_thread,
2979                                                  NULL, "srpt_thread");
2980                 if (IS_ERR(srpt_thread.thread)) {
2981                         srpt_thread.thread = NULL;
2982                         thread = 0;
2983                 }
2984         }
2985
2986         return 0;
2987
2988 out_unregister_target:
2989         /*
2990          * Note: the procfs entry is unregistered in srpt_release(), which is
2991          * called by scst_unregister_target_template().
2992          */
2993         scst_unregister_target_template(&srpt_template);
2994 out_unregister_class:
2995         class_unregister(&srpt_class);
2996 out:
2997         return ret;
2998 }
2999
3000 static void __exit srpt_cleanup_module(void)
3001 {
3002         TRACE_ENTRY();
3003
3004         if (srpt_thread.thread)
3005                 kthread_stop(srpt_thread.thread);
3006         ib_unregister_client(&srpt_client);
3007         scst_unregister_target_template(&srpt_template);
3008         class_unregister(&srpt_class);
3009
3010         TRACE_EXIT();
3011 }
3012
3013 module_init(srpt_init_module);
3014 module_exit(srpt_cleanup_module);