98fe69c809cc06a6a660352b9b0eeabe09591590
[mirror/scst/.git] / srpt / src / ib_srpt.c
1 /*
2  * Copyright (c) 2006 - 2009 Mellanox Technology Inc.  All rights reserved.
3  * Copyright (C) 2008 Vladislav Bolkhovitin <vst@vlnb.net>
4  * Copyright (C) 2008 - 2009 Bart Van Assche <bart.vanassche@gmail.com>
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  */
35
36 #include <linux/module.h>
37 #include <linux/init.h>
38 #include <linux/slab.h>
39 #include <linux/err.h>
40 #include <linux/ctype.h>
41 #include <linux/string.h>
42 #include <linux/kthread.h>
43 #include <linux/delay.h>
44 #include <asm/atomic.h>
45 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
46 #include <linux/proc_fs.h>
47 #include <linux/seq_file.h>
48 #endif
49 #include "ib_srpt.h"
50 #include "scst_debug.h"
51
52 #define CONFIG_SCST_PROC
53
54 /* Name of this kernel module. */
55 #define DRV_NAME                "ib_srpt"
56 /* Prefix for printk() kernel messages. */
57 #define LOG_PFX                 DRV_NAME ": "
58 #define DRV_VERSION             "1.0.1"
59 #define DRV_RELDATE             "July 10, 2008"
60 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
61 /* Flags to be used in SCST debug tracing statements. */
62 #define DEFAULT_SRPT_TRACE_FLAGS (TRACE_OUT_OF_MEM | TRACE_MINOR \
63                                   | TRACE_MGMT | TRACE_SPECIAL)
64 /* Name of the entry that will be created under /proc/scsi_tgt/ib_srpt. */
65 #define SRPT_PROC_TRACE_LEVEL_NAME      "trace_level"
66 #endif
67
68 #define MELLANOX_SRPT_ID_STRING "SCST SRP target"
69
70 MODULE_AUTHOR("Vu Pham");
71 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
72                    "v" DRV_VERSION " (" DRV_RELDATE ")");
73 MODULE_LICENSE("Dual BSD/GPL");
74
75 struct srpt_thread {
76         /* Protects thread_ioctx_list. */
77         spinlock_t thread_lock;
78         /* I/O contexts to be processed by the kernel thread. */
79         struct list_head thread_ioctx_list;
80         /* SRPT kernel thread. */
81         struct task_struct *thread;
82 };
83
84 /*
85  * Global Variables
86  */
87
88 static u64 srpt_service_guid;
89 /* List of srpt_device structures. */
90 static atomic_t srpt_device_count;
91 static int use_port_guid_in_session_name;
92 static int thread = 1;
93 static struct srpt_thread srpt_thread;
94 static DECLARE_WAIT_QUEUE_HEAD(ioctx_list_waitQ);
95 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
96 static unsigned long trace_flag = DEFAULT_SRPT_TRACE_FLAGS;
97 module_param(trace_flag, long, 0644);
98 MODULE_PARM_DESC(trace_flag,
99                  "Trace flags for the ib_srpt kernel module.");
100 #endif
101 #if defined(CONFIG_SCST_DEBUG)
102 static unsigned long interrupt_processing_delay_in_us;
103 module_param(interrupt_processing_delay_in_us, long, 0744);
104 MODULE_PARM_DESC(interrupt_processing_delay_in_us,
105                  "CQ completion handler interrupt delay in microseconds.");
106 static unsigned long thread_processing_delay_in_us;
107 module_param(thread_processing_delay_in_us, long, 0744);
108 MODULE_PARM_DESC(thread_processing_delay_in_us,
109                  "SRP thread processing delay in microseconds.");
110 #endif
111
112 module_param(thread, int, 0444);
113 MODULE_PARM_DESC(thread,
114                  "Executing ioctx in thread context. Default 0, i.e. soft IRQ, "
115                  "where possible.");
116
117 static unsigned int srp_max_rdma_size = 65536;
118 module_param(srp_max_rdma_size, int, 0744);
119 MODULE_PARM_DESC(thread,
120                  "Maximum size of SRP RDMA transfers for new connections.");
121
122 static unsigned int srp_max_message_size = 4096;
123 module_param(srp_max_message_size, int, 0444);
124 MODULE_PARM_DESC(thread,
125                  "Maximum size of SRP control messages in bytes.");
126
127 module_param(use_port_guid_in_session_name, bool, 0444);
128 MODULE_PARM_DESC(use_port_guid_in_session_name,
129                  "Use target port ID in the SCST session name such that"
130                  " redundant paths between multiport systems can be masked.");
131
132 static void srpt_add_one(struct ib_device *device);
133 static void srpt_remove_one(struct ib_device *device);
134 static void srpt_unregister_mad_agent(struct srpt_device *sdev);
135 #ifdef CONFIG_SCST_PROC
136 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt);
137 #endif /*CONFIG_SCST_PROC*/
138
139 static struct ib_client srpt_client = {
140         .name = DRV_NAME,
141         .add = srpt_add_one,
142         .remove = srpt_remove_one
143 };
144
145 /**
146  * Atomically set the channel state.
147  * @ch: RDMA channel.
148  * @new: new channel state.
149  */
150 static void srpt_set_channel_state(struct srpt_rdma_ch *ch,
151                                    enum rdma_ch_state new)
152 {
153         unsigned long flags;
154
155         spin_lock_irqsave(&ch->spinlock, flags);
156         ch->state = new;
157         spin_unlock_irqrestore(&ch->spinlock, flags);
158 }
159
160 /**
161  * Atomically test and set the channel state.
162  * @ch: RDMA channel.
163  * @old: channel state to compare with.
164  * @new: state to change the channel state to if the current state matches the
165  *       argument 'old'.
166  *
167  * Returns true if the channel state matched old upon entry of this function,
168  * and false otherwise.
169  */
170 static bool srpt_test_and_set_channel_state(struct srpt_rdma_ch *ch,
171                                             enum rdma_ch_state old,
172                                             enum rdma_ch_state new)
173 {
174         unsigned long flags;
175         enum rdma_ch_state cur;
176
177         spin_lock_irqsave(&ch->spinlock, flags);
178         cur = ch->state;
179         if (cur == old)
180                 ch->state = new;
181         spin_unlock_irqrestore(&ch->spinlock, flags);
182
183         return cur == old;
184 }
185
186 /*
187  * Callback function called by the InfiniBand core when an asynchronous IB
188  * event occurs. This callback may occur in interrupt context. See also
189  * section 11.5.2, Set Asynchronous Event Handler in the InfiniBand
190  * Architecture Specification.
191  */
192 static void srpt_event_handler(struct ib_event_handler *handler,
193                                struct ib_event *event)
194 {
195         struct srpt_device *sdev;
196         struct srpt_port *sport;
197
198         TRACE_ENTRY();
199
200         sdev = ib_get_client_data(event->device, &srpt_client);
201         if (!sdev || sdev->device != event->device)
202                 return;
203
204         TRACE_DBG("ASYNC event= %d on device= %s",
205                   event->event, sdev->device->name);
206
207         switch (event->event) {
208         case IB_EVENT_PORT_ERR:
209                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
210                         sport = &sdev->port[event->element.port_num - 1];
211                         sport->lid = 0;
212                         sport->sm_lid = 0;
213                 }
214                 break;
215         case IB_EVENT_PORT_ACTIVE:
216         case IB_EVENT_LID_CHANGE:
217         case IB_EVENT_PKEY_CHANGE:
218         case IB_EVENT_SM_CHANGE:
219         case IB_EVENT_CLIENT_REREGISTER:
220                 /*
221                  * Refresh port data asynchronously. Note: it is safe to call
222                  * schedule_work() even if &sport->work is already on the
223                  * global workqueue because schedule_work() tests for the
224                  * work_pending() condition before adding &sport->work to the
225                  * global work queue.
226                  */
227                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
228                         sport = &sdev->port[event->element.port_num - 1];
229                         if (!sport->lid && !sport->sm_lid)
230                                 schedule_work(&sport->work);
231                 }
232                 break;
233         default:
234                 break;
235         }
236
237         TRACE_EXIT();
238 }
239
240 /*
241  * Callback function called by the InfiniBand core for SRQ (shared receive
242  * queue) events.
243  */
244 static void srpt_srq_event(struct ib_event *event, void *ctx)
245 {
246         TRACE_ENTRY();
247
248         TRACE_DBG("SRQ event %d", event->event);
249
250         TRACE_EXIT();
251 }
252
253 /*
254  * Callback function called by the InfiniBand core for QP (queue pair) events.
255  */
256 static void srpt_qp_event(struct ib_event *event, void *ctx)
257 {
258         struct srpt_rdma_ch *ch = ctx;
259
260         TRACE_DBG("QP event %d on cm_id=%p sess_name=%s state=%d",
261                   event->event, ch->cm_id, ch->sess_name, ch->state);
262
263         switch (event->event) {
264         case IB_EVENT_COMM_EST:
265 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20) || defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
266                 ib_cm_notify(ch->cm_id, event->event);
267 #else
268                 /* Vanilla 2.6.19 kernel (or before) without OFED. */
269                 PRINT_ERROR("%s", "how to perform ib_cm_notify() on a"
270                             " vanilla 2.6.18 kernel ???");
271 #endif
272                 break;
273         case IB_EVENT_QP_LAST_WQE_REACHED:
274                 if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_LIVE,
275                                         RDMA_CHANNEL_DISCONNECTING)) {
276                         PRINT_INFO("disconnected session %s.", ch->sess_name);
277                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
278                 }
279                 break;
280         default:
281                 break;
282         }
283 }
284
285 /*
286  * Helper function for filling in an InfiniBand IOUnitInfo structure. Copies
287  * the lowest four bits of value in element slot of the array of four bit
288  * elements called c_list (controller list). The index slot is one-based.
289  *
290  * @pre 1 <= slot && 0 <= value && value < 16
291  */
292 static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
293 {
294         u16 id;
295         u8 tmp;
296
297         id = (slot - 1) / 2;
298         if (slot & 0x1) {
299                 tmp = c_list[id] & 0xf;
300                 c_list[id] = (value << 4) | tmp;
301         } else {
302                 tmp = c_list[id] & 0xf0;
303                 c_list[id] = (value & 0xf) | tmp;
304         }
305 }
306
307 /*
308  * Write InfiniBand ClassPortInfo to mad. See also section 16.3.3.1
309  * ClassPortInfo in the InfiniBand Architecture Specification.
310  */
311 static void srpt_get_class_port_info(struct ib_dm_mad *mad)
312 {
313         struct ib_class_port_info *cif;
314
315         cif = (struct ib_class_port_info *)mad->data;
316         memset(cif, 0, sizeof *cif);
317         cif->base_version = 1;
318         cif->class_version = 1;
319         cif->resp_time_value = 20;
320
321         mad->mad_hdr.status = 0;
322 }
323
324 /*
325  * Write IOUnitInfo to mad. See also section 16.3.3.3 IOUnitInfo in the
326  * InfiniBand Architecture Specification. See also section B.7,
327  * table B.6 in the T10 SRP r16a document.
328  */
329 static void srpt_get_iou(struct ib_dm_mad *mad)
330 {
331         struct ib_dm_iou_info *ioui;
332         u8 slot;
333         int i;
334
335         ioui = (struct ib_dm_iou_info *)mad->data;
336         ioui->change_id = 1;
337         ioui->max_controllers = 16;
338
339         /* set present for slot 1 and empty for the rest */
340         srpt_set_ioc(ioui->controller_list, 1, 1);
341         for (i = 1, slot = 2; i < 16; i++, slot++)
342                 srpt_set_ioc(ioui->controller_list, slot, 0);
343
344         mad->mad_hdr.status = 0;
345 }
346
347 /*
348  * Write IOControllerprofile to mad for I/O controller (sdev, slot). See also
349  * section 16.3.3.4 IOControllerProfile in the InfiniBand Architecture
350  * Specification. See also section B.7, table B.7 in the T10 SRP r16a
351  * document.
352  */
353 static void srpt_get_ioc(struct srpt_device *sdev, u32 slot,
354                          struct ib_dm_mad *mad)
355 {
356         struct ib_dm_ioc_profile *iocp;
357
358         iocp = (struct ib_dm_ioc_profile *)mad->data;
359
360         if (!slot || slot > 16) {
361                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
362                 return;
363         }
364
365         if (slot > 2) {
366                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
367                 return;
368         }
369
370         memset(iocp, 0, sizeof *iocp);
371         strcpy(iocp->id_string, MELLANOX_SRPT_ID_STRING);
372         iocp->guid = cpu_to_be64(srpt_service_guid);
373         iocp->vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
374         iocp->device_id = cpu_to_be32(sdev->dev_attr.vendor_part_id);
375         iocp->device_version = cpu_to_be16(sdev->dev_attr.hw_ver);
376         iocp->subsys_vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
377         iocp->subsys_device_id = 0x0;
378         iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
379         iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
380         iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
381         iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
382         iocp->send_queue_depth = cpu_to_be16(SRPT_SRQ_SIZE);
383         iocp->rdma_read_depth = 4;
384         iocp->send_size = cpu_to_be32(srp_max_message_size);
385         iocp->rdma_size = cpu_to_be32(min(max(srp_max_rdma_size, 256U),
386                                           1U << 24));
387         iocp->num_svc_entries = 1;
388         iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
389                 SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
390
391         mad->mad_hdr.status = 0;
392 }
393
394 /*
395  * Device management: write ServiceEntries to mad for the given slot. See also
396  * section 16.3.3.5 ServiceEntries in the InfiniBand Architecture
397  * Specification. See also section B.7, table B.8 in the T10 SRP r16a document.
398  */
399 static void srpt_get_svc_entries(u64 ioc_guid,
400                                  u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
401 {
402         struct ib_dm_svc_entries *svc_entries;
403
404         WARN_ON(!ioc_guid);
405
406         if (!slot || slot > 16) {
407                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
408                 return;
409         }
410
411         if (slot > 2 || lo > hi || hi > 1) {
412                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
413                 return;
414         }
415
416         svc_entries = (struct ib_dm_svc_entries *)mad->data;
417         memset(svc_entries, 0, sizeof *svc_entries);
418         svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid);
419         snprintf(svc_entries->service_entries[0].name,
420                  sizeof(svc_entries->service_entries[0].name),
421                  "%s%016llx",
422                  SRP_SERVICE_NAME_PREFIX,
423                  (unsigned long long)ioc_guid);
424
425         mad->mad_hdr.status = 0;
426 }
427
428 /*
429  * Actual processing of a received MAD *rq_mad received through source port *sp
430  * (MAD = InfiniBand management datagram). The response to be sent back is
431  * written to *rsp_mad.
432  */
433 static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
434                                  struct ib_dm_mad *rsp_mad)
435 {
436         u16 attr_id;
437         u32 slot;
438         u8 hi, lo;
439
440         attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
441         switch (attr_id) {
442         case DM_ATTR_CLASS_PORT_INFO:
443                 srpt_get_class_port_info(rsp_mad);
444                 break;
445         case DM_ATTR_IOU_INFO:
446                 srpt_get_iou(rsp_mad);
447                 break;
448         case DM_ATTR_IOC_PROFILE:
449                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
450                 srpt_get_ioc(sp->sdev, slot, rsp_mad);
451                 break;
452         case DM_ATTR_SVC_ENTRIES:
453                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
454                 hi = (u8) ((slot >> 8) & 0xff);
455                 lo = (u8) (slot & 0xff);
456                 slot = (u16) ((slot >> 16) & 0xffff);
457                 srpt_get_svc_entries(srpt_service_guid,
458                                      slot, hi, lo, rsp_mad);
459                 break;
460         default:
461                 rsp_mad->mad_hdr.status =
462                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
463                 break;
464         }
465 }
466
467 /*
468  * Callback function that is called by the InfiniBand core after transmission of
469  * a MAD. (MAD = management datagram; AH = address handle.)
470  */
471 static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
472                                   struct ib_mad_send_wc *mad_wc)
473 {
474         ib_destroy_ah(mad_wc->send_buf->ah);
475         ib_free_send_mad(mad_wc->send_buf);
476 }
477
478 /*
479  * Callback function that is called by the InfiniBand core after reception of
480  * a MAD (management datagram).
481  */
482 static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
483                                   struct ib_mad_recv_wc *mad_wc)
484 {
485         struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
486         struct ib_ah *ah;
487         struct ib_mad_send_buf *rsp;
488         struct ib_dm_mad *dm_mad;
489
490         if (!mad_wc || !mad_wc->recv_buf.mad)
491                 return;
492
493         ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
494                                   mad_wc->recv_buf.grh, mad_agent->port_num);
495         if (IS_ERR(ah))
496                 goto err;
497
498         BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR);
499
500         rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
501                                  mad_wc->wc->pkey_index, 0,
502                                  IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA,
503                                  GFP_KERNEL);
504         if (IS_ERR(rsp))
505                 goto err_rsp;
506
507         rsp->ah = ah;
508
509         dm_mad = rsp->mad;
510         memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof *dm_mad);
511         dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
512         dm_mad->mad_hdr.status = 0;
513
514         switch (mad_wc->recv_buf.mad->mad_hdr.method) {
515         case IB_MGMT_METHOD_GET:
516                 srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
517                 break;
518         case IB_MGMT_METHOD_SET:
519                 dm_mad->mad_hdr.status =
520                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
521                 break;
522         default:
523                 dm_mad->mad_hdr.status =
524                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
525                 break;
526         }
527
528         if (!ib_post_send_mad(rsp, NULL)) {
529                 ib_free_recv_mad(mad_wc);
530                 /* will destroy_ah & free_send_mad in send completion */
531                 return;
532         }
533
534         ib_free_send_mad(rsp);
535
536 err_rsp:
537         ib_destroy_ah(ah);
538 err:
539         ib_free_recv_mad(mad_wc);
540 }
541
542 /*
543  * Enable InfiniBand management datagram processing, update the cached sm_lid,
544  * lid and gid values, and register a callback function for processing MADs
545  * on the specified port. It is safe to call this function more than once for
546  * the same port.
547  */
548 static int srpt_refresh_port(struct srpt_port *sport)
549 {
550         struct ib_mad_reg_req reg_req;
551         struct ib_port_modify port_modify;
552         struct ib_port_attr port_attr;
553         int ret;
554
555         TRACE_ENTRY();
556
557         memset(&port_modify, 0, sizeof port_modify);
558         port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
559         port_modify.clr_port_cap_mask = 0;
560
561         ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
562         if (ret)
563                 goto err_mod_port;
564
565         ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
566         if (ret)
567                 goto err_query_port;
568
569         sport->sm_lid = port_attr.sm_lid;
570         sport->lid = port_attr.lid;
571
572         ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid);
573         if (ret)
574                 goto err_query_port;
575
576         if (!sport->mad_agent) {
577                 memset(&reg_req, 0, sizeof reg_req);
578                 reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
579                 reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
580                 set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
581                 set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
582
583                 sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
584                                                          sport->port,
585                                                          IB_QPT_GSI,
586                                                          &reg_req, 0,
587                                                          srpt_mad_send_handler,
588                                                          srpt_mad_recv_handler,
589                                                          sport);
590                 if (IS_ERR(sport->mad_agent)) {
591                         ret = PTR_ERR(sport->mad_agent);
592                         sport->mad_agent = NULL;
593                         goto err_query_port;
594                 }
595         }
596
597         TRACE_EXIT_RES(0);
598
599         return 0;
600
601 err_query_port:
602
603         port_modify.set_port_cap_mask = 0;
604         port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
605         ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
606
607 err_mod_port:
608
609         TRACE_EXIT_RES(ret);
610
611         return ret;
612 }
613
614 /*
615  * Unregister the callback function for processing MADs and disable MAD
616  * processing for all ports of the specified device. It is safe to call this
617  * function more than once for the same device.
618  */
619 static void srpt_unregister_mad_agent(struct srpt_device *sdev)
620 {
621         struct ib_port_modify port_modify = {
622                 .clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP,
623         };
624         struct srpt_port *sport;
625         int i;
626
627         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
628                 sport = &sdev->port[i - 1];
629                 WARN_ON(sport->port != i);
630                 if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
631                         PRINT_ERROR("%s", "disabling MAD processing failed.");
632                 if (sport->mad_agent) {
633                         ib_unregister_mad_agent(sport->mad_agent);
634                         sport->mad_agent = NULL;
635                 }
636         }
637 }
638
639 /**
640  * Allocate and initialize an SRPT I/O context structure.
641  */
642 static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev)
643 {
644         struct srpt_ioctx *ioctx;
645
646         ioctx = kmalloc(sizeof *ioctx, GFP_KERNEL);
647         if (!ioctx)
648                 goto out;
649
650         ioctx->buf = kzalloc(srp_max_message_size, GFP_KERNEL);
651         if (!ioctx->buf)
652                 goto out_free_ioctx;
653
654         ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf,
655                                        srp_max_message_size, DMA_BIDIRECTIONAL);
656         if (ib_dma_mapping_error(sdev->device, ioctx->dma))
657                 goto out_free_buf;
658
659         return ioctx;
660
661 out_free_buf:
662         kfree(ioctx->buf);
663 out_free_ioctx:
664         kfree(ioctx);
665 out:
666         return NULL;
667 }
668
669 /*
670  * Deallocate an SRPT I/O context structure.
671  */
672 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
673 {
674         if (!ioctx)
675                 return;
676
677         ib_dma_unmap_single(sdev->device, ioctx->dma,
678                             srp_max_message_size, DMA_BIDIRECTIONAL);
679         kfree(ioctx->buf);
680         kfree(ioctx);
681 }
682
683 /*
684  * Associate a ring of SRPT I/O context structures with the specified device.
685  */
686 static int srpt_alloc_ioctx_ring(struct srpt_device *sdev)
687 {
688         int i;
689
690         TRACE_ENTRY();
691
692         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
693                 sdev->ioctx_ring[i] = srpt_alloc_ioctx(sdev);
694
695                 if (!sdev->ioctx_ring[i])
696                         goto err;
697
698                 sdev->ioctx_ring[i]->index = i;
699         }
700
701         TRACE_EXIT_RES(0);
702
703         return 0;
704
705 err:
706         while (--i > 0) {
707                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
708                 sdev->ioctx_ring[i] = NULL;
709         }
710         TRACE_EXIT_RES(-ENOMEM);
711         return -ENOMEM;
712 }
713
714 /* Free the ring of SRPT I/O context structures. */
715 static void srpt_free_ioctx_ring(struct srpt_device *sdev)
716 {
717         int i;
718
719         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
720                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
721                 sdev->ioctx_ring[i] = NULL;
722         }
723 }
724
725 /**
726  * Set the state of a command.
727  * @new: New state to be set.
728  *
729  * Does not modify the state of aborted commands. Returns the previous command
730  * state.
731  */
732 static enum srpt_command_state srpt_set_cmd_state(struct srpt_ioctx *ioctx,
733                                                   enum srpt_command_state new)
734 {
735         enum srpt_command_state previous;
736
737         WARN_ON(!ioctx);
738         WARN_ON(new == SRPT_STATE_NEW);
739
740         previous = ioctx->state;
741         if (previous != SRPT_STATE_ABORTED)
742                 ioctx->state = new;
743
744         return previous;
745 }
746
747 /**
748  * Test and set the state of a command.
749  * @expected: State to compare against.
750  * @new: New state to be set if the current state matches 'expected'.
751  *
752  * Returns the previous command state.
753  */
754 static enum srpt_command_state
755 srpt_test_and_set_cmd_state(struct srpt_ioctx *ioctx,
756                             enum srpt_command_state expected,
757                             enum srpt_command_state new)
758 {
759         enum srpt_command_state previous;
760
761         WARN_ON(!ioctx);
762         WARN_ON(expected == SRPT_STATE_ABORTED);
763         WARN_ON(new == SRPT_STATE_NEW);
764
765         previous = ioctx->state;
766         if (previous == expected)
767                 ioctx->state = new;
768
769         return previous;
770 }
771
772 /**
773  * Post a receive request on the work queue of InfiniBand device 'sdev'.
774  */
775 static int srpt_post_recv(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
776 {
777         struct ib_sge list;
778         struct ib_recv_wr wr, *bad_wr;
779
780         wr.wr_id = ioctx->index | SRPT_OP_RECV;
781
782         list.addr = ioctx->dma;
783         list.length = srp_max_message_size;
784         list.lkey = sdev->mr->lkey;
785
786         wr.next = NULL;
787         wr.sg_list = &list;
788         wr.num_sge = 1;
789
790         return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);
791 }
792
793 /**
794  * Post an IB send request.
795  * @ch: RDMA channel to post the send request on.
796  * @ioctx: I/O context of the send request.
797  * @len: length of the request to be sent in bytes.
798  *
799  * Returns zero upon success and a non-zero value upon failure.
800  */
801 static int srpt_post_send(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
802                           int len)
803 {
804         struct ib_sge list;
805         struct ib_send_wr wr, *bad_wr;
806         struct srpt_device *sdev = ch->sport->sdev;
807
808         ib_dma_sync_single_for_device(sdev->device, ioctx->dma,
809                                       len, DMA_TO_DEVICE);
810
811         list.addr = ioctx->dma;
812         list.length = len;
813         list.lkey = sdev->mr->lkey;
814
815         wr.next = NULL;
816         wr.wr_id = ioctx->index;
817         wr.sg_list = &list;
818         wr.num_sge = 1;
819         wr.opcode = IB_WR_SEND;
820         wr.send_flags = IB_SEND_SIGNALED;
821
822         return ib_post_send(ch->qp, &wr, &bad_wr);
823 }
824
825 static int srpt_get_desc_tbl(struct srpt_ioctx *ioctx, struct srp_cmd *srp_cmd,
826                              int *ind)
827 {
828         struct srp_indirect_buf *idb;
829         struct srp_direct_buf *db;
830
831         *ind = 0;
832         if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
833             ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
834                 ioctx->n_rbuf = 1;
835                 ioctx->rbufs = &ioctx->single_rbuf;
836
837                 ib_dma_sync_single_for_cpu(ioctx->ch->sport->sdev->device,
838                                            ioctx->dma + sizeof(struct srp_cmd),
839                                            sizeof(*db), DMA_FROM_DEVICE);
840
841                 db = (void *)srp_cmd->add_data;
842                 memcpy(ioctx->rbufs, db, sizeof *db);
843                 ioctx->data_len = be32_to_cpu(db->len);
844         } else {
845                 idb = (void *)srp_cmd->add_data;
846
847                 ioctx->n_rbuf = be32_to_cpu(idb->table_desc.len) / sizeof *db;
848
849                 if (ioctx->n_rbuf >
850                     (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
851                         *ind = 1;
852                         ioctx->n_rbuf = 0;
853                         goto out;
854                 }
855
856                 if (ioctx->n_rbuf == 1)
857                         ioctx->rbufs = &ioctx->single_rbuf;
858                 else
859                         ioctx->rbufs =
860                                 kmalloc(ioctx->n_rbuf * sizeof *db, GFP_ATOMIC);
861                 if (!ioctx->rbufs) {
862                         ioctx->n_rbuf = 0;
863                         return -ENOMEM;
864                 }
865
866                 ib_dma_sync_single_for_cpu(ioctx->ch->sport->sdev->device,
867                                            ioctx->dma + sizeof(struct srp_cmd),
868                                            ioctx->n_rbuf * sizeof(*db),
869                                            DMA_FROM_DEVICE);
870
871                 db = idb->desc_list;
872                 memcpy(ioctx->rbufs, db, ioctx->n_rbuf * sizeof *db);
873                 ioctx->data_len = be32_to_cpu(idb->len);
874         }
875 out:
876         return 0;
877 }
878
879 /*
880  * Modify the attributes of queue pair 'qp': allow local write, remote read,
881  * and remote write. Also transition 'qp' to state IB_QPS_INIT.
882  */
883 static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
884 {
885         struct ib_qp_attr *attr;
886         int ret;
887
888         attr = kzalloc(sizeof *attr, GFP_KERNEL);
889         if (!attr)
890                 return -ENOMEM;
891
892         attr->qp_state = IB_QPS_INIT;
893         attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ |
894             IB_ACCESS_REMOTE_WRITE;
895         attr->port_num = ch->sport->port;
896         attr->pkey_index = 0;
897
898         ret = ib_modify_qp(qp, attr,
899                            IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
900                            IB_QP_PKEY_INDEX);
901
902         kfree(attr);
903         return ret;
904 }
905
906 /**
907  * Change the state of a channel to 'ready to receive' (RTR).
908  * @ch: channel of the queue pair.
909  * @qp: queue pair to change the state of.
910  *
911  * Returns zero upon success and a negative value upon failure.
912  *
913  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
914  * If this structure ever becomes larger, it might be necessary to allocate
915  * it dynamically instead of on the stack.
916  */
917 static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
918 {
919         struct ib_qp_attr qp_attr;
920         int attr_mask;
921         int ret;
922
923         qp_attr.qp_state = IB_QPS_RTR;
924         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
925         if (ret)
926                 goto out;
927
928         qp_attr.max_dest_rd_atomic = 4;
929
930         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
931
932 out:
933         return ret;
934 }
935
936 /**
937  * Change the state of a channel to 'ready to send' (RTS).
938  * @ch: channel of the queue pair.
939  * @qp: queue pair to change the state of.
940  *
941  * Returns zero upon success and a negative value upon failure.
942  *
943  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
944  * If this structure ever becomes larger, it might be necessary to allocate
945  * it dynamically instead of on the stack.
946  */
947 static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
948 {
949         struct ib_qp_attr qp_attr;
950         int attr_mask;
951         int ret;
952
953         qp_attr.qp_state = IB_QPS_RTS;
954         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
955         if (ret)
956                 goto out;
957
958         qp_attr.max_rd_atomic = 4;
959
960         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
961
962 out:
963         return ret;
964 }
965
966 static void srpt_reset_ioctx(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
967 {
968         int i;
969
970         if (ioctx->n_rdma_ius > 0 && ioctx->rdma_ius) {
971                 struct rdma_iu *riu = ioctx->rdma_ius;
972
973                 for (i = 0; i < ioctx->n_rdma_ius; ++i, ++riu)
974                         kfree(riu->sge);
975                 kfree(ioctx->rdma_ius);
976         }
977
978         if (ioctx->n_rbuf > 1)
979                 kfree(ioctx->rbufs);
980
981         /* If ch == NULL this means that the command has been aborted. */
982         if (!ch)
983                 return;
984
985         if (srpt_post_recv(ch->sport->sdev, ioctx))
986                 PRINT_ERROR("%s", "SRQ post_recv failed - this is serious.");
987                 /* we should queue it back to free_ioctx queue */
988         else
989                 atomic_inc(&ch->req_lim_delta);
990 }
991
992 /**
993  * Abort a command.
994  */
995 static void srpt_abort_scst_cmd(struct srpt_device *sdev,
996                                 struct scst_cmd *scmnd)
997 {
998         struct srpt_ioctx *ioctx;
999         scst_data_direction dir;
1000         enum srpt_command_state previous_state;
1001
1002         ioctx = scst_cmd_get_tgt_priv(scmnd);
1003         BUG_ON(!ioctx);
1004
1005         previous_state = srpt_set_cmd_state(ioctx, SRPT_STATE_ABORTED);
1006         if (previous_state == SRPT_STATE_ABORTED)
1007                 goto out;
1008
1009         TRACE_DBG("Aborting cmd with state %d and tag %lld",
1010                   previous_state, scst_cmd_get_tag(scmnd));
1011
1012         dir = scst_cmd_get_data_direction(scmnd);
1013         if (dir != SCST_DATA_NONE && scst_cmd_get_sg(scmnd))
1014                 ib_dma_unmap_sg(sdev->device,
1015                                 scst_cmd_get_sg(scmnd),
1016                                 scst_cmd_get_sg_cnt(scmnd),
1017                                 scst_to_tgt_dma_dir(dir));
1018
1019         switch (previous_state) {
1020         case SRPT_STATE_NEW:
1021                 break;
1022         case SRPT_STATE_NEED_DATA:
1023                 WARN_ON(scst_cmd_get_data_direction(ioctx->scmnd)
1024                         == SCST_DATA_READ);
1025                 scst_rx_data(scmnd,
1026                              SCST_RX_STATUS_ERROR,
1027                              SCST_CONTEXT_THREAD);
1028                 break;
1029         case SRPT_STATE_PROCESSED:
1030                 scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_FAILED);
1031                 WARN_ON(scmnd->state != SCST_CMD_STATE_XMIT_WAIT);
1032                 scst_tgt_cmd_done(scmnd, scst_estimate_context());
1033                 break;
1034         default:
1035                 TRACE_DBG("Aborting cmd with state %d", previous_state);
1036                 WARN_ON("ERROR: unexpected command state");
1037         }
1038
1039 out:
1040         ;
1041 }
1042
1043 /**
1044  * Abort an SCST command if the channel is has been disconnected.
1045  *
1046  * Returns true if the command has been aborted because the channel is
1047  * disconnecting, and false if the channel is not being disconnected.
1048  * Must be called from inside rdy_to_xfer, xmit_response or on_free_cmd
1049  * because any call from outside SCST's command processing callback functions
1050  * would be racy.
1051  */
1052 static bool srpt_abort_if_disconnecting(struct srpt_ioctx *ioctx)
1053 {
1054         BUG_ON(!ioctx);
1055
1056         if (ioctx->ch->state == RDMA_CHANNEL_DISCONNECTING) {
1057                 srpt_abort_scst_cmd(ioctx->ch->sport->sdev, ioctx->scmnd);
1058                 return true;
1059         }
1060         return false;
1061 }
1062
1063 static void srpt_handle_err_comp(struct srpt_rdma_ch *ch, struct ib_wc *wc)
1064 {
1065         struct srpt_ioctx *ioctx;
1066         struct srpt_device *sdev = ch->sport->sdev;
1067
1068         if (wc->wr_id & SRPT_OP_RECV) {
1069                 ioctx = sdev->ioctx_ring[wc->wr_id & ~SRPT_OP_RECV];
1070                 PRINT_ERROR("%s", "This is serious - SRQ is in bad state.");
1071         } else {
1072                 ioctx = sdev->ioctx_ring[wc->wr_id];
1073
1074                 if (ioctx->scmnd)
1075                         srpt_abort_scst_cmd(sdev, ioctx->scmnd);
1076                 else
1077                         srpt_reset_ioctx(ch, ioctx);
1078         }
1079 }
1080
1081 static void srpt_handle_send_comp(struct srpt_rdma_ch *ch,
1082                                   struct srpt_ioctx *ioctx,
1083                                   enum scst_exec_context context)
1084 {
1085         if (ioctx->scmnd) {
1086                 scst_data_direction dir =
1087                         scst_cmd_get_data_direction(ioctx->scmnd);
1088
1089                 if (dir != SCST_DATA_NONE && scst_cmd_get_sg(ioctx->scmnd))
1090                         ib_dma_unmap_sg(ch->sport->sdev->device,
1091                                         scst_cmd_get_sg(ioctx->scmnd),
1092                                         scst_cmd_get_sg_cnt(ioctx->scmnd),
1093                                         scst_to_tgt_dma_dir(dir));
1094
1095                 WARN_ON(ioctx->scmnd->state != SCST_CMD_STATE_XMIT_WAIT);
1096                 scst_tgt_cmd_done(ioctx->scmnd, context);
1097         } else
1098                 srpt_reset_ioctx(ch, ioctx);
1099 }
1100
1101 /** Process an RDMA completion notification. */
1102 static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch,
1103                                   struct srpt_ioctx *ioctx)
1104 {
1105         if (!ioctx->scmnd) {
1106                 WARN_ON("ERROR: ioctx->scmnd == NULL");
1107                 srpt_reset_ioctx(ch, ioctx);
1108                 return;
1109         }
1110
1111         /*
1112          * If an RDMA completion notification has been received for a write
1113          * command, tell SCST that processing can continue by calling
1114          * scst_rx_data().
1115          */
1116         if (srpt_test_and_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA,
1117                                 SRPT_STATE_DATA_IN) == SRPT_STATE_NEED_DATA) {
1118                 WARN_ON(scst_cmd_get_data_direction(ioctx->scmnd)
1119                         == SCST_DATA_READ);
1120                 scst_rx_data(ioctx->scmnd, SCST_RX_STATUS_SUCCESS,
1121                              scst_estimate_context());
1122         }
1123 }
1124
1125 /**
1126  * Build an SRP_RSP response.
1127  * @ch: RDMA channel through which the request has been received.
1128  * @ioctx: I/O context in which the SRP_RSP response will be built.
1129  * @s_key: sense key that will be stored in the response.
1130  * @s_code: value that will be stored in the asc_ascq field of the sense data.
1131  * @tag: tag of the request for which this response is being generated.
1132  *
1133  * Returns the size in bytes of the SRP_RSP response.
1134  *
1135  * An SRP_RSP response contains a SCSI status or service response. See also
1136  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
1137  * response. See also SPC-2 for more information about sense data.
1138  */
1139 static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
1140                               struct srpt_ioctx *ioctx, u8 s_key, u8 s_code,
1141                               u64 tag)
1142 {
1143         struct srp_rsp *srp_rsp;
1144         struct sense_data *sense;
1145         int limit_delta;
1146         int sense_data_len;
1147         int resp_len;
1148
1149         sense_data_len = (s_key == NO_SENSE) ? 0 : sizeof(*sense);
1150         resp_len = sizeof(*srp_rsp) + sense_data_len;
1151
1152         srp_rsp = ioctx->buf;
1153         memset(srp_rsp, 0, sizeof *srp_rsp);
1154
1155         limit_delta = atomic_read(&ch->req_lim_delta);
1156         atomic_sub(limit_delta, &ch->req_lim_delta);
1157
1158         srp_rsp->opcode = SRP_RSP;
1159         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1160         srp_rsp->tag = tag;
1161
1162         if (s_key != NO_SENSE) {
1163                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1164                 srp_rsp->status = SAM_STAT_CHECK_CONDITION;
1165                 srp_rsp->sense_data_len = cpu_to_be32(sense_data_len);
1166
1167                 sense = (struct sense_data *)(srp_rsp + 1);
1168                 sense->err_code = 0x70;
1169                 sense->key = s_key;
1170                 sense->asc_ascq = s_code;
1171         }
1172
1173         return resp_len;
1174 }
1175
1176 /**
1177  * Build a task management response, which is a specific SRP_RSP response.
1178  * @ch: RDMA channel through which the request has been received.
1179  * @ioctx: I/O context in which the SRP_RSP response will be built.
1180  * @rsp_code: RSP_CODE that will be stored in the response.
1181  * @tag: tag of the request for which this response is being generated.
1182  *
1183  * Returns the size in bytes of the SRP_RSP response.
1184  *
1185  * An SRP_RSP response contains a SCSI status or service response. See also
1186  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
1187  * response.
1188  */
1189 static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
1190                                   struct srpt_ioctx *ioctx, u8 rsp_code,
1191                                   u64 tag)
1192 {
1193         struct srp_rsp *srp_rsp;
1194         int limit_delta;
1195         int resp_data_len;
1196         int resp_len;
1197
1198         resp_data_len = (rsp_code == SRP_TSK_MGMT_SUCCESS) ? 0 : 4;
1199         resp_len = sizeof(*srp_rsp) + resp_data_len;
1200
1201         srp_rsp = ioctx->buf;
1202         memset(srp_rsp, 0, sizeof *srp_rsp);
1203
1204         limit_delta = atomic_read(&ch->req_lim_delta);
1205         atomic_sub(limit_delta, &ch->req_lim_delta);
1206
1207         srp_rsp->opcode = SRP_RSP;
1208         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1209         srp_rsp->tag = tag;
1210
1211         if (rsp_code != SRP_TSK_MGMT_SUCCESS) {
1212                 srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
1213                 srp_rsp->resp_data_len = cpu_to_be32(resp_data_len);
1214                 srp_rsp->data[3] = rsp_code;
1215         }
1216
1217         return resp_len;
1218 }
1219
1220 /*
1221  * Process SRP_CMD.
1222  */
1223 static int srpt_handle_cmd(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
1224 {
1225         struct scst_cmd *scmnd;
1226         struct srp_cmd *srp_cmd;
1227         struct srp_rsp *srp_rsp;
1228         scst_data_direction dir;
1229         int indirect_desc = 0;
1230         int ret;
1231         unsigned long flags;
1232
1233         srp_cmd = ioctx->buf;
1234         srp_rsp = ioctx->buf;
1235
1236         dir = SCST_DATA_NONE;
1237         if (srp_cmd->buf_fmt) {
1238                 ret = srpt_get_desc_tbl(ioctx, srp_cmd, &indirect_desc);
1239                 if (ret) {
1240                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1241                                            NO_ADD_SENSE, srp_cmd->tag);
1242                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1243                         goto err;
1244                 }
1245
1246                 if (indirect_desc) {
1247                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1248                                            NO_ADD_SENSE, srp_cmd->tag);
1249                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1250                         goto err;
1251                 }
1252
1253                 /*
1254                  * The lower four bits of the buffer format field contain the
1255                  * DATA-IN buffer descriptor format, and the highest four bits
1256                  * contain the DATA-OUT buffer descriptor format.
1257                  */
1258                 if (srp_cmd->buf_fmt & 0xf)
1259                         /* DATA-IN: transfer data from target to initiator. */
1260                         dir = SCST_DATA_READ;
1261                 else if (srp_cmd->buf_fmt >> 4)
1262                         /* DATA-OUT: transfer data from initiator to target. */
1263                         dir = SCST_DATA_WRITE;
1264         }
1265
1266         scmnd = scst_rx_cmd(ch->scst_sess, (u8 *) &srp_cmd->lun,
1267                             sizeof srp_cmd->lun, srp_cmd->cdb, 16,
1268                             thread ? SCST_NON_ATOMIC : SCST_ATOMIC);
1269         if (!scmnd) {
1270                 srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1271                                    NO_ADD_SENSE, srp_cmd->tag);
1272                 srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1273                 goto err;
1274         }
1275
1276         ioctx->scmnd = scmnd;
1277
1278         switch (srp_cmd->task_attr) {
1279         case SRP_CMD_HEAD_OF_Q:
1280                 scmnd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1281                 break;
1282         case SRP_CMD_ORDERED_Q:
1283                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1284                 break;
1285         case SRP_CMD_SIMPLE_Q:
1286                 scmnd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1287                 break;
1288         case SRP_CMD_ACA:
1289                 scmnd->queue_type = SCST_CMD_QUEUE_ACA;
1290                 break;
1291         default:
1292                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1293                 break;
1294         }
1295
1296         scst_cmd_set_tag(scmnd, srp_cmd->tag);
1297         scst_cmd_set_tgt_priv(scmnd, ioctx);
1298         scst_cmd_set_expected(scmnd, dir, ioctx->data_len);
1299
1300         spin_lock_irqsave(&ch->spinlock, flags);
1301         list_add_tail(&ioctx->scmnd_list, &ch->active_scmnd_list);
1302         ch->active_scmnd_cnt++;
1303         spin_unlock_irqrestore(&ch->spinlock, flags);
1304
1305         scst_cmd_init_done(scmnd, scst_estimate_context());
1306
1307         return 0;
1308
1309 err:
1310         WARN_ON(srp_rsp->opcode != SRP_RSP);
1311
1312         return -1;
1313 }
1314
1315 /*
1316  * Process an SRP_TSK_MGMT request.
1317  *
1318  * Returns 0 upon success and -1 upon failure.
1319  *
1320  * Each task management function is performed by calling one of the
1321  * scst_rx_mgmt_fn*() functions. These functions will either report failure
1322  * or process the task management function asynchronously. The function
1323  * srpt_tsk_mgmt_done() will be called by the SCST core upon completion of the
1324  * task management function. When srpt_handle_tsk_mgmt() reports failure
1325  * (i.e. returns -1) a response will have been built in ioctx->buf. This
1326  * information unit has to be sent back by the caller.
1327  *
1328  * For more information about SRP_TSK_MGMT information units, see also section
1329  * 6.7 in the T10 SRP r16a document.
1330  */
1331 static int srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
1332                                 struct srpt_ioctx *ioctx)
1333 {
1334         struct srp_tsk_mgmt *srp_tsk;
1335         struct srpt_mgmt_ioctx *mgmt_ioctx;
1336         int ret;
1337
1338         ib_dma_sync_single_for_cpu(ch->sport->sdev->device, ioctx->dma,
1339                                    sizeof(struct srp_tsk_mgmt),
1340                                    DMA_FROM_DEVICE);
1341
1342         srp_tsk = ioctx->buf;
1343
1344         TRACE_DBG("recv_tsk_mgmt= %d for task_tag= %lld"
1345                   " using tag= %lld cm_id= %p sess= %p",
1346                   srp_tsk->tsk_mgmt_func,
1347                   (unsigned long long) srp_tsk->task_tag,
1348                   (unsigned long long) srp_tsk->tag,
1349                   ch->cm_id, ch->scst_sess);
1350
1351         mgmt_ioctx = kmalloc(sizeof *mgmt_ioctx, GFP_ATOMIC);
1352         if (!mgmt_ioctx) {
1353                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1354                                        srp_tsk->tag);
1355                 goto err;
1356         }
1357
1358         mgmt_ioctx->ioctx = ioctx;
1359         mgmt_ioctx->ch = ch;
1360         mgmt_ioctx->tag = srp_tsk->tag;
1361
1362         switch (srp_tsk->tsk_mgmt_func) {
1363         case SRP_TSK_ABORT_TASK:
1364                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK");
1365                 ret = scst_rx_mgmt_fn_tag(ch->scst_sess,
1366                                           SCST_ABORT_TASK,
1367                                           srp_tsk->task_tag,
1368                                           thread ?
1369                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1370                                           mgmt_ioctx);
1371                 break;
1372         case SRP_TSK_ABORT_TASK_SET:
1373                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK_SET");
1374                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1375                                           SCST_ABORT_TASK_SET,
1376                                           (u8 *) &srp_tsk->lun,
1377                                           sizeof srp_tsk->lun,
1378                                           thread ?
1379                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1380                                           mgmt_ioctx);
1381                 break;
1382         case SRP_TSK_CLEAR_TASK_SET:
1383                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_TASK_SET");
1384                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1385                                           SCST_CLEAR_TASK_SET,
1386                                           (u8 *) &srp_tsk->lun,
1387                                           sizeof srp_tsk->lun,
1388                                           thread ?
1389                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1390                                           mgmt_ioctx);
1391                 break;
1392         case SRP_TSK_LUN_RESET:
1393                 TRACE_DBG("%s", "Processing SRP_TSK_LUN_RESET");
1394                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1395                                           SCST_LUN_RESET,
1396                                           (u8 *) &srp_tsk->lun,
1397                                           sizeof srp_tsk->lun,
1398                                           thread ?
1399                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1400                                           mgmt_ioctx);
1401                 break;
1402         case SRP_TSK_CLEAR_ACA:
1403                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_ACA");
1404                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1405                                           SCST_CLEAR_ACA,
1406                                           (u8 *) &srp_tsk->lun,
1407                                           sizeof srp_tsk->lun,
1408                                           thread ?
1409                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1410                                           mgmt_ioctx);
1411                 break;
1412         default:
1413                 TRACE_DBG("%s", "Unsupported task management function.");
1414                 srpt_build_tskmgmt_rsp(ch, ioctx,
1415                                        SRP_TSK_MGMT_FUNC_NOT_SUPP,
1416                                        srp_tsk->tag);
1417                 goto err;
1418         }
1419
1420         if (ret) {
1421                 TRACE_DBG("%s", "Processing task management function failed.");
1422                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1423                                        srp_tsk->tag);
1424                 goto err;
1425         }
1426
1427         WARN_ON(srp_tsk->opcode == SRP_RSP);
1428
1429         return 0;
1430
1431 err:
1432         WARN_ON(srp_tsk->opcode != SRP_RSP);
1433
1434         kfree(mgmt_ioctx);
1435         return -1;
1436 }
1437
1438 /**
1439  * Process a newly received information unit.
1440  * @ch: RDMA channel through which the information unit has been received.
1441  * @ioctx: SRPT I/O context associated with the information unit.
1442  */
1443 static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
1444                                struct srpt_ioctx *ioctx)
1445 {
1446         struct srp_cmd *srp_cmd;
1447         struct srp_rsp *srp_rsp;
1448         unsigned long flags;
1449         int len;
1450
1451         spin_lock_irqsave(&ch->spinlock, flags);
1452         if (ch->state != RDMA_CHANNEL_LIVE) {
1453                 if (ch->state == RDMA_CHANNEL_CONNECTING) {
1454                         list_add_tail(&ioctx->wait_list, &ch->cmd_wait_list);
1455                         spin_unlock_irqrestore(&ch->spinlock, flags);
1456                         return;
1457                 } else {
1458                         spin_unlock_irqrestore(&ch->spinlock, flags);
1459                         srpt_reset_ioctx(ch, ioctx);
1460                         return;
1461                 }
1462         }
1463         spin_unlock_irqrestore(&ch->spinlock, flags);
1464
1465         ib_dma_sync_single_for_cpu(ch->sport->sdev->device, ioctx->dma,
1466                                    sizeof(struct srp_cmd), DMA_FROM_DEVICE);
1467
1468         ioctx->data_len = 0;
1469         ioctx->n_rbuf = 0;
1470         ioctx->rbufs = NULL;
1471         ioctx->n_rdma = 0;
1472         ioctx->n_rdma_ius = 0;
1473         ioctx->rdma_ius = NULL;
1474         ioctx->scmnd = NULL;
1475         ioctx->ch = ch;
1476         ioctx->state = SRPT_STATE_NEW;
1477
1478         srp_cmd = ioctx->buf;
1479         srp_rsp = ioctx->buf;
1480
1481         switch (srp_cmd->opcode) {
1482         case SRP_CMD:
1483                 if (srpt_handle_cmd(ch, ioctx) < 0)
1484                         goto err;
1485                 break;
1486
1487         case SRP_TSK_MGMT:
1488                 if (srpt_handle_tsk_mgmt(ch, ioctx) < 0)
1489                         goto err;
1490                 break;
1491
1492         case SRP_I_LOGOUT:
1493         case SRP_AER_REQ:
1494         default:
1495                 srpt_build_cmd_rsp(ch, ioctx, ILLEGAL_REQUEST, INVALID_CDB,
1496                                    srp_cmd->tag);
1497                 goto err;
1498         }
1499
1500         return;
1501
1502 err:
1503         WARN_ON(srp_rsp->opcode != SRP_RSP);
1504         len = (sizeof *srp_rsp) + be32_to_cpu(srp_rsp->sense_data_len);
1505
1506         if (ch->state != RDMA_CHANNEL_LIVE) {
1507                 /* Give up if another thread modified the channel state. */
1508                 PRINT_ERROR("%s: channel is in state %d", __func__, ch->state);
1509                 srpt_reset_ioctx(ch, ioctx);
1510         } else if (srpt_post_send(ch, ioctx, len)) {
1511                 PRINT_ERROR("%s: sending SRP_RSP response failed", __func__);
1512                 srpt_reset_ioctx(ch, ioctx);
1513         }
1514 }
1515
1516 /*
1517  * Returns true if the ioctx list is non-empty or if the ib_srpt kernel thread
1518  * should stop.
1519  * @pre thread != 0
1520  */
1521 static inline int srpt_test_ioctx_list(void)
1522 {
1523         int res = (!list_empty(&srpt_thread.thread_ioctx_list) ||
1524                    unlikely(kthread_should_stop()));
1525         return res;
1526 }
1527
1528 /*
1529  * Add 'ioctx' to the tail of the ioctx list and wake up the kernel thread.
1530  *
1531  * @pre thread != 0
1532  */
1533 static inline void srpt_schedule_thread(struct srpt_ioctx *ioctx)
1534 {
1535         unsigned long flags;
1536
1537         spin_lock_irqsave(&srpt_thread.thread_lock, flags);
1538         list_add_tail(&ioctx->comp_list, &srpt_thread.thread_ioctx_list);
1539         spin_unlock_irqrestore(&srpt_thread.thread_lock, flags);
1540         wake_up(&ioctx_list_waitQ);
1541 }
1542
1543 /**
1544  * InfiniBand completion queue callback function.
1545  * @cq: completion queue.
1546  * @ctx: completion queue context, which was passed as the fourth argument of
1547  *       the function ib_create_cq().
1548  */
1549 static void srpt_completion(struct ib_cq *cq, void *ctx)
1550 {
1551         struct srpt_rdma_ch *ch = ctx;
1552         struct srpt_device *sdev = ch->sport->sdev;
1553         struct ib_wc wc;
1554         struct srpt_ioctx *ioctx;
1555
1556         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1557         while (ib_poll_cq(ch->cq, 1, &wc) > 0) {
1558                 if (wc.status) {
1559                         PRINT_ERROR("failed %s status= %d",
1560                                wc.wr_id & SRPT_OP_RECV ? "receive" : "send",
1561                                wc.status);
1562                         srpt_handle_err_comp(ch, &wc);
1563                         break;
1564                 }
1565
1566                 if (wc.wr_id & SRPT_OP_RECV) {
1567                         ioctx = sdev->ioctx_ring[wc.wr_id & ~SRPT_OP_RECV];
1568                         if (thread) {
1569                                 ioctx->ch = ch;
1570                                 ioctx->op = IB_WC_RECV;
1571                                 srpt_schedule_thread(ioctx);
1572                         } else
1573                                 srpt_handle_new_iu(ch, ioctx);
1574                         continue;
1575                 } else
1576                         ioctx = sdev->ioctx_ring[wc.wr_id];
1577
1578                 if (thread) {
1579                         ioctx->ch = ch;
1580                         ioctx->op = wc.opcode;
1581                         srpt_schedule_thread(ioctx);
1582                 } else {
1583                         switch (wc.opcode) {
1584                         case IB_WC_SEND:
1585                                 srpt_handle_send_comp(ch, ioctx,
1586                                         scst_estimate_context());
1587                                 break;
1588                         case IB_WC_RDMA_WRITE:
1589                         case IB_WC_RDMA_READ:
1590                                 srpt_handle_rdma_comp(ch, ioctx);
1591                                 break;
1592                         default:
1593                                 break;
1594                         }
1595                 }
1596
1597 #if defined(CONFIG_SCST_DEBUG)
1598                 if (interrupt_processing_delay_in_us <= MAX_UDELAY_MS * 1000)
1599                         udelay(interrupt_processing_delay_in_us);
1600 #endif
1601         }
1602 }
1603
1604 /*
1605  * Create a completion queue on the specified device.
1606  */
1607 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
1608 {
1609         struct ib_qp_init_attr *qp_init;
1610         struct srpt_device *sdev = ch->sport->sdev;
1611         int cqe;
1612         int ret;
1613
1614         qp_init = kzalloc(sizeof *qp_init, GFP_KERNEL);
1615         if (!qp_init)
1616                 return -ENOMEM;
1617
1618         /* Create a completion queue (CQ). */
1619
1620         cqe = SRPT_RQ_SIZE + SRPT_SQ_SIZE - 1;
1621 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(RHEL_RELEASE_CODE)
1622         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe);
1623 #else
1624         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe, 0);
1625 #endif
1626         if (IS_ERR(ch->cq)) {
1627                 ret = PTR_ERR(ch->cq);
1628                 PRINT_ERROR("failed to create_cq cqe= %d ret= %d", cqe, ret);
1629                 goto out;
1630         }
1631
1632         /* Request completion notification. */
1633
1634         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1635
1636         /* Create a queue pair (QP). */
1637
1638         qp_init->qp_context = (void *)ch;
1639         qp_init->event_handler = srpt_qp_event;
1640         qp_init->send_cq = ch->cq;
1641         qp_init->recv_cq = ch->cq;
1642         qp_init->srq = sdev->srq;
1643         qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
1644         qp_init->qp_type = IB_QPT_RC;
1645         qp_init->cap.max_send_wr = SRPT_SQ_SIZE;
1646         qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE;
1647
1648         ch->qp = ib_create_qp(sdev->pd, qp_init);
1649         if (IS_ERR(ch->qp)) {
1650                 ret = PTR_ERR(ch->qp);
1651                 ib_destroy_cq(ch->cq);
1652                 PRINT_ERROR("failed to create_qp ret= %d", ret);
1653                 goto out;
1654         }
1655
1656         TRACE_DBG("%s: max_cqe= %d max_sge= %d cm_id= %p",
1657                __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
1658                ch->cm_id);
1659
1660         /* Modify the attributes and the state of queue pair ch->qp. */
1661
1662         ret = srpt_init_ch_qp(ch, ch->qp);
1663         if (ret) {
1664                 ib_destroy_qp(ch->qp);
1665                 ib_destroy_cq(ch->cq);
1666                 goto out;
1667         }
1668
1669         atomic_set(&ch->req_lim_delta, SRPT_RQ_SIZE);
1670 out:
1671         kfree(qp_init);
1672         return ret;
1673 }
1674
1675 /**
1676  * Look up the RDMA channel that corresponds to the specified cm_id.
1677  *
1678  * Return NULL if no matching RDMA channel has been found.
1679  */
1680 static struct srpt_rdma_ch *srpt_find_channel(struct ib_cm_id *cm_id, bool del)
1681 {
1682         struct srpt_device *sdev = cm_id->context;
1683         struct srpt_rdma_ch *ch;
1684
1685         spin_lock_irq(&sdev->spinlock);
1686         list_for_each_entry(ch, &sdev->rch_list, list) {
1687                 if (ch->cm_id == cm_id) {
1688                         if (del)
1689                                 list_del(&ch->list);
1690                         spin_unlock_irq(&sdev->spinlock);
1691                         return ch;
1692                 }
1693         }
1694
1695         spin_unlock_irq(&sdev->spinlock);
1696
1697         return NULL;
1698 }
1699
1700 /**
1701  * Release all resources associated with the specified RDMA channel.
1702  *
1703  * Note: the caller must have removed the channel from the channel list
1704  * before calling this function.
1705  */
1706 static void srpt_release_channel(struct srpt_rdma_ch *ch, bool destroy_cmid)
1707 {
1708         unsigned long flags;
1709
1710         TRACE_ENTRY();
1711
1712         WARN_ON(srpt_find_channel(ch->cm_id, false) == ch);
1713
1714         /*
1715          * Make sure that no new commands are accepted while the channel
1716          * is being released.
1717          */
1718         srpt_set_channel_state(ch, RDMA_CHANNEL_DISCONNECTING);
1719
1720         if (ch->cm_id && destroy_cmid) {
1721                 TRACE_DBG("%s: destroy cm_id= %p", __func__, ch->cm_id);
1722                 ib_destroy_cm_id(ch->cm_id);
1723                 ch->cm_id = NULL;
1724         }
1725
1726         ib_destroy_qp(ch->qp);
1727         ib_destroy_cq(ch->cq);
1728
1729         if (ch->scst_sess) {
1730                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1731
1732                 if (ch->active_scmnd_cnt)
1733                         PRINT_INFO("Releasing session %s which still has %d"
1734                                    " active commands",
1735                                    ch->sess_name, ch->active_scmnd_cnt);
1736                 else
1737                         PRINT_INFO("Releasing session %s", ch->sess_name);
1738
1739                 spin_lock_irqsave(&ch->spinlock, flags);
1740                 list_for_each_entry_safe(ioctx, ioctx_tmp,
1741                                          &ch->active_scmnd_list, scmnd_list) {
1742                         spin_unlock_irqrestore(&ch->spinlock, flags);
1743
1744                         if (ioctx->scmnd)
1745                                 srpt_abort_scst_cmd(ch->sport->sdev,
1746                                                     ioctx->scmnd);
1747
1748                         spin_lock_irqsave(&ch->spinlock, flags);
1749                 }
1750                 WARN_ON(!list_empty(&ch->active_scmnd_list));
1751                 WARN_ON(ch->active_scmnd_cnt != 0);
1752                 spin_unlock_irqrestore(&ch->spinlock, flags);
1753
1754                 scst_unregister_session(ch->scst_sess, 0, NULL);
1755                 ch->scst_sess = NULL;
1756         }
1757
1758         kfree(ch);
1759
1760         TRACE_EXIT();
1761 }
1762
1763 static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
1764                             struct ib_cm_req_event_param *param,
1765                             void *private_data)
1766 {
1767         struct srpt_device *sdev = cm_id->context;
1768         struct srp_login_req *req;
1769         struct srp_login_rsp *rsp;
1770         struct srp_login_rej *rej;
1771         struct ib_cm_rep_param *rep_param;
1772         struct srpt_rdma_ch *ch, *tmp_ch;
1773         u32 it_iu_len;
1774         int ret = 0;
1775
1776 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
1777         WARN_ON(!sdev || !private_data);
1778         if (!sdev || !private_data)
1779                 return -EINVAL;
1780 #else
1781         if (WARN_ON(!sdev || !private_data))
1782                 return -EINVAL;
1783 #endif
1784
1785         req = (struct srp_login_req *)private_data;
1786
1787         it_iu_len = be32_to_cpu(req->req_it_iu_len);
1788
1789         PRINT_INFO("Received SRP_LOGIN_REQ with"
1790             " i_port_id 0x%llx:0x%llx, t_port_id 0x%llx:0x%llx and length %d"
1791             " on port %d (guid=0x%llx:0x%llx)",
1792             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[0]),
1793             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[8]),
1794             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[0]),
1795             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[8]),
1796             it_iu_len,
1797             param->port,
1798             (unsigned long long)be64_to_cpu(*(u64 *)
1799                                 &sdev->port[param->port - 1].gid.raw[0]),
1800             (unsigned long long)be64_to_cpu(*(u64 *)
1801                                 &sdev->port[param->port - 1].gid.raw[8]));
1802
1803         rsp = kzalloc(sizeof *rsp, GFP_KERNEL);
1804         rej = kzalloc(sizeof *rej, GFP_KERNEL);
1805         rep_param = kzalloc(sizeof *rep_param, GFP_KERNEL);
1806
1807         if (!rsp || !rej || !rep_param) {
1808                 ret = -ENOMEM;
1809                 goto out;
1810         }
1811
1812         if (it_iu_len > srp_max_message_size || it_iu_len < 64) {
1813                 rej->reason =
1814                     cpu_to_be32(SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
1815                 ret = -EINVAL;
1816                 PRINT_ERROR("rejected SRP_LOGIN_REQ because its"
1817                             " length (%d bytes) is out of range (%d .. %d)",
1818                             it_iu_len, 64, srp_max_message_size);
1819                 goto reject;
1820         }
1821
1822         if ((req->req_flags & 0x3) == SRP_MULTICHAN_SINGLE) {
1823                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
1824
1825                 spin_lock_irq(&sdev->spinlock);
1826
1827                 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
1828                         if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
1829                             && !memcmp(ch->t_port_id, req->target_port_id, 16)
1830                             && param->port == ch->sport->port
1831                             && param->listen_id == ch->sport->sdev->cm_id
1832                             && ch->cm_id) {
1833                                 enum rdma_ch_state prev_state;
1834
1835                                 /* found an existing channel */
1836                                 TRACE_DBG("Found existing channel name= %s"
1837                                           " cm_id= %p state= %d",
1838                                           ch->sess_name, ch->cm_id, ch->state);
1839
1840                                 prev_state = ch->state;
1841                                 if (ch->state == RDMA_CHANNEL_LIVE)
1842                                         ch->state = RDMA_CHANNEL_DISCONNECTING;
1843                                 else if (ch->state == RDMA_CHANNEL_CONNECTING)
1844                                         list_del(&ch->list);
1845
1846                                 spin_unlock_irq(&sdev->spinlock);
1847
1848                                 rsp->rsp_flags =
1849                                         SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
1850
1851                                 if (prev_state == RDMA_CHANNEL_LIVE) {
1852                                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
1853                                         PRINT_INFO("disconnected"
1854                                           " session %s because a new"
1855                                           " SRP_LOGIN_REQ has been received.",
1856                                           ch->sess_name);
1857                                 } else if (prev_state ==
1858                                          RDMA_CHANNEL_CONNECTING) {
1859                                         PRINT_ERROR("%s", "rejected"
1860                                           " SRP_LOGIN_REQ because another login"
1861                                           " request is being processed.");
1862                                         ib_send_cm_rej(ch->cm_id,
1863                                                        IB_CM_REJ_NO_RESOURCES,
1864                                                        NULL, 0, NULL, 0);
1865                                         srpt_release_channel(ch, 1);
1866                                 }
1867
1868                                 spin_lock_irq(&sdev->spinlock);
1869                         }
1870                 }
1871
1872                 spin_unlock_irq(&sdev->spinlock);
1873
1874         } else
1875                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
1876
1877         if (((u64) (*(u64 *) req->target_port_id) !=
1878              cpu_to_be64(srpt_service_guid)) ||
1879             ((u64) (*(u64 *) (req->target_port_id + 8)) !=
1880              cpu_to_be64(srpt_service_guid))) {
1881                 rej->reason =
1882                     cpu_to_be32(SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
1883                 ret = -ENOMEM;
1884                 PRINT_ERROR("%s", "rejected SRP_LOGIN_REQ because it"
1885                        " has an invalid target port identifier.");
1886                 goto reject;
1887         }
1888
1889         ch = kzalloc(sizeof *ch, GFP_KERNEL);
1890         if (!ch) {
1891                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1892                 PRINT_ERROR("%s",
1893                             "rejected SRP_LOGIN_REQ because out of memory.");
1894                 ret = -ENOMEM;
1895                 goto reject;
1896         }
1897
1898         spin_lock_init(&ch->spinlock);
1899         memcpy(ch->i_port_id, req->initiator_port_id, 16);
1900         memcpy(ch->t_port_id, req->target_port_id, 16);
1901         ch->sport = &sdev->port[param->port - 1];
1902         ch->cm_id = cm_id;
1903         ch->state = RDMA_CHANNEL_CONNECTING;
1904         INIT_LIST_HEAD(&ch->cmd_wait_list);
1905         INIT_LIST_HEAD(&ch->active_scmnd_list);
1906
1907         ret = srpt_create_ch_ib(ch);
1908         if (ret) {
1909                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1910                 PRINT_ERROR("%s", "rejected SRP_LOGIN_REQ because creating"
1911                             " a new RDMA channel failed.");
1912                 goto free_ch;
1913         }
1914
1915         ret = srpt_ch_qp_rtr(ch, ch->qp);
1916         if (ret) {
1917                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1918                 PRINT_ERROR("rejected SRP_LOGIN_REQ because enabling"
1919                        " RTR failed (error code = %d)", ret);
1920                 goto destroy_ib;
1921         }
1922
1923         if (use_port_guid_in_session_name) {
1924                 /*
1925                  * If the kernel module parameter use_port_guid_in_session_name
1926                  * has been specified, use a combination of the target port
1927                  * GUID and the initiator port ID as the session name. This
1928                  * was the original behavior of the SRP target implementation
1929                  * (i.e. before the SRPT was included in OFED 1.3).
1930                  */
1931                 snprintf(ch->sess_name, sizeof(ch->sess_name),
1932                          "0x%016llx%016llx",
1933                          (unsigned long long)be64_to_cpu(*(u64 *)
1934                                 &sdev->port[param->port - 1].gid.raw[8]),
1935                          (unsigned long long)be64_to_cpu(*(u64 *)
1936                                 (ch->i_port_id + 8)));
1937         } else {
1938                 /*
1939                  * Default behavior: use the initator port identifier as the
1940                  * session name.
1941                  */
1942                 snprintf(ch->sess_name, sizeof(ch->sess_name),
1943                          "0x%016llx%016llx",
1944                          (unsigned long long)be64_to_cpu(*(u64 *)ch->i_port_id),
1945                          (unsigned long long)be64_to_cpu(*(u64 *)
1946                                  (ch->i_port_id + 8)));
1947         }
1948
1949         TRACE_DBG("registering session %s", ch->sess_name);
1950
1951         BUG_ON(!sdev->scst_tgt);
1952         ch->scst_sess = scst_register_session(sdev->scst_tgt, 0, ch->sess_name,
1953                                               NULL, NULL);
1954         if (!ch->scst_sess) {
1955                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1956                 TRACE_DBG("%s", "Failed to create scst sess");
1957                 goto destroy_ib;
1958         }
1959
1960         TRACE_DBG("Establish connection sess=%p name=%s cm_id=%p",
1961                   ch->scst_sess, ch->sess_name, ch->cm_id);
1962
1963         scst_sess_set_tgt_priv(ch->scst_sess, ch);
1964
1965         /* create srp_login_response */
1966         rsp->opcode = SRP_LOGIN_RSP;
1967         rsp->tag = req->tag;
1968         rsp->max_it_iu_len = req->req_it_iu_len;
1969         rsp->max_ti_iu_len = req->req_it_iu_len;
1970         rsp->buf_fmt =
1971             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1972         rsp->req_lim_delta = cpu_to_be32(SRPT_RQ_SIZE);
1973         atomic_set(&ch->req_lim_delta, 0);
1974
1975         /* create cm reply */
1976         rep_param->qp_num = ch->qp->qp_num;
1977         rep_param->private_data = (void *)rsp;
1978         rep_param->private_data_len = sizeof *rsp;
1979         rep_param->rnr_retry_count = 7;
1980         rep_param->flow_control = 1;
1981         rep_param->failover_accepted = 0;
1982         rep_param->srq = 1;
1983         rep_param->responder_resources = 4;
1984         rep_param->initiator_depth = 4;
1985
1986         ret = ib_send_cm_rep(cm_id, rep_param);
1987         if (ret) {
1988                 PRINT_ERROR("sending SRP_LOGIN_REQ response failed"
1989                             " (error code = %d)", ret);
1990                 goto release_channel;
1991         }
1992
1993         spin_lock_irq(&sdev->spinlock);
1994         list_add_tail(&ch->list, &sdev->rch_list);
1995         spin_unlock_irq(&sdev->spinlock);
1996
1997         goto out;
1998
1999 release_channel:
2000         scst_unregister_session(ch->scst_sess, 0, NULL);
2001         ch->scst_sess = NULL;
2002
2003 destroy_ib:
2004         ib_destroy_qp(ch->qp);
2005         ib_destroy_cq(ch->cq);
2006
2007 free_ch:
2008         kfree(ch);
2009
2010 reject:
2011         rej->opcode = SRP_LOGIN_REJ;
2012         rej->tag = req->tag;
2013         rej->buf_fmt =
2014             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
2015
2016         ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
2017                              (void *)rej, sizeof *rej);
2018
2019 out:
2020         kfree(rep_param);
2021         kfree(rsp);
2022         kfree(rej);
2023
2024         return ret;
2025 }
2026
2027 /**
2028  * Release the channel with the specified cm_id.
2029  *
2030  * Returns one to indicate that the caller of srpt_cm_handler() should destroy
2031  * the cm_id.
2032  */
2033 static void srpt_find_and_release_channel(struct ib_cm_id *cm_id)
2034 {
2035         struct srpt_rdma_ch *ch;
2036
2037         ch = srpt_find_channel(cm_id, true);
2038         if (ch)
2039                 srpt_release_channel(ch, 0);
2040 }
2041
2042 static void srpt_cm_rej_recv(struct ib_cm_id *cm_id)
2043 {
2044         PRINT_INFO("Received InfiniBand REJ packet for cm_id %p.", cm_id);
2045         srpt_find_and_release_channel(cm_id);
2046 }
2047
2048 /**
2049  * Process an IB_CM_RTU_RECEIVED or IB_CM_USER_ESTABLISHED event.
2050  *
2051  * An IB_CM_RTU_RECEIVED message indicates that the connection is established
2052  * and that the recipient may begin transmitting (RTU = ready to use).
2053  */
2054 static int srpt_cm_rtu_recv(struct ib_cm_id *cm_id)
2055 {
2056         struct srpt_rdma_ch *ch;
2057         int ret;
2058
2059         ch = srpt_find_channel(cm_id, false);
2060         if (!ch)
2061                 return -EINVAL;
2062
2063         if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_CONNECTING,
2064                                             RDMA_CHANNEL_LIVE)) {
2065                 struct srpt_ioctx *ioctx, *ioctx_tmp;
2066
2067                 ret = srpt_ch_qp_rts(ch, ch->qp);
2068
2069                 list_for_each_entry_safe(ioctx, ioctx_tmp, &ch->cmd_wait_list,
2070                                          wait_list) {
2071                         list_del(&ioctx->wait_list);
2072                         srpt_handle_new_iu(ch, ioctx);
2073                 }
2074                 if (ret && srpt_test_and_set_channel_state(ch,
2075                                         RDMA_CHANNEL_LIVE,
2076                                         RDMA_CHANNEL_DISCONNECTING)) {
2077                         TRACE_DBG("cm_id=%p sess_name=%s state=%d",
2078                                   cm_id, ch->sess_name, ch->state);
2079                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
2080                 }
2081         } else if (ch->state == RDMA_CHANNEL_DISCONNECTING) {
2082                 TRACE_DBG("cm_id=%p sess_name=%s state=%d",
2083                           cm_id, ch->sess_name, ch->state);
2084                 ib_send_cm_dreq(ch->cm_id, NULL, 0);
2085                 ret = -EAGAIN;
2086         } else
2087                 ret = 0;
2088
2089         return ret;
2090 }
2091
2092 static void srpt_cm_timewait_exit(struct ib_cm_id *cm_id)
2093 {
2094         PRINT_INFO("Received InfiniBand TimeWait exit for cm_id %p.", cm_id);
2095         srpt_find_and_release_channel(cm_id);
2096 }
2097
2098 static void srpt_cm_rep_error(struct ib_cm_id *cm_id)
2099 {
2100         PRINT_INFO("Received InfiniBand REP error for cm_id %p.", cm_id);
2101         srpt_find_and_release_channel(cm_id);
2102 }
2103
2104 static int srpt_cm_dreq_recv(struct ib_cm_id *cm_id)
2105 {
2106         struct srpt_rdma_ch *ch;
2107
2108         ch = srpt_find_channel(cm_id, false);
2109         if (!ch)
2110                 return -EINVAL;
2111
2112         TRACE_DBG("%s: cm_id= %p ch->state= %d",
2113                  __func__, cm_id, ch->state);
2114
2115         switch (ch->state) {
2116         case RDMA_CHANNEL_LIVE:
2117         case RDMA_CHANNEL_CONNECTING:
2118                 ib_send_cm_drep(ch->cm_id, NULL, 0);
2119                 PRINT_INFO("Received DREQ and sent DREP for session %s.",
2120                            ch->sess_name);
2121                 break;
2122         case RDMA_CHANNEL_DISCONNECTING:
2123         default:
2124                 break;
2125         }
2126
2127         return 0;
2128 }
2129
2130 static void srpt_cm_drep_recv(struct ib_cm_id *cm_id)
2131 {
2132         PRINT_INFO("Received InfiniBand DREP message for cm_id %p.", cm_id);
2133         srpt_find_and_release_channel(cm_id);
2134 }
2135
2136 /**
2137  * IB connection manager callback function.
2138  *
2139  * A non-zero return value will make the caller destroy the CM ID.
2140  *
2141  * Note: srpt_add_one passes a struct srpt_device* as the third argument to
2142  * the ib_create_cm_id() call.
2143  */
2144 static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2145 {
2146         int ret = 0;
2147
2148         switch (event->event) {
2149         case IB_CM_REQ_RECEIVED:
2150                 ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd,
2151                                        event->private_data);
2152                 break;
2153         case IB_CM_REJ_RECEIVED:
2154                 srpt_cm_rej_recv(cm_id);
2155                 ret = -EINVAL;
2156                 break;
2157         case IB_CM_RTU_RECEIVED:
2158         case IB_CM_USER_ESTABLISHED:
2159                 ret = srpt_cm_rtu_recv(cm_id);
2160                 break;
2161         case IB_CM_DREQ_RECEIVED:
2162                 ret = srpt_cm_dreq_recv(cm_id);
2163                 break;
2164         case IB_CM_DREP_RECEIVED:
2165                 srpt_cm_drep_recv(cm_id);
2166                 ret = -EINVAL;
2167                 break;
2168         case IB_CM_TIMEWAIT_EXIT:
2169                 srpt_cm_timewait_exit(cm_id);
2170                 ret = -EINVAL;
2171                 break;
2172         case IB_CM_REP_ERROR:
2173                 srpt_cm_rep_error(cm_id);
2174                 ret = -EINVAL;
2175                 break;
2176         default:
2177                 break;
2178         }
2179
2180         return ret;
2181 }
2182
2183 static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
2184                                  struct srpt_ioctx *ioctx,
2185                                  struct scst_cmd *scmnd)
2186 {
2187         struct scatterlist *scat;
2188         scst_data_direction dir;
2189         struct rdma_iu *riu;
2190         struct srp_direct_buf *db;
2191         dma_addr_t dma_addr;
2192         struct ib_sge *sge;
2193         u64 raddr;
2194         u32 rsize;
2195         u32 tsize;
2196         u32 dma_len;
2197         int count, nrdma;
2198         int i, j, k;
2199
2200         scat = scst_cmd_get_sg(scmnd);
2201         dir = scst_cmd_get_data_direction(scmnd);
2202         WARN_ON(scat == NULL);
2203         count = ib_dma_map_sg(ch->sport->sdev->device, scat,
2204                               scst_cmd_get_sg_cnt(scmnd),
2205                               scst_to_tgt_dma_dir(dir));
2206         if (unlikely(!count))
2207                 return -EBUSY;
2208
2209         if (ioctx->rdma_ius && ioctx->n_rdma_ius)
2210                 nrdma = ioctx->n_rdma_ius;
2211         else {
2212                 nrdma = count / SRPT_DEF_SG_PER_WQE + ioctx->n_rbuf;
2213
2214                 ioctx->rdma_ius = kzalloc(nrdma * sizeof *riu,
2215                                           scst_cmd_atomic(scmnd)
2216                                           ? GFP_ATOMIC : GFP_KERNEL);
2217                 if (!ioctx->rdma_ius) {
2218                         WARN_ON(scat == NULL);
2219                         ib_dma_unmap_sg(ch->sport->sdev->device,
2220                                         scat, scst_cmd_get_sg_cnt(scmnd),
2221                                         scst_to_tgt_dma_dir(dir));
2222                         return -ENOMEM;
2223                 }
2224
2225                 ioctx->n_rdma_ius = nrdma;
2226         }
2227
2228         db = ioctx->rbufs;
2229         tsize = (dir == SCST_DATA_READ) ?
2230                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2231         dma_len = sg_dma_len(&scat[0]);
2232         riu = ioctx->rdma_ius;
2233
2234         /*
2235          * For each remote desc - calculate the #ib_sge.
2236          * If #ib_sge < SRPT_DEF_SG_PER_WQE per rdma operation then
2237          *      each remote desc rdma_iu is required a rdma wr;
2238          * else
2239          *      we need to allocate extra rdma_iu to carry extra #ib_sge in
2240          *      another rdma wr
2241          */
2242         for (i = 0, j = 0;
2243              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2244                 rsize = be32_to_cpu(db->len);
2245                 raddr = be64_to_cpu(db->va);
2246                 riu->raddr = raddr;
2247                 riu->rkey = be32_to_cpu(db->key);
2248                 riu->sge_cnt = 0;
2249
2250                 /* calculate how many sge required for this remote_buf */
2251                 while (rsize > 0 && tsize > 0) {
2252
2253                         if (rsize >= dma_len) {
2254                                 tsize -= dma_len;
2255                                 rsize -= dma_len;
2256                                 raddr += dma_len;
2257
2258                                 if (tsize > 0) {
2259                                         ++j;
2260                                         if (j < count)
2261                                                 dma_len = sg_dma_len(&scat[j]);
2262                                 }
2263                         } else {
2264                                 tsize -= rsize;
2265                                 dma_len -= rsize;
2266                                 rsize = 0;
2267                         }
2268
2269                         ++riu->sge_cnt;
2270
2271                         if (rsize > 0 && riu->sge_cnt == SRPT_DEF_SG_PER_WQE) {
2272                                 riu->sge =
2273                                     kmalloc(riu->sge_cnt * sizeof *riu->sge,
2274                                             scst_cmd_atomic(scmnd)
2275                                             ? GFP_ATOMIC : GFP_KERNEL);
2276                                 if (!riu->sge)
2277                                         goto free_mem;
2278
2279                                 ++ioctx->n_rdma;
2280                                 ++riu;
2281                                 riu->sge_cnt = 0;
2282                                 riu->raddr = raddr;
2283                                 riu->rkey = be32_to_cpu(db->key);
2284                         }
2285                 }
2286
2287                 riu->sge = kmalloc(riu->sge_cnt * sizeof *riu->sge,
2288                                    scst_cmd_atomic(scmnd)
2289                                    ? GFP_ATOMIC : GFP_KERNEL);
2290
2291                 if (!riu->sge)
2292                         goto free_mem;
2293
2294                 ++ioctx->n_rdma;
2295         }
2296
2297         db = ioctx->rbufs;
2298         scat = scst_cmd_get_sg(scmnd);
2299         tsize = (dir == SCST_DATA_READ) ?
2300                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2301         riu = ioctx->rdma_ius;
2302         dma_len = sg_dma_len(&scat[0]);
2303         dma_addr = sg_dma_address(&scat[0]);
2304
2305         /* this second loop is really mapped sg_addres to rdma_iu->ib_sge */
2306         for (i = 0, j = 0;
2307              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2308                 rsize = be32_to_cpu(db->len);
2309                 sge = riu->sge;
2310                 k = 0;
2311
2312                 while (rsize > 0 && tsize > 0) {
2313                         sge->addr = dma_addr;
2314                         sge->lkey = ch->sport->sdev->mr->lkey;
2315
2316                         if (rsize >= dma_len) {
2317                                 sge->length =
2318                                         (tsize < dma_len) ? tsize : dma_len;
2319                                 tsize -= dma_len;
2320                                 rsize -= dma_len;
2321
2322                                 if (tsize > 0) {
2323                                         ++j;
2324                                         if (j < count) {
2325                                                 dma_len = sg_dma_len(&scat[j]);
2326                                                 dma_addr =
2327                                                     sg_dma_address(&scat[j]);
2328                                         }
2329                                 }
2330                         } else {
2331                                 sge->length = (tsize < rsize) ? tsize : rsize;
2332                                 tsize -= rsize;
2333                                 dma_len -= rsize;
2334                                 dma_addr += rsize;
2335                                 rsize = 0;
2336                         }
2337
2338                         ++k;
2339                         if (k == riu->sge_cnt && rsize > 0) {
2340                                 ++riu;
2341                                 sge = riu->sge;
2342                                 k = 0;
2343                         } else if (rsize > 0)
2344                                 ++sge;
2345                 }
2346         }
2347
2348         return 0;
2349
2350 free_mem:
2351         while (ioctx->n_rdma)
2352                 kfree(ioctx->rdma_ius[ioctx->n_rdma--].sge);
2353
2354         kfree(ioctx->rdma_ius);
2355
2356         WARN_ON(scat == NULL);
2357         ib_dma_unmap_sg(ch->sport->sdev->device,
2358                         scat, scst_cmd_get_sg_cnt(scmnd),
2359                         scst_to_tgt_dma_dir(dir));
2360
2361         return -ENOMEM;
2362 }
2363
2364 static int srpt_perform_rdmas(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2365                               scst_data_direction dir)
2366 {
2367         struct ib_send_wr wr;
2368         struct ib_send_wr *bad_wr;
2369         struct rdma_iu *riu;
2370         int i;
2371         int ret = 0;
2372
2373         riu = ioctx->rdma_ius;
2374         memset(&wr, 0, sizeof wr);
2375
2376         for (i = 0; i < ioctx->n_rdma; ++i, ++riu) {
2377                 wr.opcode = (dir == SCST_DATA_READ) ?
2378                     IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;
2379                 wr.next = NULL;
2380                 wr.wr_id = ioctx->index;
2381                 wr.wr.rdma.remote_addr = riu->raddr;
2382                 wr.wr.rdma.rkey = riu->rkey;
2383                 wr.num_sge = riu->sge_cnt;
2384                 wr.sg_list = riu->sge;
2385
2386                 /* only get completion event for the last rdma wr */
2387                 if (i == (ioctx->n_rdma - 1) && dir == SCST_DATA_WRITE)
2388                         wr.send_flags = IB_SEND_SIGNALED;
2389
2390                 ret = ib_post_send(ch->qp, &wr, &bad_wr);
2391                 if (ret)
2392                         break;
2393         }
2394
2395         return ret;
2396 }
2397
2398 /*
2399  * Start data transfer between initiator and target. Must not block.
2400  */
2401 static int srpt_xfer_data(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2402                           struct scst_cmd *scmnd)
2403 {
2404         int ret;
2405
2406         ret = srpt_map_sg_to_ib_sge(ch, ioctx, scmnd);
2407         if (ret) {
2408                 PRINT_ERROR("%s[%d] ret=%d", __func__, __LINE__, ret);
2409                 ret = SCST_TGT_RES_QUEUE_FULL;
2410                 goto out;
2411         }
2412
2413         ret = srpt_perform_rdmas(ch, ioctx, scst_cmd_get_data_direction(scmnd));
2414         if (ret) {
2415                 PRINT_ERROR("%s[%d] ret=%d", __func__, __LINE__, ret);
2416                 if (ret == -EAGAIN || ret == -ENOMEM)
2417                         ret = SCST_TGT_RES_QUEUE_FULL;
2418                 else
2419                         ret = SCST_TGT_RES_FATAL_ERROR;
2420                 goto out;
2421         }
2422
2423         ret = SCST_TGT_RES_SUCCESS;
2424
2425 out:
2426         return ret;
2427 }
2428
2429 /*
2430  * Called by the SCST core to inform ib_srpt that data reception from the
2431  * initiator should start (SCST_DATA_WRITE). Must not block.
2432  */
2433 static int srpt_rdy_to_xfer(struct scst_cmd *scmnd)
2434 {
2435         struct srpt_rdma_ch *ch;
2436         struct srpt_ioctx *ioctx;
2437         int ret;
2438
2439         ioctx = scst_cmd_get_tgt_priv(scmnd);
2440         BUG_ON(!ioctx);
2441
2442         if (srpt_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA)
2443             == SRPT_STATE_ABORTED
2444             || srpt_abort_if_disconnecting(ioctx)) {
2445                 TRACE_DBG("cmd with tag %lld has been aborted",
2446                           scst_cmd_get_tag(scmnd));
2447                 ret = SCST_TGT_RES_FATAL_ERROR;
2448                 goto out;
2449         }
2450
2451         ch = ioctx->ch;
2452         WARN_ON(ch != scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd)));
2453         BUG_ON(!ch);
2454
2455         if (ch->state == RDMA_CHANNEL_DISCONNECTING) {
2456                 TRACE_DBG("cmd with tag %lld: channel disconnecting",
2457                           scst_cmd_get_tag(scmnd));
2458                 ret = SCST_TGT_RES_FATAL_ERROR;
2459                 goto out;
2460         } else if (ch->state == RDMA_CHANNEL_CONNECTING) {
2461                 ret = SCST_TGT_RES_QUEUE_FULL;
2462                 goto out;
2463         }
2464         ret = srpt_xfer_data(ch, ioctx, scmnd);
2465
2466 out:
2467         return ret;
2468 }
2469
2470 /*
2471  * Called by the SCST core. Transmits the response buffer and status held in
2472  * 'scmnd'. Must not block.
2473  */
2474 static int srpt_xmit_response(struct scst_cmd *scmnd)
2475 {
2476         struct srpt_rdma_ch *ch;
2477         struct srpt_ioctx *ioctx;
2478         struct srp_rsp *srp_rsp;
2479         u64 tag;
2480         int ret = SCST_TGT_RES_SUCCESS;
2481         int dir;
2482         int status;
2483
2484         ioctx = scst_cmd_get_tgt_priv(scmnd);
2485         BUG_ON(!ioctx);
2486
2487         if (srpt_set_cmd_state(ioctx, SRPT_STATE_PROCESSED)
2488             == SRPT_STATE_ABORTED
2489             || srpt_abort_if_disconnecting(ioctx)) {
2490                 TRACE_DBG("cmd with tag %lld has been aborted",
2491                           scst_cmd_get_tag(scmnd));
2492                 ret = SCST_TGT_RES_FATAL_ERROR;
2493                 goto out;
2494         }
2495
2496         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2497         BUG_ON(!ch);
2498
2499         tag = scst_cmd_get_tag(scmnd);
2500
2501         if (ch->state != RDMA_CHANNEL_LIVE) {
2502                 PRINT_ERROR("%s: tag= %lld channel in bad state %d",
2503                        __func__, (unsigned long long)tag, ch->state);
2504
2505                 if (ch->state == RDMA_CHANNEL_DISCONNECTING) {
2506                         TRACE_DBG("cmd with tag %lld: channel disconnecting",
2507                                   (unsigned long long)tag);
2508                         ret = SCST_TGT_RES_FATAL_ERROR;
2509                 } else if (ch->state == RDMA_CHANNEL_CONNECTING)
2510                         ret = SCST_TGT_RES_QUEUE_FULL;
2511
2512                 if (unlikely(scst_cmd_aborted(scmnd)))
2513                         goto out_aborted;
2514
2515                 goto out;
2516         }
2517
2518         if (unlikely(scst_cmd_aborted(scmnd))) {
2519                 TRACE_MGMT_DBG("%s: tag= %lld already got aborted",
2520                                __func__, (unsigned long long)tag);
2521                 goto out_aborted;
2522         }
2523
2524         dir = scst_cmd_get_data_direction(scmnd);
2525         status = scst_cmd_get_status(scmnd) & 0xff;
2526
2527         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE, NO_ADD_SENSE, tag);
2528
2529         srp_rsp = ioctx->buf;
2530
2531         if (SCST_SENSE_VALID(scst_cmd_get_sense_buffer(scmnd))) {
2532                 srp_rsp->sense_data_len = scst_cmd_get_sense_buffer_len(scmnd);
2533                 BUILD_BUG_ON(MIN_MAX_MESSAGE_SIZE <= sizeof(*srp_rsp));
2534                 WARN_ON(srp_max_message_size <= sizeof(*srp_rsp));
2535                 if (srp_rsp->sense_data_len >
2536                     (srp_max_message_size - sizeof *srp_rsp))
2537                         srp_rsp->sense_data_len =
2538                             srp_max_message_size - sizeof *srp_rsp;
2539                 WARN_ON(srp_rsp->sense_data_len <= 0);
2540
2541                 memcpy((u8 *) (srp_rsp + 1), scst_cmd_get_sense_buffer(scmnd),
2542                        srp_rsp->sense_data_len);
2543
2544                 srp_rsp->sense_data_len = cpu_to_be32(srp_rsp->sense_data_len);
2545                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
2546
2547                 if (!status)
2548                         status = SAM_STAT_CHECK_CONDITION;
2549         }
2550
2551         srp_rsp->status = status;
2552
2553         /* For read commands, transfer the data to the initiator. */
2554         if (dir == SCST_DATA_READ && scst_cmd_get_resp_data_len(scmnd)) {
2555                 ret = srpt_xfer_data(ch, ioctx, scmnd);
2556                 if (ret != SCST_TGT_RES_SUCCESS) {
2557                         PRINT_ERROR("%s: tag= %lld xfer_data failed",
2558                                     __func__, (unsigned long long)tag);
2559                         goto out;
2560                 }
2561         }
2562
2563         if (srpt_post_send(ch, ioctx,
2564                            sizeof *srp_rsp +
2565                            be32_to_cpu(srp_rsp->sense_data_len))) {
2566                 PRINT_ERROR("%s: ch->state= %d tag= %lld",
2567                             __func__, ch->state,
2568                             (unsigned long long)tag);
2569                 ret = SCST_TGT_RES_FATAL_ERROR;
2570         }
2571
2572 out:
2573         return ret;
2574
2575 out_aborted:
2576         ret = SCST_TGT_RES_SUCCESS;
2577         srpt_abort_scst_cmd(ch->sport->sdev, scmnd);
2578         goto out;
2579 }
2580
2581 /*
2582  * Called by the SCST core to inform ib_srpt that a received task management
2583  * function has been completed. Must not block.
2584  */
2585 static void srpt_tsk_mgmt_done(struct scst_mgmt_cmd *mcmnd)
2586 {
2587         struct srpt_rdma_ch *ch;
2588         struct srpt_mgmt_ioctx *mgmt_ioctx;
2589         struct srpt_ioctx *ioctx;
2590         int rsp_len;
2591
2592         mgmt_ioctx = scst_mgmt_cmd_get_tgt_priv(mcmnd);
2593         BUG_ON(!mgmt_ioctx);
2594
2595         ch = mgmt_ioctx->ch;
2596         BUG_ON(!ch);
2597
2598         ioctx = mgmt_ioctx->ioctx;
2599         BUG_ON(!ioctx);
2600
2601         TRACE_DBG("%s: tsk_mgmt_done for tag= %lld status=%d",
2602                   __func__, (unsigned long long)mgmt_ioctx->tag,
2603                   scst_mgmt_cmd_get_status(mcmnd));
2604
2605         if (srpt_set_cmd_state(ioctx, SRPT_STATE_PROCESSED)
2606             == SRPT_STATE_ABORTED
2607             || srpt_abort_if_disconnecting(ioctx))
2608                 goto out;
2609
2610         rsp_len = srpt_build_tskmgmt_rsp(ch, ioctx,
2611                                          (scst_mgmt_cmd_get_status(mcmnd) ==
2612                                           SCST_MGMT_STATUS_SUCCESS) ?
2613                                          SRP_TSK_MGMT_SUCCESS :
2614                                          SRP_TSK_MGMT_FAILED,
2615                                          mgmt_ioctx->tag);
2616         srpt_post_send(ch, ioctx, rsp_len);
2617
2618         scst_mgmt_cmd_set_tgt_priv(mcmnd, NULL);
2619
2620         kfree(mgmt_ioctx);
2621
2622 out:
2623         ;
2624 }
2625
2626 /*
2627  * Called by the SCST core to inform ib_srpt that the command 'scmnd' is about
2628  * to be freed. May be called in IRQ context.
2629  */
2630 static void srpt_on_free_cmd(struct scst_cmd *scmnd)
2631 {
2632         struct srpt_rdma_ch *ch;
2633         struct srpt_ioctx *ioctx;
2634         unsigned long flags;
2635
2636         ioctx = scst_cmd_get_tgt_priv(scmnd);
2637         BUG_ON(!ioctx);
2638
2639         srpt_set_cmd_state(ioctx, SRPT_STATE_ABORTED);
2640         ch = ioctx->ch;
2641         BUG_ON(!ch);
2642         spin_lock_irqsave(&ch->spinlock, flags);
2643         list_del(&ioctx->scmnd_list);
2644         ch->active_scmnd_cnt--;
2645         spin_unlock_irqrestore(&ch->spinlock, flags);
2646         ioctx->ch = NULL;
2647
2648         srpt_reset_ioctx(ch, ioctx);
2649         scst_cmd_set_tgt_priv(scmnd, NULL);
2650 }
2651
2652 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2653 /* A vanilla 2.6.19 or older kernel without backported OFED kernel headers. */
2654 static void srpt_refresh_port_work(void *ctx)
2655 #else
2656 static void srpt_refresh_port_work(struct work_struct *work)
2657 #endif
2658 {
2659 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2660         struct srpt_port *sport = (struct srpt_port *)ctx;
2661 #else
2662         struct srpt_port *sport = container_of(work, struct srpt_port, work);
2663 #endif
2664
2665         srpt_refresh_port(sport);
2666 }
2667
2668 /*
2669  * Called by the SCST core to detect target adapters. Returns the number of
2670  * detected target adapters.
2671  */
2672 static int srpt_detect(struct scst_tgt_template *tp)
2673 {
2674         int device_count;
2675
2676         TRACE_ENTRY();
2677
2678         device_count = atomic_read(&srpt_device_count);
2679
2680         TRACE_EXIT_RES(device_count);
2681
2682         return device_count;
2683 }
2684
2685 /*
2686  * Callback function called by the SCST core from scst_unregister() to free up
2687  * the resources associated with device scst_tgt.
2688  */
2689 static int srpt_release(struct scst_tgt *scst_tgt)
2690 {
2691         struct srpt_device *sdev = scst_tgt_get_tgt_priv(scst_tgt);
2692         struct srpt_rdma_ch *ch, *tmp_ch;
2693
2694         TRACE_ENTRY();
2695
2696         BUG_ON(!scst_tgt);
2697 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
2698         WARN_ON(!sdev);
2699         if (!sdev)
2700                 return -ENODEV;
2701 #else
2702         if (WARN_ON(!sdev))
2703                 return -ENODEV;
2704 #endif
2705
2706 #ifdef CONFIG_SCST_PROC
2707         srpt_unregister_procfs_entry(scst_tgt->tgtt);
2708 #endif /*CONFIG_SCST_PROC*/
2709
2710         spin_lock_irq(&sdev->spinlock);
2711         list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
2712                 list_del(&ch->list);
2713                 spin_unlock_irq(&sdev->spinlock);
2714                 srpt_release_channel(ch, 1);
2715                 spin_lock_irq(&sdev->spinlock);
2716         }
2717         spin_unlock_irq(&sdev->spinlock);
2718
2719         srpt_unregister_mad_agent(sdev);
2720
2721         scst_tgt_set_tgt_priv(scst_tgt, NULL);
2722
2723         TRACE_EXIT();
2724
2725         return 0;
2726 }
2727
2728 /*
2729  * Entry point for ib_srpt's kernel thread. This kernel thread is only created
2730  * when the module parameter 'thread' is not zero (the default is zero).
2731  * This thread processes the ioctx list srpt_thread.thread_ioctx_list.
2732  *
2733  * @pre thread != 0
2734  */
2735 static int srpt_ioctx_thread(void *arg)
2736 {
2737         struct srpt_ioctx *ioctx;
2738
2739         /* Hibernation / freezing of the SRPT kernel thread is not supported. */
2740         current->flags |= PF_NOFREEZE;
2741
2742         spin_lock_irq(&srpt_thread.thread_lock);
2743         while (!kthread_should_stop()) {
2744                 wait_queue_t wait;
2745                 init_waitqueue_entry(&wait, current);
2746
2747                 if (!srpt_test_ioctx_list()) {
2748                         add_wait_queue_exclusive(&ioctx_list_waitQ, &wait);
2749
2750                         for (;;) {
2751                                 set_current_state(TASK_INTERRUPTIBLE);
2752                                 if (srpt_test_ioctx_list())
2753                                         break;
2754                                 spin_unlock_irq(&srpt_thread.thread_lock);
2755                                 schedule();
2756                                 spin_lock_irq(&srpt_thread.thread_lock);
2757                         }
2758                         set_current_state(TASK_RUNNING);
2759                         remove_wait_queue(&ioctx_list_waitQ, &wait);
2760                 }
2761
2762                 while (!list_empty(&srpt_thread.thread_ioctx_list)) {
2763                         ioctx = list_entry(srpt_thread.thread_ioctx_list.next,
2764                                            struct srpt_ioctx, comp_list);
2765
2766                         list_del(&ioctx->comp_list);
2767
2768                         spin_unlock_irq(&srpt_thread.thread_lock);
2769                         switch (ioctx->op) {
2770                         case IB_WC_SEND:
2771                                 srpt_handle_send_comp(ioctx->ch, ioctx,
2772                                         SCST_CONTEXT_DIRECT);
2773                                 break;
2774                         case IB_WC_RDMA_WRITE:
2775                         case IB_WC_RDMA_READ:
2776                                 srpt_handle_rdma_comp(ioctx->ch, ioctx);
2777                                 break;
2778                         case IB_WC_RECV:
2779                                 srpt_handle_new_iu(ioctx->ch, ioctx);
2780                                 break;
2781                         default:
2782                                 break;
2783                         }
2784 #if defined(CONFIG_SCST_DEBUG)
2785                         if (thread_processing_delay_in_us
2786                             <= MAX_UDELAY_MS * 1000)
2787                                 udelay(thread_processing_delay_in_us);
2788 #endif
2789                         spin_lock_irq(&srpt_thread.thread_lock);
2790                 }
2791         }
2792         spin_unlock_irq(&srpt_thread.thread_lock);
2793
2794         return 0;
2795 }
2796
2797 /* SCST target template for the SRP target implementation. */
2798 static struct scst_tgt_template srpt_template = {
2799         .name = DRV_NAME,
2800         .sg_tablesize = SRPT_DEF_SG_TABLESIZE,
2801         .xmit_response_atomic = 1,
2802         .rdy_to_xfer_atomic = 1,
2803         .detect = srpt_detect,
2804         .release = srpt_release,
2805         .xmit_response = srpt_xmit_response,
2806         .rdy_to_xfer = srpt_rdy_to_xfer,
2807         .on_free_cmd = srpt_on_free_cmd,
2808         .task_mgmt_fn_done = srpt_tsk_mgmt_done
2809 };
2810
2811 /*
2812  * The callback function srpt_release_class_dev() is called whenever a
2813  * device is removed from the /sys/class/infiniband_srpt device class.
2814  */
2815 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2816 static void srpt_release_class_dev(struct class_device *class_dev)
2817 #else
2818 static void srpt_release_class_dev(struct device *dev)
2819 #endif
2820 {
2821 }
2822
2823 #ifdef CONFIG_SCST_PROC
2824
2825 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2826 static int srpt_trace_level_show(struct seq_file *seq, void *v)
2827 {
2828         return scst_proc_log_entry_read(seq, trace_flag, NULL);
2829 }
2830
2831 static ssize_t srpt_proc_trace_level_write(struct file *file,
2832         const char __user *buf, size_t length, loff_t *off)
2833 {
2834         return scst_proc_log_entry_write(file, buf, length, &trace_flag,
2835                 DEFAULT_SRPT_TRACE_FLAGS, NULL);
2836 }
2837
2838 static struct scst_proc_data srpt_log_proc_data = {
2839         SCST_DEF_RW_SEQ_OP(srpt_proc_trace_level_write)
2840         .show = srpt_trace_level_show,
2841 };
2842 #endif
2843
2844 #endif /* CONFIG_SCST_PROC */
2845
2846 static struct class_attribute srpt_class_attrs[] = {
2847         __ATTR_NULL,
2848 };
2849
2850 static struct class srpt_class = {
2851         .name = "infiniband_srpt",
2852 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2853         .release = srpt_release_class_dev,
2854 #else
2855         .dev_release = srpt_release_class_dev,
2856 #endif
2857         .class_attrs = srpt_class_attrs,
2858 };
2859
2860 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2861 static ssize_t show_login_info(struct class_device *class_dev, char *buf)
2862 #else
2863 static ssize_t show_login_info(struct device *dev,
2864                                struct device_attribute *attr, char *buf)
2865 #endif
2866 {
2867         struct srpt_device *sdev =
2868 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2869                 container_of(class_dev, struct srpt_device, class_dev);
2870 #else
2871                 container_of(dev, struct srpt_device, dev);
2872 #endif
2873         struct srpt_port *sport;
2874         int i;
2875         int len = 0;
2876
2877         for (i = 0; i < sdev->device->phys_port_cnt; i++) {
2878                 sport = &sdev->port[i];
2879
2880                 len += sprintf(buf + len,
2881                                "tid_ext=%016llx,ioc_guid=%016llx,pkey=ffff,"
2882                                "dgid=%04x%04x%04x%04x%04x%04x%04x%04x,"
2883                                "service_id=%016llx\n",
2884                                (unsigned long long) srpt_service_guid,
2885                                (unsigned long long) srpt_service_guid,
2886                                be16_to_cpu(((__be16 *) sport->gid.raw)[0]),
2887                                be16_to_cpu(((__be16 *) sport->gid.raw)[1]),
2888                                be16_to_cpu(((__be16 *) sport->gid.raw)[2]),
2889                                be16_to_cpu(((__be16 *) sport->gid.raw)[3]),
2890                                be16_to_cpu(((__be16 *) sport->gid.raw)[4]),
2891                                be16_to_cpu(((__be16 *) sport->gid.raw)[5]),
2892                                be16_to_cpu(((__be16 *) sport->gid.raw)[6]),
2893                                be16_to_cpu(((__be16 *) sport->gid.raw)[7]),
2894                                (unsigned long long) srpt_service_guid);
2895         }
2896
2897         return len;
2898 }
2899
2900 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2901 static CLASS_DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2902 #else
2903 static DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2904 #endif
2905
2906 /*
2907  * Callback function called by the InfiniBand core when either an InfiniBand
2908  * device has been added or during the ib_register_client() call for each
2909  * registered InfiniBand device.
2910  */
2911 static void srpt_add_one(struct ib_device *device)
2912 {
2913         struct srpt_device *sdev;
2914         struct srpt_port *sport;
2915         struct ib_srq_init_attr srq_attr;
2916         int i;
2917
2918         TRACE_ENTRY();
2919
2920         TRACE_DBG("device = %p, device->dma_ops = %p", device, device->dma_ops);
2921
2922         sdev = kzalloc(sizeof *sdev, GFP_KERNEL);
2923         if (!sdev)
2924                 return;
2925
2926         sdev->device = device;
2927
2928 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2929         sdev->class_dev.class = &srpt_class;
2930         sdev->class_dev.dev = device->dma_device;
2931         snprintf(sdev->class_dev.class_id, BUS_ID_SIZE,
2932                  "srpt-%s", device->name);
2933 #else
2934         sdev->dev.class = &srpt_class;
2935         sdev->dev.parent = device->dma_device;
2936 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
2937         snprintf(sdev->dev.bus_id, BUS_ID_SIZE, "srpt-%s", device->name);
2938 #else
2939         dev_set_name(&sdev->dev, "srpt-%s", device->name);
2940 #endif
2941 #endif
2942
2943 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2944         if (class_device_register(&sdev->class_dev))
2945                 goto free_dev;
2946         if (class_device_create_file(&sdev->class_dev,
2947                                      &class_device_attr_login_info))
2948                 goto err_dev;
2949 #else
2950         if (device_register(&sdev->dev))
2951                 goto free_dev;
2952         if (device_create_file(&sdev->dev, &dev_attr_login_info))
2953                 goto err_dev;
2954 #endif
2955
2956         if (ib_query_device(device, &sdev->dev_attr))
2957                 goto err_dev;
2958
2959         sdev->pd = ib_alloc_pd(device);
2960         if (IS_ERR(sdev->pd))
2961                 goto err_dev;
2962
2963         sdev->mr = ib_get_dma_mr(sdev->pd, IB_ACCESS_LOCAL_WRITE);
2964         if (IS_ERR(sdev->mr))
2965                 goto err_pd;
2966
2967         srq_attr.event_handler = srpt_srq_event;
2968         srq_attr.srq_context = (void *)sdev;
2969         srq_attr.attr.max_wr = min(SRPT_SRQ_SIZE, sdev->dev_attr.max_srq_wr);
2970         srq_attr.attr.max_sge = 1;
2971         srq_attr.attr.srq_limit = 0;
2972
2973         sdev->srq = ib_create_srq(sdev->pd, &srq_attr);
2974         if (IS_ERR(sdev->srq))
2975                 goto err_mr;
2976
2977         TRACE_DBG("%s: create SRQ #wr= %d max_allow=%d dev= %s",
2978                __func__, srq_attr.attr.max_wr,
2979               sdev->dev_attr.max_srq_wr, device->name);
2980
2981         if (!srpt_service_guid)
2982                 srpt_service_guid = be64_to_cpu(device->node_guid);
2983
2984         sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);
2985         if (IS_ERR(sdev->cm_id))
2986                 goto err_srq;
2987
2988         /* print out target login information */
2989         TRACE_DBG("Target login info: id_ext=%016llx,"
2990                   "ioc_guid=%016llx,pkey=ffff,service_id=%016llx",
2991                   (unsigned long long) srpt_service_guid,
2992                   (unsigned long long) srpt_service_guid,
2993                   (unsigned long long) srpt_service_guid);
2994
2995         /*
2996          * We do not have a consistent service_id (ie. also id_ext of target_id)
2997          * to identify this target. We currently use the guid of the first HCA
2998          * in the system as service_id; therefore, the target_id will change
2999          * if this HCA is gone bad and replaced by different HCA
3000          */
3001         if (ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0, NULL))
3002                 goto err_cm;
3003
3004         INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
3005                               srpt_event_handler);
3006         if (ib_register_event_handler(&sdev->event_handler))
3007                 goto err_cm;
3008
3009         if (srpt_alloc_ioctx_ring(sdev))
3010                 goto err_event;
3011
3012         INIT_LIST_HEAD(&sdev->rch_list);
3013         spin_lock_init(&sdev->spinlock);
3014
3015         for (i = 0; i < SRPT_SRQ_SIZE; ++i)
3016                 srpt_post_recv(sdev, sdev->ioctx_ring[i]);
3017
3018         ib_set_client_data(device, &srpt_client, sdev);
3019
3020         sdev->scst_tgt = scst_register(&srpt_template, NULL);
3021         if (!sdev->scst_tgt) {
3022                 PRINT_ERROR("SCST registration failed for %s.",
3023                             sdev->device->name);
3024                 goto err_ring;
3025         }
3026
3027         scst_tgt_set_tgt_priv(sdev->scst_tgt, sdev);
3028
3029         WARN_ON(sdev->device->phys_port_cnt
3030                 > sizeof(sdev->port)/sizeof(sdev->port[0]));
3031
3032         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
3033                 sport = &sdev->port[i - 1];
3034                 sport->sdev = sdev;
3035                 sport->port = i;
3036 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
3037                 /*
3038                  * A vanilla 2.6.19 or older kernel without backported OFED
3039                  * kernel headers.
3040                  */
3041                 INIT_WORK(&sport->work, srpt_refresh_port_work, sport);
3042 #else
3043                 INIT_WORK(&sport->work, srpt_refresh_port_work);
3044 #endif
3045                 if (srpt_refresh_port(sport)) {
3046                         PRINT_ERROR("MAD registration failed for %s-%d.",
3047                                     sdev->device->name, i);
3048                         goto err_refresh_port;
3049                 }
3050         }
3051
3052         atomic_inc(&srpt_device_count);
3053
3054         TRACE_EXIT();
3055
3056         return;
3057
3058 err_refresh_port:
3059         scst_unregister(sdev->scst_tgt);
3060 err_ring:
3061         ib_set_client_data(device, &srpt_client, NULL);
3062         srpt_free_ioctx_ring(sdev);
3063 err_event:
3064         ib_unregister_event_handler(&sdev->event_handler);
3065 err_cm:
3066         ib_destroy_cm_id(sdev->cm_id);
3067 err_srq:
3068         ib_destroy_srq(sdev->srq);
3069 err_mr:
3070         ib_dereg_mr(sdev->mr);
3071 err_pd:
3072         ib_dealloc_pd(sdev->pd);
3073 err_dev:
3074 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
3075         class_device_unregister(&sdev->class_dev);
3076 #else
3077         device_unregister(&sdev->dev);
3078 #endif
3079 free_dev:
3080         kfree(sdev);
3081
3082         TRACE_EXIT();
3083 }
3084
3085 /*
3086  * Callback function called by the InfiniBand core when either an InfiniBand
3087  * device has been removed or during the ib_unregister_client() call for each
3088  * registered InfiniBand device.
3089  */
3090 static void srpt_remove_one(struct ib_device *device)
3091 {
3092         int i;
3093         struct srpt_device *sdev;
3094
3095         TRACE_ENTRY();
3096
3097         sdev = ib_get_client_data(device, &srpt_client);
3098 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
3099         WARN_ON(!sdev);
3100         if (!sdev)
3101                 return;
3102 #else
3103         if (WARN_ON(!sdev))
3104                 return;
3105 #endif
3106
3107         /*
3108          * Cancel the work if it is queued. Wait until srpt_refresh_port_work()
3109          * finished if it is running.
3110          */
3111         for (i = 0; i < sdev->device->phys_port_cnt; i++)
3112 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
3113                 cancel_work_sync(&sdev->port[i].work);
3114 #else
3115                 /*
3116                  * cancel_work_sync() was introduced in kernel 2.6.22. Older
3117                  * kernels do not have a facility to cancel scheduled work.
3118                  */
3119                 PRINT_ERROR("%s",
3120                        "your kernel does not provide cancel_work_sync().");
3121 #endif
3122
3123         scst_unregister(sdev->scst_tgt);
3124         sdev->scst_tgt = NULL;
3125
3126         ib_unregister_event_handler(&sdev->event_handler);
3127         ib_destroy_cm_id(sdev->cm_id);
3128         ib_destroy_srq(sdev->srq);
3129         ib_dereg_mr(sdev->mr);
3130         ib_dealloc_pd(sdev->pd);
3131 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
3132         class_device_unregister(&sdev->class_dev);
3133 #else
3134         device_unregister(&sdev->dev);
3135 #endif
3136
3137         srpt_free_ioctx_ring(sdev);
3138         kfree(sdev);
3139
3140         TRACE_EXIT();
3141 }
3142
3143 #ifdef CONFIG_SCST_PROC
3144
3145 /**
3146  * Create procfs entries for srpt. Currently the only procfs entry created
3147  * by this function is the "trace_level" entry.
3148  */
3149 static int srpt_register_procfs_entry(struct scst_tgt_template *tgt)
3150 {
3151         int res = 0;
3152 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
3153         struct proc_dir_entry *p, *root;
3154
3155         root = scst_proc_get_tgt_root(tgt);
3156         WARN_ON(!root);
3157         if (root) {
3158                 /*
3159                  * Fill in the scst_proc_data::data pointer, which is used in
3160                  * a printk(KERN_INFO ...) statement in
3161                  * scst_proc_log_entry_write() in scst_proc.c.
3162                  */
3163                 srpt_log_proc_data.data = (char *)tgt->name;
3164                 p = scst_create_proc_entry(root, SRPT_PROC_TRACE_LEVEL_NAME,
3165                                            &srpt_log_proc_data);
3166                 if (!p)
3167                         res = -ENOMEM;
3168         } else
3169                 res = -ENOMEM;
3170
3171 #endif
3172         return res;
3173 }
3174
3175 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt)
3176 {
3177 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
3178         struct proc_dir_entry *root;
3179
3180         root = scst_proc_get_tgt_root(tgt);
3181         WARN_ON(!root);
3182         if (root)
3183                 remove_proc_entry(SRPT_PROC_TRACE_LEVEL_NAME, root);
3184 #endif
3185 }
3186
3187 #endif /*CONFIG_SCST_PROC*/
3188
3189 /*
3190  * Module initialization.
3191  *
3192  * Note: since ib_register_client() registers callback functions, and since at
3193  * least one of these callback functions (srpt_add_one()) calls SCST functions,
3194  * the SCST target template must be registered before ib_register_client() is
3195  * called.
3196  */
3197 static int __init srpt_init_module(void)
3198 {
3199         int ret;
3200
3201         ret = -EINVAL;
3202         if (srp_max_message_size < MIN_MAX_MESSAGE_SIZE) {
3203                 PRINT_ERROR("invalid value %d for kernel module parameter"
3204                             " srp_max_message_size -- must be at least %d.",
3205                             srp_max_message_size,
3206                             MIN_MAX_MESSAGE_SIZE);
3207                 goto out;
3208         }
3209
3210         ret = class_register(&srpt_class);
3211         if (ret) {
3212                 PRINT_ERROR("%s", "couldn't register class ib_srpt");
3213                 goto out;
3214         }
3215
3216         ret = scst_register_target_template(&srpt_template);
3217         if (ret < 0) {
3218                 PRINT_ERROR("%s", "couldn't register with scst");
3219                 ret = -ENODEV;
3220                 goto out_unregister_class;
3221         }
3222
3223 #ifdef CONFIG_SCST_PROC
3224         ret = srpt_register_procfs_entry(&srpt_template);
3225         if (ret) {
3226                 PRINT_ERROR("%s", "couldn't register procfs entry");
3227                 goto out_unregister_target;
3228         }
3229 #endif /*CONFIG_SCST_PROC*/
3230
3231         ret = ib_register_client(&srpt_client);
3232         if (ret) {
3233                 PRINT_ERROR("%s", "couldn't register IB client");
3234                 goto out_unregister_target;
3235         }
3236
3237         if (thread) {
3238                 spin_lock_init(&srpt_thread.thread_lock);
3239                 INIT_LIST_HEAD(&srpt_thread.thread_ioctx_list);
3240                 srpt_thread.thread = kthread_run(srpt_ioctx_thread,
3241                                                  NULL, "srpt_thread");
3242                 if (IS_ERR(srpt_thread.thread)) {
3243                         srpt_thread.thread = NULL;
3244                         thread = 0;
3245                 }
3246         }
3247
3248         return 0;
3249
3250 out_unregister_target:
3251 #ifdef CONFIG_SCST_PROC
3252         /*
3253          * Note: the procfs entry is unregistered in srpt_release(), which is
3254          * called by scst_unregister_target_template().
3255          */
3256 #endif /*CONFIG_SCST_PROC*/
3257         scst_unregister_target_template(&srpt_template);
3258 out_unregister_class:
3259         class_unregister(&srpt_class);
3260 out:
3261         return ret;
3262 }
3263
3264 static void __exit srpt_cleanup_module(void)
3265 {
3266         TRACE_ENTRY();
3267
3268         if (srpt_thread.thread)
3269                 kthread_stop(srpt_thread.thread);
3270         ib_unregister_client(&srpt_client);
3271         scst_unregister_target_template(&srpt_template);
3272         class_unregister(&srpt_class);
3273
3274         TRACE_EXIT();
3275 }
3276
3277 module_init(srpt_init_module);
3278 module_exit(srpt_cleanup_module);