- Added counter that keeps track of the number of work requests posted on
[mirror/scst/.git] / srpt / src / ib_srpt.c
1 /*
2  * Copyright (c) 2006 - 2009 Mellanox Technology Inc.  All rights reserved.
3  * Copyright (C) 2008 Vladislav Bolkhovitin <vst@vlnb.net>
4  * Copyright (C) 2008 - 2009 Bart Van Assche <bart.vanassche@gmail.com>
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  */
35
36 #include <linux/module.h>
37 #include <linux/init.h>
38 #include <linux/slab.h>
39 #include <linux/err.h>
40 #include <linux/ctype.h>
41 #include <linux/string.h>
42 #include <linux/kthread.h>
43 #include <linux/delay.h>
44 #include <asm/atomic.h>
45 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
46 #include <linux/proc_fs.h>
47 #include <linux/seq_file.h>
48 #endif
49 #include "ib_srpt.h"
50 #include "scst_debug.h"
51
52 #define CONFIG_SCST_PROC
53
54 /* Name of this kernel module. */
55 #define DRV_NAME                "ib_srpt"
56 /* Prefix for printk() kernel messages. */
57 #define LOG_PFX                 DRV_NAME ": "
58 #define DRV_VERSION             "1.0.1"
59 #define DRV_RELDATE             "July 10, 2008"
60 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
61 /* Flags to be used in SCST debug tracing statements. */
62 #define DEFAULT_SRPT_TRACE_FLAGS (TRACE_OUT_OF_MEM | TRACE_MINOR \
63                                   | TRACE_MGMT | TRACE_SPECIAL)
64 /* Name of the entry that will be created under /proc/scsi_tgt/ib_srpt. */
65 #define SRPT_PROC_TRACE_LEVEL_NAME      "trace_level"
66 #endif
67
68 #define MELLANOX_SRPT_ID_STRING "SCST SRP target"
69
70 MODULE_AUTHOR("Vu Pham");
71 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
72                    "v" DRV_VERSION " (" DRV_RELDATE ")");
73 MODULE_LICENSE("Dual BSD/GPL");
74
75 struct srpt_thread {
76         /* Protects thread_ioctx_list. */
77         spinlock_t thread_lock;
78         /* I/O contexts to be processed by the kernel thread. */
79         struct list_head thread_ioctx_list;
80         /* SRPT kernel thread. */
81         struct task_struct *thread;
82 };
83
84 /*
85  * Global Variables
86  */
87
88 static u64 srpt_service_guid;
89 /* List of srpt_device structures. */
90 static atomic_t srpt_device_count;
91 static int use_port_guid_in_session_name;
92 static int thread = 1;
93 static struct srpt_thread srpt_thread;
94 static DECLARE_WAIT_QUEUE_HEAD(ioctx_list_waitQ);
95 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
96 static unsigned long trace_flag = DEFAULT_SRPT_TRACE_FLAGS;
97 module_param(trace_flag, long, 0644);
98 MODULE_PARM_DESC(trace_flag,
99                  "Trace flags for the ib_srpt kernel module.");
100 #endif
101 #if defined(CONFIG_SCST_DEBUG)
102 static unsigned long interrupt_processing_delay_in_us;
103 module_param(interrupt_processing_delay_in_us, long, 0744);
104 MODULE_PARM_DESC(interrupt_processing_delay_in_us,
105                  "CQ completion handler interrupt delay in microseconds.");
106 static unsigned long thread_processing_delay_in_us;
107 module_param(thread_processing_delay_in_us, long, 0744);
108 MODULE_PARM_DESC(thread_processing_delay_in_us,
109                  "SRP thread processing delay in microseconds.");
110 #endif
111
112 module_param(thread, int, 0444);
113 MODULE_PARM_DESC(thread,
114                  "Executing ioctx in thread context. Default 0, i.e. soft IRQ, "
115                  "where possible.");
116
117 static unsigned int srp_max_rdma_size = 65536;
118 module_param(srp_max_rdma_size, int, 0744);
119 MODULE_PARM_DESC(thread,
120                  "Maximum size of SRP RDMA transfers for new connections.");
121
122 static unsigned int srp_max_message_size = 4096;
123 module_param(srp_max_message_size, int, 0444);
124 MODULE_PARM_DESC(thread,
125                  "Maximum size of SRP control messages in bytes.");
126
127 module_param(use_port_guid_in_session_name, bool, 0444);
128 MODULE_PARM_DESC(use_port_guid_in_session_name,
129                  "Use target port ID in the SCST session name such that"
130                  " redundant paths between multiport systems can be masked.");
131
132 static void srpt_add_one(struct ib_device *device);
133 static void srpt_remove_one(struct ib_device *device);
134 static void srpt_unregister_mad_agent(struct srpt_device *sdev);
135 #ifdef CONFIG_SCST_PROC
136 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt);
137 #endif /*CONFIG_SCST_PROC*/
138 static void srpt_unmap_sg_to_ib_sge(struct srpt_rdma_ch *ch,
139                                     struct srpt_ioctx *ioctx,
140                                     struct scst_cmd *scmnd);
141
142 static struct ib_client srpt_client = {
143         .name = DRV_NAME,
144         .add = srpt_add_one,
145         .remove = srpt_remove_one
146 };
147
148 /**
149  * Atomically set the channel state.
150  * @ch: RDMA channel.
151  * @new: new channel state.
152  */
153 static void srpt_set_channel_state(struct srpt_rdma_ch *ch,
154                                    enum rdma_ch_state new)
155 {
156         unsigned long flags;
157
158         spin_lock_irqsave(&ch->spinlock, flags);
159         ch->state = new;
160         spin_unlock_irqrestore(&ch->spinlock, flags);
161 }
162
163 /**
164  * Atomically test and set the channel state.
165  * @ch: RDMA channel.
166  * @old: channel state to compare with.
167  * @new: state to change the channel state to if the current state matches the
168  *       argument 'old'.
169  *
170  * Returns true if the channel state matched old upon entry of this function,
171  * and false otherwise.
172  */
173 static bool srpt_test_and_set_channel_state(struct srpt_rdma_ch *ch,
174                                             enum rdma_ch_state old,
175                                             enum rdma_ch_state new)
176 {
177         unsigned long flags;
178         enum rdma_ch_state cur;
179
180         spin_lock_irqsave(&ch->spinlock, flags);
181         cur = ch->state;
182         if (cur == old)
183                 ch->state = new;
184         spin_unlock_irqrestore(&ch->spinlock, flags);
185
186         return cur == old;
187 }
188
189 /*
190  * Callback function called by the InfiniBand core when an asynchronous IB
191  * event occurs. This callback may occur in interrupt context. See also
192  * section 11.5.2, Set Asynchronous Event Handler in the InfiniBand
193  * Architecture Specification.
194  */
195 static void srpt_event_handler(struct ib_event_handler *handler,
196                                struct ib_event *event)
197 {
198         struct srpt_device *sdev;
199         struct srpt_port *sport;
200
201         TRACE_ENTRY();
202
203         sdev = ib_get_client_data(event->device, &srpt_client);
204         if (!sdev || sdev->device != event->device)
205                 return;
206
207         TRACE_DBG("ASYNC event= %d on device= %s",
208                   event->event, sdev->device->name);
209
210         switch (event->event) {
211         case IB_EVENT_PORT_ERR:
212                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
213                         sport = &sdev->port[event->element.port_num - 1];
214                         sport->lid = 0;
215                         sport->sm_lid = 0;
216                 }
217                 break;
218         case IB_EVENT_PORT_ACTIVE:
219         case IB_EVENT_LID_CHANGE:
220         case IB_EVENT_PKEY_CHANGE:
221         case IB_EVENT_SM_CHANGE:
222         case IB_EVENT_CLIENT_REREGISTER:
223                 /*
224                  * Refresh port data asynchronously. Note: it is safe to call
225                  * schedule_work() even if &sport->work is already on the
226                  * global workqueue because schedule_work() tests for the
227                  * work_pending() condition before adding &sport->work to the
228                  * global work queue.
229                  */
230                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
231                         sport = &sdev->port[event->element.port_num - 1];
232                         if (!sport->lid && !sport->sm_lid)
233                                 schedule_work(&sport->work);
234                 }
235                 break;
236         default:
237                 break;
238         }
239
240         TRACE_EXIT();
241 }
242
243 /*
244  * Callback function called by the InfiniBand core for SRQ (shared receive
245  * queue) events.
246  */
247 static void srpt_srq_event(struct ib_event *event, void *ctx)
248 {
249         TRACE_ENTRY();
250
251         TRACE_DBG("SRQ event %d", event->event);
252
253         TRACE_EXIT();
254 }
255
256 /*
257  * Callback function called by the InfiniBand core for QP (queue pair) events.
258  */
259 static void srpt_qp_event(struct ib_event *event, void *ctx)
260 {
261         struct srpt_rdma_ch *ch = ctx;
262
263         TRACE_DBG("QP event %d on cm_id=%p sess_name=%s state=%d",
264                   event->event, ch->cm_id, ch->sess_name, ch->state);
265
266         switch (event->event) {
267         case IB_EVENT_COMM_EST:
268 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20) || defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
269                 ib_cm_notify(ch->cm_id, event->event);
270 #else
271                 /* Vanilla 2.6.19 kernel (or before) without OFED. */
272                 PRINT_ERROR("%s", "how to perform ib_cm_notify() on a"
273                             " vanilla 2.6.18 kernel ???");
274 #endif
275                 break;
276         case IB_EVENT_QP_LAST_WQE_REACHED:
277                 if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_LIVE,
278                                         RDMA_CHANNEL_DISCONNECTING)) {
279                         PRINT_INFO("disconnected session %s.", ch->sess_name);
280                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
281                 }
282                 break;
283         default:
284                 break;
285         }
286 }
287
288 /*
289  * Helper function for filling in an InfiniBand IOUnitInfo structure. Copies
290  * the lowest four bits of value in element slot of the array of four bit
291  * elements called c_list (controller list). The index slot is one-based.
292  *
293  * @pre 1 <= slot && 0 <= value && value < 16
294  */
295 static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
296 {
297         u16 id;
298         u8 tmp;
299
300         id = (slot - 1) / 2;
301         if (slot & 0x1) {
302                 tmp = c_list[id] & 0xf;
303                 c_list[id] = (value << 4) | tmp;
304         } else {
305                 tmp = c_list[id] & 0xf0;
306                 c_list[id] = (value & 0xf) | tmp;
307         }
308 }
309
310 /*
311  * Write InfiniBand ClassPortInfo to mad. See also section 16.3.3.1
312  * ClassPortInfo in the InfiniBand Architecture Specification.
313  */
314 static void srpt_get_class_port_info(struct ib_dm_mad *mad)
315 {
316         struct ib_class_port_info *cif;
317
318         cif = (struct ib_class_port_info *)mad->data;
319         memset(cif, 0, sizeof *cif);
320         cif->base_version = 1;
321         cif->class_version = 1;
322         cif->resp_time_value = 20;
323
324         mad->mad_hdr.status = 0;
325 }
326
327 /*
328  * Write IOUnitInfo to mad. See also section 16.3.3.3 IOUnitInfo in the
329  * InfiniBand Architecture Specification. See also section B.7,
330  * table B.6 in the T10 SRP r16a document.
331  */
332 static void srpt_get_iou(struct ib_dm_mad *mad)
333 {
334         struct ib_dm_iou_info *ioui;
335         u8 slot;
336         int i;
337
338         ioui = (struct ib_dm_iou_info *)mad->data;
339         ioui->change_id = 1;
340         ioui->max_controllers = 16;
341
342         /* set present for slot 1 and empty for the rest */
343         srpt_set_ioc(ioui->controller_list, 1, 1);
344         for (i = 1, slot = 2; i < 16; i++, slot++)
345                 srpt_set_ioc(ioui->controller_list, slot, 0);
346
347         mad->mad_hdr.status = 0;
348 }
349
350 /*
351  * Write IOControllerprofile to mad for I/O controller (sdev, slot). See also
352  * section 16.3.3.4 IOControllerProfile in the InfiniBand Architecture
353  * Specification. See also section B.7, table B.7 in the T10 SRP r16a
354  * document.
355  */
356 static void srpt_get_ioc(struct srpt_device *sdev, u32 slot,
357                          struct ib_dm_mad *mad)
358 {
359         struct ib_dm_ioc_profile *iocp;
360
361         iocp = (struct ib_dm_ioc_profile *)mad->data;
362
363         if (!slot || slot > 16) {
364                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
365                 return;
366         }
367
368         if (slot > 2) {
369                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
370                 return;
371         }
372
373         memset(iocp, 0, sizeof *iocp);
374         strcpy(iocp->id_string, MELLANOX_SRPT_ID_STRING);
375         iocp->guid = cpu_to_be64(srpt_service_guid);
376         iocp->vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
377         iocp->device_id = cpu_to_be32(sdev->dev_attr.vendor_part_id);
378         iocp->device_version = cpu_to_be16(sdev->dev_attr.hw_ver);
379         iocp->subsys_vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
380         iocp->subsys_device_id = 0x0;
381         iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
382         iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
383         iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
384         iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
385         iocp->send_queue_depth = cpu_to_be16(SRPT_SRQ_SIZE);
386         iocp->rdma_read_depth = 4;
387         iocp->send_size = cpu_to_be32(srp_max_message_size);
388         iocp->rdma_size = cpu_to_be32(min(max(srp_max_rdma_size, 256U),
389                                           1U << 24));
390         iocp->num_svc_entries = 1;
391         iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
392                 SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
393
394         mad->mad_hdr.status = 0;
395 }
396
397 /*
398  * Device management: write ServiceEntries to mad for the given slot. See also
399  * section 16.3.3.5 ServiceEntries in the InfiniBand Architecture
400  * Specification. See also section B.7, table B.8 in the T10 SRP r16a document.
401  */
402 static void srpt_get_svc_entries(u64 ioc_guid,
403                                  u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
404 {
405         struct ib_dm_svc_entries *svc_entries;
406
407         WARN_ON(!ioc_guid);
408
409         if (!slot || slot > 16) {
410                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
411                 return;
412         }
413
414         if (slot > 2 || lo > hi || hi > 1) {
415                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
416                 return;
417         }
418
419         svc_entries = (struct ib_dm_svc_entries *)mad->data;
420         memset(svc_entries, 0, sizeof *svc_entries);
421         svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid);
422         snprintf(svc_entries->service_entries[0].name,
423                  sizeof(svc_entries->service_entries[0].name),
424                  "%s%016llx",
425                  SRP_SERVICE_NAME_PREFIX,
426                  (unsigned long long)ioc_guid);
427
428         mad->mad_hdr.status = 0;
429 }
430
431 /*
432  * Actual processing of a received MAD *rq_mad received through source port *sp
433  * (MAD = InfiniBand management datagram). The response to be sent back is
434  * written to *rsp_mad.
435  */
436 static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
437                                  struct ib_dm_mad *rsp_mad)
438 {
439         u16 attr_id;
440         u32 slot;
441         u8 hi, lo;
442
443         attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
444         switch (attr_id) {
445         case DM_ATTR_CLASS_PORT_INFO:
446                 srpt_get_class_port_info(rsp_mad);
447                 break;
448         case DM_ATTR_IOU_INFO:
449                 srpt_get_iou(rsp_mad);
450                 break;
451         case DM_ATTR_IOC_PROFILE:
452                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
453                 srpt_get_ioc(sp->sdev, slot, rsp_mad);
454                 break;
455         case DM_ATTR_SVC_ENTRIES:
456                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
457                 hi = (u8) ((slot >> 8) & 0xff);
458                 lo = (u8) (slot & 0xff);
459                 slot = (u16) ((slot >> 16) & 0xffff);
460                 srpt_get_svc_entries(srpt_service_guid,
461                                      slot, hi, lo, rsp_mad);
462                 break;
463         default:
464                 rsp_mad->mad_hdr.status =
465                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
466                 break;
467         }
468 }
469
470 /*
471  * Callback function that is called by the InfiniBand core after transmission of
472  * a MAD. (MAD = management datagram; AH = address handle.)
473  */
474 static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
475                                   struct ib_mad_send_wc *mad_wc)
476 {
477         ib_destroy_ah(mad_wc->send_buf->ah);
478         ib_free_send_mad(mad_wc->send_buf);
479 }
480
481 /*
482  * Callback function that is called by the InfiniBand core after reception of
483  * a MAD (management datagram).
484  */
485 static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
486                                   struct ib_mad_recv_wc *mad_wc)
487 {
488         struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
489         struct ib_ah *ah;
490         struct ib_mad_send_buf *rsp;
491         struct ib_dm_mad *dm_mad;
492
493         if (!mad_wc || !mad_wc->recv_buf.mad)
494                 return;
495
496         ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
497                                   mad_wc->recv_buf.grh, mad_agent->port_num);
498         if (IS_ERR(ah))
499                 goto err;
500
501         BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR);
502
503         rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
504                                  mad_wc->wc->pkey_index, 0,
505                                  IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA,
506                                  GFP_KERNEL);
507         if (IS_ERR(rsp))
508                 goto err_rsp;
509
510         rsp->ah = ah;
511
512         dm_mad = rsp->mad;
513         memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof *dm_mad);
514         dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
515         dm_mad->mad_hdr.status = 0;
516
517         switch (mad_wc->recv_buf.mad->mad_hdr.method) {
518         case IB_MGMT_METHOD_GET:
519                 srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
520                 break;
521         case IB_MGMT_METHOD_SET:
522                 dm_mad->mad_hdr.status =
523                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
524                 break;
525         default:
526                 dm_mad->mad_hdr.status =
527                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
528                 break;
529         }
530
531         if (!ib_post_send_mad(rsp, NULL)) {
532                 ib_free_recv_mad(mad_wc);
533                 /* will destroy_ah & free_send_mad in send completion */
534                 return;
535         }
536
537         ib_free_send_mad(rsp);
538
539 err_rsp:
540         ib_destroy_ah(ah);
541 err:
542         ib_free_recv_mad(mad_wc);
543 }
544
545 /*
546  * Enable InfiniBand management datagram processing, update the cached sm_lid,
547  * lid and gid values, and register a callback function for processing MADs
548  * on the specified port. It is safe to call this function more than once for
549  * the same port.
550  */
551 static int srpt_refresh_port(struct srpt_port *sport)
552 {
553         struct ib_mad_reg_req reg_req;
554         struct ib_port_modify port_modify;
555         struct ib_port_attr port_attr;
556         int ret;
557
558         TRACE_ENTRY();
559
560         memset(&port_modify, 0, sizeof port_modify);
561         port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
562         port_modify.clr_port_cap_mask = 0;
563
564         ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
565         if (ret)
566                 goto err_mod_port;
567
568         ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
569         if (ret)
570                 goto err_query_port;
571
572         sport->sm_lid = port_attr.sm_lid;
573         sport->lid = port_attr.lid;
574
575         ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid);
576         if (ret)
577                 goto err_query_port;
578
579         if (!sport->mad_agent) {
580                 memset(&reg_req, 0, sizeof reg_req);
581                 reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
582                 reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
583                 set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
584                 set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
585
586                 sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
587                                                          sport->port,
588                                                          IB_QPT_GSI,
589                                                          &reg_req, 0,
590                                                          srpt_mad_send_handler,
591                                                          srpt_mad_recv_handler,
592                                                          sport);
593                 if (IS_ERR(sport->mad_agent)) {
594                         ret = PTR_ERR(sport->mad_agent);
595                         sport->mad_agent = NULL;
596                         goto err_query_port;
597                 }
598         }
599
600         TRACE_EXIT_RES(0);
601
602         return 0;
603
604 err_query_port:
605
606         port_modify.set_port_cap_mask = 0;
607         port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
608         ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
609
610 err_mod_port:
611
612         TRACE_EXIT_RES(ret);
613
614         return ret;
615 }
616
617 /*
618  * Unregister the callback function for processing MADs and disable MAD
619  * processing for all ports of the specified device. It is safe to call this
620  * function more than once for the same device.
621  */
622 static void srpt_unregister_mad_agent(struct srpt_device *sdev)
623 {
624         struct ib_port_modify port_modify = {
625                 .clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP,
626         };
627         struct srpt_port *sport;
628         int i;
629
630         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
631                 sport = &sdev->port[i - 1];
632                 WARN_ON(sport->port != i);
633                 if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
634                         PRINT_ERROR("%s", "disabling MAD processing failed.");
635                 if (sport->mad_agent) {
636                         ib_unregister_mad_agent(sport->mad_agent);
637                         sport->mad_agent = NULL;
638                 }
639         }
640 }
641
642 /**
643  * Allocate and initialize an SRPT I/O context structure.
644  */
645 static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev)
646 {
647         struct srpt_ioctx *ioctx;
648
649         ioctx = kmalloc(sizeof *ioctx, GFP_KERNEL);
650         if (!ioctx)
651                 goto out;
652
653         ioctx->buf = kzalloc(srp_max_message_size, GFP_KERNEL);
654         if (!ioctx->buf)
655                 goto out_free_ioctx;
656
657         ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf,
658                                        srp_max_message_size, DMA_BIDIRECTIONAL);
659         if (ib_dma_mapping_error(sdev->device, ioctx->dma))
660                 goto out_free_buf;
661
662         return ioctx;
663
664 out_free_buf:
665         kfree(ioctx->buf);
666 out_free_ioctx:
667         kfree(ioctx);
668 out:
669         return NULL;
670 }
671
672 /*
673  * Deallocate an SRPT I/O context structure.
674  */
675 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
676 {
677         if (!ioctx)
678                 return;
679
680         ib_dma_unmap_single(sdev->device, ioctx->dma,
681                             srp_max_message_size, DMA_BIDIRECTIONAL);
682         kfree(ioctx->buf);
683         kfree(ioctx);
684 }
685
686 /*
687  * Associate a ring of SRPT I/O context structures with the specified device.
688  */
689 static int srpt_alloc_ioctx_ring(struct srpt_device *sdev)
690 {
691         int i;
692
693         TRACE_ENTRY();
694
695         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
696                 sdev->ioctx_ring[i] = srpt_alloc_ioctx(sdev);
697
698                 if (!sdev->ioctx_ring[i])
699                         goto err;
700
701                 sdev->ioctx_ring[i]->index = i;
702         }
703
704         TRACE_EXIT_RES(0);
705
706         return 0;
707
708 err:
709         while (--i > 0) {
710                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
711                 sdev->ioctx_ring[i] = NULL;
712         }
713         TRACE_EXIT_RES(-ENOMEM);
714         return -ENOMEM;
715 }
716
717 /* Free the ring of SRPT I/O context structures. */
718 static void srpt_free_ioctx_ring(struct srpt_device *sdev)
719 {
720         int i;
721
722         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
723                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
724                 sdev->ioctx_ring[i] = NULL;
725         }
726 }
727
728 /**
729  * Set the state of a command.
730  * @new: New state to be set.
731  *
732  * Does not modify the state of aborted commands. Returns the previous command
733  * state.
734  */
735 static enum srpt_command_state srpt_set_cmd_state(struct srpt_ioctx *ioctx,
736                                                   enum srpt_command_state new)
737 {
738         enum srpt_command_state previous;
739
740         WARN_ON(!ioctx);
741         WARN_ON(new == SRPT_STATE_NEW);
742
743         previous = ioctx->state;
744         if (previous != SRPT_STATE_ABORTED)
745                 ioctx->state = new;
746
747         return previous;
748 }
749
750 /**
751  * Test and set the state of a command.
752  * @expected: State to compare against.
753  * @new: New state to be set if the current state matches 'expected'.
754  *
755  * Returns the previous command state.
756  */
757 static enum srpt_command_state
758 srpt_test_and_set_cmd_state(struct srpt_ioctx *ioctx,
759                             enum srpt_command_state expected,
760                             enum srpt_command_state new)
761 {
762         enum srpt_command_state previous;
763
764         WARN_ON(!ioctx);
765         WARN_ON(expected == SRPT_STATE_ABORTED);
766         WARN_ON(new == SRPT_STATE_NEW);
767
768         previous = ioctx->state;
769         if (previous == expected)
770                 ioctx->state = new;
771
772         return previous;
773 }
774
775 /**
776  * Post a receive request on the work queue of InfiniBand device 'sdev'.
777  */
778 static int srpt_post_recv(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
779 {
780         struct ib_sge list;
781         struct ib_recv_wr wr, *bad_wr;
782
783         wr.wr_id = ioctx->index | SRPT_OP_RECV;
784
785         list.addr = ioctx->dma;
786         list.length = srp_max_message_size;
787         list.lkey = sdev->mr->lkey;
788
789         wr.next = NULL;
790         wr.sg_list = &list;
791         wr.num_sge = 1;
792
793         return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);
794 }
795
796 /**
797  * Post an IB send request.
798  * @ch: RDMA channel to post the send request on.
799  * @ioctx: I/O context of the send request.
800  * @len: length of the request to be sent in bytes.
801  *
802  * Returns zero upon success and a non-zero value upon failure.
803  */
804 static int srpt_post_send(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
805                           int len)
806 {
807         struct ib_sge list;
808         struct ib_send_wr wr, *bad_wr;
809         struct srpt_device *sdev = ch->sport->sdev;
810         int ret;
811
812         ret = -ENOMEM;
813         if (atomic_dec_return(&ch->qp_wr_avail) < 0) {
814                 atomic_inc(&ch->qp_wr_avail);
815                 PRINT_ERROR("%s[%d]: SRQ full", __func__, __LINE__);
816                 goto out;
817         }
818
819         ib_dma_sync_single_for_device(sdev->device, ioctx->dma,
820                                       len, DMA_TO_DEVICE);
821
822         list.addr = ioctx->dma;
823         list.length = len;
824         list.lkey = sdev->mr->lkey;
825
826         wr.next = NULL;
827         wr.wr_id = ioctx->index;
828         wr.sg_list = &list;
829         wr.num_sge = 1;
830         wr.opcode = IB_WR_SEND;
831         wr.send_flags = IB_SEND_SIGNALED;
832
833         ret = ib_post_send(ch->qp, &wr, &bad_wr);
834
835 out:
836         return ret;
837 }
838
839 static int srpt_get_desc_tbl(struct srpt_ioctx *ioctx, struct srp_cmd *srp_cmd,
840                              int *ind)
841 {
842         struct srp_indirect_buf *idb;
843         struct srp_direct_buf *db;
844
845         *ind = 0;
846         if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
847             ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
848                 ioctx->n_rbuf = 1;
849                 ioctx->rbufs = &ioctx->single_rbuf;
850
851                 ib_dma_sync_single_for_cpu(ioctx->ch->sport->sdev->device,
852                                            ioctx->dma + sizeof(struct srp_cmd),
853                                            sizeof(*db), DMA_FROM_DEVICE);
854
855                 db = (void *)srp_cmd->add_data;
856                 memcpy(ioctx->rbufs, db, sizeof *db);
857                 ioctx->data_len = be32_to_cpu(db->len);
858         } else {
859                 idb = (void *)srp_cmd->add_data;
860
861                 ioctx->n_rbuf = be32_to_cpu(idb->table_desc.len) / sizeof *db;
862
863                 if (ioctx->n_rbuf >
864                     (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
865                         *ind = 1;
866                         ioctx->n_rbuf = 0;
867                         goto out;
868                 }
869
870                 if (ioctx->n_rbuf == 1)
871                         ioctx->rbufs = &ioctx->single_rbuf;
872                 else
873                         ioctx->rbufs =
874                                 kmalloc(ioctx->n_rbuf * sizeof *db, GFP_ATOMIC);
875                 if (!ioctx->rbufs) {
876                         ioctx->n_rbuf = 0;
877                         return -ENOMEM;
878                 }
879
880                 ib_dma_sync_single_for_cpu(ioctx->ch->sport->sdev->device,
881                                            ioctx->dma + sizeof(struct srp_cmd),
882                                            ioctx->n_rbuf * sizeof(*db),
883                                            DMA_FROM_DEVICE);
884
885                 db = idb->desc_list;
886                 memcpy(ioctx->rbufs, db, ioctx->n_rbuf * sizeof *db);
887                 ioctx->data_len = be32_to_cpu(idb->len);
888         }
889 out:
890         return 0;
891 }
892
893 /*
894  * Modify the attributes of queue pair 'qp': allow local write, remote read,
895  * and remote write. Also transition 'qp' to state IB_QPS_INIT.
896  */
897 static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
898 {
899         struct ib_qp_attr *attr;
900         int ret;
901
902         attr = kzalloc(sizeof *attr, GFP_KERNEL);
903         if (!attr)
904                 return -ENOMEM;
905
906         attr->qp_state = IB_QPS_INIT;
907         attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ |
908             IB_ACCESS_REMOTE_WRITE;
909         attr->port_num = ch->sport->port;
910         attr->pkey_index = 0;
911
912         ret = ib_modify_qp(qp, attr,
913                            IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
914                            IB_QP_PKEY_INDEX);
915
916         kfree(attr);
917         return ret;
918 }
919
920 /**
921  * Change the state of a channel to 'ready to receive' (RTR).
922  * @ch: channel of the queue pair.
923  * @qp: queue pair to change the state of.
924  *
925  * Returns zero upon success and a negative value upon failure.
926  *
927  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
928  * If this structure ever becomes larger, it might be necessary to allocate
929  * it dynamically instead of on the stack.
930  */
931 static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
932 {
933         struct ib_qp_attr qp_attr;
934         int attr_mask;
935         int ret;
936
937         qp_attr.qp_state = IB_QPS_RTR;
938         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
939         if (ret)
940                 goto out;
941
942         qp_attr.max_dest_rd_atomic = 4;
943
944         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
945
946 out:
947         return ret;
948 }
949
950 /**
951  * Change the state of a channel to 'ready to send' (RTS).
952  * @ch: channel of the queue pair.
953  * @qp: queue pair to change the state of.
954  *
955  * Returns zero upon success and a negative value upon failure.
956  *
957  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
958  * If this structure ever becomes larger, it might be necessary to allocate
959  * it dynamically instead of on the stack.
960  */
961 static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
962 {
963         struct ib_qp_attr qp_attr;
964         int attr_mask;
965         int ret;
966
967         qp_attr.qp_state = IB_QPS_RTS;
968         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
969         if (ret)
970                 goto out;
971
972         qp_attr.max_rd_atomic = 4;
973
974         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
975
976 out:
977         return ret;
978 }
979
980 static void srpt_reset_ioctx(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
981 {
982         int i;
983
984         if (ioctx->n_rdma_ius > 0 && ioctx->rdma_ius) {
985                 struct rdma_iu *riu = ioctx->rdma_ius;
986
987                 for (i = 0; i < ioctx->n_rdma_ius; ++i, ++riu)
988                         kfree(riu->sge);
989                 kfree(ioctx->rdma_ius);
990         }
991
992         if (ioctx->n_rbuf > 1)
993                 kfree(ioctx->rbufs);
994
995         /* If ch == NULL this means that the command has been aborted. */
996         if (!ch)
997                 return;
998
999         if (srpt_post_recv(ch->sport->sdev, ioctx))
1000                 PRINT_ERROR("%s", "SRQ post_recv failed - this is serious.");
1001                 /* we should queue it back to free_ioctx queue */
1002         else
1003                 atomic_inc(&ch->req_lim_delta);
1004 }
1005
1006 /**
1007  * Abort a command.
1008  */
1009 static void srpt_abort_scst_cmd(struct srpt_device *sdev,
1010                                 struct scst_cmd *scmnd)
1011 {
1012         struct srpt_ioctx *ioctx;
1013         scst_data_direction dir;
1014         enum srpt_command_state previous_state;
1015
1016         ioctx = scst_cmd_get_tgt_priv(scmnd);
1017         BUG_ON(!ioctx);
1018
1019         previous_state = srpt_set_cmd_state(ioctx, SRPT_STATE_ABORTED);
1020         if (previous_state == SRPT_STATE_ABORTED)
1021                 goto out;
1022
1023         TRACE_DBG("Aborting cmd with state %d and tag %lld",
1024                   previous_state, scst_cmd_get_tag(scmnd));
1025
1026         dir = scst_cmd_get_data_direction(scmnd);
1027         if (dir != SCST_DATA_NONE && scst_cmd_get_sg(scmnd))
1028                 ib_dma_unmap_sg(sdev->device,
1029                                 scst_cmd_get_sg(scmnd),
1030                                 scst_cmd_get_sg_cnt(scmnd),
1031                                 scst_to_tgt_dma_dir(dir));
1032
1033         switch (previous_state) {
1034         case SRPT_STATE_NEW:
1035                 break;
1036         case SRPT_STATE_NEED_DATA:
1037                 WARN_ON(scst_cmd_get_data_direction(ioctx->scmnd)
1038                         == SCST_DATA_READ);
1039                 scst_rx_data(scmnd,
1040                              SCST_RX_STATUS_ERROR,
1041                              SCST_CONTEXT_THREAD);
1042                 break;
1043         case SRPT_STATE_PROCESSED:
1044                 scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_FAILED);
1045                 WARN_ON(scmnd->state != SCST_CMD_STATE_XMIT_WAIT);
1046                 scst_tgt_cmd_done(scmnd, scst_estimate_context());
1047                 break;
1048         default:
1049                 TRACE_DBG("Aborting cmd with state %d", previous_state);
1050                 WARN_ON("ERROR: unexpected command state");
1051         }
1052
1053 out:
1054         ;
1055 }
1056
1057 /**
1058  * Abort an SCST command if the channel is has been disconnected.
1059  *
1060  * Returns true if the command has been aborted because the channel is
1061  * disconnecting, and false if the channel is not being disconnected.
1062  * Must be called from inside rdy_to_xfer, xmit_response or on_free_cmd
1063  * because any call from outside SCST's command processing callback functions
1064  * would be racy.
1065  */
1066 static bool srpt_abort_if_disconnecting(struct srpt_ioctx *ioctx)
1067 {
1068         BUG_ON(!ioctx);
1069
1070         if (ioctx->ch->state == RDMA_CHANNEL_DISCONNECTING) {
1071                 srpt_abort_scst_cmd(ioctx->ch->sport->sdev, ioctx->scmnd);
1072                 return true;
1073         }
1074         return false;
1075 }
1076
1077 static void srpt_handle_err_comp(struct srpt_rdma_ch *ch, struct ib_wc *wc)
1078 {
1079         struct srpt_ioctx *ioctx;
1080         struct srpt_device *sdev = ch->sport->sdev;
1081
1082         if (wc->wr_id & SRPT_OP_RECV) {
1083                 ioctx = sdev->ioctx_ring[wc->wr_id & ~SRPT_OP_RECV];
1084                 PRINT_ERROR("%s", "This is serious - SRQ is in bad state.");
1085         } else {
1086                 ioctx = sdev->ioctx_ring[wc->wr_id];
1087
1088                 if (ioctx->scmnd)
1089                         srpt_abort_scst_cmd(sdev, ioctx->scmnd);
1090                 else
1091                         srpt_reset_ioctx(ch, ioctx);
1092         }
1093 }
1094
1095 static void srpt_handle_send_comp(struct srpt_rdma_ch *ch,
1096                                   struct srpt_ioctx *ioctx,
1097                                   enum scst_exec_context context)
1098 {
1099         if (ioctx->scmnd) {
1100                 scst_data_direction dir =
1101                         scst_cmd_get_data_direction(ioctx->scmnd);
1102
1103                 if (dir != SCST_DATA_NONE && scst_cmd_get_sg(ioctx->scmnd))
1104                         ib_dma_unmap_sg(ch->sport->sdev->device,
1105                                         scst_cmd_get_sg(ioctx->scmnd),
1106                                         scst_cmd_get_sg_cnt(ioctx->scmnd),
1107                                         scst_to_tgt_dma_dir(dir));
1108
1109                 WARN_ON(ioctx->scmnd->state != SCST_CMD_STATE_XMIT_WAIT);
1110                 scst_tgt_cmd_done(ioctx->scmnd, context);
1111         } else
1112                 srpt_reset_ioctx(ch, ioctx);
1113 }
1114
1115 /** Process an RDMA completion notification. */
1116 static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch,
1117                                   struct srpt_ioctx *ioctx)
1118 {
1119         if (!ioctx->scmnd) {
1120                 WARN_ON("ERROR: ioctx->scmnd == NULL");
1121                 srpt_reset_ioctx(ch, ioctx);
1122                 return;
1123         }
1124
1125         /*
1126          * If an RDMA completion notification has been received for a write
1127          * command, tell SCST that processing can continue by calling
1128          * scst_rx_data().
1129          */
1130         if (srpt_test_and_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA,
1131                                 SRPT_STATE_DATA_IN) == SRPT_STATE_NEED_DATA) {
1132                 WARN_ON(scst_cmd_get_data_direction(ioctx->scmnd)
1133                         == SCST_DATA_READ);
1134                 scst_rx_data(ioctx->scmnd, SCST_RX_STATUS_SUCCESS,
1135                              scst_estimate_context());
1136         }
1137 }
1138
1139 /**
1140  * Build an SRP_RSP response.
1141  * @ch: RDMA channel through which the request has been received.
1142  * @ioctx: I/O context in which the SRP_RSP response will be built.
1143  * @s_key: sense key that will be stored in the response.
1144  * @s_code: value that will be stored in the asc_ascq field of the sense data.
1145  * @tag: tag of the request for which this response is being generated.
1146  *
1147  * Returns the size in bytes of the SRP_RSP response.
1148  *
1149  * An SRP_RSP response contains a SCSI status or service response. See also
1150  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
1151  * response. See also SPC-2 for more information about sense data.
1152  */
1153 static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
1154                               struct srpt_ioctx *ioctx, u8 s_key, u8 s_code,
1155                               u64 tag)
1156 {
1157         struct srp_rsp *srp_rsp;
1158         struct sense_data *sense;
1159         int limit_delta;
1160         int sense_data_len;
1161         int resp_len;
1162
1163         sense_data_len = (s_key == NO_SENSE) ? 0 : sizeof(*sense);
1164         resp_len = sizeof(*srp_rsp) + sense_data_len;
1165
1166         srp_rsp = ioctx->buf;
1167         memset(srp_rsp, 0, sizeof *srp_rsp);
1168
1169         limit_delta = atomic_read(&ch->req_lim_delta);
1170         atomic_sub(limit_delta, &ch->req_lim_delta);
1171
1172         srp_rsp->opcode = SRP_RSP;
1173         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1174         srp_rsp->tag = tag;
1175
1176         if (s_key != NO_SENSE) {
1177                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1178                 srp_rsp->status = SAM_STAT_CHECK_CONDITION;
1179                 srp_rsp->sense_data_len = cpu_to_be32(sense_data_len);
1180
1181                 sense = (struct sense_data *)(srp_rsp + 1);
1182                 sense->err_code = 0x70;
1183                 sense->key = s_key;
1184                 sense->asc_ascq = s_code;
1185         }
1186
1187         return resp_len;
1188 }
1189
1190 /**
1191  * Build a task management response, which is a specific SRP_RSP response.
1192  * @ch: RDMA channel through which the request has been received.
1193  * @ioctx: I/O context in which the SRP_RSP response will be built.
1194  * @rsp_code: RSP_CODE that will be stored in the response.
1195  * @tag: tag of the request for which this response is being generated.
1196  *
1197  * Returns the size in bytes of the SRP_RSP response.
1198  *
1199  * An SRP_RSP response contains a SCSI status or service response. See also
1200  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
1201  * response.
1202  */
1203 static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
1204                                   struct srpt_ioctx *ioctx, u8 rsp_code,
1205                                   u64 tag)
1206 {
1207         struct srp_rsp *srp_rsp;
1208         int limit_delta;
1209         int resp_data_len;
1210         int resp_len;
1211
1212         resp_data_len = (rsp_code == SRP_TSK_MGMT_SUCCESS) ? 0 : 4;
1213         resp_len = sizeof(*srp_rsp) + resp_data_len;
1214
1215         srp_rsp = ioctx->buf;
1216         memset(srp_rsp, 0, sizeof *srp_rsp);
1217
1218         limit_delta = atomic_read(&ch->req_lim_delta);
1219         atomic_sub(limit_delta, &ch->req_lim_delta);
1220
1221         srp_rsp->opcode = SRP_RSP;
1222         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
1223         srp_rsp->tag = tag;
1224
1225         if (rsp_code != SRP_TSK_MGMT_SUCCESS) {
1226                 srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
1227                 srp_rsp->resp_data_len = cpu_to_be32(resp_data_len);
1228                 srp_rsp->data[3] = rsp_code;
1229         }
1230
1231         return resp_len;
1232 }
1233
1234 /*
1235  * Process SRP_CMD.
1236  */
1237 static int srpt_handle_cmd(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
1238 {
1239         struct scst_cmd *scmnd;
1240         struct srp_cmd *srp_cmd;
1241         struct srp_rsp *srp_rsp;
1242         scst_data_direction dir;
1243         int indirect_desc = 0;
1244         int ret;
1245         unsigned long flags;
1246
1247         srp_cmd = ioctx->buf;
1248         srp_rsp = ioctx->buf;
1249
1250         dir = SCST_DATA_NONE;
1251         if (srp_cmd->buf_fmt) {
1252                 ret = srpt_get_desc_tbl(ioctx, srp_cmd, &indirect_desc);
1253                 if (ret) {
1254                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1255                                            NO_ADD_SENSE, srp_cmd->tag);
1256                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1257                         goto err;
1258                 }
1259
1260                 if (indirect_desc) {
1261                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1262                                            NO_ADD_SENSE, srp_cmd->tag);
1263                         srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1264                         goto err;
1265                 }
1266
1267                 /*
1268                  * The lower four bits of the buffer format field contain the
1269                  * DATA-IN buffer descriptor format, and the highest four bits
1270                  * contain the DATA-OUT buffer descriptor format.
1271                  */
1272                 if (srp_cmd->buf_fmt & 0xf)
1273                         /* DATA-IN: transfer data from target to initiator. */
1274                         dir = SCST_DATA_READ;
1275                 else if (srp_cmd->buf_fmt >> 4)
1276                         /* DATA-OUT: transfer data from initiator to target. */
1277                         dir = SCST_DATA_WRITE;
1278         }
1279
1280         scmnd = scst_rx_cmd(ch->scst_sess, (u8 *) &srp_cmd->lun,
1281                             sizeof srp_cmd->lun, srp_cmd->cdb, 16,
1282                             thread ? SCST_NON_ATOMIC : SCST_ATOMIC);
1283         if (!scmnd) {
1284                 srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
1285                                    NO_ADD_SENSE, srp_cmd->tag);
1286                 srp_rsp->status = SAM_STAT_TASK_SET_FULL;
1287                 goto err;
1288         }
1289
1290         ioctx->scmnd = scmnd;
1291
1292         switch (srp_cmd->task_attr) {
1293         case SRP_CMD_HEAD_OF_Q:
1294                 scmnd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1295                 break;
1296         case SRP_CMD_ORDERED_Q:
1297                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1298                 break;
1299         case SRP_CMD_SIMPLE_Q:
1300                 scmnd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1301                 break;
1302         case SRP_CMD_ACA:
1303                 scmnd->queue_type = SCST_CMD_QUEUE_ACA;
1304                 break;
1305         default:
1306                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1307                 break;
1308         }
1309
1310         scst_cmd_set_tag(scmnd, srp_cmd->tag);
1311         scst_cmd_set_tgt_priv(scmnd, ioctx);
1312         scst_cmd_set_expected(scmnd, dir, ioctx->data_len);
1313
1314         spin_lock_irqsave(&ch->spinlock, flags);
1315         list_add_tail(&ioctx->scmnd_list, &ch->active_scmnd_list);
1316         ch->active_scmnd_cnt++;
1317         spin_unlock_irqrestore(&ch->spinlock, flags);
1318
1319         scst_cmd_init_done(scmnd, scst_estimate_context());
1320
1321         return 0;
1322
1323 err:
1324         WARN_ON(srp_rsp->opcode != SRP_RSP);
1325
1326         return -1;
1327 }
1328
1329 /*
1330  * Process an SRP_TSK_MGMT request.
1331  *
1332  * Returns 0 upon success and -1 upon failure.
1333  *
1334  * Each task management function is performed by calling one of the
1335  * scst_rx_mgmt_fn*() functions. These functions will either report failure
1336  * or process the task management function asynchronously. The function
1337  * srpt_tsk_mgmt_done() will be called by the SCST core upon completion of the
1338  * task management function. When srpt_handle_tsk_mgmt() reports failure
1339  * (i.e. returns -1) a response will have been built in ioctx->buf. This
1340  * information unit has to be sent back by the caller.
1341  *
1342  * For more information about SRP_TSK_MGMT information units, see also section
1343  * 6.7 in the T10 SRP r16a document.
1344  */
1345 static int srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
1346                                 struct srpt_ioctx *ioctx)
1347 {
1348         struct srp_tsk_mgmt *srp_tsk;
1349         struct srpt_mgmt_ioctx *mgmt_ioctx;
1350         int ret;
1351
1352         ib_dma_sync_single_for_cpu(ch->sport->sdev->device, ioctx->dma,
1353                                    sizeof(struct srp_tsk_mgmt),
1354                                    DMA_FROM_DEVICE);
1355
1356         srp_tsk = ioctx->buf;
1357
1358         TRACE_DBG("recv_tsk_mgmt= %d for task_tag= %lld"
1359                   " using tag= %lld cm_id= %p sess= %p",
1360                   srp_tsk->tsk_mgmt_func,
1361                   (unsigned long long) srp_tsk->task_tag,
1362                   (unsigned long long) srp_tsk->tag,
1363                   ch->cm_id, ch->scst_sess);
1364
1365         mgmt_ioctx = kmalloc(sizeof *mgmt_ioctx, GFP_ATOMIC);
1366         if (!mgmt_ioctx) {
1367                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1368                                        srp_tsk->tag);
1369                 goto err;
1370         }
1371
1372         mgmt_ioctx->ioctx = ioctx;
1373         mgmt_ioctx->ch = ch;
1374         mgmt_ioctx->tag = srp_tsk->tag;
1375
1376         switch (srp_tsk->tsk_mgmt_func) {
1377         case SRP_TSK_ABORT_TASK:
1378                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK");
1379                 ret = scst_rx_mgmt_fn_tag(ch->scst_sess,
1380                                           SCST_ABORT_TASK,
1381                                           srp_tsk->task_tag,
1382                                           thread ?
1383                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1384                                           mgmt_ioctx);
1385                 break;
1386         case SRP_TSK_ABORT_TASK_SET:
1387                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK_SET");
1388                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1389                                           SCST_ABORT_TASK_SET,
1390                                           (u8 *) &srp_tsk->lun,
1391                                           sizeof srp_tsk->lun,
1392                                           thread ?
1393                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1394                                           mgmt_ioctx);
1395                 break;
1396         case SRP_TSK_CLEAR_TASK_SET:
1397                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_TASK_SET");
1398                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1399                                           SCST_CLEAR_TASK_SET,
1400                                           (u8 *) &srp_tsk->lun,
1401                                           sizeof srp_tsk->lun,
1402                                           thread ?
1403                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1404                                           mgmt_ioctx);
1405                 break;
1406         case SRP_TSK_LUN_RESET:
1407                 TRACE_DBG("%s", "Processing SRP_TSK_LUN_RESET");
1408                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1409                                           SCST_LUN_RESET,
1410                                           (u8 *) &srp_tsk->lun,
1411                                           sizeof srp_tsk->lun,
1412                                           thread ?
1413                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1414                                           mgmt_ioctx);
1415                 break;
1416         case SRP_TSK_CLEAR_ACA:
1417                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_ACA");
1418                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1419                                           SCST_CLEAR_ACA,
1420                                           (u8 *) &srp_tsk->lun,
1421                                           sizeof srp_tsk->lun,
1422                                           thread ?
1423                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1424                                           mgmt_ioctx);
1425                 break;
1426         default:
1427                 TRACE_DBG("%s", "Unsupported task management function.");
1428                 srpt_build_tskmgmt_rsp(ch, ioctx,
1429                                        SRP_TSK_MGMT_FUNC_NOT_SUPP,
1430                                        srp_tsk->tag);
1431                 goto err;
1432         }
1433
1434         if (ret) {
1435                 TRACE_DBG("%s", "Processing task management function failed.");
1436                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
1437                                        srp_tsk->tag);
1438                 goto err;
1439         }
1440
1441         WARN_ON(srp_tsk->opcode == SRP_RSP);
1442
1443         return 0;
1444
1445 err:
1446         WARN_ON(srp_tsk->opcode != SRP_RSP);
1447
1448         kfree(mgmt_ioctx);
1449         return -1;
1450 }
1451
1452 /**
1453  * Process a newly received information unit.
1454  * @ch: RDMA channel through which the information unit has been received.
1455  * @ioctx: SRPT I/O context associated with the information unit.
1456  */
1457 static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
1458                                struct srpt_ioctx *ioctx)
1459 {
1460         struct srp_cmd *srp_cmd;
1461         struct srp_rsp *srp_rsp;
1462         unsigned long flags;
1463         int len;
1464
1465         spin_lock_irqsave(&ch->spinlock, flags);
1466         if (ch->state != RDMA_CHANNEL_LIVE) {
1467                 if (ch->state == RDMA_CHANNEL_CONNECTING) {
1468                         list_add_tail(&ioctx->wait_list, &ch->cmd_wait_list);
1469                         spin_unlock_irqrestore(&ch->spinlock, flags);
1470                         return;
1471                 } else {
1472                         spin_unlock_irqrestore(&ch->spinlock, flags);
1473                         srpt_reset_ioctx(ch, ioctx);
1474                         return;
1475                 }
1476         }
1477         spin_unlock_irqrestore(&ch->spinlock, flags);
1478
1479         ib_dma_sync_single_for_cpu(ch->sport->sdev->device, ioctx->dma,
1480                                    sizeof(struct srp_cmd), DMA_FROM_DEVICE);
1481
1482         ioctx->data_len = 0;
1483         ioctx->n_rbuf = 0;
1484         ioctx->rbufs = NULL;
1485         ioctx->n_rdma = 0;
1486         ioctx->n_rdma_ius = 0;
1487         ioctx->rdma_ius = NULL;
1488         ioctx->scmnd = NULL;
1489         ioctx->ch = ch;
1490         ioctx->state = SRPT_STATE_NEW;
1491
1492         srp_cmd = ioctx->buf;
1493         srp_rsp = ioctx->buf;
1494
1495         switch (srp_cmd->opcode) {
1496         case SRP_CMD:
1497                 if (srpt_handle_cmd(ch, ioctx) < 0)
1498                         goto err;
1499                 break;
1500
1501         case SRP_TSK_MGMT:
1502                 if (srpt_handle_tsk_mgmt(ch, ioctx) < 0)
1503                         goto err;
1504                 break;
1505
1506         case SRP_I_LOGOUT:
1507         case SRP_AER_REQ:
1508         default:
1509                 srpt_build_cmd_rsp(ch, ioctx, ILLEGAL_REQUEST, INVALID_CDB,
1510                                    srp_cmd->tag);
1511                 goto err;
1512         }
1513
1514         return;
1515
1516 err:
1517         WARN_ON(srp_rsp->opcode != SRP_RSP);
1518         len = (sizeof *srp_rsp) + be32_to_cpu(srp_rsp->sense_data_len);
1519
1520         if (ch->state != RDMA_CHANNEL_LIVE) {
1521                 /* Give up if another thread modified the channel state. */
1522                 PRINT_ERROR("%s: channel is in state %d", __func__, ch->state);
1523                 srpt_reset_ioctx(ch, ioctx);
1524         } else if (srpt_post_send(ch, ioctx, len)) {
1525                 PRINT_ERROR("%s: sending SRP_RSP response failed", __func__);
1526                 srpt_reset_ioctx(ch, ioctx);
1527         }
1528 }
1529
1530 /*
1531  * Returns true if the ioctx list is non-empty or if the ib_srpt kernel thread
1532  * should stop.
1533  * @pre thread != 0
1534  */
1535 static inline int srpt_test_ioctx_list(void)
1536 {
1537         int res = (!list_empty(&srpt_thread.thread_ioctx_list) ||
1538                    unlikely(kthread_should_stop()));
1539         return res;
1540 }
1541
1542 /*
1543  * Add 'ioctx' to the tail of the ioctx list and wake up the kernel thread.
1544  *
1545  * @pre thread != 0
1546  */
1547 static inline void srpt_schedule_thread(struct srpt_ioctx *ioctx)
1548 {
1549         unsigned long flags;
1550
1551         spin_lock_irqsave(&srpt_thread.thread_lock, flags);
1552         list_add_tail(&ioctx->comp_list, &srpt_thread.thread_ioctx_list);
1553         spin_unlock_irqrestore(&srpt_thread.thread_lock, flags);
1554         wake_up(&ioctx_list_waitQ);
1555 }
1556
1557 /**
1558  * InfiniBand completion queue callback function.
1559  * @cq: completion queue.
1560  * @ctx: completion queue context, which was passed as the fourth argument of
1561  *       the function ib_create_cq().
1562  */
1563 static void srpt_completion(struct ib_cq *cq, void *ctx)
1564 {
1565         struct srpt_rdma_ch *ch = ctx;
1566         struct srpt_device *sdev = ch->sport->sdev;
1567         struct ib_wc wc;
1568         struct srpt_ioctx *ioctx;
1569
1570         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1571         while (ib_poll_cq(ch->cq, 1, &wc) > 0) {
1572                 if (wc.status) {
1573                         PRINT_ERROR("failed %s status= %d",
1574                                wc.wr_id & SRPT_OP_RECV ? "receive" : "send",
1575                                wc.status);
1576                         srpt_handle_err_comp(ch, &wc);
1577                         break;
1578                 }
1579
1580                 if (wc.wr_id & SRPT_OP_RECV) {
1581                         ioctx = sdev->ioctx_ring[wc.wr_id & ~SRPT_OP_RECV];
1582                         if (thread) {
1583                                 ioctx->ch = ch;
1584                                 ioctx->op = IB_WC_RECV;
1585                                 srpt_schedule_thread(ioctx);
1586                         } else
1587                                 srpt_handle_new_iu(ch, ioctx);
1588                         continue;
1589                 } else {
1590                         ioctx = sdev->ioctx_ring[wc.wr_id];
1591                         if (wc.opcode == IB_WC_SEND)
1592                                 atomic_inc(&ch->qp_wr_avail);
1593                         else {
1594                                 WARN_ON(wc.opcode != IB_WC_RDMA_READ);
1595                                 WARN_ON(ioctx->n_rdma <= 0);
1596                                 atomic_add(ioctx->n_rdma,
1597                                            &ch->qp_wr_avail);
1598                         }
1599                 }
1600
1601                 if (thread) {
1602                         ioctx->ch = ch;
1603                         ioctx->op = wc.opcode;
1604                         srpt_schedule_thread(ioctx);
1605                 } else {
1606                         switch (wc.opcode) {
1607                         case IB_WC_SEND:
1608                                 srpt_handle_send_comp(ch, ioctx,
1609                                         scst_estimate_context());
1610                                 break;
1611                         case IB_WC_RDMA_WRITE:
1612                         case IB_WC_RDMA_READ:
1613                                 srpt_handle_rdma_comp(ch, ioctx);
1614                                 break;
1615                         default:
1616                                 break;
1617                         }
1618                 }
1619
1620 #if defined(CONFIG_SCST_DEBUG)
1621                 if (interrupt_processing_delay_in_us <= MAX_UDELAY_MS * 1000)
1622                         udelay(interrupt_processing_delay_in_us);
1623 #endif
1624         }
1625 }
1626
1627 /*
1628  * Create a completion queue on the specified device.
1629  */
1630 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
1631 {
1632         struct ib_qp_init_attr *qp_init;
1633         struct srpt_device *sdev = ch->sport->sdev;
1634         int cqe;
1635         int ret;
1636
1637         qp_init = kzalloc(sizeof *qp_init, GFP_KERNEL);
1638         if (!qp_init)
1639                 return -ENOMEM;
1640
1641         /* Create a completion queue (CQ). */
1642
1643         cqe = SRPT_RQ_SIZE + SRPT_SQ_SIZE - 1;
1644 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(RHEL_RELEASE_CODE)
1645         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe);
1646 #else
1647         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe, 0);
1648 #endif
1649         if (IS_ERR(ch->cq)) {
1650                 ret = PTR_ERR(ch->cq);
1651                 PRINT_ERROR("failed to create_cq cqe= %d ret= %d", cqe, ret);
1652                 goto out;
1653         }
1654
1655         /* Request completion notification. */
1656
1657         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1658
1659         /* Create a queue pair (QP). */
1660
1661         qp_init->qp_context = (void *)ch;
1662         qp_init->event_handler = srpt_qp_event;
1663         qp_init->send_cq = ch->cq;
1664         qp_init->recv_cq = ch->cq;
1665         qp_init->srq = sdev->srq;
1666         qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
1667         qp_init->qp_type = IB_QPT_RC;
1668         qp_init->cap.max_send_wr = SRPT_SQ_SIZE;
1669         qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE;
1670
1671         ch->qp = ib_create_qp(sdev->pd, qp_init);
1672         if (IS_ERR(ch->qp)) {
1673                 ret = PTR_ERR(ch->qp);
1674                 ib_destroy_cq(ch->cq);
1675                 PRINT_ERROR("failed to create_qp ret= %d", ret);
1676                 goto out;
1677         }
1678
1679         atomic_set(&ch->qp_wr_avail, qp_init->cap.max_send_wr);
1680
1681         TRACE_DBG("%s: max_cqe= %d max_sge= %d cm_id= %p",
1682                __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
1683                ch->cm_id);
1684
1685         /* Modify the attributes and the state of queue pair ch->qp. */
1686
1687         ret = srpt_init_ch_qp(ch, ch->qp);
1688         if (ret) {
1689                 ib_destroy_qp(ch->qp);
1690                 ib_destroy_cq(ch->cq);
1691                 goto out;
1692         }
1693
1694         atomic_set(&ch->req_lim_delta, SRPT_RQ_SIZE);
1695 out:
1696         kfree(qp_init);
1697         return ret;
1698 }
1699
1700 /**
1701  * Look up the RDMA channel that corresponds to the specified cm_id.
1702  *
1703  * Return NULL if no matching RDMA channel has been found.
1704  */
1705 static struct srpt_rdma_ch *srpt_find_channel(struct ib_cm_id *cm_id, bool del)
1706 {
1707         struct srpt_device *sdev = cm_id->context;
1708         struct srpt_rdma_ch *ch;
1709
1710         spin_lock_irq(&sdev->spinlock);
1711         list_for_each_entry(ch, &sdev->rch_list, list) {
1712                 if (ch->cm_id == cm_id) {
1713                         if (del)
1714                                 list_del(&ch->list);
1715                         spin_unlock_irq(&sdev->spinlock);
1716                         return ch;
1717                 }
1718         }
1719
1720         spin_unlock_irq(&sdev->spinlock);
1721
1722         return NULL;
1723 }
1724
1725 /**
1726  * Release all resources associated with the specified RDMA channel.
1727  *
1728  * Note: the caller must have removed the channel from the channel list
1729  * before calling this function.
1730  */
1731 static void srpt_release_channel(struct srpt_rdma_ch *ch, bool destroy_cmid)
1732 {
1733         unsigned long flags;
1734
1735         TRACE_ENTRY();
1736
1737         WARN_ON(srpt_find_channel(ch->cm_id, false) == ch);
1738
1739         /*
1740          * Make sure that no new commands are accepted while the channel
1741          * is being released.
1742          */
1743         srpt_set_channel_state(ch, RDMA_CHANNEL_DISCONNECTING);
1744
1745         if (ch->cm_id && destroy_cmid) {
1746                 TRACE_DBG("%s: destroy cm_id= %p", __func__, ch->cm_id);
1747                 ib_destroy_cm_id(ch->cm_id);
1748                 ch->cm_id = NULL;
1749         }
1750
1751         ib_destroy_qp(ch->qp);
1752         ib_destroy_cq(ch->cq);
1753
1754         if (ch->scst_sess) {
1755                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1756
1757                 if (ch->active_scmnd_cnt)
1758                         PRINT_INFO("Releasing session %s which still has %d"
1759                                    " active commands",
1760                                    ch->sess_name, ch->active_scmnd_cnt);
1761                 else
1762                         PRINT_INFO("Releasing session %s", ch->sess_name);
1763
1764                 spin_lock_irqsave(&ch->spinlock, flags);
1765                 list_for_each_entry_safe(ioctx, ioctx_tmp,
1766                                          &ch->active_scmnd_list, scmnd_list) {
1767                         spin_unlock_irqrestore(&ch->spinlock, flags);
1768
1769                         if (ioctx->scmnd)
1770                                 srpt_abort_scst_cmd(ch->sport->sdev,
1771                                                     ioctx->scmnd);
1772
1773                         spin_lock_irqsave(&ch->spinlock, flags);
1774                 }
1775                 WARN_ON(!list_empty(&ch->active_scmnd_list));
1776                 WARN_ON(ch->active_scmnd_cnt != 0);
1777                 spin_unlock_irqrestore(&ch->spinlock, flags);
1778
1779                 scst_unregister_session(ch->scst_sess, 0, NULL);
1780                 ch->scst_sess = NULL;
1781         }
1782
1783         kfree(ch);
1784
1785         TRACE_EXIT();
1786 }
1787
1788 static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
1789                             struct ib_cm_req_event_param *param,
1790                             void *private_data)
1791 {
1792         struct srpt_device *sdev = cm_id->context;
1793         struct srp_login_req *req;
1794         struct srp_login_rsp *rsp;
1795         struct srp_login_rej *rej;
1796         struct ib_cm_rep_param *rep_param;
1797         struct srpt_rdma_ch *ch, *tmp_ch;
1798         u32 it_iu_len;
1799         int ret = 0;
1800
1801 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
1802         WARN_ON(!sdev || !private_data);
1803         if (!sdev || !private_data)
1804                 return -EINVAL;
1805 #else
1806         if (WARN_ON(!sdev || !private_data))
1807                 return -EINVAL;
1808 #endif
1809
1810         req = (struct srp_login_req *)private_data;
1811
1812         it_iu_len = be32_to_cpu(req->req_it_iu_len);
1813
1814         PRINT_INFO("Received SRP_LOGIN_REQ with"
1815             " i_port_id 0x%llx:0x%llx, t_port_id 0x%llx:0x%llx and length %d"
1816             " on port %d (guid=0x%llx:0x%llx)",
1817             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[0]),
1818             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[8]),
1819             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[0]),
1820             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[8]),
1821             it_iu_len,
1822             param->port,
1823             (unsigned long long)be64_to_cpu(*(u64 *)
1824                                 &sdev->port[param->port - 1].gid.raw[0]),
1825             (unsigned long long)be64_to_cpu(*(u64 *)
1826                                 &sdev->port[param->port - 1].gid.raw[8]));
1827
1828         rsp = kzalloc(sizeof *rsp, GFP_KERNEL);
1829         rej = kzalloc(sizeof *rej, GFP_KERNEL);
1830         rep_param = kzalloc(sizeof *rep_param, GFP_KERNEL);
1831
1832         if (!rsp || !rej || !rep_param) {
1833                 ret = -ENOMEM;
1834                 goto out;
1835         }
1836
1837         if (it_iu_len > srp_max_message_size || it_iu_len < 64) {
1838                 rej->reason =
1839                     cpu_to_be32(SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
1840                 ret = -EINVAL;
1841                 PRINT_ERROR("rejected SRP_LOGIN_REQ because its"
1842                             " length (%d bytes) is out of range (%d .. %d)",
1843                             it_iu_len, 64, srp_max_message_size);
1844                 goto reject;
1845         }
1846
1847         if ((req->req_flags & 0x3) == SRP_MULTICHAN_SINGLE) {
1848                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
1849
1850                 spin_lock_irq(&sdev->spinlock);
1851
1852                 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
1853                         if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
1854                             && !memcmp(ch->t_port_id, req->target_port_id, 16)
1855                             && param->port == ch->sport->port
1856                             && param->listen_id == ch->sport->sdev->cm_id
1857                             && ch->cm_id) {
1858                                 enum rdma_ch_state prev_state;
1859
1860                                 /* found an existing channel */
1861                                 TRACE_DBG("Found existing channel name= %s"
1862                                           " cm_id= %p state= %d",
1863                                           ch->sess_name, ch->cm_id, ch->state);
1864
1865                                 prev_state = ch->state;
1866                                 if (ch->state == RDMA_CHANNEL_LIVE)
1867                                         ch->state = RDMA_CHANNEL_DISCONNECTING;
1868                                 else if (ch->state == RDMA_CHANNEL_CONNECTING)
1869                                         list_del(&ch->list);
1870
1871                                 spin_unlock_irq(&sdev->spinlock);
1872
1873                                 rsp->rsp_flags =
1874                                         SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
1875
1876                                 if (prev_state == RDMA_CHANNEL_LIVE) {
1877                                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
1878                                         PRINT_INFO("disconnected"
1879                                           " session %s because a new"
1880                                           " SRP_LOGIN_REQ has been received.",
1881                                           ch->sess_name);
1882                                 } else if (prev_state ==
1883                                          RDMA_CHANNEL_CONNECTING) {
1884                                         PRINT_ERROR("%s", "rejected"
1885                                           " SRP_LOGIN_REQ because another login"
1886                                           " request is being processed.");
1887                                         ib_send_cm_rej(ch->cm_id,
1888                                                        IB_CM_REJ_NO_RESOURCES,
1889                                                        NULL, 0, NULL, 0);
1890                                         srpt_release_channel(ch, 1);
1891                                 }
1892
1893                                 spin_lock_irq(&sdev->spinlock);
1894                         }
1895                 }
1896
1897                 spin_unlock_irq(&sdev->spinlock);
1898
1899         } else
1900                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
1901
1902         if (((u64) (*(u64 *) req->target_port_id) !=
1903              cpu_to_be64(srpt_service_guid)) ||
1904             ((u64) (*(u64 *) (req->target_port_id + 8)) !=
1905              cpu_to_be64(srpt_service_guid))) {
1906                 rej->reason =
1907                     cpu_to_be32(SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
1908                 ret = -ENOMEM;
1909                 PRINT_ERROR("%s", "rejected SRP_LOGIN_REQ because it"
1910                        " has an invalid target port identifier.");
1911                 goto reject;
1912         }
1913
1914         ch = kzalloc(sizeof *ch, GFP_KERNEL);
1915         if (!ch) {
1916                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1917                 PRINT_ERROR("%s",
1918                             "rejected SRP_LOGIN_REQ because out of memory.");
1919                 ret = -ENOMEM;
1920                 goto reject;
1921         }
1922
1923         spin_lock_init(&ch->spinlock);
1924         memcpy(ch->i_port_id, req->initiator_port_id, 16);
1925         memcpy(ch->t_port_id, req->target_port_id, 16);
1926         ch->sport = &sdev->port[param->port - 1];
1927         ch->cm_id = cm_id;
1928         ch->state = RDMA_CHANNEL_CONNECTING;
1929         INIT_LIST_HEAD(&ch->cmd_wait_list);
1930         INIT_LIST_HEAD(&ch->active_scmnd_list);
1931
1932         ret = srpt_create_ch_ib(ch);
1933         if (ret) {
1934                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1935                 PRINT_ERROR("%s", "rejected SRP_LOGIN_REQ because creating"
1936                             " a new RDMA channel failed.");
1937                 goto free_ch;
1938         }
1939
1940         ret = srpt_ch_qp_rtr(ch, ch->qp);
1941         if (ret) {
1942                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1943                 PRINT_ERROR("rejected SRP_LOGIN_REQ because enabling"
1944                        " RTR failed (error code = %d)", ret);
1945                 goto destroy_ib;
1946         }
1947
1948         if (use_port_guid_in_session_name) {
1949                 /*
1950                  * If the kernel module parameter use_port_guid_in_session_name
1951                  * has been specified, use a combination of the target port
1952                  * GUID and the initiator port ID as the session name. This
1953                  * was the original behavior of the SRP target implementation
1954                  * (i.e. before the SRPT was included in OFED 1.3).
1955                  */
1956                 snprintf(ch->sess_name, sizeof(ch->sess_name),
1957                          "0x%016llx%016llx",
1958                          (unsigned long long)be64_to_cpu(*(u64 *)
1959                                 &sdev->port[param->port - 1].gid.raw[8]),
1960                          (unsigned long long)be64_to_cpu(*(u64 *)
1961                                 (ch->i_port_id + 8)));
1962         } else {
1963                 /*
1964                  * Default behavior: use the initator port identifier as the
1965                  * session name.
1966                  */
1967                 snprintf(ch->sess_name, sizeof(ch->sess_name),
1968                          "0x%016llx%016llx",
1969                          (unsigned long long)be64_to_cpu(*(u64 *)ch->i_port_id),
1970                          (unsigned long long)be64_to_cpu(*(u64 *)
1971                                  (ch->i_port_id + 8)));
1972         }
1973
1974         TRACE_DBG("registering session %s", ch->sess_name);
1975
1976         BUG_ON(!sdev->scst_tgt);
1977         ch->scst_sess = scst_register_session(sdev->scst_tgt, 0, ch->sess_name,
1978                                               NULL, NULL);
1979         if (!ch->scst_sess) {
1980                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1981                 TRACE_DBG("%s", "Failed to create scst sess");
1982                 goto destroy_ib;
1983         }
1984
1985         TRACE_DBG("Establish connection sess=%p name=%s cm_id=%p",
1986                   ch->scst_sess, ch->sess_name, ch->cm_id);
1987
1988         scst_sess_set_tgt_priv(ch->scst_sess, ch);
1989
1990         /* create srp_login_response */
1991         rsp->opcode = SRP_LOGIN_RSP;
1992         rsp->tag = req->tag;
1993         rsp->max_it_iu_len = req->req_it_iu_len;
1994         rsp->max_ti_iu_len = req->req_it_iu_len;
1995         rsp->buf_fmt =
1996             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1997         rsp->req_lim_delta = cpu_to_be32(SRPT_RQ_SIZE);
1998         atomic_set(&ch->req_lim_delta, 0);
1999
2000         /* create cm reply */
2001         rep_param->qp_num = ch->qp->qp_num;
2002         rep_param->private_data = (void *)rsp;
2003         rep_param->private_data_len = sizeof *rsp;
2004         rep_param->rnr_retry_count = 7;
2005         rep_param->flow_control = 1;
2006         rep_param->failover_accepted = 0;
2007         rep_param->srq = 1;
2008         rep_param->responder_resources = 4;
2009         rep_param->initiator_depth = 4;
2010
2011         ret = ib_send_cm_rep(cm_id, rep_param);
2012         if (ret) {
2013                 PRINT_ERROR("sending SRP_LOGIN_REQ response failed"
2014                             " (error code = %d)", ret);
2015                 goto release_channel;
2016         }
2017
2018         spin_lock_irq(&sdev->spinlock);
2019         list_add_tail(&ch->list, &sdev->rch_list);
2020         spin_unlock_irq(&sdev->spinlock);
2021
2022         goto out;
2023
2024 release_channel:
2025         scst_unregister_session(ch->scst_sess, 0, NULL);
2026         ch->scst_sess = NULL;
2027
2028 destroy_ib:
2029         ib_destroy_qp(ch->qp);
2030         ib_destroy_cq(ch->cq);
2031
2032 free_ch:
2033         kfree(ch);
2034
2035 reject:
2036         rej->opcode = SRP_LOGIN_REJ;
2037         rej->tag = req->tag;
2038         rej->buf_fmt =
2039             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
2040
2041         ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
2042                              (void *)rej, sizeof *rej);
2043
2044 out:
2045         kfree(rep_param);
2046         kfree(rsp);
2047         kfree(rej);
2048
2049         return ret;
2050 }
2051
2052 /**
2053  * Release the channel with the specified cm_id.
2054  *
2055  * Returns one to indicate that the caller of srpt_cm_handler() should destroy
2056  * the cm_id.
2057  */
2058 static void srpt_find_and_release_channel(struct ib_cm_id *cm_id)
2059 {
2060         struct srpt_rdma_ch *ch;
2061
2062         ch = srpt_find_channel(cm_id, true);
2063         if (ch)
2064                 srpt_release_channel(ch, 0);
2065 }
2066
2067 static void srpt_cm_rej_recv(struct ib_cm_id *cm_id)
2068 {
2069         PRINT_INFO("Received InfiniBand REJ packet for cm_id %p.", cm_id);
2070         srpt_find_and_release_channel(cm_id);
2071 }
2072
2073 /**
2074  * Process an IB_CM_RTU_RECEIVED or IB_CM_USER_ESTABLISHED event.
2075  *
2076  * An IB_CM_RTU_RECEIVED message indicates that the connection is established
2077  * and that the recipient may begin transmitting (RTU = ready to use).
2078  */
2079 static int srpt_cm_rtu_recv(struct ib_cm_id *cm_id)
2080 {
2081         struct srpt_rdma_ch *ch;
2082         int ret;
2083
2084         ch = srpt_find_channel(cm_id, false);
2085         if (!ch)
2086                 return -EINVAL;
2087
2088         if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_CONNECTING,
2089                                             RDMA_CHANNEL_LIVE)) {
2090                 struct srpt_ioctx *ioctx, *ioctx_tmp;
2091
2092                 ret = srpt_ch_qp_rts(ch, ch->qp);
2093
2094                 list_for_each_entry_safe(ioctx, ioctx_tmp, &ch->cmd_wait_list,
2095                                          wait_list) {
2096                         list_del(&ioctx->wait_list);
2097                         srpt_handle_new_iu(ch, ioctx);
2098                 }
2099                 if (ret && srpt_test_and_set_channel_state(ch,
2100                                         RDMA_CHANNEL_LIVE,
2101                                         RDMA_CHANNEL_DISCONNECTING)) {
2102                         TRACE_DBG("cm_id=%p sess_name=%s state=%d",
2103                                   cm_id, ch->sess_name, ch->state);
2104                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
2105                 }
2106         } else if (ch->state == RDMA_CHANNEL_DISCONNECTING) {
2107                 TRACE_DBG("cm_id=%p sess_name=%s state=%d",
2108                           cm_id, ch->sess_name, ch->state);
2109                 ib_send_cm_dreq(ch->cm_id, NULL, 0);
2110                 ret = -EAGAIN;
2111         } else
2112                 ret = 0;
2113
2114         return ret;
2115 }
2116
2117 static void srpt_cm_timewait_exit(struct ib_cm_id *cm_id)
2118 {
2119         PRINT_INFO("Received InfiniBand TimeWait exit for cm_id %p.", cm_id);
2120         srpt_find_and_release_channel(cm_id);
2121 }
2122
2123 static void srpt_cm_rep_error(struct ib_cm_id *cm_id)
2124 {
2125         PRINT_INFO("Received InfiniBand REP error for cm_id %p.", cm_id);
2126         srpt_find_and_release_channel(cm_id);
2127 }
2128
2129 static int srpt_cm_dreq_recv(struct ib_cm_id *cm_id)
2130 {
2131         struct srpt_rdma_ch *ch;
2132
2133         ch = srpt_find_channel(cm_id, false);
2134         if (!ch)
2135                 return -EINVAL;
2136
2137         TRACE_DBG("%s: cm_id= %p ch->state= %d",
2138                  __func__, cm_id, ch->state);
2139
2140         switch (ch->state) {
2141         case RDMA_CHANNEL_LIVE:
2142         case RDMA_CHANNEL_CONNECTING:
2143                 ib_send_cm_drep(ch->cm_id, NULL, 0);
2144                 PRINT_INFO("Received DREQ and sent DREP for session %s.",
2145                            ch->sess_name);
2146                 break;
2147         case RDMA_CHANNEL_DISCONNECTING:
2148         default:
2149                 break;
2150         }
2151
2152         return 0;
2153 }
2154
2155 static void srpt_cm_drep_recv(struct ib_cm_id *cm_id)
2156 {
2157         PRINT_INFO("Received InfiniBand DREP message for cm_id %p.", cm_id);
2158         srpt_find_and_release_channel(cm_id);
2159 }
2160
2161 /**
2162  * IB connection manager callback function.
2163  *
2164  * A non-zero return value will make the caller destroy the CM ID.
2165  *
2166  * Note: srpt_add_one passes a struct srpt_device* as the third argument to
2167  * the ib_create_cm_id() call.
2168  */
2169 static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2170 {
2171         int ret = 0;
2172
2173         switch (event->event) {
2174         case IB_CM_REQ_RECEIVED:
2175                 ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd,
2176                                        event->private_data);
2177                 break;
2178         case IB_CM_REJ_RECEIVED:
2179                 srpt_cm_rej_recv(cm_id);
2180                 ret = -EINVAL;
2181                 break;
2182         case IB_CM_RTU_RECEIVED:
2183         case IB_CM_USER_ESTABLISHED:
2184                 ret = srpt_cm_rtu_recv(cm_id);
2185                 break;
2186         case IB_CM_DREQ_RECEIVED:
2187                 ret = srpt_cm_dreq_recv(cm_id);
2188                 break;
2189         case IB_CM_DREP_RECEIVED:
2190                 srpt_cm_drep_recv(cm_id);
2191                 ret = -EINVAL;
2192                 break;
2193         case IB_CM_TIMEWAIT_EXIT:
2194                 srpt_cm_timewait_exit(cm_id);
2195                 ret = -EINVAL;
2196                 break;
2197         case IB_CM_REP_ERROR:
2198                 srpt_cm_rep_error(cm_id);
2199                 ret = -EINVAL;
2200                 break;
2201         default:
2202                 break;
2203         }
2204
2205         return ret;
2206 }
2207
2208 static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
2209                                  struct srpt_ioctx *ioctx,
2210                                  struct scst_cmd *scmnd)
2211 {
2212         struct scatterlist *scat;
2213         scst_data_direction dir;
2214         struct rdma_iu *riu;
2215         struct srp_direct_buf *db;
2216         dma_addr_t dma_addr;
2217         struct ib_sge *sge;
2218         u64 raddr;
2219         u32 rsize;
2220         u32 tsize;
2221         u32 dma_len;
2222         int count, nrdma;
2223         int i, j, k;
2224
2225         scat = scst_cmd_get_sg(scmnd);
2226         dir = scst_cmd_get_data_direction(scmnd);
2227         WARN_ON(scat == NULL);
2228         count = ib_dma_map_sg(ch->sport->sdev->device, scat,
2229                               scst_cmd_get_sg_cnt(scmnd),
2230                               scst_to_tgt_dma_dir(dir));
2231         if (unlikely(!count))
2232                 return -EBUSY;
2233
2234         if (ioctx->rdma_ius && ioctx->n_rdma_ius)
2235                 nrdma = ioctx->n_rdma_ius;
2236         else {
2237                 nrdma = count / SRPT_DEF_SG_PER_WQE + ioctx->n_rbuf;
2238
2239                 ioctx->rdma_ius = kzalloc(nrdma * sizeof *riu,
2240                                           scst_cmd_atomic(scmnd)
2241                                           ? GFP_ATOMIC : GFP_KERNEL);
2242                 if (!ioctx->rdma_ius) {
2243                         WARN_ON(scat == NULL);
2244                         ib_dma_unmap_sg(ch->sport->sdev->device,
2245                                         scat, scst_cmd_get_sg_cnt(scmnd),
2246                                         scst_to_tgt_dma_dir(dir));
2247                         return -ENOMEM;
2248                 }
2249
2250                 ioctx->n_rdma_ius = nrdma;
2251         }
2252
2253         db = ioctx->rbufs;
2254         tsize = (dir == SCST_DATA_READ) ?
2255                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2256         dma_len = sg_dma_len(&scat[0]);
2257         riu = ioctx->rdma_ius;
2258
2259         /*
2260          * For each remote desc - calculate the #ib_sge.
2261          * If #ib_sge < SRPT_DEF_SG_PER_WQE per rdma operation then
2262          *      each remote desc rdma_iu is required a rdma wr;
2263          * else
2264          *      we need to allocate extra rdma_iu to carry extra #ib_sge in
2265          *      another rdma wr
2266          */
2267         for (i = 0, j = 0;
2268              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2269                 rsize = be32_to_cpu(db->len);
2270                 raddr = be64_to_cpu(db->va);
2271                 riu->raddr = raddr;
2272                 riu->rkey = be32_to_cpu(db->key);
2273                 riu->sge_cnt = 0;
2274
2275                 /* calculate how many sge required for this remote_buf */
2276                 while (rsize > 0 && tsize > 0) {
2277
2278                         if (rsize >= dma_len) {
2279                                 tsize -= dma_len;
2280                                 rsize -= dma_len;
2281                                 raddr += dma_len;
2282
2283                                 if (tsize > 0) {
2284                                         ++j;
2285                                         if (j < count)
2286                                                 dma_len = sg_dma_len(&scat[j]);
2287                                 }
2288                         } else {
2289                                 tsize -= rsize;
2290                                 dma_len -= rsize;
2291                                 rsize = 0;
2292                         }
2293
2294                         ++riu->sge_cnt;
2295
2296                         if (rsize > 0 && riu->sge_cnt == SRPT_DEF_SG_PER_WQE) {
2297                                 ++ioctx->n_rdma;
2298                                 riu->sge =
2299                                     kmalloc(riu->sge_cnt * sizeof *riu->sge,
2300                                             scst_cmd_atomic(scmnd)
2301                                             ? GFP_ATOMIC : GFP_KERNEL);
2302                                 if (!riu->sge)
2303                                         goto free_mem;
2304
2305                                 ++riu;
2306                                 riu->sge_cnt = 0;
2307                                 riu->raddr = raddr;
2308                                 riu->rkey = be32_to_cpu(db->key);
2309                         }
2310                 }
2311
2312                 ++ioctx->n_rdma;
2313                 riu->sge = kmalloc(riu->sge_cnt * sizeof *riu->sge,
2314                                    scst_cmd_atomic(scmnd)
2315                                    ? GFP_ATOMIC : GFP_KERNEL);
2316                 if (!riu->sge)
2317                         goto free_mem;
2318         }
2319
2320         db = ioctx->rbufs;
2321         scat = scst_cmd_get_sg(scmnd);
2322         tsize = (dir == SCST_DATA_READ) ?
2323                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2324         riu = ioctx->rdma_ius;
2325         dma_len = sg_dma_len(&scat[0]);
2326         dma_addr = sg_dma_address(&scat[0]);
2327
2328         /* this second loop is really mapped sg_addres to rdma_iu->ib_sge */
2329         for (i = 0, j = 0;
2330              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2331                 rsize = be32_to_cpu(db->len);
2332                 sge = riu->sge;
2333                 k = 0;
2334
2335                 while (rsize > 0 && tsize > 0) {
2336                         sge->addr = dma_addr;
2337                         sge->lkey = ch->sport->sdev->mr->lkey;
2338
2339                         if (rsize >= dma_len) {
2340                                 sge->length =
2341                                         (tsize < dma_len) ? tsize : dma_len;
2342                                 tsize -= dma_len;
2343                                 rsize -= dma_len;
2344
2345                                 if (tsize > 0) {
2346                                         ++j;
2347                                         if (j < count) {
2348                                                 dma_len = sg_dma_len(&scat[j]);
2349                                                 dma_addr =
2350                                                     sg_dma_address(&scat[j]);
2351                                         }
2352                                 }
2353                         } else {
2354                                 sge->length = (tsize < rsize) ? tsize : rsize;
2355                                 tsize -= rsize;
2356                                 dma_len -= rsize;
2357                                 dma_addr += rsize;
2358                                 rsize = 0;
2359                         }
2360
2361                         ++k;
2362                         if (k == riu->sge_cnt && rsize > 0) {
2363                                 ++riu;
2364                                 sge = riu->sge;
2365                                 k = 0;
2366                         } else if (rsize > 0)
2367                                 ++sge;
2368                 }
2369         }
2370
2371         return 0;
2372
2373 free_mem:
2374         srpt_unmap_sg_to_ib_sge(ch, ioctx, scmnd);
2375
2376         return -ENOMEM;
2377 }
2378
2379 static void srpt_unmap_sg_to_ib_sge(struct srpt_rdma_ch *ch,
2380                                     struct srpt_ioctx *ioctx,
2381                                     struct scst_cmd *scmnd)
2382 {
2383         struct scatterlist *scat;
2384         scst_data_direction dir;
2385
2386         TRACE_ENTRY();
2387
2388         scat = scst_cmd_get_sg(scmnd);
2389
2390         TRACE_DBG("n_rdma = %d; rdma_ius = %p; scat = %p\n",
2391                   ioctx->n_rdma, ioctx->rdma_ius, scat);
2392
2393         BUG_ON(ioctx->n_rdma && !ioctx->rdma_ius);
2394
2395         while (ioctx->n_rdma)
2396                 kfree(ioctx->rdma_ius[--ioctx->n_rdma].sge);
2397
2398         kfree(ioctx->rdma_ius);
2399         ioctx->rdma_ius = NULL;
2400
2401         if (scat) {
2402                 dir = scst_cmd_get_data_direction(scmnd);
2403                 ib_dma_unmap_sg(ch->sport->sdev->device,
2404                                 scat, scst_cmd_get_sg_cnt(scmnd),
2405                                 scst_to_tgt_dma_dir(dir));
2406         }
2407
2408         TRACE_EXIT();
2409 }
2410
2411 static int srpt_perform_rdmas(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2412                               scst_data_direction dir)
2413 {
2414         struct ib_send_wr wr;
2415         struct ib_send_wr *bad_wr;
2416         struct rdma_iu *riu;
2417         int i;
2418         int ret;
2419         int srq_wr_avail;
2420
2421         if (dir == SCST_DATA_WRITE) {
2422                 ret = -ENOMEM;
2423                 srq_wr_avail = atomic_sub_return(ioctx->n_rdma,
2424                                                  &ch->qp_wr_avail);
2425                 if (srq_wr_avail < 0) {
2426                         atomic_add(ioctx->n_rdma, &ch->qp_wr_avail);
2427                         PRINT_INFO("%s[%d]: SRQ full", __func__, __LINE__);
2428                         goto out;
2429                 }
2430         }
2431
2432         ret = 0;
2433         riu = ioctx->rdma_ius;
2434         memset(&wr, 0, sizeof wr);
2435
2436         for (i = 0; i < ioctx->n_rdma; ++i, ++riu) {
2437                 wr.opcode = (dir == SCST_DATA_READ) ?
2438                     IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;
2439                 wr.next = NULL;
2440                 wr.wr_id = ioctx->index;
2441                 wr.wr.rdma.remote_addr = riu->raddr;
2442                 wr.wr.rdma.rkey = riu->rkey;
2443                 wr.num_sge = riu->sge_cnt;
2444                 wr.sg_list = riu->sge;
2445
2446                 /* only get completion event for the last rdma wr */
2447                 if (i == (ioctx->n_rdma - 1) && dir == SCST_DATA_WRITE)
2448                         wr.send_flags = IB_SEND_SIGNALED;
2449
2450                 ret = ib_post_send(ch->qp, &wr, &bad_wr);
2451                 if (ret)
2452                         goto out;
2453         }
2454
2455 out:
2456         return ret;
2457 }
2458
2459 /*
2460  * Start data transfer between initiator and target. Must not block.
2461  */
2462 static int srpt_xfer_data(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2463                           struct scst_cmd *scmnd)
2464 {
2465         int ret;
2466
2467         ret = srpt_map_sg_to_ib_sge(ch, ioctx, scmnd);
2468         if (ret) {
2469                 PRINT_ERROR("%s[%d] ret=%d", __func__, __LINE__, ret);
2470                 ret = SCST_TGT_RES_QUEUE_FULL;
2471                 goto out;
2472         }
2473
2474         ret = srpt_perform_rdmas(ch, ioctx, scst_cmd_get_data_direction(scmnd));
2475         if (ret) {
2476                 if (ret == -EAGAIN || ret == -ENOMEM) {
2477                         PRINT_INFO("%s[%d] queue full -- ret=%d",
2478                                    __func__, __LINE__, ret);
2479                         ret = SCST_TGT_RES_QUEUE_FULL;
2480                 } else {
2481                         PRINT_ERROR("%s[%d] fatal error -- ret=%d",
2482                                     __func__, __LINE__, ret);
2483                         ret = SCST_TGT_RES_FATAL_ERROR;
2484                 }
2485                 goto out_unmap;
2486         }
2487
2488         ret = SCST_TGT_RES_SUCCESS;
2489
2490 out:
2491         return ret;
2492 out_unmap:
2493         srpt_unmap_sg_to_ib_sge(ch, ioctx, scmnd);
2494         goto out;
2495 }
2496
2497 /*
2498  * Called by the SCST core to inform ib_srpt that data reception from the
2499  * initiator should start (SCST_DATA_WRITE). Must not block.
2500  */
2501 static int srpt_rdy_to_xfer(struct scst_cmd *scmnd)
2502 {
2503         struct srpt_rdma_ch *ch;
2504         struct srpt_ioctx *ioctx;
2505         int ret;
2506
2507         ioctx = scst_cmd_get_tgt_priv(scmnd);
2508         BUG_ON(!ioctx);
2509
2510         if (srpt_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA)
2511             == SRPT_STATE_ABORTED
2512             || srpt_abort_if_disconnecting(ioctx)) {
2513                 TRACE_DBG("cmd with tag %lld has been aborted",
2514                           scst_cmd_get_tag(scmnd));
2515                 ret = SCST_TGT_RES_FATAL_ERROR;
2516                 goto out;
2517         }
2518
2519         ch = ioctx->ch;
2520         WARN_ON(ch != scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd)));
2521         BUG_ON(!ch);
2522
2523         if (ch->state == RDMA_CHANNEL_DISCONNECTING) {
2524                 TRACE_DBG("cmd with tag %lld: channel disconnecting",
2525                           scst_cmd_get_tag(scmnd));
2526                 ret = SCST_TGT_RES_FATAL_ERROR;
2527                 goto out;
2528         } else if (ch->state == RDMA_CHANNEL_CONNECTING) {
2529                 ret = SCST_TGT_RES_QUEUE_FULL;
2530                 goto out;
2531         }
2532         ret = srpt_xfer_data(ch, ioctx, scmnd);
2533
2534 out:
2535         return ret;
2536 }
2537
2538 /*
2539  * Called by the SCST core. Transmits the response buffer and status held in
2540  * 'scmnd'. Must not block.
2541  */
2542 static int srpt_xmit_response(struct scst_cmd *scmnd)
2543 {
2544         struct srpt_rdma_ch *ch;
2545         struct srpt_ioctx *ioctx;
2546         struct srp_rsp *srp_rsp;
2547         u64 tag;
2548         int ret = SCST_TGT_RES_SUCCESS;
2549         int dir;
2550         int status;
2551
2552         ioctx = scst_cmd_get_tgt_priv(scmnd);
2553         BUG_ON(!ioctx);
2554
2555         if (srpt_set_cmd_state(ioctx, SRPT_STATE_PROCESSED)
2556             == SRPT_STATE_ABORTED
2557             || srpt_abort_if_disconnecting(ioctx)) {
2558                 TRACE_DBG("cmd with tag %lld has been aborted",
2559                           scst_cmd_get_tag(scmnd));
2560                 ret = SCST_TGT_RES_FATAL_ERROR;
2561                 goto out;
2562         }
2563
2564         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2565         BUG_ON(!ch);
2566
2567         tag = scst_cmd_get_tag(scmnd);
2568
2569         if (ch->state != RDMA_CHANNEL_LIVE) {
2570                 PRINT_ERROR("%s: tag= %lld channel in bad state %d",
2571                        __func__, (unsigned long long)tag, ch->state);
2572
2573                 if (ch->state == RDMA_CHANNEL_DISCONNECTING) {
2574                         TRACE_DBG("cmd with tag %lld: channel disconnecting",
2575                                   (unsigned long long)tag);
2576                         ret = SCST_TGT_RES_FATAL_ERROR;
2577                 } else if (ch->state == RDMA_CHANNEL_CONNECTING)
2578                         ret = SCST_TGT_RES_QUEUE_FULL;
2579
2580                 if (unlikely(scst_cmd_aborted(scmnd)))
2581                         goto out_aborted;
2582
2583                 goto out;
2584         }
2585
2586         if (unlikely(scst_cmd_aborted(scmnd))) {
2587                 TRACE_MGMT_DBG("%s: tag= %lld already got aborted",
2588                                __func__, (unsigned long long)tag);
2589                 goto out_aborted;
2590         }
2591
2592         dir = scst_cmd_get_data_direction(scmnd);
2593         status = scst_cmd_get_status(scmnd) & 0xff;
2594
2595         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE, NO_ADD_SENSE, tag);
2596
2597         srp_rsp = ioctx->buf;
2598
2599         if (SCST_SENSE_VALID(scst_cmd_get_sense_buffer(scmnd))) {
2600                 srp_rsp->sense_data_len = scst_cmd_get_sense_buffer_len(scmnd);
2601                 BUILD_BUG_ON(MIN_MAX_MESSAGE_SIZE <= sizeof(*srp_rsp));
2602                 WARN_ON(srp_max_message_size <= sizeof(*srp_rsp));
2603                 if (srp_rsp->sense_data_len >
2604                     (srp_max_message_size - sizeof *srp_rsp))
2605                         srp_rsp->sense_data_len =
2606                             srp_max_message_size - sizeof *srp_rsp;
2607                 WARN_ON(srp_rsp->sense_data_len <= 0);
2608
2609                 memcpy((u8 *) (srp_rsp + 1), scst_cmd_get_sense_buffer(scmnd),
2610                        srp_rsp->sense_data_len);
2611
2612                 srp_rsp->sense_data_len = cpu_to_be32(srp_rsp->sense_data_len);
2613                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
2614
2615                 if (!status)
2616                         status = SAM_STAT_CHECK_CONDITION;
2617         }
2618
2619         srp_rsp->status = status;
2620
2621         /* For read commands, transfer the data to the initiator. */
2622         if (dir == SCST_DATA_READ && scst_cmd_get_resp_data_len(scmnd)) {
2623                 ret = srpt_xfer_data(ch, ioctx, scmnd);
2624                 if (ret != SCST_TGT_RES_SUCCESS) {
2625                         PRINT_ERROR("%s: tag= %lld xfer_data failed",
2626                                     __func__, (unsigned long long)tag);
2627                         goto out;
2628                 }
2629         }
2630
2631         if (srpt_post_send(ch, ioctx,
2632                            sizeof *srp_rsp +
2633                            be32_to_cpu(srp_rsp->sense_data_len))) {
2634                 PRINT_ERROR("%s: ch->state= %d tag= %lld",
2635                             __func__, ch->state,
2636                             (unsigned long long)tag);
2637                 ret = SCST_TGT_RES_FATAL_ERROR;
2638         }
2639
2640 out:
2641         return ret;
2642
2643 out_aborted:
2644         ret = SCST_TGT_RES_SUCCESS;
2645         srpt_abort_scst_cmd(ch->sport->sdev, scmnd);
2646         goto out;
2647 }
2648
2649 /*
2650  * Called by the SCST core to inform ib_srpt that a received task management
2651  * function has been completed. Must not block.
2652  */
2653 static void srpt_tsk_mgmt_done(struct scst_mgmt_cmd *mcmnd)
2654 {
2655         struct srpt_rdma_ch *ch;
2656         struct srpt_mgmt_ioctx *mgmt_ioctx;
2657         struct srpt_ioctx *ioctx;
2658         int rsp_len;
2659
2660         mgmt_ioctx = scst_mgmt_cmd_get_tgt_priv(mcmnd);
2661         BUG_ON(!mgmt_ioctx);
2662
2663         ch = mgmt_ioctx->ch;
2664         BUG_ON(!ch);
2665
2666         ioctx = mgmt_ioctx->ioctx;
2667         BUG_ON(!ioctx);
2668
2669         TRACE_DBG("%s: tsk_mgmt_done for tag= %lld status=%d",
2670                   __func__, (unsigned long long)mgmt_ioctx->tag,
2671                   scst_mgmt_cmd_get_status(mcmnd));
2672
2673         if (srpt_set_cmd_state(ioctx, SRPT_STATE_PROCESSED)
2674             == SRPT_STATE_ABORTED
2675             || srpt_abort_if_disconnecting(ioctx))
2676                 goto out;
2677
2678         rsp_len = srpt_build_tskmgmt_rsp(ch, ioctx,
2679                                          (scst_mgmt_cmd_get_status(mcmnd) ==
2680                                           SCST_MGMT_STATUS_SUCCESS) ?
2681                                          SRP_TSK_MGMT_SUCCESS :
2682                                          SRP_TSK_MGMT_FAILED,
2683                                          mgmt_ioctx->tag);
2684         srpt_post_send(ch, ioctx, rsp_len);
2685
2686         scst_mgmt_cmd_set_tgt_priv(mcmnd, NULL);
2687
2688         kfree(mgmt_ioctx);
2689
2690 out:
2691         ;
2692 }
2693
2694 /*
2695  * Called by the SCST core to inform ib_srpt that the command 'scmnd' is about
2696  * to be freed. May be called in IRQ context.
2697  */
2698 static void srpt_on_free_cmd(struct scst_cmd *scmnd)
2699 {
2700         struct srpt_rdma_ch *ch;
2701         struct srpt_ioctx *ioctx;
2702         unsigned long flags;
2703
2704         ioctx = scst_cmd_get_tgt_priv(scmnd);
2705         BUG_ON(!ioctx);
2706
2707         srpt_set_cmd_state(ioctx, SRPT_STATE_ABORTED);
2708         ch = ioctx->ch;
2709         BUG_ON(!ch);
2710         spin_lock_irqsave(&ch->spinlock, flags);
2711         list_del(&ioctx->scmnd_list);
2712         ch->active_scmnd_cnt--;
2713         spin_unlock_irqrestore(&ch->spinlock, flags);
2714         ioctx->ch = NULL;
2715
2716         srpt_reset_ioctx(ch, ioctx);
2717         scst_cmd_set_tgt_priv(scmnd, NULL);
2718 }
2719
2720 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2721 /* A vanilla 2.6.19 or older kernel without backported OFED kernel headers. */
2722 static void srpt_refresh_port_work(void *ctx)
2723 #else
2724 static void srpt_refresh_port_work(struct work_struct *work)
2725 #endif
2726 {
2727 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2728         struct srpt_port *sport = (struct srpt_port *)ctx;
2729 #else
2730         struct srpt_port *sport = container_of(work, struct srpt_port, work);
2731 #endif
2732
2733         srpt_refresh_port(sport);
2734 }
2735
2736 /*
2737  * Called by the SCST core to detect target adapters. Returns the number of
2738  * detected target adapters.
2739  */
2740 static int srpt_detect(struct scst_tgt_template *tp)
2741 {
2742         int device_count;
2743
2744         TRACE_ENTRY();
2745
2746         device_count = atomic_read(&srpt_device_count);
2747
2748         TRACE_EXIT_RES(device_count);
2749
2750         return device_count;
2751 }
2752
2753 /*
2754  * Callback function called by the SCST core from scst_unregister() to free up
2755  * the resources associated with device scst_tgt.
2756  */
2757 static int srpt_release(struct scst_tgt *scst_tgt)
2758 {
2759         struct srpt_device *sdev = scst_tgt_get_tgt_priv(scst_tgt);
2760         struct srpt_rdma_ch *ch, *tmp_ch;
2761
2762         TRACE_ENTRY();
2763
2764         BUG_ON(!scst_tgt);
2765 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
2766         WARN_ON(!sdev);
2767         if (!sdev)
2768                 return -ENODEV;
2769 #else
2770         if (WARN_ON(!sdev))
2771                 return -ENODEV;
2772 #endif
2773
2774 #ifdef CONFIG_SCST_PROC
2775         srpt_unregister_procfs_entry(scst_tgt->tgtt);
2776 #endif /*CONFIG_SCST_PROC*/
2777
2778         spin_lock_irq(&sdev->spinlock);
2779         list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
2780                 list_del(&ch->list);
2781                 spin_unlock_irq(&sdev->spinlock);
2782                 srpt_release_channel(ch, 1);
2783                 spin_lock_irq(&sdev->spinlock);
2784         }
2785         spin_unlock_irq(&sdev->spinlock);
2786
2787         srpt_unregister_mad_agent(sdev);
2788
2789         scst_tgt_set_tgt_priv(scst_tgt, NULL);
2790
2791         TRACE_EXIT();
2792
2793         return 0;
2794 }
2795
2796 /*
2797  * Entry point for ib_srpt's kernel thread. This kernel thread is only created
2798  * when the module parameter 'thread' is not zero (the default is zero).
2799  * This thread processes the ioctx list srpt_thread.thread_ioctx_list.
2800  *
2801  * @pre thread != 0
2802  */
2803 static int srpt_ioctx_thread(void *arg)
2804 {
2805         struct srpt_ioctx *ioctx;
2806
2807         /* Hibernation / freezing of the SRPT kernel thread is not supported. */
2808         current->flags |= PF_NOFREEZE;
2809
2810         spin_lock_irq(&srpt_thread.thread_lock);
2811         while (!kthread_should_stop()) {
2812                 wait_queue_t wait;
2813                 init_waitqueue_entry(&wait, current);
2814
2815                 if (!srpt_test_ioctx_list()) {
2816                         add_wait_queue_exclusive(&ioctx_list_waitQ, &wait);
2817
2818                         for (;;) {
2819                                 set_current_state(TASK_INTERRUPTIBLE);
2820                                 if (srpt_test_ioctx_list())
2821                                         break;
2822                                 spin_unlock_irq(&srpt_thread.thread_lock);
2823                                 schedule();
2824                                 spin_lock_irq(&srpt_thread.thread_lock);
2825                         }
2826                         set_current_state(TASK_RUNNING);
2827                         remove_wait_queue(&ioctx_list_waitQ, &wait);
2828                 }
2829
2830                 while (!list_empty(&srpt_thread.thread_ioctx_list)) {
2831                         ioctx = list_entry(srpt_thread.thread_ioctx_list.next,
2832                                            struct srpt_ioctx, comp_list);
2833
2834                         list_del(&ioctx->comp_list);
2835
2836                         spin_unlock_irq(&srpt_thread.thread_lock);
2837                         switch (ioctx->op) {
2838                         case IB_WC_SEND:
2839                                 srpt_handle_send_comp(ioctx->ch, ioctx,
2840                                         SCST_CONTEXT_DIRECT);
2841                                 break;
2842                         case IB_WC_RDMA_WRITE:
2843                         case IB_WC_RDMA_READ:
2844                                 srpt_handle_rdma_comp(ioctx->ch, ioctx);
2845                                 break;
2846                         case IB_WC_RECV:
2847                                 srpt_handle_new_iu(ioctx->ch, ioctx);
2848                                 break;
2849                         default:
2850                                 break;
2851                         }
2852 #if defined(CONFIG_SCST_DEBUG)
2853                         if (thread_processing_delay_in_us
2854                             <= MAX_UDELAY_MS * 1000)
2855                                 udelay(thread_processing_delay_in_us);
2856 #endif
2857                         spin_lock_irq(&srpt_thread.thread_lock);
2858                 }
2859         }
2860         spin_unlock_irq(&srpt_thread.thread_lock);
2861
2862         return 0;
2863 }
2864
2865 /* SCST target template for the SRP target implementation. */
2866 static struct scst_tgt_template srpt_template = {
2867         .name = DRV_NAME,
2868         .sg_tablesize = SRPT_DEF_SG_TABLESIZE,
2869         .xmit_response_atomic = 1,
2870         .rdy_to_xfer_atomic = 1,
2871         .detect = srpt_detect,
2872         .release = srpt_release,
2873         .xmit_response = srpt_xmit_response,
2874         .rdy_to_xfer = srpt_rdy_to_xfer,
2875         .on_free_cmd = srpt_on_free_cmd,
2876         .task_mgmt_fn_done = srpt_tsk_mgmt_done
2877 };
2878
2879 /*
2880  * The callback function srpt_release_class_dev() is called whenever a
2881  * device is removed from the /sys/class/infiniband_srpt device class.
2882  * Although this function has been left empty, a release function has been
2883  * defined such that upon module removal no complaint is logged about a
2884  * missing release function.
2885  */
2886 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2887 static void srpt_release_class_dev(struct class_device *class_dev)
2888 #else
2889 static void srpt_release_class_dev(struct device *dev)
2890 #endif
2891 {
2892 }
2893
2894 #ifdef CONFIG_SCST_PROC
2895
2896 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2897 static int srpt_trace_level_show(struct seq_file *seq, void *v)
2898 {
2899         return scst_proc_log_entry_read(seq, trace_flag, NULL);
2900 }
2901
2902 static ssize_t srpt_proc_trace_level_write(struct file *file,
2903         const char __user *buf, size_t length, loff_t *off)
2904 {
2905         return scst_proc_log_entry_write(file, buf, length, &trace_flag,
2906                 DEFAULT_SRPT_TRACE_FLAGS, NULL);
2907 }
2908
2909 static struct scst_proc_data srpt_log_proc_data = {
2910         SCST_DEF_RW_SEQ_OP(srpt_proc_trace_level_write)
2911         .show = srpt_trace_level_show,
2912 };
2913 #endif
2914
2915 #endif /* CONFIG_SCST_PROC */
2916
2917 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2918 static ssize_t show_login_info(struct class_device *class_dev, char *buf)
2919 #else
2920 static ssize_t show_login_info(struct device *dev,
2921                                struct device_attribute *attr, char *buf)
2922 #endif
2923 {
2924         struct srpt_device *sdev =
2925 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2926                 container_of(class_dev, struct srpt_device, class_dev);
2927 #else
2928                 container_of(dev, struct srpt_device, dev);
2929 #endif
2930         struct srpt_port *sport;
2931         int i;
2932         int len = 0;
2933
2934         for (i = 0; i < sdev->device->phys_port_cnt; i++) {
2935                 sport = &sdev->port[i];
2936
2937                 len += sprintf(buf + len,
2938                                "tid_ext=%016llx,ioc_guid=%016llx,pkey=ffff,"
2939                                "dgid=%04x%04x%04x%04x%04x%04x%04x%04x,"
2940                                "service_id=%016llx\n",
2941                                (unsigned long long) srpt_service_guid,
2942                                (unsigned long long) srpt_service_guid,
2943                                be16_to_cpu(((__be16 *) sport->gid.raw)[0]),
2944                                be16_to_cpu(((__be16 *) sport->gid.raw)[1]),
2945                                be16_to_cpu(((__be16 *) sport->gid.raw)[2]),
2946                                be16_to_cpu(((__be16 *) sport->gid.raw)[3]),
2947                                be16_to_cpu(((__be16 *) sport->gid.raw)[4]),
2948                                be16_to_cpu(((__be16 *) sport->gid.raw)[5]),
2949                                be16_to_cpu(((__be16 *) sport->gid.raw)[6]),
2950                                be16_to_cpu(((__be16 *) sport->gid.raw)[7]),
2951                                (unsigned long long) srpt_service_guid);
2952         }
2953
2954         return len;
2955 }
2956
2957 static struct class_attribute srpt_class_attrs[] = {
2958         __ATTR_NULL,
2959 };
2960
2961 static struct device_attribute srpt_dev_attrs[] = {
2962         __ATTR(login_info,   S_IRUGO, show_login_info,   NULL),
2963         __ATTR_NULL,
2964 };
2965
2966 static struct class srpt_class = {
2967         .name        = "infiniband_srpt",
2968 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2969         .release = srpt_release_class_dev,
2970 #else
2971         .dev_release = srpt_release_class_dev,
2972 #endif
2973         .class_attrs = srpt_class_attrs,
2974         .dev_attrs   = srpt_dev_attrs,
2975 };
2976
2977 /*
2978  * Callback function called by the InfiniBand core when either an InfiniBand
2979  * device has been added or during the ib_register_client() call for each
2980  * registered InfiniBand device.
2981  */
2982 static void srpt_add_one(struct ib_device *device)
2983 {
2984         struct srpt_device *sdev;
2985         struct srpt_port *sport;
2986         struct ib_srq_init_attr srq_attr;
2987         int i;
2988
2989         TRACE_ENTRY();
2990
2991         TRACE_DBG("device = %p, device->dma_ops = %p", device, device->dma_ops);
2992
2993         sdev = kzalloc(sizeof *sdev, GFP_KERNEL);
2994         if (!sdev)
2995                 return;
2996
2997         sdev->device = device;
2998
2999 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
3000         sdev->class_dev.class = &srpt_class;
3001         sdev->class_dev.dev = device->dma_device;
3002         snprintf(sdev->class_dev.class_id, BUS_ID_SIZE,
3003                  "srpt-%s", device->name);
3004 #else
3005         sdev->dev.class = &srpt_class;
3006         sdev->dev.parent = device->dma_device;
3007 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
3008         snprintf(sdev->dev.bus_id, BUS_ID_SIZE, "srpt-%s", device->name);
3009 #else
3010         dev_set_name(&sdev->dev, "srpt-%s", device->name);
3011 #endif
3012 #endif
3013
3014 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
3015         if (class_device_register(&sdev->class_dev))
3016                 goto free_dev;
3017 #else
3018         if (device_register(&sdev->dev))
3019                 goto free_dev;
3020 #endif
3021
3022         if (ib_query_device(device, &sdev->dev_attr))
3023                 goto err_dev;
3024
3025         sdev->pd = ib_alloc_pd(device);
3026         if (IS_ERR(sdev->pd))
3027                 goto err_dev;
3028
3029         sdev->mr = ib_get_dma_mr(sdev->pd, IB_ACCESS_LOCAL_WRITE);
3030         if (IS_ERR(sdev->mr))
3031                 goto err_pd;
3032
3033         srq_attr.event_handler = srpt_srq_event;
3034         srq_attr.srq_context = (void *)sdev;
3035         srq_attr.attr.max_wr = min(SRPT_SRQ_SIZE, sdev->dev_attr.max_srq_wr);
3036         srq_attr.attr.max_sge = 1;
3037         srq_attr.attr.srq_limit = 0;
3038
3039         sdev->srq = ib_create_srq(sdev->pd, &srq_attr);
3040         if (IS_ERR(sdev->srq))
3041                 goto err_mr;
3042
3043         TRACE_DBG("%s: create SRQ #wr= %d max_allow=%d dev= %s",
3044                __func__, srq_attr.attr.max_wr,
3045               sdev->dev_attr.max_srq_wr, device->name);
3046
3047         if (!srpt_service_guid)
3048                 srpt_service_guid = be64_to_cpu(device->node_guid);
3049
3050         sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);
3051         if (IS_ERR(sdev->cm_id))
3052                 goto err_srq;
3053
3054         /* print out target login information */
3055         TRACE_DBG("Target login info: id_ext=%016llx,"
3056                   "ioc_guid=%016llx,pkey=ffff,service_id=%016llx",
3057                   (unsigned long long) srpt_service_guid,
3058                   (unsigned long long) srpt_service_guid,
3059                   (unsigned long long) srpt_service_guid);
3060
3061         /*
3062          * We do not have a consistent service_id (ie. also id_ext of target_id)
3063          * to identify this target. We currently use the guid of the first HCA
3064          * in the system as service_id; therefore, the target_id will change
3065          * if this HCA is gone bad and replaced by different HCA
3066          */
3067         if (ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0, NULL))
3068                 goto err_cm;
3069
3070         INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
3071                               srpt_event_handler);
3072         if (ib_register_event_handler(&sdev->event_handler))
3073                 goto err_cm;
3074
3075         if (srpt_alloc_ioctx_ring(sdev))
3076                 goto err_event;
3077
3078         INIT_LIST_HEAD(&sdev->rch_list);
3079         spin_lock_init(&sdev->spinlock);
3080
3081         for (i = 0; i < SRPT_SRQ_SIZE; ++i)
3082                 srpt_post_recv(sdev, sdev->ioctx_ring[i]);
3083
3084         ib_set_client_data(device, &srpt_client, sdev);
3085
3086         sdev->scst_tgt = scst_register(&srpt_template, NULL);
3087         if (!sdev->scst_tgt) {
3088                 PRINT_ERROR("SCST registration failed for %s.",
3089                             sdev->device->name);
3090                 goto err_ring;
3091         }
3092
3093         scst_tgt_set_tgt_priv(sdev->scst_tgt, sdev);
3094
3095         WARN_ON(sdev->device->phys_port_cnt
3096                 > sizeof(sdev->port)/sizeof(sdev->port[0]));
3097
3098         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
3099                 sport = &sdev->port[i - 1];
3100                 sport->sdev = sdev;
3101                 sport->port = i;
3102 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
3103                 /*
3104                  * A vanilla 2.6.19 or older kernel without backported OFED
3105                  * kernel headers.
3106                  */
3107                 INIT_WORK(&sport->work, srpt_refresh_port_work, sport);
3108 #else
3109                 INIT_WORK(&sport->work, srpt_refresh_port_work);
3110 #endif
3111                 if (srpt_refresh_port(sport)) {
3112                         PRINT_ERROR("MAD registration failed for %s-%d.",
3113                                     sdev->device->name, i);
3114                         goto err_refresh_port;
3115                 }
3116         }
3117
3118         atomic_inc(&srpt_device_count);
3119
3120         TRACE_EXIT();
3121
3122         return;
3123
3124 err_refresh_port:
3125         scst_unregister(sdev->scst_tgt);
3126 err_ring:
3127         ib_set_client_data(device, &srpt_client, NULL);
3128         srpt_free_ioctx_ring(sdev);
3129 err_event:
3130         ib_unregister_event_handler(&sdev->event_handler);
3131 err_cm:
3132         ib_destroy_cm_id(sdev->cm_id);
3133 err_srq:
3134         ib_destroy_srq(sdev->srq);
3135 err_mr:
3136         ib_dereg_mr(sdev->mr);
3137 err_pd:
3138         ib_dealloc_pd(sdev->pd);
3139 err_dev:
3140 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
3141         class_device_unregister(&sdev->class_dev);
3142 #else
3143         device_unregister(&sdev->dev);
3144 #endif
3145 free_dev:
3146         kfree(sdev);
3147
3148         TRACE_EXIT();
3149 }
3150
3151 /*
3152  * Callback function called by the InfiniBand core when either an InfiniBand
3153  * device has been removed or during the ib_unregister_client() call for each
3154  * registered InfiniBand device.
3155  */
3156 static void srpt_remove_one(struct ib_device *device)
3157 {
3158         int i;
3159         struct srpt_device *sdev;
3160
3161         TRACE_ENTRY();
3162
3163         sdev = ib_get_client_data(device, &srpt_client);
3164 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
3165         WARN_ON(!sdev);
3166         if (!sdev)
3167                 return;
3168 #else
3169         if (WARN_ON(!sdev))
3170                 return;
3171 #endif
3172
3173         /*
3174          * Cancel the work if it is queued. Wait until srpt_refresh_port_work()
3175          * finished if it is running.
3176          */
3177         for (i = 0; i < sdev->device->phys_port_cnt; i++)
3178 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
3179                 cancel_work_sync(&sdev->port[i].work);
3180 #else
3181                 /*
3182                  * cancel_work_sync() was introduced in kernel 2.6.22. Older
3183                  * kernels do not have a facility to cancel scheduled work.
3184                  */
3185                 PRINT_ERROR("%s",
3186                        "your kernel does not provide cancel_work_sync().");
3187 #endif
3188
3189         scst_unregister(sdev->scst_tgt);
3190         sdev->scst_tgt = NULL;
3191
3192         ib_unregister_event_handler(&sdev->event_handler);
3193         ib_destroy_cm_id(sdev->cm_id);
3194         ib_destroy_srq(sdev->srq);
3195         ib_dereg_mr(sdev->mr);
3196         ib_dealloc_pd(sdev->pd);
3197 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
3198         class_device_unregister(&sdev->class_dev);
3199 #else
3200         device_unregister(&sdev->dev);
3201 #endif
3202
3203         srpt_free_ioctx_ring(sdev);
3204         kfree(sdev);
3205
3206         TRACE_EXIT();
3207 }
3208
3209 #ifdef CONFIG_SCST_PROC
3210
3211 /**
3212  * Create procfs entries for srpt. Currently the only procfs entry created
3213  * by this function is the "trace_level" entry.
3214  */
3215 static int srpt_register_procfs_entry(struct scst_tgt_template *tgt)
3216 {
3217         int res = 0;
3218 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
3219         struct proc_dir_entry *p, *root;
3220
3221         root = scst_proc_get_tgt_root(tgt);
3222         WARN_ON(!root);
3223         if (root) {
3224                 /*
3225                  * Fill in the scst_proc_data::data pointer, which is used in
3226                  * a printk(KERN_INFO ...) statement in
3227                  * scst_proc_log_entry_write() in scst_proc.c.
3228                  */
3229                 srpt_log_proc_data.data = (char *)tgt->name;
3230                 p = scst_create_proc_entry(root, SRPT_PROC_TRACE_LEVEL_NAME,
3231                                            &srpt_log_proc_data);
3232                 if (!p)
3233                         res = -ENOMEM;
3234         } else
3235                 res = -ENOMEM;
3236
3237 #endif
3238         return res;
3239 }
3240
3241 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt)
3242 {
3243 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
3244         struct proc_dir_entry *root;
3245
3246         root = scst_proc_get_tgt_root(tgt);
3247         WARN_ON(!root);
3248         if (root)
3249                 remove_proc_entry(SRPT_PROC_TRACE_LEVEL_NAME, root);
3250 #endif
3251 }
3252
3253 #endif /*CONFIG_SCST_PROC*/
3254
3255 /*
3256  * Module initialization.
3257  *
3258  * Note: since ib_register_client() registers callback functions, and since at
3259  * least one of these callback functions (srpt_add_one()) calls SCST functions,
3260  * the SCST target template must be registered before ib_register_client() is
3261  * called.
3262  */
3263 static int __init srpt_init_module(void)
3264 {
3265         int ret;
3266
3267         ret = -EINVAL;
3268         if (srp_max_message_size < MIN_MAX_MESSAGE_SIZE) {
3269                 PRINT_ERROR("invalid value %d for kernel module parameter"
3270                             " srp_max_message_size -- must be at least %d.",
3271                             srp_max_message_size,
3272                             MIN_MAX_MESSAGE_SIZE);
3273                 goto out;
3274         }
3275
3276         ret = class_register(&srpt_class);
3277         if (ret) {
3278                 PRINT_ERROR("%s", "couldn't register class ib_srpt");
3279                 goto out;
3280         }
3281
3282         ret = scst_register_target_template(&srpt_template);
3283         if (ret < 0) {
3284                 PRINT_ERROR("%s", "couldn't register with scst");
3285                 ret = -ENODEV;
3286                 goto out_unregister_class;
3287         }
3288
3289 #ifdef CONFIG_SCST_PROC
3290         ret = srpt_register_procfs_entry(&srpt_template);
3291         if (ret) {
3292                 PRINT_ERROR("%s", "couldn't register procfs entry");
3293                 goto out_unregister_target;
3294         }
3295 #endif /*CONFIG_SCST_PROC*/
3296
3297         ret = ib_register_client(&srpt_client);
3298         if (ret) {
3299                 PRINT_ERROR("%s", "couldn't register IB client");
3300                 goto out_unregister_target;
3301         }
3302
3303         if (thread) {
3304                 spin_lock_init(&srpt_thread.thread_lock);
3305                 INIT_LIST_HEAD(&srpt_thread.thread_ioctx_list);
3306                 srpt_thread.thread = kthread_run(srpt_ioctx_thread,
3307                                                  NULL, "srpt_thread");
3308                 if (IS_ERR(srpt_thread.thread)) {
3309                         srpt_thread.thread = NULL;
3310                         thread = 0;
3311                 }
3312         }
3313
3314         return 0;
3315
3316 out_unregister_target:
3317 #ifdef CONFIG_SCST_PROC
3318         /*
3319          * Note: the procfs entry is unregistered in srpt_release(), which is
3320          * called by scst_unregister_target_template().
3321          */
3322 #endif /*CONFIG_SCST_PROC*/
3323         scst_unregister_target_template(&srpt_template);
3324 out_unregister_class:
3325         class_unregister(&srpt_class);
3326 out:
3327         return ret;
3328 }
3329
3330 static void __exit srpt_cleanup_module(void)
3331 {
3332         TRACE_ENTRY();
3333
3334         if (srpt_thread.thread)
3335                 kthread_stop(srpt_thread.thread);
3336         ib_unregister_client(&srpt_client);
3337         scst_unregister_target_template(&srpt_template);
3338         class_unregister(&srpt_class);
3339
3340         TRACE_EXIT();
3341 }
3342
3343 module_init(srpt_init_module);
3344 module_exit(srpt_cleanup_module);