Removed SRP_CRED_REQ support again.
[mirror/scst/.git] / srpt / src / ib_srpt.c
1 /*
2  * Copyright (c) 2006 - 2009 Mellanox Technology Inc.  All rights reserved.
3  * Copyright (C) 2008 Vladislav Bolkhovitin <vst@vlnb.net>
4  * Copyright (C) 2008 - 2009 Bart Van Assche <bart.vanassche@gmail.com>
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  */
35
36 #include <linux/module.h>
37 #include <linux/init.h>
38 #include <linux/slab.h>
39 #include <linux/err.h>
40 #include <linux/ctype.h>
41 #include <linux/string.h>
42 #include <linux/kthread.h>
43 #include <linux/delay.h>
44 #include <asm/atomic.h>
45 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
46 #include <linux/proc_fs.h>
47 #include <linux/seq_file.h>
48 #endif
49 #include "ib_srpt.h"
50 #define LOG_PREFIX "ib_srpt" /* Prefix for SCST tracing macros. */
51 #include "scst_debug.h"
52
53 #define CONFIG_SCST_PROC
54
55 /* Name of this kernel module. */
56 #define DRV_NAME                "ib_srpt"
57 #define DRV_VERSION             "1.0.1"
58 #define DRV_RELDATE             "July 10, 2008"
59 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
60 /* Flags to be used in SCST debug tracing statements. */
61 #define DEFAULT_SRPT_TRACE_FLAGS (TRACE_OUT_OF_MEM | TRACE_MINOR \
62                                   | TRACE_MGMT | TRACE_SPECIAL)
63 /* Name of the entry that will be created under /proc/scsi_tgt/ib_srpt. */
64 #define SRPT_PROC_TRACE_LEVEL_NAME      "trace_level"
65 #endif
66
67 #define MELLANOX_SRPT_ID_STRING "SCST SRP target"
68
69 MODULE_AUTHOR("Vu Pham");
70 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
71                    "v" DRV_VERSION " (" DRV_RELDATE ")");
72 MODULE_LICENSE("Dual BSD/GPL");
73
74 struct srpt_thread {
75         /* Protects thread_ioctx_list. */
76         spinlock_t thread_lock;
77         /* I/O contexts to be processed by the kernel thread. */
78         struct list_head thread_ioctx_list;
79         /* SRPT kernel thread. */
80         struct task_struct *thread;
81 };
82
83 /*
84  * Global Variables
85  */
86
87 static u64 srpt_service_guid;
88 /* List of srpt_device structures. */
89 static atomic_t srpt_device_count;
90 static int use_port_guid_in_session_name;
91 static int thread = 1;
92 static struct srpt_thread srpt_thread;
93 static DECLARE_WAIT_QUEUE_HEAD(ioctx_list_waitQ);
94 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
95 static unsigned long trace_flag = DEFAULT_SRPT_TRACE_FLAGS;
96 module_param(trace_flag, long, 0644);
97 MODULE_PARM_DESC(trace_flag,
98                  "Trace flags for the ib_srpt kernel module.");
99 #endif
100 #if defined(CONFIG_SCST_DEBUG)
101 static unsigned long interrupt_processing_delay_in_us;
102 module_param(interrupt_processing_delay_in_us, long, 0744);
103 MODULE_PARM_DESC(interrupt_processing_delay_in_us,
104                  "CQ completion handler interrupt delay in microseconds.");
105 static unsigned long thread_processing_delay_in_us;
106 module_param(thread_processing_delay_in_us, long, 0744);
107 MODULE_PARM_DESC(thread_processing_delay_in_us,
108                  "SRP thread processing delay in microseconds.");
109 #endif
110
111 module_param(thread, int, 0444);
112 MODULE_PARM_DESC(thread,
113                  "Executing ioctx in thread context. Default 0, i.e. soft IRQ, "
114                  "where possible.");
115
116 static unsigned int srp_max_rdma_size = DEFAULT_MAX_RDMA_SIZE;
117 module_param(srp_max_rdma_size, int, 0744);
118 MODULE_PARM_DESC(thread,
119                  "Maximum size of SRP RDMA transfers for new connections.");
120
121 static unsigned int srp_max_message_size = DEFAULT_MAX_MESSAGE_SIZE;
122 module_param(srp_max_message_size, int, 0444);
123 MODULE_PARM_DESC(thread,
124                  "Maximum size of SRP control messages in bytes.");
125
126 module_param(use_port_guid_in_session_name, bool, 0444);
127 MODULE_PARM_DESC(use_port_guid_in_session_name,
128                  "Use target port ID in the SCST session name such that"
129                  " redundant paths between multiport systems can be masked.");
130
131 static void srpt_add_one(struct ib_device *device);
132 static void srpt_remove_one(struct ib_device *device);
133 static void srpt_unregister_mad_agent(struct srpt_device *sdev);
134 #ifdef CONFIG_SCST_PROC
135 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt);
136 #endif /*CONFIG_SCST_PROC*/
137 static void srpt_unmap_sg_to_ib_sge(struct srpt_rdma_ch *ch,
138                                     struct srpt_ioctx *ioctx);
139 static void srpt_release_channel(struct scst_session *scst_sess);
140
141 static struct ib_client srpt_client = {
142         .name = DRV_NAME,
143         .add = srpt_add_one,
144         .remove = srpt_remove_one
145 };
146
147 /**
148  * Atomically test and set the channel state.
149  * @ch: RDMA channel.
150  * @old: channel state to compare with.
151  * @new: state to change the channel state to if the current state matches the
152  *       argument 'old'.
153  *
154  * Returns the previous channel state.
155  */
156 static enum rdma_ch_state
157 srpt_test_and_set_channel_state(struct srpt_rdma_ch *ch,
158                                 enum rdma_ch_state old,
159                                 enum rdma_ch_state new)
160 {
161         return atomic_cmpxchg(&ch->state, old, new);
162 }
163
164 /*
165  * Callback function called by the InfiniBand core when an asynchronous IB
166  * event occurs. This callback may occur in interrupt context. See also
167  * section 11.5.2, Set Asynchronous Event Handler in the InfiniBand
168  * Architecture Specification.
169  */
170 static void srpt_event_handler(struct ib_event_handler *handler,
171                                struct ib_event *event)
172 {
173         struct srpt_device *sdev;
174         struct srpt_port *sport;
175
176         TRACE_ENTRY();
177
178         sdev = ib_get_client_data(event->device, &srpt_client);
179         if (!sdev || sdev->device != event->device)
180                 return;
181
182         TRACE_DBG("ASYNC event= %d on device= %s",
183                   event->event, sdev->device->name);
184
185         switch (event->event) {
186         case IB_EVENT_PORT_ERR:
187                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
188                         sport = &sdev->port[event->element.port_num - 1];
189                         sport->lid = 0;
190                         sport->sm_lid = 0;
191                 }
192                 break;
193         case IB_EVENT_PORT_ACTIVE:
194         case IB_EVENT_LID_CHANGE:
195         case IB_EVENT_PKEY_CHANGE:
196         case IB_EVENT_SM_CHANGE:
197         case IB_EVENT_CLIENT_REREGISTER:
198                 /*
199                  * Refresh port data asynchronously. Note: it is safe to call
200                  * schedule_work() even if &sport->work is already on the
201                  * global workqueue because schedule_work() tests for the
202                  * work_pending() condition before adding &sport->work to the
203                  * global work queue.
204                  */
205                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
206                         sport = &sdev->port[event->element.port_num - 1];
207                         if (!sport->lid && !sport->sm_lid)
208                                 schedule_work(&sport->work);
209                 }
210                 break;
211         default:
212                 PRINT_ERROR("received unrecognized IB event %d", event->event);
213                 break;
214         }
215
216         TRACE_EXIT();
217 }
218
219 /*
220  * Callback function called by the InfiniBand core for SRQ (shared receive
221  * queue) events.
222  */
223 static void srpt_srq_event(struct ib_event *event, void *ctx)
224 {
225         TRACE_ENTRY();
226
227         TRACE_DBG("SRQ event %d", event->event);
228
229         TRACE_EXIT();
230 }
231
232 /*
233  * Callback function called by the InfiniBand core for QP (queue pair) events.
234  */
235 static void srpt_qp_event(struct ib_event *event, struct srpt_rdma_ch *ch)
236 {
237         TRACE_DBG("QP event %d on cm_id=%p sess_name=%s state=%d",
238                   event->event, ch->cm_id, ch->sess_name,
239                   atomic_read(&ch->state));
240
241         switch (event->event) {
242         case IB_EVENT_COMM_EST:
243 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20) || defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
244                 ib_cm_notify(ch->cm_id, event->event);
245 #else
246                 /* Vanilla 2.6.19 kernel (or before) without OFED. */
247                 PRINT_ERROR("%s", "how to perform ib_cm_notify() on a"
248                             " vanilla 2.6.18 kernel ???");
249 #endif
250                 break;
251         case IB_EVENT_QP_LAST_WQE_REACHED:
252                 if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_LIVE,
253                         RDMA_CHANNEL_DISCONNECTING) == RDMA_CHANNEL_LIVE) {
254                         PRINT_INFO("disconnected session %s.", ch->sess_name);
255                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
256                 }
257                 break;
258         default:
259                 PRINT_ERROR("received unrecognized IB QP event %d",
260                             event->event);
261                 break;
262         }
263 }
264
265 /*
266  * Helper function for filling in an InfiniBand IOUnitInfo structure. Copies
267  * the lowest four bits of value in element slot of the array of four bit
268  * elements called c_list (controller list). The index slot is one-based.
269  *
270  * @pre 1 <= slot && 0 <= value && value < 16
271  */
272 static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
273 {
274         u16 id;
275         u8 tmp;
276
277         id = (slot - 1) / 2;
278         if (slot & 0x1) {
279                 tmp = c_list[id] & 0xf;
280                 c_list[id] = (value << 4) | tmp;
281         } else {
282                 tmp = c_list[id] & 0xf0;
283                 c_list[id] = (value & 0xf) | tmp;
284         }
285 }
286
287 /*
288  * Write InfiniBand ClassPortInfo to mad. See also section 16.3.3.1
289  * ClassPortInfo in the InfiniBand Architecture Specification.
290  */
291 static void srpt_get_class_port_info(struct ib_dm_mad *mad)
292 {
293         struct ib_class_port_info *cif;
294
295         cif = (struct ib_class_port_info *)mad->data;
296         memset(cif, 0, sizeof *cif);
297         cif->base_version = 1;
298         cif->class_version = 1;
299         cif->resp_time_value = 20;
300
301         mad->mad_hdr.status = 0;
302 }
303
304 /*
305  * Write IOUnitInfo to mad. See also section 16.3.3.3 IOUnitInfo in the
306  * InfiniBand Architecture Specification. See also section B.7,
307  * table B.6 in the T10 SRP r16a document.
308  */
309 static void srpt_get_iou(struct ib_dm_mad *mad)
310 {
311         struct ib_dm_iou_info *ioui;
312         u8 slot;
313         int i;
314
315         ioui = (struct ib_dm_iou_info *)mad->data;
316         ioui->change_id = 1;
317         ioui->max_controllers = 16;
318
319         /* set present for slot 1 and empty for the rest */
320         srpt_set_ioc(ioui->controller_list, 1, 1);
321         for (i = 1, slot = 2; i < 16; i++, slot++)
322                 srpt_set_ioc(ioui->controller_list, slot, 0);
323
324         mad->mad_hdr.status = 0;
325 }
326
327 /*
328  * Write IOControllerprofile to mad for I/O controller (sdev, slot). See also
329  * section 16.3.3.4 IOControllerProfile in the InfiniBand Architecture
330  * Specification. See also section B.7, table B.7 in the T10 SRP r16a
331  * document.
332  */
333 static void srpt_get_ioc(struct srpt_device *sdev, u32 slot,
334                          struct ib_dm_mad *mad)
335 {
336         struct ib_dm_ioc_profile *iocp;
337
338         iocp = (struct ib_dm_ioc_profile *)mad->data;
339
340         if (!slot || slot > 16) {
341                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
342                 return;
343         }
344
345         if (slot > 2) {
346                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
347                 return;
348         }
349
350         memset(iocp, 0, sizeof *iocp);
351         strcpy(iocp->id_string, MELLANOX_SRPT_ID_STRING);
352         iocp->guid = cpu_to_be64(srpt_service_guid);
353         iocp->vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
354         iocp->device_id = cpu_to_be32(sdev->dev_attr.vendor_part_id);
355         iocp->device_version = cpu_to_be16(sdev->dev_attr.hw_ver);
356         iocp->subsys_vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
357         iocp->subsys_device_id = 0x0;
358         iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
359         iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
360         iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
361         iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
362         iocp->send_queue_depth = cpu_to_be16(SRPT_SRQ_SIZE);
363         iocp->rdma_read_depth = 4;
364         iocp->send_size = cpu_to_be32(srp_max_message_size);
365         iocp->rdma_size = cpu_to_be32(min(max(srp_max_rdma_size, 256U),
366                                           1U << 24));
367         iocp->num_svc_entries = 1;
368         iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
369                 SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
370
371         mad->mad_hdr.status = 0;
372 }
373
374 /*
375  * Device management: write ServiceEntries to mad for the given slot. See also
376  * section 16.3.3.5 ServiceEntries in the InfiniBand Architecture
377  * Specification. See also section B.7, table B.8 in the T10 SRP r16a document.
378  */
379 static void srpt_get_svc_entries(u64 ioc_guid,
380                                  u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
381 {
382         struct ib_dm_svc_entries *svc_entries;
383
384         WARN_ON(!ioc_guid);
385
386         if (!slot || slot > 16) {
387                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
388                 return;
389         }
390
391         if (slot > 2 || lo > hi || hi > 1) {
392                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
393                 return;
394         }
395
396         svc_entries = (struct ib_dm_svc_entries *)mad->data;
397         memset(svc_entries, 0, sizeof *svc_entries);
398         svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid);
399         snprintf(svc_entries->service_entries[0].name,
400                  sizeof(svc_entries->service_entries[0].name),
401                  "%s%016llx",
402                  SRP_SERVICE_NAME_PREFIX,
403                  (unsigned long long)ioc_guid);
404
405         mad->mad_hdr.status = 0;
406 }
407
408 /*
409  * Actual processing of a received MAD *rq_mad received through source port *sp
410  * (MAD = InfiniBand management datagram). The response to be sent back is
411  * written to *rsp_mad.
412  */
413 static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
414                                  struct ib_dm_mad *rsp_mad)
415 {
416         u16 attr_id;
417         u32 slot;
418         u8 hi, lo;
419
420         attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
421         switch (attr_id) {
422         case DM_ATTR_CLASS_PORT_INFO:
423                 srpt_get_class_port_info(rsp_mad);
424                 break;
425         case DM_ATTR_IOU_INFO:
426                 srpt_get_iou(rsp_mad);
427                 break;
428         case DM_ATTR_IOC_PROFILE:
429                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
430                 srpt_get_ioc(sp->sdev, slot, rsp_mad);
431                 break;
432         case DM_ATTR_SVC_ENTRIES:
433                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
434                 hi = (u8) ((slot >> 8) & 0xff);
435                 lo = (u8) (slot & 0xff);
436                 slot = (u16) ((slot >> 16) & 0xffff);
437                 srpt_get_svc_entries(srpt_service_guid,
438                                      slot, hi, lo, rsp_mad);
439                 break;
440         default:
441                 rsp_mad->mad_hdr.status =
442                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
443                 break;
444         }
445 }
446
447 /*
448  * Callback function that is called by the InfiniBand core after transmission of
449  * a MAD. (MAD = management datagram; AH = address handle.)
450  */
451 static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
452                                   struct ib_mad_send_wc *mad_wc)
453 {
454         ib_destroy_ah(mad_wc->send_buf->ah);
455         ib_free_send_mad(mad_wc->send_buf);
456 }
457
458 /*
459  * Callback function that is called by the InfiniBand core after reception of
460  * a MAD (management datagram).
461  */
462 static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
463                                   struct ib_mad_recv_wc *mad_wc)
464 {
465         struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
466         struct ib_ah *ah;
467         struct ib_mad_send_buf *rsp;
468         struct ib_dm_mad *dm_mad;
469
470         if (!mad_wc || !mad_wc->recv_buf.mad)
471                 return;
472
473         ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
474                                   mad_wc->recv_buf.grh, mad_agent->port_num);
475         if (IS_ERR(ah))
476                 goto err;
477
478         BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR);
479
480         rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
481                                  mad_wc->wc->pkey_index, 0,
482                                  IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA,
483                                  GFP_KERNEL);
484         if (IS_ERR(rsp))
485                 goto err_rsp;
486
487         rsp->ah = ah;
488
489         dm_mad = rsp->mad;
490         memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof *dm_mad);
491         dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
492         dm_mad->mad_hdr.status = 0;
493
494         switch (mad_wc->recv_buf.mad->mad_hdr.method) {
495         case IB_MGMT_METHOD_GET:
496                 srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
497                 break;
498         case IB_MGMT_METHOD_SET:
499                 dm_mad->mad_hdr.status =
500                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
501                 break;
502         default:
503                 dm_mad->mad_hdr.status =
504                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
505                 break;
506         }
507
508         if (!ib_post_send_mad(rsp, NULL)) {
509                 ib_free_recv_mad(mad_wc);
510                 /* will destroy_ah & free_send_mad in send completion */
511                 return;
512         }
513
514         ib_free_send_mad(rsp);
515
516 err_rsp:
517         ib_destroy_ah(ah);
518 err:
519         ib_free_recv_mad(mad_wc);
520 }
521
522 /*
523  * Enable InfiniBand management datagram processing, update the cached sm_lid,
524  * lid and gid values, and register a callback function for processing MADs
525  * on the specified port. It is safe to call this function more than once for
526  * the same port.
527  */
528 static int srpt_refresh_port(struct srpt_port *sport)
529 {
530         struct ib_mad_reg_req reg_req;
531         struct ib_port_modify port_modify;
532         struct ib_port_attr port_attr;
533         int ret;
534
535         TRACE_ENTRY();
536
537         memset(&port_modify, 0, sizeof port_modify);
538         port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
539         port_modify.clr_port_cap_mask = 0;
540
541         ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
542         if (ret)
543                 goto err_mod_port;
544
545         ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
546         if (ret)
547                 goto err_query_port;
548
549         sport->sm_lid = port_attr.sm_lid;
550         sport->lid = port_attr.lid;
551
552         ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid);
553         if (ret)
554                 goto err_query_port;
555
556         if (!sport->mad_agent) {
557                 memset(&reg_req, 0, sizeof reg_req);
558                 reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
559                 reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
560                 set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
561                 set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
562
563                 sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
564                                                          sport->port,
565                                                          IB_QPT_GSI,
566                                                          &reg_req, 0,
567                                                          srpt_mad_send_handler,
568                                                          srpt_mad_recv_handler,
569                                                          sport);
570                 if (IS_ERR(sport->mad_agent)) {
571                         ret = PTR_ERR(sport->mad_agent);
572                         sport->mad_agent = NULL;
573                         goto err_query_port;
574                 }
575         }
576
577         TRACE_EXIT_RES(0);
578
579         return 0;
580
581 err_query_port:
582
583         port_modify.set_port_cap_mask = 0;
584         port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
585         ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
586
587 err_mod_port:
588
589         TRACE_EXIT_RES(ret);
590
591         return ret;
592 }
593
594 /*
595  * Unregister the callback function for processing MADs and disable MAD
596  * processing for all ports of the specified device. It is safe to call this
597  * function more than once for the same device.
598  */
599 static void srpt_unregister_mad_agent(struct srpt_device *sdev)
600 {
601         struct ib_port_modify port_modify = {
602                 .clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP,
603         };
604         struct srpt_port *sport;
605         int i;
606
607         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
608                 sport = &sdev->port[i - 1];
609                 WARN_ON(sport->port != i);
610                 if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
611                         PRINT_ERROR("%s", "disabling MAD processing failed.");
612                 if (sport->mad_agent) {
613                         ib_unregister_mad_agent(sport->mad_agent);
614                         sport->mad_agent = NULL;
615                 }
616         }
617 }
618
619 /**
620  * Allocate and initialize an SRPT I/O context structure.
621  */
622 static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev)
623 {
624         struct srpt_ioctx *ioctx;
625
626         ioctx = kmalloc(sizeof *ioctx, GFP_KERNEL);
627         if (!ioctx)
628                 goto out;
629
630         ioctx->buf = kzalloc(srp_max_message_size, GFP_KERNEL);
631         if (!ioctx->buf)
632                 goto out_free_ioctx;
633
634         ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf,
635                                        srp_max_message_size, DMA_BIDIRECTIONAL);
636         if (ib_dma_mapping_error(sdev->device, ioctx->dma))
637                 goto out_free_buf;
638
639         return ioctx;
640
641 out_free_buf:
642         kfree(ioctx->buf);
643 out_free_ioctx:
644         kfree(ioctx);
645 out:
646         return NULL;
647 }
648
649 /*
650  * Deallocate an SRPT I/O context structure.
651  */
652 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
653 {
654         if (!ioctx)
655                 return;
656
657         ib_dma_unmap_single(sdev->device, ioctx->dma,
658                             srp_max_message_size, DMA_BIDIRECTIONAL);
659         kfree(ioctx->buf);
660         kfree(ioctx);
661 }
662
663 /**
664  * srpt_alloc_ioctx_ring() -- allocate a ring of SRPT I/O context structures.
665  * @sdev: device to allocate the I/O context ring for.
666  * @ioctx_ring: pointer to an array of I/O contexts.
667  * @ring_size: number of elements in the I/O context ring.
668  * @flags: flags to be set in the ring index.
669  */
670 static int srpt_alloc_ioctx_ring(struct srpt_device *sdev,
671                                  struct srpt_ioctx **ioctx_ring,
672                                  int ring_size,
673                                  u32 flags)
674 {
675         int res;
676         int i;
677
678         TRACE_ENTRY();
679
680         res = -ENOMEM;
681         for (i = 0; i < ring_size; ++i) {
682                 ioctx_ring[i] = srpt_alloc_ioctx(sdev);
683
684                 if (!ioctx_ring[i])
685                         goto err;
686
687                 WARN_ON(i & flags);
688                 ioctx_ring[i]->index = i | flags;
689         }
690         res = 0;
691         goto out;
692
693 err:
694         while (--i > 0) {
695                 srpt_free_ioctx(sdev, ioctx_ring[i]);
696                 ioctx_ring[i] = NULL;
697         }
698 out:
699         TRACE_EXIT_RES(res);
700         return res;
701 }
702
703 /* Free the ring of SRPT I/O context structures. */
704 static void srpt_free_ioctx_ring(struct srpt_device *sdev,
705                                  struct srpt_ioctx **ioctx_ring,
706                                  int ring_size)
707 {
708         int i;
709
710         for (i = 0; i < ring_size; ++i) {
711                 srpt_free_ioctx(sdev, ioctx_ring[i]);
712                 ioctx_ring[i] = NULL;
713         }
714 }
715
716 /**
717  * Set the state of a command.
718  * @new: New state to be set.
719  *
720  * Does not modify the state of aborted commands. Returns the previous command
721  * state.
722  */
723 static enum srpt_command_state srpt_set_cmd_state(struct srpt_ioctx *ioctx,
724                                                   enum srpt_command_state new)
725 {
726         enum srpt_command_state previous;
727
728         WARN_ON(!ioctx);
729         WARN_ON(new == SRPT_STATE_NEW);
730
731         do {
732                 previous = atomic_read(&ioctx->state);
733         } while (previous != SRPT_STATE_ABORTED
734                && atomic_cmpxchg(&ioctx->state, previous, new) != previous);
735
736         return previous;
737 }
738
739 /**
740  * Test and set the state of a command.
741  * @old: State to compare against.
742  * @new: New state to be set if the current state matches 'old'.
743  *
744  * Returns the previous command state.
745  */
746 static enum srpt_command_state
747 srpt_test_and_set_cmd_state(struct srpt_ioctx *ioctx,
748                             enum srpt_command_state old,
749                             enum srpt_command_state new)
750 {
751         WARN_ON(!ioctx);
752         WARN_ON(old == SRPT_STATE_ABORTED);
753         WARN_ON(new == SRPT_STATE_NEW);
754
755         return atomic_cmpxchg(&ioctx->state, old, new);
756 }
757
758 /**
759  * Post a receive request on the work queue of InfiniBand device 'sdev'.
760  */
761 static int srpt_post_recv(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
762 {
763         struct ib_sge list;
764         struct ib_recv_wr wr, *bad_wr;
765
766         wr.wr_id = ioctx->index | SRPT_OP_RECV;
767
768         list.addr = ioctx->dma;
769         list.length = srp_max_message_size;
770         list.lkey = sdev->mr->lkey;
771
772         wr.next = NULL;
773         wr.sg_list = &list;
774         wr.num_sge = 1;
775
776         return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);
777 }
778
779 /**
780  * Post an IB send request.
781  * @ch: RDMA channel to post the send request on.
782  * @ioctx: I/O context of the send request.
783  * @len: length of the request to be sent in bytes.
784  *
785  * Returns zero upon success and a non-zero value upon failure.
786  */
787 static int srpt_post_send(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
788                           int len)
789 {
790         struct ib_sge list;
791         struct ib_send_wr wr, *bad_wr;
792         struct srpt_device *sdev = ch->sport->sdev;
793         int ret;
794
795         ret = -ENOMEM;
796         if (atomic_dec_return(&ch->qp_wr_avail) < 0) {
797                 PRINT_ERROR("%s[%d]: SRQ full", __func__, __LINE__);
798                 goto out;
799         }
800
801         ib_dma_sync_single_for_device(sdev->device, ioctx->dma,
802                                       len, DMA_TO_DEVICE);
803
804         list.addr = ioctx->dma;
805         list.length = len;
806         list.lkey = sdev->mr->lkey;
807
808         wr.next = NULL;
809         wr.wr_id = ioctx->index;
810         wr.sg_list = &list;
811         wr.num_sge = 1;
812         wr.opcode = IB_WR_SEND;
813         wr.send_flags = IB_SEND_SIGNALED;
814
815         ret = ib_post_send(ch->qp, &wr, &bad_wr);
816
817 out:
818         if (ret < 0)
819                 atomic_inc(&ch->qp_wr_avail);
820         return ret;
821 }
822
823 /**
824  * srpt_get_desc_tbl() - Parse the data descriptors of an SRP_CMD request.
825  * @ioctx: Pointer to the I/O context associated with the request.
826  * @srp_cmd: Pointer to the SRP_CMD request data.
827  * @dir: Pointer to the variable to which the transfer direction will be
828  *   written.
829  * @data_len: Pointer to the variable to which the total data length of all
830  *   descriptors in the SRP_CMD request will be written.
831  *
832  * This function initializes ioctx->nrbuf and ioctx->r_bufs.
833  *
834  * Returns -EINVAL when the SRP_CMD request contains inconsistent descriptors;
835  * -ENOMEM when memory allocation fails and zero upon success.
836  */
837 static int srpt_get_desc_tbl(struct srpt_ioctx *ioctx, struct srp_cmd *srp_cmd,
838                              scst_data_direction *dir, u64 *data_len)
839 {
840         struct srp_indirect_buf *idb;
841         struct srp_direct_buf *db;
842         unsigned add_cdb_offset;
843         int ret;
844
845         /*
846          * The pointer computations below will only be compiled correctly
847          * if srp_cmd::add_data is declared as s8*, u8*, s8[] or u8[], so check
848          * whether srp_cmd::add_data has been declared as a byte pointer.
849          */
850 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 31)
851         BUILD_BUG_ON(!__same_type(srp_cmd->add_data[0], (s8)0)
852                      && !__same_type(srp_cmd->add_data[0], (u8)0));
853 #else
854         /* Note: the __same_type() macro has been introduced in kernel 2.6.31.*/
855 #endif
856
857         BUG_ON(!dir);
858         BUG_ON(!data_len);
859
860         ret = 0;
861         *data_len = 0;
862
863         /*
864          * The lower four bits of the buffer format field contain the DATA-IN
865          * buffer descriptor format, and the highest four bits contain the
866          * DATA-OUT buffer descriptor format.
867          */
868         *dir = SCST_DATA_NONE;
869         if (srp_cmd->buf_fmt & 0xf)
870                 /* DATA-IN: transfer data from target to initiator. */
871                 *dir = SCST_DATA_READ;
872         else if (srp_cmd->buf_fmt >> 4)
873                 /* DATA-OUT: transfer data from initiator to target. */
874                 *dir = SCST_DATA_WRITE;
875
876         /*
877          * According to the SRP spec, the lower two bits of the 'ADDITIONAL
878          * CDB LENGTH' field are reserved and the size in bytes of this field
879          * is four times the value specified in bits 3..7. Hence the "& ~3".
880          */
881         add_cdb_offset = srp_cmd->add_cdb_len & ~3;
882         if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
883             ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
884                 ioctx->n_rbuf = 1;
885                 ioctx->rbufs = &ioctx->single_rbuf;
886
887                 db = (struct srp_direct_buf *)(srp_cmd->add_data
888                                                + add_cdb_offset);
889                 memcpy(ioctx->rbufs, db, sizeof *db);
890                 *data_len = be32_to_cpu(db->len);
891         } else if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_INDIRECT) ||
892                    ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_INDIRECT)) {
893                 idb = (struct srp_indirect_buf *)(srp_cmd->add_data
894                                                   + add_cdb_offset);
895
896                 ioctx->n_rbuf = be32_to_cpu(idb->table_desc.len) / sizeof *db;
897
898                 if (ioctx->n_rbuf >
899                     (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
900                         PRINT_ERROR("received corrupt SRP_CMD request"
901                                     " (%u out + %u in != %u / %zu)",
902                                     srp_cmd->data_out_desc_cnt,
903                                     srp_cmd->data_in_desc_cnt,
904                                     be32_to_cpu(idb->table_desc.len),
905                                     sizeof(*db));
906                         ioctx->n_rbuf = 0;
907                         ret = -EINVAL;
908                         goto out;
909                 }
910
911                 if (ioctx->n_rbuf == 1)
912                         ioctx->rbufs = &ioctx->single_rbuf;
913                 else {
914                         ioctx->rbufs =
915                                 kmalloc(ioctx->n_rbuf * sizeof *db, GFP_ATOMIC);
916                         if (!ioctx->rbufs) {
917                                 ioctx->n_rbuf = 0;
918                                 ret = -ENOMEM;
919                                 goto out;
920                         }
921                 }
922
923                 db = idb->desc_list;
924                 memcpy(ioctx->rbufs, db, ioctx->n_rbuf * sizeof *db);
925                 *data_len = be32_to_cpu(idb->len);
926         }
927 out:
928         return ret;
929 }
930
931 /*
932  * Modify the attributes of queue pair 'qp': allow local write, remote read,
933  * and remote write. Also transition 'qp' to state IB_QPS_INIT.
934  */
935 static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
936 {
937         struct ib_qp_attr *attr;
938         int ret;
939
940         attr = kzalloc(sizeof *attr, GFP_KERNEL);
941         if (!attr)
942                 return -ENOMEM;
943
944         attr->qp_state = IB_QPS_INIT;
945         attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ |
946             IB_ACCESS_REMOTE_WRITE;
947         attr->port_num = ch->sport->port;
948         attr->pkey_index = 0;
949
950         ret = ib_modify_qp(qp, attr,
951                            IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
952                            IB_QP_PKEY_INDEX);
953
954         kfree(attr);
955         return ret;
956 }
957
958 /**
959  * Change the state of a channel to 'ready to receive' (RTR).
960  * @ch: channel of the queue pair.
961  * @qp: queue pair to change the state of.
962  *
963  * Returns zero upon success and a negative value upon failure.
964  *
965  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
966  * If this structure ever becomes larger, it might be necessary to allocate
967  * it dynamically instead of on the stack.
968  */
969 static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
970 {
971         struct ib_qp_attr qp_attr;
972         int attr_mask;
973         int ret;
974
975         qp_attr.qp_state = IB_QPS_RTR;
976         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
977         if (ret)
978                 goto out;
979
980         qp_attr.max_dest_rd_atomic = 4;
981
982         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
983
984 out:
985         return ret;
986 }
987
988 /**
989  * Change the state of a channel to 'ready to send' (RTS).
990  * @ch: channel of the queue pair.
991  * @qp: queue pair to change the state of.
992  *
993  * Returns zero upon success and a negative value upon failure.
994  *
995  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
996  * If this structure ever becomes larger, it might be necessary to allocate
997  * it dynamically instead of on the stack.
998  */
999 static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
1000 {
1001         struct ib_qp_attr qp_attr;
1002         int attr_mask;
1003         int ret;
1004
1005         qp_attr.qp_state = IB_QPS_RTS;
1006         ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
1007         if (ret)
1008                 goto out;
1009
1010         qp_attr.max_rd_atomic = 4;
1011
1012         ret = ib_modify_qp(qp, &qp_attr, attr_mask);
1013
1014 out:
1015         return ret;
1016 }
1017
1018 /**
1019  * srpt_req_lim_delta() - Compute by how much req_lim changed since the
1020  * last time this function has been called. This value is necessary for
1021  * filling in the REQUEST LIMIT DELTA field of an SRP_RSP response.
1022  *
1023  * Side Effect: Modifies ch->last_response_req_lim.
1024  */
1025 static int srpt_req_lim_delta(struct srpt_rdma_ch *ch)
1026 {
1027         int req_lim;
1028         int req_lim_delta;
1029
1030         req_lim = atomic_read(&ch->req_lim);
1031         req_lim_delta = req_lim - atomic_read(&ch->last_response_req_lim);
1032         atomic_add(req_lim_delta, &ch->last_response_req_lim);
1033         return req_lim_delta;
1034 }
1035
1036 static void srpt_reset_ioctx(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
1037 {
1038         srpt_unmap_sg_to_ib_sge(ch, ioctx);
1039
1040         if (ioctx->n_rbuf > 1) {
1041                 kfree(ioctx->rbufs);
1042                 ioctx->rbufs = NULL;
1043         }
1044
1045         WARN_ON(!ch);
1046         if (!ch)
1047                 return;
1048
1049         if (srpt_post_recv(ch->sport->sdev, ioctx))
1050                 PRINT_ERROR("%s", "SRQ post_recv failed - this is serious.");
1051                 /* we should queue it back to free_ioctx queue */
1052         else {
1053                 int req_lim;
1054
1055                 req_lim = atomic_inc_return(&ch->req_lim);
1056                 if (req_lim < 0 || req_lim > SRPT_RQ_SIZE)
1057                         PRINT_ERROR("internal error: req_lim = %d out of range"
1058                                     "%d .. %d", req_lim, 0, SRPT_RQ_SIZE);
1059         }
1060 }
1061
1062 /**
1063  * Abort a command.
1064  */
1065 static void srpt_abort_scst_cmd(struct srpt_device *sdev,
1066                                 struct scst_cmd *scmnd)
1067 {
1068         struct srpt_ioctx *ioctx;
1069         scst_data_direction dir;
1070         enum srpt_command_state previous_state;
1071
1072         TRACE_ENTRY();
1073
1074         ioctx = scst_cmd_get_tgt_priv(scmnd);
1075         BUG_ON(!ioctx);
1076
1077         previous_state = srpt_set_cmd_state(ioctx, SRPT_STATE_ABORTED);
1078         if (previous_state == SRPT_STATE_ABORTED)
1079                 goto out;
1080
1081         TRACE_DBG("Aborting cmd with state %d and tag %lld",
1082                   previous_state, scst_cmd_get_tag(scmnd));
1083
1084         dir = scst_cmd_get_data_direction(scmnd);
1085         if (dir != SCST_DATA_NONE && scst_cmd_get_sg(scmnd))
1086                 ib_dma_unmap_sg(sdev->device,
1087                                 scst_cmd_get_sg(scmnd),
1088                                 scst_cmd_get_sg_cnt(scmnd),
1089                                 scst_to_tgt_dma_dir(dir));
1090
1091         switch (previous_state) {
1092         case SRPT_STATE_NEW:
1093                 scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_ABORTED);
1094                 break;
1095         case SRPT_STATE_NEED_DATA:
1096                 WARN_ON(scst_cmd_get_data_direction(ioctx->scmnd)
1097                         == SCST_DATA_READ);
1098                 scst_rx_data(scmnd, SCST_RX_STATUS_ERROR,
1099                              scst_estimate_context());
1100                 break;
1101         case SRPT_STATE_DATA_IN:
1102         case SRPT_STATE_PROCESSED:
1103                 scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_ABORTED);
1104                 break;
1105         default:
1106                 TRACE_DBG("Aborting cmd with state %d", previous_state);
1107                 WARN_ON("ERROR: unexpected command state");
1108         }
1109         scst_tgt_cmd_done(scmnd, scst_estimate_context());
1110
1111 out:
1112         ;
1113
1114         TRACE_EXIT();
1115 }
1116
1117 static void srpt_handle_err_comp(struct srpt_rdma_ch *ch, struct ib_wc *wc)
1118 {
1119         struct srpt_ioctx *ioctx;
1120         struct srpt_device *sdev = ch->sport->sdev;
1121
1122         if (wc->wr_id & SRPT_OP_RECV) {
1123                 ioctx = sdev->ioctx_ring[wc->wr_id & ~SRPT_OP_RECV];
1124                 PRINT_ERROR("%s", "This is serious - SRQ is in bad state.");
1125         } else {
1126                 ioctx = sdev->ioctx_ring[wc->wr_id];
1127
1128                 if (ioctx->scmnd)
1129                         srpt_abort_scst_cmd(sdev, ioctx->scmnd);
1130                 else
1131                         srpt_reset_ioctx(ch, ioctx);
1132         }
1133 }
1134
1135 /** Process an IB send completion notification. */
1136 static void srpt_handle_send_comp(struct srpt_rdma_ch *ch,
1137                                   struct srpt_ioctx *ioctx,
1138                                   enum scst_exec_context context)
1139 {
1140         if (ioctx->scmnd) {
1141                 scst_data_direction dir =
1142                         scst_cmd_get_data_direction(ioctx->scmnd);
1143
1144                 if (dir != SCST_DATA_NONE && scst_cmd_get_sg(ioctx->scmnd))
1145                         ib_dma_unmap_sg(ch->sport->sdev->device,
1146                                         scst_cmd_get_sg(ioctx->scmnd),
1147                                         scst_cmd_get_sg_cnt(ioctx->scmnd),
1148                                         scst_to_tgt_dma_dir(dir));
1149                 scst_tgt_cmd_done(ioctx->scmnd, context);
1150         } else
1151                 srpt_reset_ioctx(ch, ioctx);
1152 }
1153
1154 /** Process an IB RDMA completion notification. */
1155 static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch,
1156                                   struct srpt_ioctx *ioctx)
1157 {
1158         if (!ioctx->scmnd) {
1159                 WARN_ON("ERROR: ioctx->scmnd == NULL");
1160                 srpt_reset_ioctx(ch, ioctx);
1161                 return;
1162         }
1163
1164         /*
1165          * If an RDMA completion notification has been received for a write
1166          * command, tell SCST that processing can continue by calling
1167          * scst_rx_data().
1168          */
1169         if (srpt_test_and_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA,
1170                                 SRPT_STATE_DATA_IN) == SRPT_STATE_NEED_DATA) {
1171                 WARN_ON(scst_cmd_get_data_direction(ioctx->scmnd)
1172                         == SCST_DATA_READ);
1173                 scst_rx_data(ioctx->scmnd, SCST_RX_STATUS_SUCCESS,
1174                              scst_estimate_context());
1175         }
1176 }
1177
1178 /**
1179  * srpt_build_cmd_rsp() - Build an SRP_RSP response.
1180  * @ch: RDMA channel through which the request has been received.
1181  * @ioctx: I/O context associated with the SRP_CMD request. The response will
1182  *   be built in the buffer ioctx->buf points at and hence this function will
1183  *   overwrite the request data.
1184  * @tag: tag of the request for which this response is being generated.
1185  * @status: value for the STATUS field of the SRP_RSP information unit.
1186  * @sense_data: pointer to sense data to be included in the response.
1187  * @sense_data_len: length in bytes of the sense data.
1188  *
1189  * Returns the size in bytes of the SRP_RSP response.
1190  *
1191  * An SRP_RSP response contains a SCSI status or service response. See also
1192  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
1193  * response. See also SPC-2 for more information about sense data.
1194  */
1195 static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
1196                               struct srpt_ioctx *ioctx, s32 req_lim_delta,
1197                               u64 tag, int status,
1198                               const u8 *sense_data, int sense_data_len)
1199 {
1200         struct srp_rsp *srp_rsp;
1201         int max_sense_len;
1202
1203         /*
1204          * The lowest bit of all SAM-3 status codes is zero (see also
1205          * paragraph 5.3 in SAM-3).
1206          */
1207         WARN_ON(status & 1);
1208
1209         srp_rsp = ioctx->buf;
1210         BUG_ON(!srp_rsp);
1211         memset(srp_rsp, 0, sizeof *srp_rsp);
1212
1213         srp_rsp->opcode = SRP_RSP;
1214         /*
1215          * Copy the SCSOLNT or UCSOLNT bit from the request to the SOLNT bit
1216          * of the response.
1217          */
1218         srp_rsp->sol_not
1219                 = (ioctx->sol_not
1220                    & (status == SAM_STAT_GOOD ? SRP_SCSOLNT : SRP_UCSOLNT))
1221                 ? SRP_SOLNT : 0;
1222         srp_rsp->req_lim_delta = cpu_to_be32(req_lim_delta);
1223         srp_rsp->tag = tag;
1224
1225         if (SCST_SENSE_VALID(sense_data)) {
1226                 BUILD_BUG_ON(MIN_MAX_MESSAGE_SIZE <= sizeof(*srp_rsp));
1227                 max_sense_len = ch->max_ti_iu_len - sizeof(*srp_rsp);
1228                 if (sense_data_len > max_sense_len) {
1229                         PRINT_WARNING("truncated sense data from %d to %d"
1230                                 " bytes", sense_data_len,
1231                                 max_sense_len);
1232                         sense_data_len = max_sense_len;
1233                 }
1234
1235                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1236                 srp_rsp->status = status;
1237                 srp_rsp->sense_data_len = cpu_to_be32(sense_data_len);
1238                 memcpy(srp_rsp + 1, sense_data, sense_data_len);
1239         } else
1240                 sense_data_len = 0;
1241
1242         return sizeof(*srp_rsp) + sense_data_len;
1243 }
1244
1245 /**
1246  * Build a task management response, which is a specific SRP_RSP response.
1247  * @ch: RDMA channel through which the request has been received.
1248  * @ioctx: I/O context in which the SRP_RSP response will be built.
1249  * @rsp_code: RSP_CODE that will be stored in the response.
1250  * @tag: tag of the request for which this response is being generated.
1251  *
1252  * Returns the size in bytes of the SRP_RSP response.
1253  *
1254  * An SRP_RSP response contains a SCSI status or service response. See also
1255  * section 6.9 in the T10 SRP r16a document for the format of an SRP_RSP
1256  * response.
1257  */
1258 static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
1259                                   struct srpt_ioctx *ioctx, s32 req_lim_delta,
1260                                   u8 rsp_code, u64 tag)
1261 {
1262         struct srp_rsp *srp_rsp;
1263         int resp_data_len;
1264         int resp_len;
1265
1266         resp_data_len = (rsp_code == SRP_TSK_MGMT_SUCCESS) ? 0 : 4;
1267         resp_len = sizeof(*srp_rsp) + resp_data_len;
1268
1269         srp_rsp = ioctx->buf;
1270         memset(srp_rsp, 0, sizeof *srp_rsp);
1271
1272         srp_rsp->opcode = SRP_RSP;
1273         /*
1274          * Copy the SCSOLNT or UCSOLNT bit from the request to the SOLNT bit
1275          * of the response.
1276          */
1277         srp_rsp->sol_not
1278                 = (ioctx->sol_not
1279                    & (rsp_code == SRP_TSK_MGMT_SUCCESS
1280                       ? SRP_SCSOLNT : SRP_UCSOLNT))
1281                 ? SRP_SOLNT : 0;
1282         srp_rsp->req_lim_delta = cpu_to_be32(req_lim_delta);
1283         srp_rsp->tag = tag;
1284
1285         if (rsp_code != SRP_TSK_MGMT_SUCCESS) {
1286                 srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
1287                 srp_rsp->resp_data_len = cpu_to_be32(resp_data_len);
1288                 srp_rsp->data[3] = rsp_code;
1289         }
1290
1291         return resp_len;
1292 }
1293
1294 /*
1295  * Process SRP_CMD.
1296  */
1297 static int srpt_handle_cmd(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
1298 {
1299         struct scst_cmd *scmnd;
1300         struct srp_cmd *srp_cmd;
1301         scst_data_direction dir;
1302         u64 data_len;
1303         int ret;
1304
1305         srp_cmd = ioctx->buf;
1306
1307         scmnd = scst_rx_cmd(ch->scst_sess, (u8 *) &srp_cmd->lun,
1308                             sizeof srp_cmd->lun, srp_cmd->cdb, 16,
1309                             thread ? SCST_NON_ATOMIC : SCST_ATOMIC);
1310         if (!scmnd)
1311                 goto err;
1312
1313         ioctx->scmnd = scmnd;
1314
1315         ret = srpt_get_desc_tbl(ioctx, srp_cmd, &dir, &data_len);
1316         if (ret) {
1317                 scst_set_cmd_error(scmnd,
1318                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1319                 goto err;
1320         }
1321
1322         switch (srp_cmd->task_attr) {
1323         case SRP_CMD_HEAD_OF_Q:
1324                 scmnd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1325                 break;
1326         case SRP_CMD_ORDERED_Q:
1327                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1328                 break;
1329         case SRP_CMD_SIMPLE_Q:
1330                 scmnd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1331                 break;
1332         case SRP_CMD_ACA:
1333                 scmnd->queue_type = SCST_CMD_QUEUE_ACA;
1334                 break;
1335         default:
1336                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
1337                 break;
1338         }
1339
1340         scst_cmd_set_tag(scmnd, srp_cmd->tag);
1341         scst_cmd_set_tgt_priv(scmnd, ioctx);
1342         scst_cmd_set_expected(scmnd, dir, data_len);
1343         scst_cmd_init_done(scmnd, scst_estimate_context());
1344
1345         return 0;
1346
1347 err:
1348         return -1;
1349 }
1350
1351 /*
1352  * srpt_handle_tsk_mgmt() - Process an SRP_TSK_MGMT information unit.
1353  *
1354  * Returns SRP_TSK_MGMT_SUCCESS upon success.
1355  *
1356  * Each task management function is performed by calling one of the
1357  * scst_rx_mgmt_fn*() functions. These functions will either report failure
1358  * or process the task management function asynchronously. The function
1359  * srpt_tsk_mgmt_done() will be called by the SCST core upon completion of the
1360  * task management function. When srpt_handle_tsk_mgmt() reports failure
1361  * (i.e. returns -1) a response will have been built in ioctx->buf. This
1362  * information unit has to be sent back by the caller.
1363  *
1364  * For more information about SRP_TSK_MGMT information units, see also section
1365  * 6.7 in the T10 SRP r16a document.
1366  */
1367 static u8 srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
1368                                struct srpt_ioctx *ioctx)
1369 {
1370         struct srp_tsk_mgmt *srp_tsk;
1371         struct srpt_mgmt_ioctx *mgmt_ioctx;
1372         int ret;
1373         u8 srp_tsk_mgmt_status;
1374
1375         srp_tsk = ioctx->buf;
1376
1377         TRACE_DBG("recv_tsk_mgmt= %d for task_tag= %lld"
1378                   " using tag= %lld cm_id= %p sess= %p",
1379                   srp_tsk->tsk_mgmt_func,
1380                   (unsigned long long) srp_tsk->task_tag,
1381                   (unsigned long long) srp_tsk->tag,
1382                   ch->cm_id, ch->scst_sess);
1383
1384         srp_tsk_mgmt_status = SRP_TSK_MGMT_FAILED;
1385         mgmt_ioctx = kmalloc(sizeof *mgmt_ioctx, GFP_ATOMIC);
1386         if (!mgmt_ioctx)
1387                 goto err;
1388
1389         mgmt_ioctx->ioctx = ioctx;
1390         mgmt_ioctx->ch = ch;
1391         mgmt_ioctx->tag = srp_tsk->tag;
1392
1393         switch (srp_tsk->tsk_mgmt_func) {
1394         case SRP_TSK_ABORT_TASK:
1395                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK");
1396                 ret = scst_rx_mgmt_fn_tag(ch->scst_sess,
1397                                           SCST_ABORT_TASK,
1398                                           srp_tsk->task_tag,
1399                                           thread ?
1400                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1401                                           mgmt_ioctx);
1402                 break;
1403         case SRP_TSK_ABORT_TASK_SET:
1404                 TRACE_DBG("%s", "Processing SRP_TSK_ABORT_TASK_SET");
1405                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1406                                           SCST_ABORT_TASK_SET,
1407                                           (u8 *) &srp_tsk->lun,
1408                                           sizeof srp_tsk->lun,
1409                                           thread ?
1410                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1411                                           mgmt_ioctx);
1412                 break;
1413         case SRP_TSK_CLEAR_TASK_SET:
1414                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_TASK_SET");
1415                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1416                                           SCST_CLEAR_TASK_SET,
1417                                           (u8 *) &srp_tsk->lun,
1418                                           sizeof srp_tsk->lun,
1419                                           thread ?
1420                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1421                                           mgmt_ioctx);
1422                 break;
1423         case SRP_TSK_LUN_RESET:
1424                 TRACE_DBG("%s", "Processing SRP_TSK_LUN_RESET");
1425                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1426                                           SCST_LUN_RESET,
1427                                           (u8 *) &srp_tsk->lun,
1428                                           sizeof srp_tsk->lun,
1429                                           thread ?
1430                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1431                                           mgmt_ioctx);
1432                 break;
1433         case SRP_TSK_CLEAR_ACA:
1434                 TRACE_DBG("%s", "Processing SRP_TSK_CLEAR_ACA");
1435                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1436                                           SCST_CLEAR_ACA,
1437                                           (u8 *) &srp_tsk->lun,
1438                                           sizeof srp_tsk->lun,
1439                                           thread ?
1440                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1441                                           mgmt_ioctx);
1442                 break;
1443         default:
1444                 TRACE_DBG("%s", "Unsupported task management function.");
1445                 srp_tsk_mgmt_status = SRP_TSK_MGMT_FUNC_NOT_SUPP;
1446                 goto err;
1447         }
1448
1449         if (ret) {
1450                 TRACE_DBG("Processing task management function failed"
1451                           " (ret = %d).", ret);
1452                 goto err;
1453         }
1454         return SRP_TSK_MGMT_SUCCESS;
1455
1456 err:
1457         kfree(mgmt_ioctx);
1458         return srp_tsk_mgmt_status;
1459 }
1460
1461 /**
1462  * Process a newly received information unit.
1463  * @ch: RDMA channel through which the information unit has been received.
1464  * @ioctx: SRPT I/O context associated with the information unit.
1465  */
1466 static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
1467                                struct srpt_ioctx *ioctx)
1468 {
1469         struct srp_cmd *srp_cmd;
1470         enum rdma_ch_state ch_state;
1471         u8 srp_response_status;
1472         u8 srp_tsk_mgmt_status;
1473         int len;
1474
1475         /*
1476          * A quote from SAM-3, paragraph 4.9.6: "Any command that is not
1477          * relayed to a dependent logical unit shall be terminated with a
1478          * CHECK CONDITION status. The sense key shall be set to ILLEGAL
1479          * REQUEST and the additional sense code shall be set to INVALID
1480          * COMMAND OPERATION CODE. If a task management function cannot be
1481          * relayed to a dependent logical unit, a service response of SERVICE
1482          * DELIVERY OR TARGET FAILURE shall be returned."
1483          */
1484
1485         srp_response_status = SAM_STAT_BUSY;
1486         /* To keep the compiler happy. */
1487         srp_tsk_mgmt_status = -1;
1488
1489         ch_state = atomic_read(&ch->state);
1490         if (ch_state == RDMA_CHANNEL_CONNECTING) {
1491                 list_add_tail(&ioctx->wait_list, &ch->cmd_wait_list);
1492                 return;
1493         } else if (ch_state == RDMA_CHANNEL_DISCONNECTING) {
1494                 srpt_reset_ioctx(ch, ioctx);
1495                 return;
1496         }
1497
1498         WARN_ON(ch_state != RDMA_CHANNEL_LIVE);
1499
1500         ib_dma_sync_single_for_cpu(ch->sport->sdev->device,
1501                                    ioctx->dma, srp_max_message_size,
1502                                    DMA_FROM_DEVICE);
1503
1504         srp_cmd = ioctx->buf;
1505
1506         ioctx->n_rbuf = 0;
1507         ioctx->rbufs = NULL;
1508         ioctx->n_rdma = 0;
1509         ioctx->n_rdma_ius = 0;
1510         ioctx->rdma_ius = NULL;
1511         ioctx->scmnd = NULL;
1512         ioctx->ch = ch;
1513         ioctx->sol_not = srp_cmd->sol_not;
1514         atomic_set(&ioctx->state, SRPT_STATE_NEW);
1515
1516         switch (srp_cmd->opcode) {
1517         case SRP_CMD:
1518                 if (srpt_handle_cmd(ch, ioctx) < 0) {
1519                         if (ioctx->scmnd)
1520                                 srp_response_status =
1521                                         scst_cmd_get_status(ioctx->scmnd);
1522                         goto err;
1523                 }
1524                 break;
1525
1526         case SRP_TSK_MGMT:
1527                 srp_tsk_mgmt_status = srpt_handle_tsk_mgmt(ch, ioctx);
1528                 if (srp_tsk_mgmt_status != SRP_TSK_MGMT_SUCCESS)
1529                         goto err;
1530                 break;
1531
1532         case SRP_CRED_RSP:
1533                 TRACE_DBG("%s", "received SRP_CRED_RSP");
1534                 srpt_reset_ioctx(ch, ioctx);
1535                 break;
1536
1537         case SRP_AER_RSP:
1538                 TRACE_DBG("%s", "received SRP_AER_RSP");
1539                 srpt_reset_ioctx(ch, ioctx);
1540                 break;
1541
1542         case SRP_I_LOGOUT:
1543         default:
1544                 goto err;
1545         }
1546
1547         return;
1548
1549 err:
1550         ch_state = atomic_read(&ch->state);
1551         if (ch_state != RDMA_CHANNEL_LIVE) {
1552                 /* Give up if another thread modified the channel state. */
1553                 PRINT_ERROR("%s: channel is in state %d", __func__, ch_state);
1554                 srpt_reset_ioctx(ch, ioctx);
1555         } else {
1556                 s32 req_lim_delta;
1557
1558                 req_lim_delta = srpt_req_lim_delta(ch) + 1;
1559                 if (srp_cmd->opcode == SRP_TSK_MGMT) {
1560                         len = srpt_build_tskmgmt_rsp(ch, ioctx, req_lim_delta,
1561                                      srp_tsk_mgmt_status,
1562                                      ((struct srp_tsk_mgmt *)srp_cmd)->tag);
1563                 } else if (ioctx->scmnd)
1564                         len = srpt_build_cmd_rsp(ch, ioctx, req_lim_delta,
1565                                 srp_cmd->tag, srp_response_status,
1566                                 scst_cmd_get_sense_buffer(ioctx->scmnd),
1567                                 scst_cmd_get_sense_buffer_len(ioctx->scmnd));
1568                 else {
1569                         len = srpt_build_cmd_rsp(ch, ioctx, srp_cmd->tag,
1570                                                  req_lim_delta,
1571                                                  srp_response_status,
1572                                                  NULL, 0);
1573                 }
1574                 if (srpt_post_send(ch, ioctx, len)) {
1575                         PRINT_ERROR("%s", "Sending SRP_RSP response failed.");
1576                         atomic_sub(req_lim_delta, &ch->last_response_req_lim);
1577                         srpt_reset_ioctx(ch, ioctx);
1578                 }
1579         }
1580 }
1581
1582 /*
1583  * Returns true if the ioctx list is non-empty or if the ib_srpt kernel thread
1584  * should stop.
1585  * @pre thread != 0
1586  */
1587 static inline int srpt_test_ioctx_list(void)
1588 {
1589         int res = (!list_empty(&srpt_thread.thread_ioctx_list) ||
1590                    unlikely(kthread_should_stop()));
1591         return res;
1592 }
1593
1594 /*
1595  * Add 'ioctx' to the tail of the ioctx list and wake up the kernel thread.
1596  *
1597  * @pre thread != 0
1598  */
1599 static inline void srpt_schedule_thread(struct srpt_ioctx *ioctx)
1600 {
1601         unsigned long flags;
1602
1603         spin_lock_irqsave(&srpt_thread.thread_lock, flags);
1604         list_add_tail(&ioctx->comp_list, &srpt_thread.thread_ioctx_list);
1605         spin_unlock_irqrestore(&srpt_thread.thread_lock, flags);
1606         wake_up(&ioctx_list_waitQ);
1607 }
1608
1609 /**
1610  * InfiniBand completion queue callback function.
1611  * @cq: completion queue.
1612  * @ctx: completion queue context, which was passed as the fourth argument of
1613  *       the function ib_create_cq().
1614  */
1615 static void srpt_completion(struct ib_cq *cq, void *ctx)
1616 {
1617         struct srpt_rdma_ch *ch = ctx;
1618         struct srpt_device *sdev = ch->sport->sdev;
1619         struct ib_wc wc;
1620         struct srpt_ioctx *ioctx;
1621
1622         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1623         while (ib_poll_cq(ch->cq, 1, &wc) > 0) {
1624                 if (wc.status) {
1625                         PRINT_INFO("%s failed with status %d",
1626                                    wc.wr_id & SRPT_OP_RECV
1627                                    ? "receiving"
1628                                    : "sending response",
1629                                    wc.status);
1630                         srpt_handle_err_comp(ch, &wc);
1631                         continue;
1632                 }
1633
1634                 if (wc.wr_id & SRPT_OP_RECV) {
1635                         int req_lim;
1636
1637                         req_lim = atomic_dec_return(&ch->req_lim);
1638                         if (req_lim < 0)
1639                                 PRINT_ERROR("internal error: req_lim = %d < 0",
1640                                             req_lim);
1641                         ioctx = sdev->ioctx_ring[wc.wr_id & ~SRPT_OP_RECV];
1642                         if (thread) {
1643                                 ioctx->ch = ch;
1644                                 ioctx->op = IB_WC_RECV;
1645                                 srpt_schedule_thread(ioctx);
1646                         } else
1647                                 srpt_handle_new_iu(ch, ioctx);
1648                 } else {
1649                         ioctx = sdev->ioctx_ring[wc.wr_id];
1650                         if (wc.opcode == IB_WC_SEND)
1651                                 atomic_inc(&ch->qp_wr_avail);
1652                         else {
1653                                 WARN_ON(wc.opcode != IB_WC_RDMA_READ);
1654                                 WARN_ON(ioctx->n_rdma <= 0);
1655                                 atomic_add(ioctx->n_rdma,
1656                                            &ch->qp_wr_avail);
1657                         }
1658                         if (thread) {
1659                                 ioctx->ch = ch;
1660                                 ioctx->op = wc.opcode;
1661                                 srpt_schedule_thread(ioctx);
1662                         } else {
1663                                 switch (wc.opcode) {
1664                                 case IB_WC_SEND:
1665                                         srpt_handle_send_comp(ch, ioctx,
1666                                                 scst_estimate_context());
1667                                         break;
1668                                 case IB_WC_RDMA_WRITE:
1669                                 case IB_WC_RDMA_READ:
1670                                         srpt_handle_rdma_comp(ch, ioctx);
1671                                         break;
1672                                 default:
1673                                         PRINT_ERROR("received unrecognized"
1674                                                     " IB WC opcode %d",
1675                                                     wc.opcode);
1676                                         break;
1677                                 }
1678                         }
1679                 }
1680
1681 #if defined(CONFIG_SCST_DEBUG)
1682                 if (interrupt_processing_delay_in_us <= MAX_UDELAY_MS * 1000)
1683                         udelay(interrupt_processing_delay_in_us);
1684 #endif
1685         }
1686 }
1687
1688 /*
1689  * Create a completion queue on the specified device.
1690  */
1691 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
1692 {
1693         struct ib_qp_init_attr *qp_init;
1694         struct srpt_device *sdev = ch->sport->sdev;
1695         int cqe;
1696         int ret;
1697
1698         qp_init = kzalloc(sizeof *qp_init, GFP_KERNEL);
1699         if (!qp_init)
1700                 return -ENOMEM;
1701
1702         /* Create a completion queue (CQ). */
1703
1704         cqe = SRPT_RQ_SIZE + SRPT_SQ_SIZE - 1;
1705 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(RHEL_RELEASE_CODE)
1706         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe);
1707 #else
1708         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe, 0);
1709 #endif
1710         if (IS_ERR(ch->cq)) {
1711                 ret = PTR_ERR(ch->cq);
1712                 PRINT_ERROR("failed to create_cq cqe= %d ret= %d", cqe, ret);
1713                 goto out;
1714         }
1715
1716         /* Request completion notification. */
1717
1718         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1719
1720         /* Create a queue pair (QP). */
1721
1722         qp_init->qp_context = (void *)ch;
1723         qp_init->event_handler
1724                 = (void(*)(struct ib_event *, void*))srpt_qp_event;
1725         qp_init->send_cq = ch->cq;
1726         qp_init->recv_cq = ch->cq;
1727         qp_init->srq = sdev->srq;
1728         qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
1729         qp_init->qp_type = IB_QPT_RC;
1730         qp_init->cap.max_send_wr = SRPT_SQ_SIZE;
1731         qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE;
1732
1733         ch->qp = ib_create_qp(sdev->pd, qp_init);
1734         if (IS_ERR(ch->qp)) {
1735                 ret = PTR_ERR(ch->qp);
1736                 ib_destroy_cq(ch->cq);
1737                 PRINT_ERROR("failed to create_qp ret= %d", ret);
1738                 goto out;
1739         }
1740
1741         atomic_set(&ch->qp_wr_avail, qp_init->cap.max_send_wr);
1742
1743         TRACE_DBG("%s: max_cqe= %d max_sge= %d cm_id= %p",
1744                __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
1745                ch->cm_id);
1746
1747         /* Modify the attributes and the state of queue pair ch->qp. */
1748
1749         ret = srpt_init_ch_qp(ch, ch->qp);
1750         if (ret) {
1751                 ib_destroy_qp(ch->qp);
1752                 ib_destroy_cq(ch->cq);
1753                 goto out;
1754         }
1755
1756 out:
1757         kfree(qp_init);
1758         return ret;
1759 }
1760
1761 /**
1762  * Release the channel corresponding to the specified cm_id.
1763  *
1764  * Note: must be called from inside srpt_cm_handler to avoid a race between
1765  * accessing sdev->spinlock and the call to kfree(sdev) in srpt_remove_one()
1766  * (the caller of srpt_cm_handler holds the cm_id spinlock;
1767  * srpt_remove_one() waits until all SCST sessions for the associated
1768  * IB device have been unregistered and SCST session registration involves
1769  * a call to ib_destroy_cm_id(), which locks the cm_id spinlock and hence
1770  * waits until this function has finished).
1771  */
1772 static void srpt_release_channel_by_cmid(struct ib_cm_id *cm_id)
1773 {
1774         struct srpt_device *sdev;
1775         struct srpt_rdma_ch *ch;
1776         bool found;
1777
1778         TRACE_ENTRY();
1779
1780         sdev = cm_id->context;
1781         BUG_ON(!sdev);
1782         found = false;
1783         spin_lock_irq(&sdev->spinlock);
1784         list_for_each_entry(ch, &sdev->rch_list, list) {
1785                 if (ch->cm_id == cm_id) {
1786                         list_del(&ch->list);
1787                         atomic_set(&ch->state, RDMA_CHANNEL_DISCONNECTING);
1788                         found = true;
1789                         break;
1790                 }
1791         }
1792         spin_unlock_irq(&sdev->spinlock);
1793
1794         if (found)
1795                 scst_unregister_session(ch->scst_sess, 0, srpt_release_channel);
1796
1797         TRACE_EXIT();
1798 }
1799
1800 /**
1801  * Look up the RDMA channel that corresponds to the specified cm_id.
1802  *
1803  * Return NULL if no matching RDMA channel has been found.
1804  */
1805 static struct srpt_rdma_ch *srpt_find_channel(struct srpt_device *sdev,
1806                                               struct ib_cm_id *cm_id)
1807 {
1808         struct srpt_rdma_ch *ch;
1809         bool found;
1810
1811         BUG_ON(!sdev);
1812         found = false;
1813         spin_lock_irq(&sdev->spinlock);
1814         list_for_each_entry(ch, &sdev->rch_list, list) {
1815                 if (ch->cm_id == cm_id) {
1816                         found = true;
1817                         break;
1818                 }
1819         }
1820         spin_unlock_irq(&sdev->spinlock);
1821
1822         return found ? ch : NULL;
1823 }
1824
1825 /**
1826  * Release all resources associated with an RDMA channel.
1827  *
1828  * Notes:
1829  * - The caller must have removed the channel from the channel list before
1830  *   calling this function.
1831  * - Must be called as a callback function via scst_unregister_session(). Never
1832  *   call this function directly because doing so would trigger several race
1833  *   conditions.
1834  * - Do not access ch->sport or ch->sport->sdev in this function because the
1835  *   memory that was allocated for the sport and/or sdev data structures may
1836  *   already have been freed at the time this function is called.
1837  */
1838 static void srpt_release_channel(struct scst_session *scst_sess)
1839 {
1840         struct srpt_rdma_ch *ch;
1841
1842         TRACE_ENTRY();
1843
1844         ch = scst_sess_get_tgt_priv(scst_sess);
1845         BUG_ON(!ch);
1846         WARN_ON(atomic_read(&ch->state) != RDMA_CHANNEL_DISCONNECTING);
1847
1848         TRACE_DBG("destroying cm_id %p", ch->cm_id);
1849         BUG_ON(!ch->cm_id);
1850         ib_destroy_cm_id(ch->cm_id);
1851
1852         ib_destroy_qp(ch->qp);
1853         ib_destroy_cq(ch->cq);
1854         kfree(ch);
1855
1856         TRACE_EXIT();
1857 }
1858
1859 /**
1860  * Process the event IB_CM_REQ_RECEIVED.
1861  *
1862  * Ownership of the cm_id is transferred to the SCST session if this functions
1863  * returns zero. Otherwise the caller remains the owner of cm_id.
1864  */
1865 static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
1866                             struct ib_cm_req_event_param *param,
1867                             void *private_data)
1868 {
1869         struct srpt_device *sdev = cm_id->context;
1870         struct srp_login_req *req;
1871         struct srp_login_rsp *rsp;
1872         struct srp_login_rej *rej;
1873         struct ib_cm_rep_param *rep_param;
1874         struct srpt_rdma_ch *ch, *tmp_ch;
1875         u32 it_iu_len;
1876         int ret = 0;
1877
1878 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
1879         WARN_ON(!sdev || !private_data);
1880         if (!sdev || !private_data)
1881                 return -EINVAL;
1882 #else
1883         if (WARN_ON(!sdev || !private_data))
1884                 return -EINVAL;
1885 #endif
1886
1887         req = (struct srp_login_req *)private_data;
1888
1889         it_iu_len = be32_to_cpu(req->req_it_iu_len);
1890
1891         PRINT_INFO("Received SRP_LOGIN_REQ with"
1892             " i_port_id 0x%llx:0x%llx, t_port_id 0x%llx:0x%llx and it_iu_len %d"
1893             " on port %d (guid=0x%llx:0x%llx)",
1894             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[0]),
1895             (unsigned long long)be64_to_cpu(*(u64 *)&req->initiator_port_id[8]),
1896             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[0]),
1897             (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[8]),
1898             it_iu_len,
1899             param->port,
1900             (unsigned long long)be64_to_cpu(*(u64 *)
1901                                 &sdev->port[param->port - 1].gid.raw[0]),
1902             (unsigned long long)be64_to_cpu(*(u64 *)
1903                                 &sdev->port[param->port - 1].gid.raw[8]));
1904
1905         rsp = kzalloc(sizeof *rsp, GFP_KERNEL);
1906         rej = kzalloc(sizeof *rej, GFP_KERNEL);
1907         rep_param = kzalloc(sizeof *rep_param, GFP_KERNEL);
1908
1909         if (!rsp || !rej || !rep_param) {
1910                 ret = -ENOMEM;
1911                 goto out;
1912         }
1913
1914         if (it_iu_len > srp_max_message_size || it_iu_len < 64) {
1915                 rej->reason =
1916                     cpu_to_be32(SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
1917                 ret = -EINVAL;
1918                 PRINT_ERROR("rejected SRP_LOGIN_REQ because its"
1919                             " length (%d bytes) is out of range (%d .. %d)",
1920                             it_iu_len, 64, srp_max_message_size);
1921                 goto reject;
1922         }
1923
1924         if ((req->req_flags & SRP_MTCH_ACTION) == SRP_MULTICHAN_SINGLE) {
1925                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
1926
1927                 spin_lock_irq(&sdev->spinlock);
1928
1929                 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
1930                         if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
1931                             && !memcmp(ch->t_port_id, req->target_port_id, 16)
1932                             && param->port == ch->sport->port
1933                             && param->listen_id == ch->sport->sdev->cm_id
1934                             && ch->cm_id) {
1935                                 enum rdma_ch_state prev_state;
1936
1937                                 /* found an existing channel */
1938                                 TRACE_DBG("Found existing channel name= %s"
1939                                           " cm_id= %p state= %d",
1940                                           ch->sess_name, ch->cm_id,
1941                                           atomic_read(&ch->state));
1942
1943                                 prev_state = atomic_xchg(&ch->state,
1944                                                 RDMA_CHANNEL_DISCONNECTING);
1945                                 if (prev_state == RDMA_CHANNEL_CONNECTING)
1946                                         list_del(&ch->list);
1947
1948                                 spin_unlock_irq(&sdev->spinlock);
1949
1950                                 rsp->rsp_flags =
1951                                         SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
1952
1953                                 if (prev_state == RDMA_CHANNEL_LIVE) {
1954                                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
1955                                         PRINT_INFO("disconnected"
1956                                           " session %s because a new"
1957                                           " SRP_LOGIN_REQ has been received.",
1958                                           ch->sess_name);
1959                                 } else if (prev_state ==
1960                                          RDMA_CHANNEL_CONNECTING) {
1961                                         PRINT_ERROR("%s", "rejected"
1962                                           " SRP_LOGIN_REQ because another login"
1963                                           " request is being processed.");
1964                                         ib_send_cm_rej(ch->cm_id,
1965                                                        IB_CM_REJ_NO_RESOURCES,
1966                                                        NULL, 0, NULL, 0);
1967                                         scst_unregister_session(ch->scst_sess,
1968                                                         0,
1969                                                         srpt_release_channel);
1970                                 }
1971
1972                                 spin_lock_irq(&sdev->spinlock);
1973                         }
1974                 }
1975
1976                 spin_unlock_irq(&sdev->spinlock);
1977
1978         } else
1979                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
1980
1981         if (((u64) (*(u64 *) req->target_port_id) !=
1982              cpu_to_be64(srpt_service_guid)) ||
1983             ((u64) (*(u64 *) (req->target_port_id + 8)) !=
1984              cpu_to_be64(srpt_service_guid))) {
1985                 rej->reason =
1986                     cpu_to_be32(SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
1987                 ret = -ENOMEM;
1988                 PRINT_ERROR("%s", "rejected SRP_LOGIN_REQ because it"
1989                        " has an invalid target port identifier.");
1990                 goto reject;
1991         }
1992
1993         ch = kzalloc(sizeof *ch, GFP_KERNEL);
1994         if (!ch) {
1995                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1996                 PRINT_ERROR("%s",
1997                             "rejected SRP_LOGIN_REQ because out of memory.");
1998                 ret = -ENOMEM;
1999                 goto reject;
2000         }
2001
2002         memcpy(ch->i_port_id, req->initiator_port_id, 16);
2003         memcpy(ch->t_port_id, req->target_port_id, 16);
2004         ch->sport = &sdev->port[param->port - 1];
2005         ch->losolnt = req->req_flags & SRP_LOSOLNT ? 1 : 0;
2006         ch->crsolnt = req->req_flags & SRP_CRSOLNT ? 1 : 0;
2007         ch->aesolnt = req->req_flags & SRP_AESOLNT ? 1 : 0;
2008         ch->cm_id = cm_id;
2009         atomic_set(&ch->state, RDMA_CHANNEL_CONNECTING);
2010         INIT_LIST_HEAD(&ch->cmd_wait_list);
2011
2012         ret = srpt_create_ch_ib(ch);
2013         if (ret) {
2014                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2015                 PRINT_ERROR("%s", "rejected SRP_LOGIN_REQ because creating"
2016                             " a new RDMA channel failed.");
2017                 goto free_ch;
2018         }
2019
2020         ret = srpt_ch_qp_rtr(ch, ch->qp);
2021         if (ret) {
2022                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2023                 PRINT_ERROR("rejected SRP_LOGIN_REQ because enabling"
2024                        " RTR failed (error code = %d)", ret);
2025                 goto destroy_ib;
2026         }
2027
2028         if (use_port_guid_in_session_name) {
2029                 /*
2030                  * If the kernel module parameter use_port_guid_in_session_name
2031                  * has been specified, use a combination of the target port
2032                  * GUID and the initiator port ID as the session name. This
2033                  * was the original behavior of the SRP target implementation
2034                  * (i.e. before the SRPT was included in OFED 1.3).
2035                  */
2036                 snprintf(ch->sess_name, sizeof(ch->sess_name),
2037                          "0x%016llx%016llx",
2038                          (unsigned long long)be64_to_cpu(*(u64 *)
2039                                 &sdev->port[param->port - 1].gid.raw[8]),
2040                          (unsigned long long)be64_to_cpu(*(u64 *)
2041                                 (ch->i_port_id + 8)));
2042         } else {
2043                 /*
2044                  * Default behavior: use the initator port identifier as the
2045                  * session name.
2046                  */
2047                 snprintf(ch->sess_name, sizeof(ch->sess_name),
2048                          "0x%016llx%016llx",
2049                          (unsigned long long)be64_to_cpu(*(u64 *)ch->i_port_id),
2050                          (unsigned long long)be64_to_cpu(*(u64 *)
2051                                  (ch->i_port_id + 8)));
2052         }
2053
2054         TRACE_DBG("registering session %s", ch->sess_name);
2055
2056         BUG_ON(!sdev->scst_tgt);
2057         ch->scst_sess = scst_register_session(sdev->scst_tgt, 0, ch->sess_name,
2058                                               NULL, NULL);
2059         if (!ch->scst_sess) {
2060                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
2061                 TRACE_DBG("%s", "Failed to create scst sess");
2062                 goto destroy_ib;
2063         }
2064
2065         TRACE_DBG("Establish connection sess=%p name=%s cm_id=%p",
2066                   ch->scst_sess, ch->sess_name, ch->cm_id);
2067
2068         scst_sess_set_tgt_priv(ch->scst_sess, ch);
2069
2070         /* create srp_login_response */
2071         rsp->opcode = SRP_LOGIN_RSP;
2072         rsp->tag = req->tag;
2073         rsp->max_it_iu_len = req->req_it_iu_len;
2074         rsp->max_ti_iu_len = req->req_it_iu_len;
2075         ch->max_ti_iu_len = req->req_it_iu_len;
2076         rsp->buf_fmt =
2077             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
2078         rsp->req_lim_delta = cpu_to_be32(SRPT_RQ_SIZE);
2079         atomic_set(&ch->req_lim, SRPT_RQ_SIZE);
2080         atomic_set(&ch->last_response_req_lim, SRPT_RQ_SIZE);
2081
2082         /* create cm reply */
2083         rep_param->qp_num = ch->qp->qp_num;
2084         rep_param->private_data = (void *)rsp;
2085         rep_param->private_data_len = sizeof *rsp;
2086         rep_param->rnr_retry_count = 7;
2087         rep_param->flow_control = 1;
2088         rep_param->failover_accepted = 0;
2089         rep_param->srq = 1;
2090         rep_param->responder_resources = 4;
2091         rep_param->initiator_depth = 4;
2092
2093         ret = ib_send_cm_rep(cm_id, rep_param);
2094         if (ret) {
2095                 PRINT_ERROR("sending SRP_LOGIN_REQ response failed"
2096                             " (error code = %d)", ret);
2097                 goto release_channel;
2098         }
2099
2100         spin_lock_irq(&sdev->spinlock);
2101         list_add_tail(&ch->list, &sdev->rch_list);
2102         spin_unlock_irq(&sdev->spinlock);
2103
2104         goto out;
2105
2106 release_channel:
2107         atomic_set(&ch->state, RDMA_CHANNEL_DISCONNECTING);
2108         scst_unregister_session(ch->scst_sess, 0, NULL);
2109         ch->scst_sess = NULL;
2110
2111 destroy_ib:
2112         ib_destroy_qp(ch->qp);
2113         ib_destroy_cq(ch->cq);
2114
2115 free_ch:
2116         kfree(ch);
2117
2118 reject:
2119         rej->opcode = SRP_LOGIN_REJ;
2120         rej->tag = req->tag;
2121         rej->buf_fmt =
2122             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
2123
2124         ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
2125                              (void *)rej, sizeof *rej);
2126
2127 out:
2128         kfree(rep_param);
2129         kfree(rsp);
2130         kfree(rej);
2131
2132         return ret;
2133 }
2134
2135 static void srpt_cm_rej_recv(struct ib_cm_id *cm_id)
2136 {
2137         PRINT_INFO("Received InfiniBand REJ packet for cm_id %p.", cm_id);
2138         srpt_release_channel_by_cmid(cm_id);
2139 }
2140
2141 /**
2142  * Process an IB_CM_RTU_RECEIVED or IB_CM_USER_ESTABLISHED event.
2143  *
2144  * An IB_CM_RTU_RECEIVED message indicates that the connection is established
2145  * and that the recipient may begin transmitting (RTU = ready to use).
2146  */
2147 static void srpt_cm_rtu_recv(struct ib_cm_id *cm_id)
2148 {
2149         struct srpt_rdma_ch *ch;
2150         int ret;
2151
2152         ch = srpt_find_channel(cm_id->context, cm_id);
2153         WARN_ON(!ch);
2154         if (!ch)
2155                 goto out;
2156
2157         if (srpt_test_and_set_channel_state(ch, RDMA_CHANNEL_CONNECTING,
2158                         RDMA_CHANNEL_LIVE) == RDMA_CHANNEL_CONNECTING) {
2159                 struct srpt_ioctx *ioctx, *ioctx_tmp;
2160
2161                 ret = srpt_ch_qp_rts(ch, ch->qp);
2162
2163                 list_for_each_entry_safe(ioctx, ioctx_tmp, &ch->cmd_wait_list,
2164                                          wait_list) {
2165                         list_del(&ioctx->wait_list);
2166                         srpt_handle_new_iu(ch, ioctx);
2167                 }
2168                 if (ret && srpt_test_and_set_channel_state(ch,
2169                         RDMA_CHANNEL_LIVE,
2170                         RDMA_CHANNEL_DISCONNECTING) == RDMA_CHANNEL_LIVE) {
2171                         TRACE_DBG("cm_id=%p sess_name=%s state=%d",
2172                                   cm_id, ch->sess_name,
2173                                   atomic_read(&ch->state));
2174                         ib_send_cm_dreq(ch->cm_id, NULL, 0);
2175                 }
2176         }
2177
2178 out:
2179         ;
2180 }
2181
2182 static void srpt_cm_timewait_exit(struct ib_cm_id *cm_id)
2183 {
2184         PRINT_INFO("Received InfiniBand TimeWait exit for cm_id %p.", cm_id);
2185         srpt_release_channel_by_cmid(cm_id);
2186 }
2187
2188 static void srpt_cm_rep_error(struct ib_cm_id *cm_id)
2189 {
2190         PRINT_INFO("Received InfiniBand REP error for cm_id %p.", cm_id);
2191         srpt_release_channel_by_cmid(cm_id);
2192 }
2193
2194 static void srpt_cm_dreq_recv(struct ib_cm_id *cm_id)
2195 {
2196         struct srpt_rdma_ch *ch;
2197
2198         ch = srpt_find_channel(cm_id->context, cm_id);
2199         WARN_ON(!ch);
2200         if (!ch)
2201                 goto out;
2202
2203         TRACE_DBG("cm_id= %p ch->state= %d", cm_id, atomic_read(&ch->state));
2204
2205         switch (atomic_read(&ch->state)) {
2206         case RDMA_CHANNEL_LIVE:
2207         case RDMA_CHANNEL_CONNECTING:
2208                 ib_send_cm_drep(ch->cm_id, NULL, 0);
2209                 PRINT_INFO("Received DREQ and sent DREP for session %s.",
2210                            ch->sess_name);
2211                 break;
2212         case RDMA_CHANNEL_DISCONNECTING:
2213         default:
2214                 break;
2215         }
2216
2217 out:
2218         ;
2219 }
2220
2221 static void srpt_cm_drep_recv(struct ib_cm_id *cm_id)
2222 {
2223         PRINT_INFO("Received InfiniBand DREP message for cm_id %p.", cm_id);
2224         srpt_release_channel_by_cmid(cm_id);
2225 }
2226
2227 /**
2228  * IB connection manager callback function.
2229  *
2230  * A non-zero return value will cause the caller destroy the CM ID.
2231  *
2232  * Note: srpt_cm_handler() must only return a non-zero value when transferring
2233  * ownership of the cm_id to a channel by srpt_cm_req_recv() failed. Returning
2234  * a non-zero value in any other case will trigger a race with the
2235  * ib_destroy_cm_id() call in srpt_release_channel().
2236  */
2237 static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2238 {
2239         int ret;
2240
2241         ret = 0;
2242         switch (event->event) {
2243         case IB_CM_REQ_RECEIVED:
2244                 ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd,
2245                                        event->private_data);
2246                 break;
2247         case IB_CM_REJ_RECEIVED:
2248                 srpt_cm_rej_recv(cm_id);
2249                 break;
2250         case IB_CM_RTU_RECEIVED:
2251         case IB_CM_USER_ESTABLISHED:
2252                 srpt_cm_rtu_recv(cm_id);
2253                 break;
2254         case IB_CM_DREQ_RECEIVED:
2255                 srpt_cm_dreq_recv(cm_id);
2256                 break;
2257         case IB_CM_DREP_RECEIVED:
2258                 srpt_cm_drep_recv(cm_id);
2259                 break;
2260         case IB_CM_TIMEWAIT_EXIT:
2261                 srpt_cm_timewait_exit(cm_id);
2262                 break;
2263         case IB_CM_REP_ERROR:
2264                 srpt_cm_rep_error(cm_id);
2265                 break;
2266         default:
2267                 PRINT_ERROR("received unrecognized IB CM event %d",
2268                             event->event);
2269                 break;
2270         }
2271
2272         return ret;
2273 }
2274
2275 static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
2276                                  struct srpt_ioctx *ioctx,
2277                                  struct scst_cmd *scmnd)
2278 {
2279         struct scatterlist *scat;
2280         scst_data_direction dir;
2281         struct rdma_iu *riu;
2282         struct srp_direct_buf *db;
2283         dma_addr_t dma_addr;
2284         struct ib_sge *sge;
2285         u64 raddr;
2286         u32 rsize;
2287         u32 tsize;
2288         u32 dma_len;
2289         int count, nrdma;
2290         int i, j, k;
2291
2292         scat = scst_cmd_get_sg(scmnd);
2293         dir = scst_cmd_get_data_direction(scmnd);
2294         WARN_ON(scat == NULL);
2295         count = ib_dma_map_sg(ch->sport->sdev->device, scat,
2296                               scst_cmd_get_sg_cnt(scmnd),
2297                               scst_to_tgt_dma_dir(dir));
2298         if (unlikely(!count))
2299                 return -EBUSY;
2300
2301         if (ioctx->rdma_ius && ioctx->n_rdma_ius)
2302                 nrdma = ioctx->n_rdma_ius;
2303         else {
2304                 nrdma = count / SRPT_DEF_SG_PER_WQE + ioctx->n_rbuf;
2305
2306                 ioctx->rdma_ius = kzalloc(nrdma * sizeof *riu,
2307                                           scst_cmd_atomic(scmnd)
2308                                           ? GFP_ATOMIC : GFP_KERNEL);
2309                 if (!ioctx->rdma_ius) {
2310                         WARN_ON(scat == NULL);
2311                         ib_dma_unmap_sg(ch->sport->sdev->device,
2312                                         scat, scst_cmd_get_sg_cnt(scmnd),
2313                                         scst_to_tgt_dma_dir(dir));
2314                         return -ENOMEM;
2315                 }
2316
2317                 ioctx->n_rdma_ius = nrdma;
2318         }
2319
2320         db = ioctx->rbufs;
2321         tsize = (dir == SCST_DATA_READ) ?
2322                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2323         dma_len = sg_dma_len(&scat[0]);
2324         riu = ioctx->rdma_ius;
2325
2326         /*
2327          * For each remote desc - calculate the #ib_sge.
2328          * If #ib_sge < SRPT_DEF_SG_PER_WQE per rdma operation then
2329          *      each remote desc rdma_iu is required a rdma wr;
2330          * else
2331          *      we need to allocate extra rdma_iu to carry extra #ib_sge in
2332          *      another rdma wr
2333          */
2334         for (i = 0, j = 0;
2335              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2336                 rsize = be32_to_cpu(db->len);
2337                 raddr = be64_to_cpu(db->va);
2338                 riu->raddr = raddr;
2339                 riu->rkey = be32_to_cpu(db->key);
2340                 riu->sge_cnt = 0;
2341
2342                 /* calculate how many sge required for this remote_buf */
2343                 while (rsize > 0 && tsize > 0) {
2344
2345                         if (rsize >= dma_len) {
2346                                 tsize -= dma_len;
2347                                 rsize -= dma_len;
2348                                 raddr += dma_len;
2349
2350                                 if (tsize > 0) {
2351                                         ++j;
2352                                         if (j < count)
2353                                                 dma_len = sg_dma_len(&scat[j]);
2354                                 }
2355                         } else {
2356                                 tsize -= rsize;
2357                                 dma_len -= rsize;
2358                                 rsize = 0;
2359                         }
2360
2361                         ++riu->sge_cnt;
2362
2363                         if (rsize > 0 && riu->sge_cnt == SRPT_DEF_SG_PER_WQE) {
2364                                 ++ioctx->n_rdma;
2365                                 riu->sge =
2366                                     kmalloc(riu->sge_cnt * sizeof *riu->sge,
2367                                             scst_cmd_atomic(scmnd)
2368                                             ? GFP_ATOMIC : GFP_KERNEL);
2369                                 if (!riu->sge)
2370                                         goto free_mem;
2371
2372                                 ++riu;
2373                                 riu->sge_cnt = 0;
2374                                 riu->raddr = raddr;
2375                                 riu->rkey = be32_to_cpu(db->key);
2376                         }
2377                 }
2378
2379                 ++ioctx->n_rdma;
2380                 riu->sge = kmalloc(riu->sge_cnt * sizeof *riu->sge,
2381                                    scst_cmd_atomic(scmnd)
2382                                    ? GFP_ATOMIC : GFP_KERNEL);
2383                 if (!riu->sge)
2384                         goto free_mem;
2385         }
2386
2387         db = ioctx->rbufs;
2388         scat = scst_cmd_get_sg(scmnd);
2389         tsize = (dir == SCST_DATA_READ) ?
2390                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
2391         riu = ioctx->rdma_ius;
2392         dma_len = sg_dma_len(&scat[0]);
2393         dma_addr = sg_dma_address(&scat[0]);
2394
2395         /* this second loop is really mapped sg_addres to rdma_iu->ib_sge */
2396         for (i = 0, j = 0;
2397              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
2398                 rsize = be32_to_cpu(db->len);
2399                 sge = riu->sge;
2400                 k = 0;
2401
2402                 while (rsize > 0 && tsize > 0) {
2403                         sge->addr = dma_addr;
2404                         sge->lkey = ch->sport->sdev->mr->lkey;
2405
2406                         if (rsize >= dma_len) {
2407                                 sge->length =
2408                                         (tsize < dma_len) ? tsize : dma_len;
2409                                 tsize -= dma_len;
2410                                 rsize -= dma_len;
2411
2412                                 if (tsize > 0) {
2413                                         ++j;
2414                                         if (j < count) {
2415                                                 dma_len = sg_dma_len(&scat[j]);
2416                                                 dma_addr =
2417                                                     sg_dma_address(&scat[j]);
2418                                         }
2419                                 }
2420                         } else {
2421                                 sge->length = (tsize < rsize) ? tsize : rsize;
2422                                 tsize -= rsize;
2423                                 dma_len -= rsize;
2424                                 dma_addr += rsize;
2425                                 rsize = 0;
2426                         }
2427
2428                         ++k;
2429                         if (k == riu->sge_cnt && rsize > 0) {
2430                                 ++riu;
2431                                 sge = riu->sge;
2432                                 k = 0;
2433                         } else if (rsize > 0)
2434                                 ++sge;
2435                 }
2436         }
2437
2438         return 0;
2439
2440 free_mem:
2441         srpt_unmap_sg_to_ib_sge(ch, ioctx);
2442
2443         return -ENOMEM;
2444 }
2445
2446 static void srpt_unmap_sg_to_ib_sge(struct srpt_rdma_ch *ch,
2447                                     struct srpt_ioctx *ioctx)
2448 {
2449         struct scst_cmd *scmnd;
2450         struct scatterlist *scat;
2451         scst_data_direction dir;
2452
2453         BUG_ON(ioctx->n_rdma && !ioctx->rdma_ius);
2454
2455         while (ioctx->n_rdma)
2456                 kfree(ioctx->rdma_ius[--ioctx->n_rdma].sge);
2457
2458         kfree(ioctx->rdma_ius);
2459         ioctx->rdma_ius = NULL;
2460
2461         scmnd = ioctx->scmnd;
2462         if (scmnd) {
2463                 BUG_ON(ioctx != scst_cmd_get_tgt_priv(scmnd));
2464                 scat = scst_cmd_get_sg(scmnd);
2465                 if (scat) {
2466                         dir = scst_cmd_get_data_direction(scmnd);
2467                         ib_dma_unmap_sg(ch->sport->sdev->device,
2468                                         scat, scst_cmd_get_sg_cnt(scmnd),
2469                                         scst_to_tgt_dma_dir(dir));
2470                 }
2471         }
2472 }
2473
2474 static int srpt_perform_rdmas(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2475                               scst_data_direction dir)
2476 {
2477         struct ib_send_wr wr;
2478         struct ib_send_wr *bad_wr;
2479         struct rdma_iu *riu;
2480         int i;
2481         int ret;
2482         int srq_wr_avail;
2483
2484         if (dir == SCST_DATA_WRITE) {
2485                 ret = -ENOMEM;
2486                 srq_wr_avail = atomic_sub_return(ioctx->n_rdma,
2487                                                  &ch->qp_wr_avail);
2488                 if (srq_wr_avail < 0) {
2489                         atomic_add(ioctx->n_rdma, &ch->qp_wr_avail);
2490                         PRINT_INFO("%s[%d]: SRQ full", __func__, __LINE__);
2491                         goto out;
2492                 }
2493         }
2494
2495         ret = 0;
2496         riu = ioctx->rdma_ius;
2497         memset(&wr, 0, sizeof wr);
2498
2499         for (i = 0; i < ioctx->n_rdma; ++i, ++riu) {
2500                 wr.opcode = (dir == SCST_DATA_READ) ?
2501                     IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;
2502                 wr.next = NULL;
2503                 wr.wr_id = ioctx->index;
2504                 wr.wr.rdma.remote_addr = riu->raddr;
2505                 wr.wr.rdma.rkey = riu->rkey;
2506                 wr.num_sge = riu->sge_cnt;
2507                 wr.sg_list = riu->sge;
2508
2509                 /* only get completion event for the last rdma wr */
2510                 if (i == (ioctx->n_rdma - 1) && dir == SCST_DATA_WRITE)
2511                         wr.send_flags = IB_SEND_SIGNALED;
2512
2513                 ret = ib_post_send(ch->qp, &wr, &bad_wr);
2514                 if (ret)
2515                         goto out;
2516         }
2517
2518 out:
2519         return ret;
2520 }
2521
2522 /*
2523  * Start data transfer between initiator and target. Must not block.
2524  */
2525 static int srpt_xfer_data(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
2526                           struct scst_cmd *scmnd)
2527 {
2528         int ret;
2529
2530         ret = srpt_map_sg_to_ib_sge(ch, ioctx, scmnd);
2531         if (ret) {
2532                 PRINT_ERROR("%s[%d] ret=%d", __func__, __LINE__, ret);
2533                 ret = SCST_TGT_RES_QUEUE_FULL;
2534                 goto out;
2535         }
2536
2537         ret = srpt_perform_rdmas(ch, ioctx, scst_cmd_get_data_direction(scmnd));
2538         if (ret) {
2539                 if (ret == -EAGAIN || ret == -ENOMEM) {
2540                         PRINT_INFO("%s[%d] queue full -- ret=%d",
2541                                    __func__, __LINE__, ret);
2542                         ret = SCST_TGT_RES_QUEUE_FULL;
2543                 } else {
2544                         PRINT_ERROR("%s[%d] fatal error -- ret=%d",
2545                                     __func__, __LINE__, ret);
2546                         ret = SCST_TGT_RES_FATAL_ERROR;
2547                 }
2548                 goto out_unmap;
2549         }
2550
2551         ret = SCST_TGT_RES_SUCCESS;
2552
2553 out:
2554         return ret;
2555 out_unmap:
2556         srpt_unmap_sg_to_ib_sge(ch, ioctx);
2557         goto out;
2558 }
2559
2560 /*
2561  * Called by the SCST core to inform ib_srpt that data reception from the
2562  * initiator should start (SCST_DATA_WRITE). Must not block.
2563  */
2564 static int srpt_rdy_to_xfer(struct scst_cmd *scmnd)
2565 {
2566         struct srpt_rdma_ch *ch;
2567         struct srpt_ioctx *ioctx;
2568         enum rdma_ch_state ch_state;
2569         int ret;
2570
2571         ioctx = scst_cmd_get_tgt_priv(scmnd);
2572         BUG_ON(!ioctx);
2573
2574         WARN_ON(srpt_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA)
2575                 == SRPT_STATE_ABORTED);
2576
2577         ch = ioctx->ch;
2578         WARN_ON(ch != scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd)));
2579         BUG_ON(!ch);
2580
2581         ch_state = atomic_read(&ch->state);
2582         if (ch_state == RDMA_CHANNEL_DISCONNECTING) {
2583                 TRACE_DBG("cmd with tag %lld: channel disconnecting",
2584                           scst_cmd_get_tag(scmnd));
2585                 ret = SCST_TGT_RES_FATAL_ERROR;
2586                 goto out;
2587         } else if (ch_state == RDMA_CHANNEL_CONNECTING) {
2588                 ret = SCST_TGT_RES_QUEUE_FULL;
2589                 goto out;
2590         }
2591         ret = srpt_xfer_data(ch, ioctx, scmnd);
2592
2593 out:
2594         return ret;
2595 }
2596
2597 /**
2598  * srpt_xmit_response() - SCST callback function that transmits the response
2599  * to a SCSI command.
2600  *
2601  * Must not block.
2602  */
2603 static int srpt_xmit_response(struct scst_cmd *scmnd)
2604 {
2605         struct srpt_rdma_ch *ch;
2606         struct srpt_ioctx *ioctx;
2607         s32 req_lim_delta;
2608         int ret = SCST_TGT_RES_SUCCESS;
2609         int dir;
2610         int resp_len;
2611
2612         ioctx = scst_cmd_get_tgt_priv(scmnd);
2613         BUG_ON(!ioctx);
2614
2615         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2616         BUG_ON(!ch);
2617
2618         if (unlikely(scst_cmd_aborted(scmnd))) {
2619                 TRACE_DBG("cmd with tag %lld has been aborted",
2620                           scst_cmd_get_tag(scmnd));
2621                 srpt_abort_scst_cmd(ch->sport->sdev, scmnd);
2622                 ret = SCST_TGT_RES_SUCCESS;
2623                 goto out;
2624         }
2625
2626         if (srpt_set_cmd_state(ioctx, SRPT_STATE_PROCESSED)
2627             == SRPT_STATE_ABORTED) {
2628                 ret = SCST_TGT_RES_SUCCESS;
2629                 goto out;
2630         }
2631
2632         dir = scst_cmd_get_data_direction(scmnd);
2633
2634         /* For read commands, transfer the data to the initiator. */
2635         if (dir == SCST_DATA_READ && scst_cmd_get_resp_data_len(scmnd)) {
2636                 ret = srpt_xfer_data(ch, ioctx, scmnd);
2637                 if (ret != SCST_TGT_RES_SUCCESS) {
2638                         PRINT_ERROR("%s: tag= %lld xfer_data failed",
2639                                     __func__,
2640                                     (unsigned long long)
2641                                     scst_cmd_get_tag(scmnd));
2642                         goto out;
2643                 }
2644         }
2645
2646         scst_check_convert_sense(scmnd);
2647
2648         req_lim_delta = srpt_req_lim_delta(ch) + 1;
2649         resp_len = srpt_build_cmd_rsp(ch, ioctx, req_lim_delta,
2650                                       scst_cmd_get_tag(scmnd),
2651                                       scst_cmd_get_status(scmnd),
2652                                       scst_cmd_get_sense_buffer(scmnd),
2653                                       scst_cmd_get_sense_buffer_len(scmnd));
2654
2655         if (srpt_post_send(ch, ioctx, resp_len)) {
2656                 PRINT_ERROR("%s[%d]: ch->state= %d tag= %lld",
2657                             __func__, __LINE__, atomic_read(&ch->state),
2658                             (unsigned long long)scst_cmd_get_tag(scmnd));
2659                 atomic_sub(req_lim_delta, &ch->last_response_req_lim);
2660                 ret = SCST_TGT_RES_FATAL_ERROR;
2661         }
2662
2663 out:
2664         return ret;
2665 }
2666
2667 /**
2668  * srpt_tsk_mgmt_done() - SCST callback function that sends back the response
2669  * for a task management request.
2670  *
2671  * Must not block.
2672  */
2673 static void srpt_tsk_mgmt_done(struct scst_mgmt_cmd *mcmnd)
2674 {
2675         struct srpt_rdma_ch *ch;
2676         struct srpt_mgmt_ioctx *mgmt_ioctx;
2677         struct srpt_ioctx *ioctx;
2678         s32 req_lim_delta;
2679         int rsp_len;
2680
2681         mgmt_ioctx = scst_mgmt_cmd_get_tgt_priv(mcmnd);
2682         BUG_ON(!mgmt_ioctx);
2683
2684         ch = mgmt_ioctx->ch;
2685         BUG_ON(!ch);
2686
2687         ioctx = mgmt_ioctx->ioctx;
2688         BUG_ON(!ioctx);
2689
2690         TRACE_DBG("%s: tsk_mgmt_done for tag= %lld status=%d",
2691                   __func__, (unsigned long long)mgmt_ioctx->tag,
2692                   scst_mgmt_cmd_get_status(mcmnd));
2693
2694         if (srpt_set_cmd_state(ioctx, SRPT_STATE_PROCESSED)
2695             == SRPT_STATE_ABORTED)
2696                 goto out;
2697
2698         req_lim_delta = srpt_req_lim_delta(ch) + 1;
2699         rsp_len = srpt_build_tskmgmt_rsp(ch, ioctx, req_lim_delta,
2700                                          (scst_mgmt_cmd_get_status(mcmnd) ==
2701                                           SCST_MGMT_STATUS_SUCCESS) ?
2702                                          SRP_TSK_MGMT_SUCCESS :
2703                                          SRP_TSK_MGMT_FAILED,
2704                                          mgmt_ioctx->tag);
2705         if (srpt_post_send(ch, ioctx, rsp_len)) {
2706                 PRINT_ERROR("%s", "Sending SRP_RSP response failed.");
2707                 atomic_sub(req_lim_delta, &ch->last_response_req_lim);
2708         }
2709
2710         scst_mgmt_cmd_set_tgt_priv(mcmnd, NULL);
2711
2712         kfree(mgmt_ioctx);
2713
2714 out:
2715         ;
2716 }
2717
2718 /*
2719  * Called by the SCST core to inform ib_srpt that the command 'scmnd' is about
2720  * to be freed. May be called in IRQ context.
2721  */
2722 static void srpt_on_free_cmd(struct scst_cmd *scmnd)
2723 {
2724         struct srpt_rdma_ch *ch;
2725         struct srpt_ioctx *ioctx;
2726
2727         ioctx = scst_cmd_get_tgt_priv(scmnd);
2728         BUG_ON(!ioctx);
2729
2730         ch = ioctx->ch;
2731         BUG_ON(!ch);
2732
2733         scst_cmd_set_tgt_priv(scmnd, NULL);
2734         srpt_set_cmd_state(ioctx, SRPT_STATE_ABORTED);
2735         ioctx->scmnd = NULL;
2736         ioctx->ch = NULL;
2737         srpt_reset_ioctx(ch, ioctx);
2738 }
2739
2740 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2741 /* A vanilla 2.6.19 or older kernel without backported OFED kernel headers. */
2742 static void srpt_refresh_port_work(void *ctx)
2743 #else
2744 static void srpt_refresh_port_work(struct work_struct *work)
2745 #endif
2746 {
2747 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2748         struct srpt_port *sport = (struct srpt_port *)ctx;
2749 #else
2750         struct srpt_port *sport = container_of(work, struct srpt_port, work);
2751 #endif
2752
2753         srpt_refresh_port(sport);
2754 }
2755
2756 /*
2757  * Called by the SCST core to detect target adapters. Returns the number of
2758  * detected target adapters.
2759  */
2760 static int srpt_detect(struct scst_tgt_template *tp)
2761 {
2762         int device_count;
2763
2764         TRACE_ENTRY();
2765
2766         device_count = atomic_read(&srpt_device_count);
2767
2768         TRACE_EXIT_RES(device_count);
2769
2770         return device_count;
2771 }
2772
2773 /*
2774  * Callback function called by the SCST core from scst_unregister() to free up
2775  * the resources associated with device scst_tgt.
2776  */
2777 static int srpt_release(struct scst_tgt *scst_tgt)
2778 {
2779         struct srpt_device *sdev = scst_tgt_get_tgt_priv(scst_tgt);
2780         struct srpt_rdma_ch *ch, *tmp_ch;
2781
2782         TRACE_ENTRY();
2783
2784         BUG_ON(!scst_tgt);
2785 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
2786         WARN_ON(!sdev);
2787         if (!sdev)
2788                 return -ENODEV;
2789 #else
2790         if (WARN_ON(!sdev))
2791                 return -ENODEV;
2792 #endif
2793
2794 #ifdef CONFIG_SCST_PROC
2795         srpt_unregister_procfs_entry(scst_tgt->tgtt);
2796 #endif /*CONFIG_SCST_PROC*/
2797
2798         spin_lock_irq(&sdev->spinlock);
2799         list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
2800                 list_del(&ch->list);
2801                 atomic_set(&ch->state, RDMA_CHANNEL_DISCONNECTING);
2802                 spin_unlock_irq(&sdev->spinlock);
2803                 scst_unregister_session(ch->scst_sess, true,
2804                                         srpt_release_channel);
2805                 spin_lock_irq(&sdev->spinlock);
2806         }
2807         spin_unlock_irq(&sdev->spinlock);
2808
2809         scst_tgt_set_tgt_priv(scst_tgt, NULL);
2810
2811         TRACE_EXIT();
2812
2813         return 0;
2814 }
2815
2816 /*
2817  * Entry point for ib_srpt's kernel thread. This kernel thread is only created
2818  * when the module parameter 'thread' is not zero (the default is zero).
2819  * This thread processes the ioctx list srpt_thread.thread_ioctx_list.
2820  *
2821  * @pre thread != 0
2822  */
2823 static int srpt_ioctx_thread(void *arg)
2824 {
2825         struct srpt_ioctx *ioctx;
2826
2827         /* Hibernation / freezing of the SRPT kernel thread is not supported. */
2828         current->flags |= PF_NOFREEZE;
2829
2830         spin_lock_irq(&srpt_thread.thread_lock);
2831         while (!kthread_should_stop()) {
2832                 wait_queue_t wait;
2833                 init_waitqueue_entry(&wait, current);
2834
2835                 if (!srpt_test_ioctx_list()) {
2836                         add_wait_queue_exclusive(&ioctx_list_waitQ, &wait);
2837
2838                         for (;;) {
2839                                 set_current_state(TASK_INTERRUPTIBLE);
2840                                 if (srpt_test_ioctx_list())
2841                                         break;
2842                                 spin_unlock_irq(&srpt_thread.thread_lock);
2843                                 schedule();
2844                                 spin_lock_irq(&srpt_thread.thread_lock);
2845                         }
2846                         set_current_state(TASK_RUNNING);
2847                         remove_wait_queue(&ioctx_list_waitQ, &wait);
2848                 }
2849
2850                 while (!list_empty(&srpt_thread.thread_ioctx_list)) {
2851                         ioctx = list_entry(srpt_thread.thread_ioctx_list.next,
2852                                            struct srpt_ioctx, comp_list);
2853
2854                         list_del(&ioctx->comp_list);
2855
2856                         spin_unlock_irq(&srpt_thread.thread_lock);
2857                         switch (ioctx->op) {
2858                         case IB_WC_SEND:
2859                                 srpt_handle_send_comp(ioctx->ch, ioctx,
2860                                         SCST_CONTEXT_DIRECT);
2861                                 break;
2862                         case IB_WC_RDMA_WRITE:
2863                         case IB_WC_RDMA_READ:
2864                                 srpt_handle_rdma_comp(ioctx->ch, ioctx);
2865                                 break;
2866                         case IB_WC_RECV:
2867                                 srpt_handle_new_iu(ioctx->ch, ioctx);
2868                                 break;
2869                         default:
2870                                 PRINT_ERROR("received unrecognized WC opcode"
2871                                             " %d", ioctx->op);
2872                                 break;
2873                         }
2874 #if defined(CONFIG_SCST_DEBUG)
2875                         if (thread_processing_delay_in_us
2876                             <= MAX_UDELAY_MS * 1000)
2877                                 udelay(thread_processing_delay_in_us);
2878 #endif
2879                         spin_lock_irq(&srpt_thread.thread_lock);
2880                 }
2881         }
2882         spin_unlock_irq(&srpt_thread.thread_lock);
2883
2884         return 0;
2885 }
2886
2887 /* SCST target template for the SRP target implementation. */
2888 static struct scst_tgt_template srpt_template = {
2889         .name = DRV_NAME,
2890         .sg_tablesize = SRPT_DEF_SG_TABLESIZE,
2891         .xmit_response_atomic = 1,
2892         .rdy_to_xfer_atomic = 1,
2893         .detect = srpt_detect,
2894         .release = srpt_release,
2895         .xmit_response = srpt_xmit_response,
2896         .rdy_to_xfer = srpt_rdy_to_xfer,
2897         .on_free_cmd = srpt_on_free_cmd,
2898         .task_mgmt_fn_done = srpt_tsk_mgmt_done
2899 };
2900
2901 /*
2902  * The callback function srpt_release_class_dev() is called whenever a
2903  * device is removed from the /sys/class/infiniband_srpt device class.
2904  * Although this function has been left empty, a release function has been
2905  * defined such that upon module removal no complaint is logged about a
2906  * missing release function.
2907  */
2908 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2909 static void srpt_release_class_dev(struct class_device *class_dev)
2910 #else
2911 static void srpt_release_class_dev(struct device *dev)
2912 #endif
2913 {
2914 }
2915
2916 #ifdef CONFIG_SCST_PROC
2917
2918 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2919 static int srpt_trace_level_show(struct seq_file *seq, void *v)
2920 {
2921         return scst_proc_log_entry_read(seq, trace_flag, NULL);
2922 }
2923
2924 static ssize_t srpt_proc_trace_level_write(struct file *file,
2925         const char __user *buf, size_t length, loff_t *off)
2926 {
2927         return scst_proc_log_entry_write(file, buf, length, &trace_flag,
2928                 DEFAULT_SRPT_TRACE_FLAGS, NULL);
2929 }
2930
2931 static struct scst_proc_data srpt_log_proc_data = {
2932         SCST_DEF_RW_SEQ_OP(srpt_proc_trace_level_write)
2933         .show = srpt_trace_level_show,
2934 };
2935 #endif
2936
2937 #endif /* CONFIG_SCST_PROC */
2938
2939 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2940 static ssize_t show_login_info(struct class_device *class_dev, char *buf)
2941 #else
2942 static ssize_t show_login_info(struct device *dev,
2943                                struct device_attribute *attr, char *buf)
2944 #endif
2945 {
2946         struct srpt_device *sdev =
2947 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2948                 container_of(class_dev, struct srpt_device, class_dev);
2949 #else
2950                 container_of(dev, struct srpt_device, dev);
2951 #endif
2952         struct srpt_port *sport;
2953         int i;
2954         int len = 0;
2955
2956         for (i = 0; i < sdev->device->phys_port_cnt; i++) {
2957                 sport = &sdev->port[i];
2958
2959                 len += sprintf(buf + len,
2960                                "tid_ext=%016llx,ioc_guid=%016llx,pkey=ffff,"
2961                                "dgid=%04x%04x%04x%04x%04x%04x%04x%04x,"
2962                                "service_id=%016llx\n",
2963                                (unsigned long long) srpt_service_guid,
2964                                (unsigned long long) srpt_service_guid,
2965                                be16_to_cpu(((__be16 *) sport->gid.raw)[0]),
2966                                be16_to_cpu(((__be16 *) sport->gid.raw)[1]),
2967                                be16_to_cpu(((__be16 *) sport->gid.raw)[2]),
2968                                be16_to_cpu(((__be16 *) sport->gid.raw)[3]),
2969                                be16_to_cpu(((__be16 *) sport->gid.raw)[4]),
2970                                be16_to_cpu(((__be16 *) sport->gid.raw)[5]),
2971                                be16_to_cpu(((__be16 *) sport->gid.raw)[6]),
2972                                be16_to_cpu(((__be16 *) sport->gid.raw)[7]),
2973                                (unsigned long long) srpt_service_guid);
2974         }
2975
2976         return len;
2977 }
2978
2979 static struct class_attribute srpt_class_attrs[] = {
2980         __ATTR_NULL,
2981 };
2982
2983 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2984 static struct class_device_attribute srpt_dev_attrs[] = {
2985 #else
2986 static struct device_attribute srpt_dev_attrs[] = {
2987 #endif
2988         __ATTR(login_info, S_IRUGO, show_login_info, NULL),
2989         __ATTR_NULL,
2990 };
2991
2992 static struct class srpt_class = {
2993         .name        = "infiniband_srpt",
2994 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2995         .release = srpt_release_class_dev,
2996 #else
2997         .dev_release = srpt_release_class_dev,
2998 #endif
2999         .class_attrs = srpt_class_attrs,
3000 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
3001         .class_dev_attrs = srpt_dev_attrs,
3002 #else
3003         .dev_attrs   = srpt_dev_attrs,
3004 #endif
3005 };
3006
3007 /*
3008  * Callback function called by the InfiniBand core when either an InfiniBand
3009  * device has been added or during the ib_register_client() call for each
3010  * registered InfiniBand device.
3011  */
3012 static void srpt_add_one(struct ib_device *device)
3013 {
3014         struct srpt_device *sdev;
3015         struct srpt_port *sport;
3016         struct ib_srq_init_attr srq_attr;
3017         int i;
3018
3019         TRACE_ENTRY();
3020
3021         TRACE_DBG("device = %p, device->dma_ops = %p", device, device->dma_ops);
3022
3023         sdev = kzalloc(sizeof *sdev, GFP_KERNEL);
3024         if (!sdev)
3025                 return;
3026
3027         sdev->device = device;
3028
3029 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
3030         sdev->class_dev.class = &srpt_class;
3031         sdev->class_dev.dev = device->dma_device;
3032         snprintf(sdev->class_dev.class_id, BUS_ID_SIZE,
3033                  "srpt-%s", device->name);
3034 #else
3035         sdev->dev.class = &srpt_class;
3036         sdev->dev.parent = device->dma_device;
3037 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30)
3038         snprintf(sdev->dev.bus_id, BUS_ID_SIZE, "srpt-%s", device->name);
3039 #else
3040         dev_set_name(&sdev->dev, "srpt-%s", device->name);
3041 #endif
3042 #endif
3043
3044 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
3045         if (class_device_register(&sdev->class_dev))
3046                 goto free_dev;
3047 #else
3048         if (device_register(&sdev->dev))
3049                 goto free_dev;
3050 #endif
3051
3052         if (ib_query_device(device, &sdev->dev_attr))
3053                 goto err_dev;
3054
3055         sdev->pd = ib_alloc_pd(device);
3056         if (IS_ERR(sdev->pd))
3057                 goto err_dev;
3058
3059         sdev->mr = ib_get_dma_mr(sdev->pd, IB_ACCESS_LOCAL_WRITE);
3060         if (IS_ERR(sdev->mr))
3061                 goto err_pd;
3062
3063         srq_attr.event_handler = srpt_srq_event;
3064         srq_attr.srq_context = (void *)sdev;
3065         srq_attr.attr.max_wr = min(SRPT_SRQ_SIZE, sdev->dev_attr.max_srq_wr);
3066         srq_attr.attr.max_sge = 1;
3067         srq_attr.attr.srq_limit = 0;
3068
3069         sdev->srq = ib_create_srq(sdev->pd, &srq_attr);
3070         if (IS_ERR(sdev->srq))
3071                 goto err_mr;
3072
3073         TRACE_DBG("%s: create SRQ #wr= %d max_allow=%d dev= %s",
3074                __func__, srq_attr.attr.max_wr,
3075               sdev->dev_attr.max_srq_wr, device->name);
3076
3077         if (!srpt_service_guid)
3078                 srpt_service_guid = be64_to_cpu(device->node_guid);
3079
3080         sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);
3081         if (IS_ERR(sdev->cm_id))
3082                 goto err_srq;
3083
3084         /* print out target login information */
3085         TRACE_DBG("Target login info: id_ext=%016llx,"
3086                   "ioc_guid=%016llx,pkey=ffff,service_id=%016llx",
3087                   (unsigned long long) srpt_service_guid,
3088                   (unsigned long long) srpt_service_guid,
3089                   (unsigned long long) srpt_service_guid);
3090
3091         /*
3092          * We do not have a consistent service_id (ie. also id_ext of target_id)
3093          * to identify this target. We currently use the guid of the first HCA
3094          * in the system as service_id; therefore, the target_id will change
3095          * if this HCA is gone bad and replaced by different HCA
3096          */
3097         if (ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0, NULL))
3098                 goto err_cm;
3099
3100         INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
3101                               srpt_event_handler);
3102         if (ib_register_event_handler(&sdev->event_handler))
3103                 goto err_cm;
3104
3105         if (srpt_alloc_ioctx_ring(sdev, sdev->ioctx_ring,
3106                                   ARRAY_SIZE(sdev->ioctx_ring), 0))
3107                 goto err_event;
3108
3109         INIT_LIST_HEAD(&sdev->rch_list);
3110         spin_lock_init(&sdev->spinlock);
3111
3112         for (i = 0; i < SRPT_SRQ_SIZE; ++i)
3113                 srpt_post_recv(sdev, sdev->ioctx_ring[i]);
3114
3115         ib_set_client_data(device, &srpt_client, sdev);
3116
3117         sdev->scst_tgt = scst_register(&srpt_template, NULL);
3118         if (!sdev->scst_tgt) {
3119                 PRINT_ERROR("SCST registration failed for %s.",
3120                             sdev->device->name);
3121                 goto err_ring;
3122         }
3123
3124         scst_tgt_set_tgt_priv(sdev->scst_tgt, sdev);
3125
3126         WARN_ON(sdev->device->phys_port_cnt
3127                 > sizeof(sdev->port)/sizeof(sdev->port[0]));
3128
3129         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
3130                 sport = &sdev->port[i - 1];
3131                 sport->sdev = sdev;
3132                 sport->port = i;
3133 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
3134                 /*
3135                  * A vanilla 2.6.19 or older kernel without backported OFED
3136                  * kernel headers.
3137                  */
3138                 INIT_WORK(&sport->work, srpt_refresh_port_work, sport);
3139 #else
3140                 INIT_WORK(&sport->work, srpt_refresh_port_work);
3141 #endif
3142                 if (srpt_refresh_port(sport)) {
3143                         PRINT_ERROR("MAD registration failed for %s-%d.",
3144                                     sdev->device->name, i);
3145                         goto err_refresh_port;
3146                 }
3147         }
3148
3149         atomic_inc(&srpt_device_count);
3150
3151         TRACE_EXIT();
3152
3153         return;
3154
3155 err_refresh_port:
3156         scst_unregister(sdev->scst_tgt);
3157 err_ring:
3158         ib_set_client_data(device, &srpt_client, NULL);
3159         srpt_free_ioctx_ring(sdev, sdev->ioctx_ring,
3160                              ARRAY_SIZE(sdev->ioctx_ring));
3161 err_event:
3162         ib_unregister_event_handler(&sdev->event_handler);
3163 err_cm:
3164         ib_destroy_cm_id(sdev->cm_id);
3165 err_srq:
3166         ib_destroy_srq(sdev->srq);
3167 err_mr:
3168         ib_dereg_mr(sdev->mr);
3169 err_pd:
3170         ib_dealloc_pd(sdev->pd);
3171 err_dev:
3172 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
3173         class_device_unregister(&sdev->class_dev);
3174 #else
3175         device_unregister(&sdev->dev);
3176 #endif
3177 free_dev:
3178         kfree(sdev);
3179
3180         TRACE_EXIT();
3181 }
3182
3183 /*
3184  * Callback function called by the InfiniBand core when either an InfiniBand
3185  * device has been removed or during the ib_unregister_client() call for each
3186  * registered InfiniBand device.
3187  */
3188 static void srpt_remove_one(struct ib_device *device)
3189 {
3190         int i;
3191         struct srpt_device *sdev;
3192
3193         TRACE_ENTRY();
3194
3195         sdev = ib_get_client_data(device, &srpt_client);
3196 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
3197         WARN_ON(!sdev);
3198         if (!sdev)
3199                 return;
3200 #else
3201         if (WARN_ON(!sdev))
3202                 return;
3203 #endif
3204
3205         srpt_unregister_mad_agent(sdev);
3206
3207         ib_unregister_event_handler(&sdev->event_handler);
3208
3209         /* Cancel any work queued by the just unregistered IB event handler. */
3210         for (i = 0; i < sdev->device->phys_port_cnt; i++)
3211 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
3212                 cancel_work_sync(&sdev->port[i].work);
3213 #else
3214                 /*
3215                  * cancel_work_sync() was introduced in kernel 2.6.22. Older
3216                  * kernels do not have a facility to cancel scheduled work, so
3217                  * wait until the scheduled work finished.
3218                  */
3219                 flush_workqueue(&sdev->port[i].work);
3220 #endif
3221
3222         ib_destroy_cm_id(sdev->cm_id);
3223         ib_destroy_srq(sdev->srq);
3224         ib_dereg_mr(sdev->mr);
3225         ib_dealloc_pd(sdev->pd);
3226
3227 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
3228         class_device_unregister(&sdev->class_dev);
3229 #else
3230         device_unregister(&sdev->dev);
3231 #endif
3232
3233         /*
3234          * Unregistering an SCST target must happen after destroying sdev->cm_id
3235          * such that no new SRP_LOGIN_REQ information units can arrive while
3236          * destroying the SCST target.
3237          */
3238         scst_unregister(sdev->scst_tgt);
3239         sdev->scst_tgt = NULL;
3240
3241         srpt_free_ioctx_ring(sdev, sdev->ioctx_ring,
3242                              ARRAY_SIZE(sdev->ioctx_ring));
3243         kfree(sdev);
3244
3245         TRACE_EXIT();
3246 }
3247
3248 #ifdef CONFIG_SCST_PROC
3249
3250 /**
3251  * Create procfs entries for srpt. Currently the only procfs entry created
3252  * by this function is the "trace_level" entry.
3253  */
3254 static int srpt_register_procfs_entry(struct scst_tgt_template *tgt)
3255 {
3256         int res = 0;
3257 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
3258         struct proc_dir_entry *p, *root;
3259
3260         root = scst_proc_get_tgt_root(tgt);
3261         WARN_ON(!root);
3262         if (root) {
3263                 /*
3264                  * Fill in the scst_proc_data::data pointer, which is used in
3265                  * a printk(KERN_INFO ...) statement in
3266                  * scst_proc_log_entry_write() in scst_proc.c.
3267                  */
3268                 srpt_log_proc_data.data = (char *)tgt->name;
3269                 p = scst_create_proc_entry(root, SRPT_PROC_TRACE_LEVEL_NAME,
3270                                            &srpt_log_proc_data);
3271                 if (!p)
3272                         res = -ENOMEM;
3273         } else
3274                 res = -ENOMEM;
3275
3276 #endif
3277         return res;
3278 }
3279
3280 static void srpt_unregister_procfs_entry(struct scst_tgt_template *tgt)
3281 {
3282 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
3283         struct proc_dir_entry *root;
3284
3285         root = scst_proc_get_tgt_root(tgt);
3286         WARN_ON(!root);
3287         if (root)
3288                 remove_proc_entry(SRPT_PROC_TRACE_LEVEL_NAME, root);
3289 #endif
3290 }
3291
3292 #endif /*CONFIG_SCST_PROC*/
3293
3294 /*
3295  * Module initialization.
3296  *
3297  * Note: since ib_register_client() registers callback functions, and since at
3298  * least one of these callback functions (srpt_add_one()) calls SCST functions,
3299  * the SCST target template must be registered before ib_register_client() is
3300  * called.
3301  */
3302 static int __init srpt_init_module(void)
3303 {
3304         int ret;
3305
3306         ret = -EINVAL;
3307         if (srp_max_message_size < MIN_MAX_MESSAGE_SIZE) {
3308                 PRINT_ERROR("invalid value %d for kernel module parameter"
3309                             " srp_max_message_size -- must be at least %d.",
3310                             srp_max_message_size,
3311                             MIN_MAX_MESSAGE_SIZE);
3312                 goto out;
3313         }
3314
3315         ret = class_register(&srpt_class);
3316         if (ret) {
3317                 PRINT_ERROR("%s", "couldn't register class ib_srpt");
3318                 goto out;
3319         }
3320
3321         ret = scst_register_target_template(&srpt_template);
3322         if (ret < 0) {
3323                 PRINT_ERROR("%s", "couldn't register with scst");
3324                 ret = -ENODEV;
3325                 goto out_unregister_class;
3326         }
3327
3328 #ifdef CONFIG_SCST_PROC
3329         ret = srpt_register_procfs_entry(&srpt_template);
3330         if (ret) {
3331                 PRINT_ERROR("%s", "couldn't register procfs entry");
3332                 goto out_unregister_target;
3333         }
3334 #endif /*CONFIG_SCST_PROC*/
3335
3336         ret = ib_register_client(&srpt_client);
3337         if (ret) {
3338                 PRINT_ERROR("%s", "couldn't register IB client");
3339                 goto out_unregister_target;
3340         }
3341
3342         if (thread) {
3343                 spin_lock_init(&srpt_thread.thread_lock);
3344                 INIT_LIST_HEAD(&srpt_thread.thread_ioctx_list);
3345                 srpt_thread.thread = kthread_run(srpt_ioctx_thread,
3346                                                  NULL, "srpt_thread");
3347                 if (IS_ERR(srpt_thread.thread)) {
3348                         srpt_thread.thread = NULL;
3349                         thread = 0;
3350                 }
3351         }
3352
3353         return 0;
3354
3355 out_unregister_target:
3356 #ifdef CONFIG_SCST_PROC
3357         /*
3358          * Note: the procfs entry is unregistered in srpt_release(), which is
3359          * called by scst_unregister_target_template().
3360          */
3361 #endif /*CONFIG_SCST_PROC*/
3362         scst_unregister_target_template(&srpt_template);
3363 out_unregister_class:
3364         class_unregister(&srpt_class);
3365 out:
3366         return ret;
3367 }
3368
3369 static void __exit srpt_cleanup_module(void)
3370 {
3371         TRACE_ENTRY();
3372
3373         ib_unregister_client(&srpt_client);
3374         scst_unregister_target_template(&srpt_template);
3375         if (srpt_thread.thread)
3376                 kthread_stop(srpt_thread.thread);
3377         class_unregister(&srpt_class);
3378
3379         TRACE_EXIT();
3380 }
3381
3382 module_init(srpt_init_module);
3383 module_exit(srpt_cleanup_module);