Added more comments.
[mirror/scst/.git] / srpt / src / ib_srpt.c
1 /*
2  * Copyright (c) 2006 - 2009 Mellanox Technology Inc.  All rights reserved.
3  * Copyright (C) 2008 Vladislav Bolkhovitin <vst@vlnb.net>
4  * Copyright (C) 2008 - 2009 Bart Van Assche <bart.vanassche@gmail.com>
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  */
35
36 #include <linux/module.h>
37 #include <linux/init.h>
38 #include <linux/slab.h>
39 #include <linux/err.h>
40 #include <linux/string.h>
41 #include <linux/kthread.h>
42
43 #include <asm/atomic.h>
44
45 #include "ib_srpt.h"
46
47 /* Name of this kernel module. */
48 #define DRV_NAME                "ib_srpt"
49 /* Prefix for printk() kernel messages. */
50 #define PFX                     DRV_NAME ": "
51 #define DRV_VERSION             "1.0.1"
52 #define DRV_RELDATE             "July 10, 2008"
53
54 #define MELLANOX_SRPT_ID_STRING "Mellanox OFED SRP target"
55
56 MODULE_AUTHOR("Vu Pham");
57 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
58                    "v" DRV_VERSION " (" DRV_RELDATE ")");
59 MODULE_LICENSE("Dual BSD/GPL");
60
61 struct srpt_thread {
62         /* Protects thread_ioctx_list. */
63         spinlock_t thread_lock;
64         /* I/O contexts to be processed by the kernel thread. */
65         struct list_head thread_ioctx_list;
66         /* SRPT kernel thread. */
67         struct task_struct *thread;
68 };
69
70 static u64 mellanox_ioc_guid;
71 /* List of srpt_device structures. */
72 static struct list_head srpt_devices;
73 static int thread;
74 static struct srpt_thread srpt_thread;
75 static DECLARE_WAIT_QUEUE_HEAD(ioctx_list_waitQ);
76
77 module_param(thread, int, 0444);
78 MODULE_PARM_DESC(thread,
79                  "Executing ioctx in thread context. Default 0, i.e. soft IRQ, "
80                  "where possible");
81
82 static void srpt_add_one(struct ib_device *device);
83 static void srpt_remove_one(struct ib_device *device);
84 static int srpt_disconnect_channel(struct srpt_rdma_ch *ch, int dreq);
85
86 static struct ib_client srpt_client = {
87         .name = DRV_NAME,
88         .add = srpt_add_one,
89         .remove = srpt_remove_one
90 };
91
92 /*
93  * Callback function called by the InfiniBand core when an asynchronous IB
94  * event occurs. This callback may occur in interrupt context. See also
95  * section 11.5.2, Set Asynchronous Event Handler in the InfiniBand
96  * Architecture Specification.
97  */
98 static void srpt_event_handler(struct ib_event_handler *handler,
99                                struct ib_event *event)
100 {
101         struct srpt_device *sdev =
102             ib_get_client_data(event->device, &srpt_client);
103         struct srpt_port *sport;
104
105         if (!sdev || sdev->device != event->device)
106                 return;
107
108         printk(KERN_WARNING PFX "ASYNC event= %d on device= %s\n",
109                 event->event, sdev->device->name);
110
111         switch (event->event) {
112         case IB_EVENT_PORT_ERR:
113                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
114                         sport = &sdev->port[event->element.port_num - 1];
115                         sport->lid = 0;
116                         sport->sm_lid = 0;
117                 }
118                 break;
119         case IB_EVENT_PORT_ACTIVE:
120         case IB_EVENT_LID_CHANGE:
121         case IB_EVENT_PKEY_CHANGE:
122         case IB_EVENT_SM_CHANGE:
123         case IB_EVENT_CLIENT_REREGISTER:
124                 if (event->element.port_num <= sdev->device->phys_port_cnt) {
125                         sport = &sdev->port[event->element.port_num - 1];
126                         if (!sport->lid && !sport->sm_lid)
127                                 schedule_work(&sport->work);
128                 }
129                 break;
130         default:
131                 break;
132         }
133
134 }
135
136 /*
137  * Callback function called by the InfiniBand core for SRQ (shared receive
138  * queue) events.
139  */
140 static void srpt_srq_event(struct ib_event *event, void *ctx)
141 {
142         printk(KERN_WARNING PFX "SRQ event %d\n", event->event);
143 }
144
145 /*
146  * Callback function called by the InfiniBand core for QP (queue pair) events.
147  */
148 static void srpt_qp_event(struct ib_event *event, void *ctx)
149 {
150         struct srpt_rdma_ch *ch = ctx;
151
152         printk(KERN_WARNING PFX
153                "QP event %d on cm_id=%p sess_name=%s state=%d\n",
154                event->event, ch->cm_id, ch->sess_name, ch->state);
155
156         switch (event->event) {
157         case IB_EVENT_COMM_EST:
158                 ib_cm_notify(ch->cm_id, event->event);
159                 break;
160         case IB_EVENT_QP_LAST_WQE_REACHED:
161                 if (ch->state == RDMA_CHANNEL_LIVE) {
162                         printk(KERN_WARNING PFX
163                                "Schedule CM_DISCONNECT_WORK\n");
164                         srpt_disconnect_channel(ch, 1);
165                 }
166                 break;
167         default:
168                 break;
169         }
170 }
171
172 /*
173  * Helper function for filling in an InfiniBand IOUnitInfo structure. Copies
174  * the lowest four bits of value in element slot of the array of four bit
175  * elements called c_list (controller list). The index slot is one-based.
176  *
177  * @pre 1 <= slot && 0 <= value && value < 16
178  */
179 static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
180 {
181         u16 id;
182         u8 tmp;
183
184         id = (slot - 1) / 2;
185         if (slot & 0x1) {
186                 tmp = c_list[id] & 0xf;
187                 c_list[id] = (value << 4) | tmp;
188         } else {
189                 tmp = c_list[id] & 0xf0;
190                 c_list[id] = (value & 0xf) | tmp;
191         }
192 }
193
194 /*
195  * Write InfiniBand ClassPortInfo to mad. See also section 16.3.3.1
196  * ClassPortInfo in the InfiniBand Architecture Specification.
197  */
198 static void srpt_get_class_port_info(struct ib_dm_mad *mad)
199 {
200         struct ib_class_port_info *cif;
201
202         cif = (struct ib_class_port_info *)mad->data;
203         memset(cif, 0, sizeof *cif);
204         cif->base_version = 1;
205         cif->class_version = 1;
206         cif->resp_time_value = 20;
207
208         mad->mad_hdr.status = 0;
209 }
210
211 /*
212  * Write IOUnitInfo to mad. See also section 16.3.3.3 IOUnitInfo in the
213  * InfiniBand Architecture Specification. See also section B.7,
214  * table B.6 in the T10 SRP r16a document.
215  */
216 static void srpt_get_iou(struct ib_dm_mad *mad)
217 {
218         struct ib_dm_iou_info *ioui;
219         u8 slot;
220         int i;
221
222         ioui = (struct ib_dm_iou_info *)mad->data;
223         ioui->change_id = 1;
224         ioui->max_controllers = 16;
225
226         /* set present for slot 1 and empty for the rest */
227         srpt_set_ioc(ioui->controller_list, 1, 1);
228         for (i = 1, slot = 2; i < 16; i++, slot++)
229                 srpt_set_ioc(ioui->controller_list, slot, 0);
230
231         mad->mad_hdr.status = 0;
232 }
233
234 /*
235  * Write IOControllerprofile to mad for I/O controller (sdev, slot). See also
236  * section 16.3.3.4 IOControllerProfile in the InfiniBand Architecture
237  * Specification. See also section B.7, table B.7 in the T10 SRP r16a
238  * document.
239  */
240 static void srpt_get_ioc(struct srpt_device *sdev, u32 slot,
241                          struct ib_dm_mad *mad)
242 {
243         struct ib_dm_ioc_profile *iocp;
244
245         iocp = (struct ib_dm_ioc_profile *)mad->data;
246
247         if (!slot || slot > 16) {
248                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
249                 return;
250         }
251
252         if (slot > 2) {
253                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
254                 return;
255         }
256
257         memset(iocp, 0, sizeof *iocp);
258         strcpy(iocp->id_string, MELLANOX_SRPT_ID_STRING);
259         iocp->guid = cpu_to_be64(mellanox_ioc_guid);
260         iocp->vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
261         iocp->device_id = cpu_to_be32(sdev->dev_attr.vendor_part_id);
262         iocp->device_version = cpu_to_be16(sdev->dev_attr.hw_ver);
263         iocp->subsys_vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
264         iocp->subsys_device_id = 0x0;
265         iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
266         iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
267         iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
268         iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
269         iocp->send_queue_depth = cpu_to_be16(SRPT_SRQ_SIZE);
270         iocp->rdma_read_depth = 4;
271         iocp->send_size = cpu_to_be32(MAX_MESSAGE_SIZE);
272         iocp->rdma_size = cpu_to_be32(MAX_RDMA_SIZE);
273         iocp->num_svc_entries = 1;
274         iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
275             SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
276
277         mad->mad_hdr.status = 0;
278 }
279
280 /*
281  * Device management: write ServiceEntries to mad for the given slot. See also
282  * section 16.3.3.5 ServiceEntries in the InfiniBand Architecture
283  * Specification. See also section B.7, table B.8 in the T10 SRP r16a document.
284  */
285 static void srpt_get_svc_entries(u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
286 {
287         struct ib_dm_svc_entries *svc_entries;
288
289         if (!slot || slot > 16) {
290                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
291                 return;
292         }
293
294         if (slot > 2 || lo > hi || hi > 1) {
295                 mad->mad_hdr.status = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
296                 return;
297         }
298
299         svc_entries = (struct ib_dm_svc_entries *)mad->data;
300         memset(svc_entries, 0, sizeof *svc_entries);
301         svc_entries->service_entries[0].id = cpu_to_be64(mellanox_ioc_guid);
302         sprintf(svc_entries->service_entries[0].name, "%s%016llx",
303                 SRP_SERVICE_NAME_PREFIX, (unsigned long long)mellanox_ioc_guid);
304
305         mad->mad_hdr.status = 0;
306 }
307
308 /*
309  * Actual processing of a received MAD *rq_mad received through source port *sp
310  * (MAD = InfiniBand management datagram). The response to be sent back is
311  * written to *rsp_mad.
312  */
313 static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
314                                  struct ib_dm_mad *rsp_mad)
315 {
316         u16 attr_id;
317         u32 slot;
318         u8 hi, lo;
319
320         attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
321         switch (attr_id) {
322         case DM_ATTR_CLASS_PORT_INFO:
323                 srpt_get_class_port_info(rsp_mad);
324                 break;
325         case DM_ATTR_IOU_INFO:
326                 srpt_get_iou(rsp_mad);
327                 break;
328         case DM_ATTR_IOC_PROFILE:
329                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
330                 srpt_get_ioc(sp->sdev, slot, rsp_mad);
331                 break;
332         case DM_ATTR_SVC_ENTRIES:
333                 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
334                 hi = (u8) ((slot >> 8) & 0xff);
335                 lo = (u8) (slot & 0xff);
336                 slot = (u16) ((slot >> 16) & 0xffff);
337                 srpt_get_svc_entries(slot, hi, lo, rsp_mad);
338                 break;
339         default:
340                 rsp_mad->mad_hdr.status =
341                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
342                 break;
343         }
344 }
345
346 /*
347  * Callback function that is called by the InfiniBand core after transmission of
348  * a MAD. (MAD = management datagram; AH = address handle.)
349  */
350 static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
351                                   struct ib_mad_send_wc *mad_wc)
352 {
353         ib_destroy_ah(mad_wc->send_buf->ah);
354         ib_free_send_mad(mad_wc->send_buf);
355 }
356
357 /*
358  * Callback function that is called by the InfiniBand core after reception of
359  * a MAD (management datagram).
360  */
361 static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
362                                   struct ib_mad_recv_wc *mad_wc)
363 {
364         struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
365         struct ib_ah *ah;
366         struct ib_mad_send_buf *rsp;
367         struct ib_dm_mad *dm_mad;
368
369         if (!mad_wc || !mad_wc->recv_buf.mad)
370                 return;
371
372         ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
373                                   mad_wc->recv_buf.grh, mad_agent->port_num);
374         if (IS_ERR(ah))
375                 goto err;
376
377         BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR);
378
379         rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
380                                  mad_wc->wc->pkey_index, 0,
381                                  IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA,
382                                  GFP_KERNEL);
383         if (IS_ERR(rsp))
384                 goto err_rsp;
385
386         rsp->ah = ah;
387
388         dm_mad = rsp->mad;
389         memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof *dm_mad);
390         dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
391         dm_mad->mad_hdr.status = 0;
392
393         switch (mad_wc->recv_buf.mad->mad_hdr.method) {
394         case IB_MGMT_METHOD_GET:
395                 srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
396                 break;
397         case IB_MGMT_METHOD_SET:
398                 dm_mad->mad_hdr.status =
399                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
400                 break;
401         default:
402                 dm_mad->mad_hdr.status =
403                     cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
404                 break;
405         }
406
407         if (!ib_post_send_mad(rsp, NULL)) {
408                 ib_free_recv_mad(mad_wc);
409                 /* will destroy_ah & free_send_mad in send completion */
410                 return;
411         }
412
413         ib_free_send_mad(rsp);
414
415 err_rsp:
416         ib_destroy_ah(ah);
417 err:
418         ib_free_recv_mad(mad_wc);
419 }
420
421 /*
422  * Register InfiniBand management datagram callback functions for the specified
423  * port.
424  */
425 static int srpt_refresh_port(struct srpt_port *sport)
426 {
427         struct ib_mad_reg_req reg_req;
428         struct ib_port_modify port_modify;
429         struct ib_port_attr port_attr;
430         int ret;
431
432         memset(&port_modify, 0, sizeof port_modify);
433         port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
434         port_modify.clr_port_cap_mask = 0;
435
436         ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
437         if (ret)
438                 goto err_mod_port;
439
440         ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
441         if (ret)
442                 goto err_query_port;
443
444         sport->sm_lid = port_attr.sm_lid;
445         sport->lid = port_attr.lid;
446
447         ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid);
448         if (ret)
449                 goto err_query_port;
450
451         if (!sport->mad_agent) {
452                 memset(&reg_req, 0, sizeof reg_req);
453                 reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
454                 reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
455                 set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
456                 set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
457
458                 sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
459                                                          sport->port,
460                                                          IB_QPT_GSI,
461                                                          &reg_req, 0,
462                                                          srpt_mad_send_handler,
463                                                          srpt_mad_recv_handler,
464                                                          sport);
465                 if (IS_ERR(sport->mad_agent))
466                         goto err_query_port;
467         }
468
469         return 0;
470
471 err_query_port:
472
473         port_modify.set_port_cap_mask = 0;
474         port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
475         ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
476
477 err_mod_port:
478
479         return ret;
480 }
481
482 /*
483  * Allocate and initialize an SRPT I/O context structure.
484  */
485 static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev)
486 {
487         struct srpt_ioctx *ioctx;
488
489         ioctx = kmalloc(sizeof *ioctx, GFP_KERNEL);
490         if (!ioctx)
491                 goto out;
492
493         ioctx->buf = kzalloc(MAX_MESSAGE_SIZE, GFP_KERNEL);
494         if (!ioctx->buf)
495                 goto out_free_ioctx;
496
497         ioctx->dma = dma_map_single(sdev->device->dma_device, ioctx->buf,
498                                     MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
499 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
500         if (dma_mapping_error(sdev->device->dma_device, ioctx->dma))
501 #else
502         if (dma_mapping_error(ioctx->dma))
503 #endif
504                 goto out_free_buf;
505
506         return ioctx;
507
508 out_free_buf:
509         kfree(ioctx->buf);
510 out_free_ioctx:
511         kfree(ioctx);
512 out:
513         return NULL;
514 }
515
516 /*
517  * Deallocate an SRPT I/O context structure.
518  */
519 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
520 {
521         if (!ioctx)
522                 return;
523
524         dma_unmap_single(sdev->device->dma_device, ioctx->dma,
525                          MAX_MESSAGE_SIZE, DMA_BIDIRECTIONAL);
526         kfree(ioctx->buf);
527         kfree(ioctx);
528 }
529
530 /*
531  * Associate a ring of SRPT I/O context structures with the specified device.
532  */
533 static int srpt_alloc_ioctx_ring(struct srpt_device *sdev)
534 {
535         int i;
536
537         for (i = 0; i < SRPT_SRQ_SIZE; ++i) {
538                 sdev->ioctx_ring[i] = srpt_alloc_ioctx(sdev);
539
540                 if (!sdev->ioctx_ring[i])
541                         goto err;
542
543                 sdev->ioctx_ring[i]->index = i;
544         }
545
546         return 0;
547
548 err:
549         while (--i > 0) {
550                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
551                 sdev->ioctx_ring[i] = NULL;
552         }
553         return -ENOMEM;
554 }
555
556 /*
557  * Post a receive request on the work queue of InfiniBand device 'sdev'.
558  */
559 static int srpt_post_recv(struct srpt_device *sdev, struct srpt_ioctx *ioctx)
560 {
561         struct ib_sge list;
562         struct ib_recv_wr wr, *bad_wr;
563
564         wr.wr_id = ioctx->index | SRPT_OP_RECV;
565
566         list.addr = ioctx->dma;
567         list.length = MAX_MESSAGE_SIZE;
568         list.lkey = sdev->mr->lkey;
569
570         wr.next = NULL;
571         wr.sg_list = &list;
572         wr.num_sge = 1;
573
574         return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);
575 }
576
577 /*
578  * Post a send request on the SRPT RDMA channel 'ch'.
579  */
580 static int srpt_post_send(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
581                           int len)
582 {
583         struct ib_sge list;
584         struct ib_send_wr wr, *bad_wr;
585         struct srpt_device *sdev = ch->sport->sdev;
586
587         dma_sync_single_for_device(sdev->device->dma_device, ioctx->dma,
588                                    MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
589
590         list.addr = ioctx->dma;
591         list.length = len;
592         list.lkey = sdev->mr->lkey;
593
594         wr.next = NULL;
595         wr.wr_id = ioctx->index;
596         wr.sg_list = &list;
597         wr.num_sge = 1;
598         wr.opcode = IB_WR_SEND;
599         wr.send_flags = IB_SEND_SIGNALED;
600
601         return ib_post_send(ch->qp, &wr, &bad_wr);
602 }
603
604 static int srpt_get_desc_tbl(struct srpt_ioctx *ioctx, struct srp_cmd *srp_cmd,
605                              int *ind)
606 {
607         struct srp_indirect_buf *idb;
608         struct srp_direct_buf *db;
609
610         *ind = 0;
611         if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
612             ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
613                 ioctx->n_rbuf = 1;
614                 ioctx->rbufs = &ioctx->single_rbuf;
615
616                 db = (void *)srp_cmd->add_data;
617                 memcpy(ioctx->rbufs, db, sizeof *db);
618                 ioctx->data_len = be32_to_cpu(db->len);
619         } else {
620                 idb = (void *)srp_cmd->add_data;
621
622                 ioctx->n_rbuf = be32_to_cpu(idb->table_desc.len) / sizeof *db;
623
624                 if (ioctx->n_rbuf >
625                     (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
626                         *ind = 1;
627                         ioctx->n_rbuf = 0;
628                         goto out;
629                 }
630
631                 if (ioctx->n_rbuf == 1)
632                         ioctx->rbufs = &ioctx->single_rbuf;
633                 else
634                         ioctx->rbufs =
635                                 kmalloc(ioctx->n_rbuf * sizeof *db, GFP_ATOMIC);
636                 if (!ioctx->rbufs) {
637                         ioctx->n_rbuf = 0;
638                         return -ENOMEM;
639                 }
640
641                 db = idb->desc_list;
642                 memcpy(ioctx->rbufs, db, ioctx->n_rbuf * sizeof *db);
643                 ioctx->data_len = be32_to_cpu(idb->len);
644         }
645 out:
646         return 0;
647 }
648
649 /*
650  * Modify the attributes of queue pair 'qp': allow local write, remote read,
651  * and remote write. Also transition 'qp' to state IB_QPS_INIT.
652  */
653 static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
654 {
655         struct ib_qp_attr *attr;
656         int ret;
657
658         attr = kzalloc(sizeof *attr, GFP_KERNEL);
659         if (!attr)
660                 return -ENOMEM;
661
662         attr->qp_state = IB_QPS_INIT;
663         attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ |
664             IB_ACCESS_REMOTE_WRITE;
665         attr->port_num = ch->sport->port;
666         attr->pkey_index = 0;
667
668         ret = ib_modify_qp(qp, attr,
669                            IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
670                            IB_QP_PKEY_INDEX);
671
672         kfree(attr);
673         return ret;
674 }
675
676 static int srpt_ch_qp_rtr_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp,
677                               enum ib_qp_state qp_state)
678 {
679         struct ib_qp_attr *qp_attr;
680         int attr_mask;
681         int ret;
682
683         qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
684         if (!qp_attr)
685                 return -ENOMEM;
686
687         qp_attr->qp_state = qp_state;
688         ret = ib_cm_init_qp_attr(ch->cm_id, qp_attr, &attr_mask);
689         if (ret)
690                 goto out;
691
692         if (qp_state == IB_QPS_RTR)
693                 qp_attr->max_dest_rd_atomic = 4;
694         else
695                 qp_attr->max_rd_atomic = 4;
696
697         ret = ib_modify_qp(qp, qp_attr, attr_mask);
698
699 out:
700         kfree(qp_attr);
701         return ret;
702 }
703
704 static void srpt_reset_ioctx(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
705 {
706         int i;
707
708         if (ioctx->n_rdma_ius > 0 && ioctx->rdma_ius) {
709                 struct rdma_iu *riu = ioctx->rdma_ius;
710
711                 for (i = 0; i < ioctx->n_rdma_ius; ++i, ++riu)
712                         kfree(riu->sge);
713                 kfree(ioctx->rdma_ius);
714         }
715
716         if (ioctx->n_rbuf > 1)
717                 kfree(ioctx->rbufs);
718
719         if (srpt_post_recv(ch->sport->sdev, ioctx))
720                 printk(KERN_ERR PFX "SRQ post_recv failed - this is serious\n");
721                 /* we should queue it back to free_ioctx queue */
722         else
723                 atomic_inc(&ch->req_lim_delta);
724 }
725
726 static void srpt_handle_err_comp(struct srpt_rdma_ch *ch, struct ib_wc *wc)
727 {
728         struct srpt_ioctx *ioctx;
729         struct srpt_device *sdev = ch->sport->sdev;
730         scst_data_direction dir = SCST_DATA_NONE;
731
732         if (wc->wr_id & SRPT_OP_RECV) {
733                 ioctx = sdev->ioctx_ring[wc->wr_id & ~SRPT_OP_RECV];
734                 printk(KERN_ERR PFX "This is serious - SRQ is in bad state\n");
735         } else {
736                 ioctx = sdev->ioctx_ring[wc->wr_id];
737
738                 if (ioctx->scmnd) {
739                         struct scst_cmd *scmnd = ioctx->scmnd;
740
741                         dir = scst_cmd_get_data_direction(scmnd);
742
743                         if (dir == SCST_DATA_NONE)
744                                 scst_tgt_cmd_done(scmnd,
745                                         scst_estimate_context());
746                         else {
747                                 dma_unmap_sg(sdev->device->dma_device,
748                                              scst_cmd_get_sg(scmnd),
749                                              scst_cmd_get_sg_cnt(scmnd),
750                                              scst_to_tgt_dma_dir(dir));
751
752                                 if (scmnd->state == SCST_CMD_STATE_DATA_WAIT)
753                                         scst_rx_data(scmnd,
754                                                      SCST_RX_STATUS_ERROR,
755                                                      SCST_CONTEXT_THREAD);
756                                 else if (scmnd->state == SCST_CMD_STATE_XMIT_WAIT)
757                                         scst_tgt_cmd_done(scmnd,
758                                                 scst_estimate_context());
759                         }
760                 } else
761                         srpt_reset_ioctx(ch, ioctx);
762         }
763 }
764
765 static void srpt_handle_send_comp(struct srpt_rdma_ch *ch,
766                                   struct srpt_ioctx *ioctx,
767                                   enum scst_exec_context context)
768 {
769         if (ioctx->scmnd) {
770                 scst_data_direction dir =
771                         scst_cmd_get_data_direction(ioctx->scmnd);
772
773                 if (dir != SCST_DATA_NONE)
774                         dma_unmap_sg(ch->sport->sdev->device->dma_device,
775                                      scst_cmd_get_sg(ioctx->scmnd),
776                                      scst_cmd_get_sg_cnt(ioctx->scmnd),
777                                      scst_to_tgt_dma_dir(dir));
778
779                 scst_tgt_cmd_done(ioctx->scmnd, context);
780         } else
781                 srpt_reset_ioctx(ch, ioctx);
782 }
783
784 static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch,
785                                   struct srpt_ioctx *ioctx)
786 {
787         if (!ioctx->scmnd) {
788                 srpt_reset_ioctx(ch, ioctx);
789                 return;
790         }
791
792         if (scst_cmd_get_data_direction(ioctx->scmnd) == SCST_DATA_WRITE)
793                 scst_rx_data(ioctx->scmnd, SCST_RX_STATUS_SUCCESS,
794                         scst_estimate_context());
795 }
796
797 static void srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
798                                struct srpt_ioctx *ioctx, u8 s_key, u8 s_code,
799                                u64 tag)
800 {
801         struct srp_rsp *srp_rsp;
802         struct sense_data *sense;
803         int limit_delta;
804
805         srp_rsp = ioctx->buf;
806         memset(srp_rsp, 0, sizeof *srp_rsp);
807
808         limit_delta = atomic_read(&ch->req_lim_delta);
809         atomic_sub(limit_delta, &ch->req_lim_delta);
810
811         srp_rsp->opcode = SRP_RSP;
812         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
813         srp_rsp->tag = tag;
814
815         if (s_key != NO_SENSE) {
816                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
817                 srp_rsp->status = SAM_STAT_CHECK_CONDITION;
818                 srp_rsp->sense_data_len =
819                     cpu_to_be32(sizeof *sense + (sizeof *sense % 4));
820
821                 sense = (struct sense_data *)(srp_rsp + 1);
822                 sense->err_code = 0x70;
823                 sense->key = s_key;
824                 sense->asc_ascq = s_code;
825         }
826 }
827
828 static void srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
829                                    struct srpt_ioctx *ioctx, u8 rsp_code,
830                                    u64 tag)
831 {
832         struct srp_rsp *srp_rsp;
833         int limit_delta;
834
835         dma_sync_single_for_cpu(ch->sport->sdev->device->dma_device, ioctx->dma,
836                                 MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
837
838         srp_rsp = ioctx->buf;
839         memset(srp_rsp, 0, sizeof *srp_rsp);
840
841         limit_delta = atomic_read(&ch->req_lim_delta);
842         atomic_sub(limit_delta, &ch->req_lim_delta);
843
844         srp_rsp->opcode = SRP_RSP;
845         srp_rsp->req_lim_delta = cpu_to_be32(limit_delta);
846         srp_rsp->tag = tag;
847
848         if (rsp_code != SRP_TSK_MGMT_SUCCESS) {
849                 srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
850                 srp_rsp->resp_data_len = cpu_to_be32(4);
851                 srp_rsp->data[3] = rsp_code;
852         }
853 }
854
855 /*
856  * Process SRP_CMD.
857  */
858 static int srpt_handle_cmd(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx)
859 {
860         struct scst_cmd *scmnd = NULL;
861         struct srp_cmd *srp_cmd = NULL;
862         scst_data_direction dir = SCST_DATA_NONE;
863         int indirect_desc = 0;
864         int ret;
865
866         srp_cmd = ioctx->buf;
867
868         if (srp_cmd->buf_fmt) {
869                 ret = srpt_get_desc_tbl(ioctx, srp_cmd, &indirect_desc);
870                 if (ret) {
871                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
872                                            NO_ADD_SENSE, srp_cmd->tag);
873                         ((struct srp_rsp *)ioctx->buf)->status =
874                                         SAM_STAT_TASK_SET_FULL;
875                         goto send_rsp;
876                 }
877
878                 if (indirect_desc) {
879                         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
880                                            NO_ADD_SENSE, srp_cmd->tag);
881                         ((struct srp_rsp *)ioctx->buf)->status =
882                                         SAM_STAT_TASK_SET_FULL;
883                         goto send_rsp;
884                 }
885
886                 if (srp_cmd->buf_fmt & 0xf)
887                         dir = SCST_DATA_READ;
888                 else if (srp_cmd->buf_fmt >> 4)
889                         dir = SCST_DATA_WRITE;
890                 else
891                         dir = SCST_DATA_NONE;
892         } else
893                 dir = SCST_DATA_NONE;
894
895         scmnd = scst_rx_cmd(ch->scst_sess, (u8 *) &srp_cmd->lun,
896                             sizeof srp_cmd->lun, srp_cmd->cdb, 16,
897                             thread ? SCST_NON_ATOMIC : SCST_ATOMIC);
898         if (!scmnd) {
899                 srpt_build_cmd_rsp(ch, ioctx, NO_SENSE,
900                                    NO_ADD_SENSE, srp_cmd->tag);
901                 ((struct srp_rsp *)ioctx->buf)->status =
902                         SAM_STAT_TASK_SET_FULL;
903                 goto send_rsp;
904         }
905
906         ioctx->scmnd = scmnd;
907
908         switch (srp_cmd->task_attr) {
909         case SRP_CMD_HEAD_OF_Q:
910                 scmnd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
911                 break;
912         case SRP_CMD_ORDERED_Q:
913                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
914                 break;
915         case SRP_CMD_SIMPLE_Q:
916                 scmnd->queue_type = SCST_CMD_QUEUE_SIMPLE;
917                 break;
918         case SRP_CMD_ACA:
919                 scmnd->queue_type = SCST_CMD_QUEUE_ACA;
920                 break;
921         default:
922                 scmnd->queue_type = SCST_CMD_QUEUE_ORDERED;
923                 break;
924         }
925
926         scst_cmd_set_tag(scmnd, srp_cmd->tag);
927         scst_cmd_set_tgt_priv(scmnd, ioctx);
928         scst_cmd_set_expected(scmnd, dir, ioctx->data_len);
929
930         spin_lock_irq(&ch->spinlock);
931         list_add_tail(&ioctx->scmnd_list, &ch->active_scmnd_list);
932         ch->active_scmnd_cnt++;
933         spin_unlock_irq(&ch->spinlock);
934
935         scst_cmd_init_done(scmnd, scst_estimate_context());
936
937         return 0;
938
939 send_rsp:
940         return -1;
941 }
942
943 /*
944  * Process SRP_TSK_MGMT. See also table 19 in the T10 SRP r16a document.
945  */
946 static int srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
947                                 struct srpt_ioctx *ioctx)
948 {
949         struct srp_tsk_mgmt *srp_tsk = NULL;
950         struct srpt_mgmt_ioctx *mgmt_ioctx;
951         int ret;
952
953         srp_tsk = ioctx->buf;
954
955         printk(KERN_WARNING PFX
956                "recv_tsk_mgmt= %d for task_tag= %lld"
957                " using tag= %lld cm_id= %p sess= %p\n",
958                srp_tsk->tsk_mgmt_func,
959                (unsigned long long) srp_tsk->task_tag,
960                (unsigned long long) srp_tsk->tag,
961                ch->cm_id, ch->scst_sess);
962
963         mgmt_ioctx = kmalloc(sizeof *mgmt_ioctx, GFP_ATOMIC);
964         if (!mgmt_ioctx) {
965                 srpt_build_tskmgmt_rsp(ch, ioctx, SRP_TSK_MGMT_FAILED,
966                                        srp_tsk->tag);
967                 goto send_rsp;
968         }
969
970         mgmt_ioctx->ioctx = ioctx;
971         mgmt_ioctx->ch = ch;
972         mgmt_ioctx->tag = srp_tsk->tag;
973
974         switch (srp_tsk->tsk_mgmt_func) {
975         case SRP_TSK_ABORT_TASK:
976                 ret = scst_rx_mgmt_fn_tag(ch->scst_sess,
977                                           SCST_ABORT_TASK,
978                                           srp_tsk->task_tag,
979                                           thread ?
980                                           SCST_NON_ATOMIC : SCST_ATOMIC,
981                                           mgmt_ioctx);
982                 break;
983         case SRP_TSK_ABORT_TASK_SET:
984                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
985                                           SCST_ABORT_TASK_SET,
986                                           (u8 *) &srp_tsk->lun,
987                                           sizeof srp_tsk->lun,
988                                           thread ?
989                                           SCST_NON_ATOMIC : SCST_ATOMIC,
990                                           mgmt_ioctx);
991                 break;
992         case SRP_TSK_CLEAR_TASK_SET:
993                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
994                                           SCST_CLEAR_TASK_SET,
995                                           (u8 *) &srp_tsk->lun,
996                                           sizeof srp_tsk->lun,
997                                           thread ?
998                                           SCST_NON_ATOMIC : SCST_ATOMIC,
999                                           mgmt_ioctx);
1000                 break;
1001 #if 0
1002         case SRP_TSK_LUN_RESET:
1003                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1004                                           SCST_LUN_RESET,
1005                                           (u8 *) &srp_tsk->lun,
1006                                           sizeof srp_tsk->lun,
1007                                           thread ?
1008                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1009                                           mgmt_ioctx);
1010                 break;
1011 #endif
1012         case SRP_TSK_CLEAR_ACA:
1013                 ret = scst_rx_mgmt_fn_lun(ch->scst_sess,
1014                                           SCST_CLEAR_ACA,
1015                                           (u8 *) &srp_tsk->lun,
1016                                           sizeof srp_tsk->lun,
1017                                           thread ?
1018                                           SCST_NON_ATOMIC : SCST_ATOMIC,
1019                                           mgmt_ioctx);
1020                 break;
1021         default:
1022                 srpt_build_tskmgmt_rsp(ch, ioctx,
1023                                        SRP_TSK_MGMT_FUNC_NOT_SUPP,
1024                                        srp_tsk->tag);
1025                 goto send_rsp;
1026         }
1027         return 0;
1028
1029 send_rsp:
1030         return -1;
1031 }
1032
1033 static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
1034                                struct srpt_ioctx *ioctx)
1035 {
1036         u8 op;
1037
1038         if (ch->state != RDMA_CHANNEL_LIVE) {
1039                 if (ch->state == RDMA_CHANNEL_CONNECTING) {
1040                         spin_lock_irq(&ch->spinlock);
1041                         list_add_tail(&ioctx->wait_list, &ch->cmd_wait_list);
1042                         spin_unlock_irq(&ch->spinlock);
1043                 } else
1044                         srpt_reset_ioctx(ch, ioctx);
1045
1046                 return;
1047         }
1048
1049         dma_sync_single_for_cpu(ch->sport->sdev->device->dma_device, ioctx->dma,
1050                                 MAX_MESSAGE_SIZE, DMA_FROM_DEVICE);
1051
1052         ioctx->data_len = 0;
1053         ioctx->n_rbuf = 0;
1054         ioctx->rbufs = NULL;
1055         ioctx->n_rdma = 0;
1056         ioctx->n_rdma_ius = 0;
1057         ioctx->rdma_ius = NULL;
1058         ioctx->scmnd = NULL;
1059
1060         op = *(u8 *) ioctx->buf;
1061         switch (op) {
1062         case SRP_CMD:
1063                 if (srpt_handle_cmd(ch, ioctx) < 0)
1064                         goto send_rsp;
1065                 break;
1066
1067         case SRP_TSK_MGMT:
1068                 if (srpt_handle_tsk_mgmt(ch, ioctx) < 0)
1069                         goto send_rsp;
1070                 break;
1071
1072         case SRP_I_LOGOUT:
1073         case SRP_AER_REQ:
1074         default:
1075                 srpt_build_cmd_rsp(ch, ioctx, ILLEGAL_REQUEST, INVALID_CDB,
1076                                    ((struct srp_cmd *)ioctx->buf)->tag);
1077
1078                 goto send_rsp;
1079         }
1080
1081         dma_sync_single_for_device(ch->sport->sdev->device->dma_device,
1082                                    ioctx->dma, MAX_MESSAGE_SIZE,
1083                                    DMA_FROM_DEVICE);
1084
1085         return;
1086
1087 send_rsp:
1088         if (ch->state != RDMA_CHANNEL_LIVE ||
1089             srpt_post_send(ch, ioctx,
1090                            sizeof(struct srp_rsp) +
1091                            be32_to_cpu(((struct srp_rsp *)ioctx->buf)->
1092                                        sense_data_len)))
1093                 srpt_reset_ioctx(ch, ioctx);
1094 }
1095
1096 /*
1097  * Returns true if the ioctx list is non-empty or if the ib_srpt kernel thread
1098  * should stop.
1099  * @pre thread != 0
1100  */
1101 static inline int srpt_test_ioctx_list(void)
1102 {
1103         int res = (!list_empty(&srpt_thread.thread_ioctx_list) ||
1104                    unlikely(kthread_should_stop()));
1105         return res;
1106 }
1107
1108 /*
1109  * Add 'ioctx' to the tail of the ioctx list and wake up the kernel thread.
1110  *
1111  * @pre thread != 0
1112  */
1113 static inline void srpt_schedule_thread(struct srpt_ioctx *ioctx)
1114 {
1115         unsigned long flags;
1116
1117         spin_lock_irqsave(&srpt_thread.thread_lock, flags);
1118         list_add_tail(&ioctx->comp_list, &srpt_thread.thread_ioctx_list);
1119         spin_unlock_irqrestore(&srpt_thread.thread_lock, flags);
1120         wake_up(&ioctx_list_waitQ);
1121 }
1122
1123 /*
1124  * InfiniBand CQ (completion queue) event handler for asynchronous events not
1125  * associated with a completion.
1126  */
1127 static void srpt_completion(struct ib_cq *cq, void *ctx)
1128 {
1129         struct srpt_rdma_ch *ch = ctx;
1130         struct srpt_device *sdev = ch->sport->sdev;
1131         struct ib_wc wc;
1132         struct srpt_ioctx *ioctx;
1133
1134         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1135         while (ib_poll_cq(ch->cq, 1, &wc) > 0) {
1136                 if (wc.status) {
1137                         printk(KERN_ERR PFX "failed %s status= %d\n",
1138                                wc.wr_id & SRPT_OP_RECV ? "receive" : "send",
1139                                wc.status);
1140                         srpt_handle_err_comp(ch, &wc);
1141                         break;
1142                 }
1143
1144                 if (wc.wr_id & SRPT_OP_RECV) {
1145                         ioctx = sdev->ioctx_ring[wc.wr_id & ~SRPT_OP_RECV];
1146                         if (thread) {
1147                                 ioctx->ch = ch;
1148                                 ioctx->op = IB_WC_RECV;
1149                                 srpt_schedule_thread(ioctx);
1150                         } else
1151                                 srpt_handle_new_iu(ch, ioctx);
1152                         continue;
1153                 } else
1154                         ioctx = sdev->ioctx_ring[wc.wr_id];
1155
1156                 if (thread) {
1157                         ioctx->ch = ch;
1158                         ioctx->op = wc.opcode;
1159                         srpt_schedule_thread(ioctx);
1160                 } else {
1161                         switch (wc.opcode) {
1162                         case IB_WC_SEND:
1163                                 srpt_handle_send_comp(ch, ioctx,
1164                                         scst_estimate_context());
1165                                 break;
1166                         case IB_WC_RDMA_WRITE:
1167                         case IB_WC_RDMA_READ:
1168                                 srpt_handle_rdma_comp(ch, ioctx);
1169                                 break;
1170                         default:
1171                                 break;
1172                         }
1173                 }
1174         }
1175 }
1176
1177 /*
1178  * Create a completion queue on the specified device.
1179  */
1180 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
1181 {
1182         struct ib_qp_init_attr *qp_init;
1183         struct srpt_device *sdev = ch->sport->sdev;
1184         int cqe;
1185         int ret;
1186
1187         qp_init = kzalloc(sizeof *qp_init, GFP_KERNEL);
1188         if (!qp_init)
1189                 return -ENOMEM;
1190
1191         /* Create a completion queue (CQ). */
1192
1193         cqe = SRPT_RQ_SIZE + SRPT_SQ_SIZE - 1;
1194 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(RHEL_RELEASE_CODE)
1195         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe);
1196 #else
1197         ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, cqe, 0);
1198 #endif
1199         if (IS_ERR(ch->cq)) {
1200                 ret = PTR_ERR(ch->cq);
1201                 printk(KERN_ERR PFX "failed to create_cq cqe= %d ret= %d\n",
1202                         cqe, ret);
1203                 goto out;
1204         }
1205
1206         /* Request completion notification. */
1207
1208         ib_req_notify_cq(ch->cq, IB_CQ_NEXT_COMP);
1209
1210         /* Create a queue pair (QP). */
1211
1212         qp_init->qp_context = (void *)ch;
1213         qp_init->event_handler = srpt_qp_event;
1214         qp_init->send_cq = ch->cq;
1215         qp_init->recv_cq = ch->cq;
1216         qp_init->srq = sdev->srq;
1217         qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
1218         qp_init->qp_type = IB_QPT_RC;
1219         qp_init->cap.max_send_wr = SRPT_SQ_SIZE;
1220         qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE;
1221
1222         ch->qp = ib_create_qp(sdev->pd, qp_init);
1223         if (IS_ERR(ch->qp)) {
1224                 ret = PTR_ERR(ch->qp);
1225                 ib_destroy_cq(ch->cq);
1226                 printk(KERN_ERR PFX "failed to create_qp ret= %d\n", ret);
1227                 goto out;
1228         }
1229
1230         printk(KERN_DEBUG PFX "%s: max_cqe= %d max_sge= %d cm_id= %p\n",
1231                __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
1232                ch->cm_id);
1233
1234         /* Modify the attributes and the state of queue pair ch->qp. */
1235
1236         ret = srpt_init_ch_qp(ch, ch->qp);
1237         if (ret) {
1238                 ib_destroy_qp(ch->qp);
1239                 ib_destroy_cq(ch->cq);
1240                 goto out;
1241         }
1242
1243         atomic_set(&ch->req_lim_delta, SRPT_RQ_SIZE);
1244 out:
1245         kfree(qp_init);
1246         return ret;
1247 }
1248
1249 static struct srpt_rdma_ch *srpt_find_channel(struct ib_cm_id *cm_id)
1250 {
1251         struct srpt_device *sdev = cm_id->context;
1252         struct srpt_rdma_ch *ch, *tmp_ch;
1253
1254         spin_lock_irq(&sdev->spinlock);
1255         list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
1256                 if (ch->cm_id == cm_id) {
1257                         spin_unlock_irq(&sdev->spinlock);
1258                         return ch;
1259                 }
1260         }
1261
1262         spin_unlock_irq(&sdev->spinlock);
1263
1264         return NULL;
1265 }
1266
1267 static int srpt_release_channel(struct srpt_rdma_ch *ch, int destroy_cmid)
1268 {
1269         spin_lock_irq(&ch->sport->sdev->spinlock);
1270         list_del(&ch->list);
1271         spin_unlock_irq(&ch->sport->sdev->spinlock);
1272
1273         if (ch->cm_id && destroy_cmid) {
1274                 printk(KERN_WARNING PFX
1275                        "%s: destroy cm_id= %p\n", __func__, ch->cm_id);
1276                 ib_destroy_cm_id(ch->cm_id);
1277                 ch->cm_id = NULL;
1278         }
1279
1280         ib_destroy_qp(ch->qp);
1281         ib_destroy_cq(ch->cq);
1282
1283         if (ch->scst_sess) {
1284                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1285
1286                 printk(KERN_WARNING PFX
1287                        "%s: release sess= %p sess_name= %s active_cmd= %d\n",
1288                        __func__, ch->scst_sess, ch->sess_name,
1289                        ch->active_scmnd_cnt);
1290
1291                 list_for_each_entry_safe(ioctx, ioctx_tmp,
1292                                          &ch->active_scmnd_list, scmnd_list) {
1293                         list_del(&ioctx->scmnd_list);
1294                         ch->active_scmnd_cnt--;
1295                 }
1296
1297                 scst_unregister_session(ch->scst_sess, 0, NULL);
1298                 ch->scst_sess = NULL;
1299         }
1300
1301         kfree(ch);
1302
1303         return destroy_cmid ? 0 : 1;
1304 }
1305
1306 static int srpt_disconnect_channel(struct srpt_rdma_ch *ch, int dreq)
1307 {
1308         spin_lock_irq(&ch->spinlock);
1309         ch->state = RDMA_CHANNEL_DISCONNECTING;
1310         spin_unlock_irq(&ch->spinlock);
1311
1312         if (dreq)
1313                 ib_send_cm_dreq(ch->cm_id, NULL, 0);
1314         else
1315                 ib_send_cm_drep(ch->cm_id, NULL, 0);
1316
1317         return 0;
1318 }
1319
1320 static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
1321                             struct ib_cm_req_event_param *param,
1322                             void *private_data)
1323 {
1324         struct srpt_device *sdev = cm_id->context;
1325         struct srp_login_req *req;
1326         struct srp_login_rsp *rsp;
1327         struct srp_login_rej *rej;
1328         struct ib_cm_rep_param *rep_param;
1329         struct srpt_rdma_ch *ch, *tmp_ch;
1330         u32 it_iu_len;
1331         int ret = 0;
1332
1333         if (!sdev || !private_data)
1334                 return -EINVAL;
1335
1336         rsp = kzalloc(sizeof *rsp, GFP_KERNEL);
1337         rej = kzalloc(sizeof *rej, GFP_KERNEL);
1338         rep_param = kzalloc(sizeof *rep_param, GFP_KERNEL);
1339
1340         if (!rsp || !rej || !rep_param) {
1341                 ret = -ENOMEM;
1342                 goto out;
1343         }
1344
1345         req = (struct srp_login_req *)private_data;
1346
1347         it_iu_len = be32_to_cpu(req->req_it_iu_len);
1348
1349         printk(KERN_DEBUG PFX
1350                "Host login i_port_id=0x%llx:0x%llx t_port_id=0x%llx:0x%llx"
1351                " it_iu_len=%d\n",
1352                (unsigned long long)
1353                be64_to_cpu(*(u64 *)&req->initiator_port_id[0]),
1354                (unsigned long long)
1355                be64_to_cpu(*(u64 *)&req->initiator_port_id[8]),
1356                (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[0]),
1357                (unsigned long long)be64_to_cpu(*(u64 *)&req->target_port_id[8]),
1358                it_iu_len);
1359
1360         if (it_iu_len > MAX_MESSAGE_SIZE || it_iu_len < 64) {
1361                 rej->reason =
1362                     cpu_to_be32(SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
1363                 ret = -EINVAL;
1364                 printk(KERN_WARNING PFX
1365                        "Reject invalid it_iu_len=%d\n", it_iu_len);
1366                 goto reject;
1367         }
1368
1369         if ((req->req_flags & 0x3) == SRP_MULTICHAN_SINGLE) {
1370                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
1371
1372                 spin_lock_irq(&sdev->spinlock);
1373
1374                 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
1375                         if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
1376                             && !memcmp(ch->t_port_id, req->target_port_id, 16)
1377                             && param->port == ch->sport->port
1378                             && param->listen_id == ch->sport->sdev->cm_id
1379                             && ch->cm_id) {
1380                                 /* found an existing channel */
1381                                 printk(KERN_WARNING PFX
1382                                        "Found existing channel name= %s"
1383                                        " cm_id= %p state= %d\n",
1384                                        ch->sess_name, ch->cm_id, ch->state);
1385
1386                                 spin_unlock_irq(&sdev->spinlock);
1387
1388                                 rsp->rsp_flags =
1389                                     SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
1390
1391                                 if (ch->state == RDMA_CHANNEL_LIVE)
1392                                         srpt_disconnect_channel(ch, 1);
1393                                 else if (ch->state == RDMA_CHANNEL_CONNECTING) {
1394                                         ib_send_cm_rej(ch->cm_id,
1395                                                        IB_CM_REJ_NO_RESOURCES,
1396                                                        NULL, 0, NULL, 0);
1397                                         srpt_release_channel(ch, 1);
1398                                 }
1399
1400                                 spin_lock_irq(&sdev->spinlock);
1401                         }
1402                 }
1403
1404                 spin_unlock_irq(&sdev->spinlock);
1405
1406         } else
1407                 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
1408
1409         if (((u64) (*(u64 *) req->target_port_id) !=
1410              cpu_to_be64(mellanox_ioc_guid)) ||
1411             ((u64) (*(u64 *) (req->target_port_id + 8)) !=
1412              cpu_to_be64(mellanox_ioc_guid))) {
1413                 rej->reason =
1414                     cpu_to_be32(SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
1415                 ret = -ENOMEM;
1416                 printk(KERN_WARNING PFX "Reject invalid target_port_id\n");
1417                 goto reject;
1418         }
1419
1420         ch = kzalloc(sizeof *ch, GFP_KERNEL);
1421         if (!ch) {
1422                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1423                 printk(KERN_WARNING PFX "Reject failed allocate rdma_ch\n");
1424                 ret = -ENOMEM;
1425                 goto reject;
1426         }
1427
1428         spin_lock_init(&ch->spinlock);
1429         memcpy(ch->i_port_id, req->initiator_port_id, 16);
1430         memcpy(ch->t_port_id, req->target_port_id, 16);
1431         ch->sport = &sdev->port[param->port - 1];
1432         ch->cm_id = cm_id;
1433         ch->state = RDMA_CHANNEL_CONNECTING;
1434         INIT_LIST_HEAD(&ch->cmd_wait_list);
1435         INIT_LIST_HEAD(&ch->active_scmnd_list);
1436
1437         ret = srpt_create_ch_ib(ch);
1438         if (ret) {
1439                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1440                 printk(KERN_WARNING PFX "Reject failed to create rdma_ch\n");
1441                 goto free_ch;
1442         }
1443
1444         ret = srpt_ch_qp_rtr_rts(ch, ch->qp, IB_QPS_RTR);
1445         if (ret) {
1446                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1447                 printk(KERN_WARNING PFX
1448                        "Reject failed qp to rtr/rts ret=%d\n", ret);
1449                 goto destroy_ib;
1450         }
1451
1452         sprintf(ch->sess_name, "0x%016llx%016llx",
1453                 (unsigned long long)be64_to_cpu(*(u64 *)ch->i_port_id),
1454                 (unsigned long long)be64_to_cpu(*(u64 *)(ch->i_port_id + 8)));
1455
1456         ch->scst_sess = scst_register_session(sdev->scst_tgt, 0, ch->sess_name,
1457                                   NULL, NULL);
1458         if (!ch->scst_sess) {
1459                 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1460                 printk(KERN_WARNING PFX "Failed to create scst sess\n");
1461                 goto destroy_ib;
1462         }
1463
1464         spin_lock_irq(&sdev->spinlock);
1465         list_add_tail(&ch->list, &sdev->rch_list);
1466         spin_unlock_irq(&sdev->spinlock);
1467
1468         printk(KERN_DEBUG PFX "Establish connection sess=%p name=%s cm_id=%p\n",
1469                ch->scst_sess, ch->sess_name, ch->cm_id);
1470
1471         scst_sess_set_tgt_priv(ch->scst_sess, ch);
1472
1473         /* create srp_login_response */
1474         rsp->opcode = SRP_LOGIN_RSP;
1475         rsp->tag = req->tag;
1476         rsp->max_it_iu_len = req->req_it_iu_len;
1477         rsp->max_ti_iu_len = req->req_it_iu_len;
1478         rsp->buf_fmt =
1479             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1480         rsp->req_lim_delta = cpu_to_be32(SRPT_RQ_SIZE);
1481         atomic_set(&ch->req_lim_delta, 0);
1482
1483         /* create cm reply */
1484         rep_param->qp_num = ch->qp->qp_num;
1485         rep_param->private_data = (void *)rsp;
1486         rep_param->private_data_len = sizeof *rsp;
1487         rep_param->rnr_retry_count = 7;
1488         rep_param->flow_control = 1;
1489         rep_param->failover_accepted = 0;
1490         rep_param->srq = 1;
1491         rep_param->responder_resources = 4;
1492         rep_param->initiator_depth = 4;
1493
1494         ret = ib_send_cm_rep(cm_id, rep_param);
1495         if (ret)
1496                 srpt_release_channel(ch, 0);
1497
1498         goto out;
1499
1500 destroy_ib:
1501         ib_destroy_qp(ch->qp);
1502         ib_destroy_cq(ch->cq);
1503
1504 free_ch:
1505         kfree(ch);
1506
1507 reject:
1508         rej->opcode = SRP_LOGIN_REJ;
1509         rej->tag = req->tag;
1510         rej->buf_fmt =
1511             cpu_to_be16(SRP_BUF_FORMAT_DIRECT | SRP_BUF_FORMAT_INDIRECT);
1512
1513         ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
1514                              (void *)rej, sizeof *rej);
1515
1516 out:
1517         kfree(rep_param);
1518         kfree(rsp);
1519         kfree(rej);
1520
1521         return ret;
1522 }
1523
1524 static int srpt_find_and_release_channel(struct ib_cm_id *cm_id)
1525 {
1526         struct srpt_rdma_ch *ch;
1527
1528         ch = srpt_find_channel(cm_id);
1529         if (!ch)
1530                 return -EINVAL;
1531
1532         return srpt_release_channel(ch, 0);
1533 }
1534
1535 static int srpt_cm_rej_recv(struct ib_cm_id *cm_id)
1536 {
1537         printk(KERN_DEBUG PFX "%s: cm_id=%p\n", __func__, cm_id);
1538         return srpt_find_and_release_channel(cm_id);
1539 }
1540
1541 static int srpt_cm_rtu_recv(struct ib_cm_id *cm_id)
1542 {
1543         struct srpt_rdma_ch *ch;
1544         int ret;
1545
1546         ch = srpt_find_channel(cm_id);
1547         if (!ch)
1548                 return -EINVAL;
1549
1550         if (ch->state == RDMA_CHANNEL_CONNECTING) {
1551                 struct srpt_ioctx *ioctx, *ioctx_tmp;
1552
1553                 spin_lock_irq(&ch->spinlock);
1554                 ch->state = RDMA_CHANNEL_LIVE;
1555                 spin_unlock_irq(&ch->spinlock);
1556                 ret = srpt_ch_qp_rtr_rts(ch, ch->qp, IB_QPS_RTS);
1557
1558                 list_for_each_entry_safe(ioctx, ioctx_tmp, &ch->cmd_wait_list,
1559                                          wait_list) {
1560                         list_del(&ioctx->wait_list);
1561                         srpt_handle_new_iu(ch, ioctx);
1562                 }
1563         } else if (ch->state == RDMA_CHANNEL_DISCONNECTING)
1564                 ret = -EAGAIN;
1565         else
1566                 ret = 0;
1567
1568         if (ret) {
1569                 printk(KERN_ERR PFX "cm_id=%p sess_name=%s state=%d\n",
1570                        cm_id, ch->sess_name, ch->state);
1571                 srpt_disconnect_channel(ch, 1);
1572         }
1573
1574         return ret;
1575 }
1576
1577 static int srpt_cm_timewait_exit(struct ib_cm_id *cm_id)
1578 {
1579         printk(KERN_DEBUG PFX "%s: cm_id=%p\n", __func__, cm_id);
1580         return srpt_find_and_release_channel(cm_id);
1581 }
1582
1583 static int srpt_cm_rep_error(struct ib_cm_id *cm_id)
1584 {
1585         printk(KERN_DEBUG PFX "%s: cm_id=%p\n", __func__, cm_id);
1586         return srpt_find_and_release_channel(cm_id);
1587 }
1588
1589 static int srpt_cm_dreq_recv(struct ib_cm_id *cm_id)
1590 {
1591         struct srpt_rdma_ch *ch;
1592         int ret = 0;
1593
1594         ch = srpt_find_channel(cm_id);
1595
1596         if (!ch)
1597                 return -EINVAL;
1598
1599         printk(KERN_DEBUG PFX "%s: cm_id= %p ch->state= %d\n",
1600                  __func__, cm_id, ch->state);
1601
1602         switch (ch->state) {
1603         case RDMA_CHANNEL_LIVE:
1604         case RDMA_CHANNEL_CONNECTING:
1605                 ret = srpt_disconnect_channel(ch, 0);
1606                 break;
1607         case RDMA_CHANNEL_DISCONNECTING:
1608         default:
1609                 break;
1610         }
1611
1612         return ret;
1613 }
1614
1615 static int srpt_cm_drep_recv(struct ib_cm_id *cm_id)
1616 {
1617         printk(KERN_DEBUG PFX "%s: cm_id=%p\n", __func__, cm_id);
1618         return srpt_find_and_release_channel(cm_id);
1619 }
1620
1621 static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
1622 {
1623         int ret = 0;
1624
1625         switch (event->event) {
1626         case IB_CM_REQ_RECEIVED:
1627                 ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd,
1628                                        event->private_data);
1629                 break;
1630         case IB_CM_REJ_RECEIVED:
1631                 ret = srpt_cm_rej_recv(cm_id);
1632                 break;
1633         case IB_CM_RTU_RECEIVED:
1634         case IB_CM_USER_ESTABLISHED:
1635                 ret = srpt_cm_rtu_recv(cm_id);
1636                 break;
1637         case IB_CM_DREQ_RECEIVED:
1638                 ret = srpt_cm_dreq_recv(cm_id);
1639                 break;
1640         case IB_CM_DREP_RECEIVED:
1641                 ret = srpt_cm_drep_recv(cm_id);
1642                 break;
1643         case IB_CM_TIMEWAIT_EXIT:
1644                 ret = srpt_cm_timewait_exit(cm_id);
1645                 break;
1646         case IB_CM_REP_ERROR:
1647                 ret = srpt_cm_rep_error(cm_id);
1648                 break;
1649         default:
1650                 break;
1651         }
1652
1653         return ret;
1654 }
1655
1656 static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
1657                                  struct srpt_ioctx *ioctx,
1658                                  struct scst_cmd *scmnd)
1659 {
1660         struct scatterlist *scat;
1661         scst_data_direction dir;
1662         struct rdma_iu *riu;
1663         struct srp_direct_buf *db;
1664         dma_addr_t dma_addr;
1665         struct ib_sge *sge;
1666         u64 raddr;
1667         u32 rsize;
1668         u32 tsize;
1669         u32 dma_len;
1670         int count, nrdma;
1671         int i, j, k;
1672
1673         scat = scst_cmd_get_sg(scmnd);
1674         dir = scst_cmd_get_data_direction(scmnd);
1675         count = dma_map_sg(ch->sport->sdev->device->dma_device, scat,
1676                            scst_cmd_get_sg_cnt(scmnd),
1677                            scst_to_tgt_dma_dir(dir));
1678         if (unlikely(!count))
1679                 return -EBUSY;
1680
1681         if (ioctx->rdma_ius && ioctx->n_rdma_ius)
1682                 nrdma = ioctx->n_rdma_ius;
1683         else {
1684                 nrdma = count / SRPT_DEF_SG_PER_WQE + ioctx->n_rbuf;
1685
1686                 ioctx->rdma_ius = kzalloc(nrdma * sizeof *riu,
1687                                           scst_cmd_atomic(scmnd)
1688                                           ? GFP_ATOMIC : GFP_KERNEL);
1689                 if (!ioctx->rdma_ius) {
1690                         dma_unmap_sg(ch->sport->sdev->device->dma_device,
1691                                      scat, scst_cmd_get_sg_cnt(scmnd),
1692                                      scst_to_tgt_dma_dir(dir));
1693                         return -ENOMEM;
1694                 }
1695
1696                 ioctx->n_rdma_ius = nrdma;
1697         }
1698
1699         db = ioctx->rbufs;
1700         tsize = (dir == SCST_DATA_READ) ?
1701                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
1702         dma_len = sg_dma_len(&scat[0]);
1703         riu = ioctx->rdma_ius;
1704
1705         /*
1706          * For each remote desc - calculate the #ib_sge.
1707          * If #ib_sge < SRPT_DEF_SG_PER_WQE per rdma operation then
1708          *      each remote desc rdma_iu is required a rdma wr;
1709          * else
1710          *      we need to allocate extra rdma_iu to carry extra #ib_sge in
1711          *      another rdma wr
1712          */
1713         for (i = 0, j = 0;
1714              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
1715                 rsize = be32_to_cpu(db->len);
1716                 raddr = be64_to_cpu(db->va);
1717                 riu->raddr = raddr;
1718                 riu->rkey = be32_to_cpu(db->key);
1719                 riu->sge_cnt = 0;
1720
1721                 /* calculate how many sge required for this remote_buf */
1722                 while (rsize > 0 && tsize > 0) {
1723
1724                         if (rsize >= dma_len) {
1725                                 tsize -= dma_len;
1726                                 rsize -= dma_len;
1727                                 raddr += dma_len;
1728
1729                                 if (tsize > 0) {
1730                                         ++j;
1731                                         if (j < count)
1732                                                 dma_len = sg_dma_len(&scat[j]);
1733                                 }
1734                         } else {
1735                                 tsize -= rsize;
1736                                 dma_len -= rsize;
1737                                 rsize = 0;
1738                         }
1739
1740                         ++riu->sge_cnt;
1741
1742                         if (rsize > 0 && riu->sge_cnt == SRPT_DEF_SG_PER_WQE) {
1743                                 riu->sge =
1744                                     kmalloc(riu->sge_cnt * sizeof *riu->sge,
1745                                             scst_cmd_atomic(scmnd)
1746                                             ? GFP_ATOMIC : GFP_KERNEL);
1747                                 if (!riu->sge)
1748                                         goto free_mem;
1749
1750                                 ++ioctx->n_rdma;
1751                                 ++riu;
1752                                 riu->sge_cnt = 0;
1753                                 riu->raddr = raddr;
1754                                 riu->rkey = be32_to_cpu(db->key);
1755                         }
1756                 }
1757
1758                 riu->sge = kmalloc(riu->sge_cnt * sizeof *riu->sge,
1759                                    scst_cmd_atomic(scmnd)
1760                                    ? GFP_ATOMIC : GFP_KERNEL);
1761
1762                 if (!riu->sge)
1763                         goto free_mem;
1764
1765                 ++ioctx->n_rdma;
1766         }
1767
1768         db = ioctx->rbufs;
1769         scat = scst_cmd_get_sg(scmnd);
1770         tsize = (dir == SCST_DATA_READ) ?
1771                 scst_cmd_get_resp_data_len(scmnd) : scst_cmd_get_bufflen(scmnd);
1772         riu = ioctx->rdma_ius;
1773         dma_len = sg_dma_len(&scat[0]);
1774         dma_addr = sg_dma_address(&scat[0]);
1775
1776         /* this second loop is really mapped sg_addres to rdma_iu->ib_sge */
1777         for (i = 0, j = 0;
1778              j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
1779                 rsize = be32_to_cpu(db->len);
1780                 sge = riu->sge;
1781                 k = 0;
1782
1783                 while (rsize > 0 && tsize > 0) {
1784                         sge->addr = dma_addr;
1785                         sge->lkey = ch->sport->sdev->mr->lkey;
1786
1787                         if (rsize >= dma_len) {
1788                                 sge->length =
1789                                         (tsize < dma_len) ? tsize : dma_len;
1790                                 tsize -= dma_len;
1791                                 rsize -= dma_len;
1792
1793                                 if (tsize > 0) {
1794                                         ++j;
1795                                         if (j < count) {
1796                                                 dma_len = sg_dma_len(&scat[j]);
1797                                                 dma_addr =
1798                                                     sg_dma_address(&scat[j]);
1799                                         }
1800                                 }
1801                         } else {
1802                                 sge->length = (tsize < rsize) ? tsize : rsize;
1803                                 tsize -= rsize;
1804                                 dma_len -= rsize;
1805                                 dma_addr += rsize;
1806                                 rsize = 0;
1807                         }
1808
1809                         ++k;
1810                         if (k == riu->sge_cnt && rsize > 0) {
1811                                 ++riu;
1812                                 sge = riu->sge;
1813                                 k = 0;
1814                         } else if (rsize > 0)
1815                                 ++sge;
1816                 }
1817         }
1818
1819         return 0;
1820
1821 free_mem:
1822         while (ioctx->n_rdma)
1823                 kfree(ioctx->rdma_ius[ioctx->n_rdma--].sge);
1824
1825         kfree(ioctx->rdma_ius);
1826
1827         dma_unmap_sg(ch->sport->sdev->device->dma_device,
1828                      scat, scst_cmd_get_sg_cnt(scmnd),
1829                      scst_to_tgt_dma_dir(dir));
1830
1831         return -ENOMEM;
1832 }
1833
1834 static int srpt_perform_rdmas(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
1835                               scst_data_direction dir)
1836 {
1837         struct ib_send_wr wr;
1838         struct ib_send_wr *bad_wr;
1839         struct rdma_iu *riu;
1840         int i;
1841         int ret = 0;
1842
1843         riu = ioctx->rdma_ius;
1844         memset(&wr, 0, sizeof wr);
1845
1846         for (i = 0; i < ioctx->n_rdma; ++i, ++riu) {
1847                 wr.opcode = (dir == SCST_DATA_READ) ?
1848                     IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;
1849                 wr.next = NULL;
1850                 wr.wr_id = ioctx->index;
1851                 wr.wr.rdma.remote_addr = riu->raddr;
1852                 wr.wr.rdma.rkey = riu->rkey;
1853                 wr.num_sge = riu->sge_cnt;
1854                 wr.sg_list = riu->sge;
1855
1856                 /* only get completion event for the last rdma wr */
1857                 if (i == (ioctx->n_rdma - 1) && dir == SCST_DATA_WRITE)
1858                         wr.send_flags = IB_SEND_SIGNALED;
1859
1860                 ret = ib_post_send(ch->qp, &wr, &bad_wr);
1861                 if (ret)
1862                         break;
1863         }
1864
1865         return ret;
1866 }
1867
1868 /*
1869  * Start data reception. Must not block.
1870  */
1871 static int srpt_xfer_data(struct srpt_rdma_ch *ch, struct srpt_ioctx *ioctx,
1872                           struct scst_cmd *scmnd)
1873 {
1874         int ret;
1875
1876         ret = srpt_map_sg_to_ib_sge(ch, ioctx, scmnd);
1877         if (ret) {
1878                 printk(KERN_ERR PFX "%s[%d] ret=%d\n", __func__, __LINE__, ret);
1879                 ret = SCST_TGT_RES_QUEUE_FULL;
1880                 goto out;
1881         }
1882
1883         ret = srpt_perform_rdmas(ch, ioctx, scst_cmd_get_data_direction(scmnd));
1884         if (ret) {
1885                 printk(KERN_ERR PFX "%s[%d] ret=%d\n", __func__, __LINE__, ret);
1886                 if (ret == -EAGAIN || ret == -ENOMEM)
1887                         ret = SCST_TGT_RES_QUEUE_FULL;
1888                 else
1889                         ret = SCST_TGT_RES_FATAL_ERROR;
1890                 goto out;
1891         }
1892
1893         ret = SCST_TGT_RES_SUCCESS;
1894
1895 out:
1896         return ret;
1897 }
1898
1899 /*
1900  * Called by the SCST core to inform ib_srpt that data reception should start.
1901  * Must not block.
1902  */
1903 static int srpt_rdy_to_xfer(struct scst_cmd *scmnd)
1904 {
1905         struct srpt_rdma_ch *ch;
1906         struct srpt_ioctx *ioctx;
1907
1908         ioctx = scst_cmd_get_tgt_priv(scmnd);
1909         BUG_ON(!ioctx);
1910
1911         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
1912         BUG_ON(!ch);
1913
1914         if (ch->state == RDMA_CHANNEL_DISCONNECTING)
1915                 return SCST_TGT_RES_FATAL_ERROR;
1916         else if (ch->state == RDMA_CHANNEL_CONNECTING)
1917                 return SCST_TGT_RES_QUEUE_FULL;
1918
1919         return srpt_xfer_data(ch, ioctx, scmnd);
1920 }
1921
1922 /*
1923  * Called by the SCST core. Transmits the response buffer and status held in
1924  * 'scmnd'. Must not block.
1925  */
1926 static int srpt_xmit_response(struct scst_cmd *scmnd)
1927 {
1928         struct srpt_rdma_ch *ch;
1929         struct srpt_ioctx *ioctx;
1930         struct srp_rsp *srp_rsp;
1931         u64 tag;
1932         int ret = SCST_TGT_RES_SUCCESS;
1933         int dir;
1934         int status;
1935
1936         ioctx = scst_cmd_get_tgt_priv(scmnd);
1937         BUG_ON(!ioctx);
1938
1939         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
1940         BUG_ON(!ch);
1941
1942         tag = scst_cmd_get_tag(scmnd);
1943
1944         if (ch->state != RDMA_CHANNEL_LIVE) {
1945                 printk(KERN_ERR PFX
1946                        "%s: tag= %lld channel in bad state %d\n",
1947                        __func__, (unsigned long long)tag, ch->state);
1948
1949                 if (ch->state == RDMA_CHANNEL_DISCONNECTING)
1950                         ret = SCST_TGT_RES_FATAL_ERROR;
1951                 else if (ch->state == RDMA_CHANNEL_CONNECTING)
1952                         ret = SCST_TGT_RES_QUEUE_FULL;
1953
1954                 if (unlikely(scst_cmd_aborted(scmnd)))
1955                         goto out_aborted;
1956
1957                 goto out;
1958         }
1959
1960         dma_sync_single_for_cpu(ch->sport->sdev->device->dma_device, ioctx->dma,
1961                                 MAX_MESSAGE_SIZE, DMA_TO_DEVICE);
1962
1963         srp_rsp = ioctx->buf;
1964
1965         if (unlikely(scst_cmd_aborted(scmnd))) {
1966                 printk(KERN_ERR PFX
1967                        "%s: tag= %lld already get aborted\n",
1968                        __func__, (unsigned long long)tag);
1969                 goto out_aborted;
1970         }
1971
1972         dir = scst_cmd_get_data_direction(scmnd);
1973         status = scst_cmd_get_status(scmnd) & 0xff;
1974
1975         srpt_build_cmd_rsp(ch, ioctx, NO_SENSE, NO_ADD_SENSE, tag);
1976
1977         if (SCST_SENSE_VALID(scst_cmd_get_sense_buffer(scmnd))) {
1978                 srp_rsp->sense_data_len = scst_cmd_get_sense_buffer_len(scmnd);
1979                 if (srp_rsp->sense_data_len >
1980                     (MAX_MESSAGE_SIZE - sizeof *srp_rsp))
1981                         srp_rsp->sense_data_len =
1982                             MAX_MESSAGE_SIZE - sizeof *srp_rsp;
1983
1984                 memcpy((u8 *) (srp_rsp + 1), scst_cmd_get_sense_buffer(scmnd),
1985                        srp_rsp->sense_data_len);
1986
1987                 srp_rsp->sense_data_len = cpu_to_be32(srp_rsp->sense_data_len);
1988                 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1989
1990                 if (!status)
1991                         status = SAM_STAT_CHECK_CONDITION;
1992         }
1993
1994         srp_rsp->status = status;
1995
1996         /* transfer read data if any */
1997         if (dir == SCST_DATA_READ && scst_cmd_get_resp_data_len(scmnd)) {
1998                 ret = srpt_xfer_data(ch, ioctx, scmnd);
1999                 if (ret != SCST_TGT_RES_SUCCESS) {
2000                         printk(KERN_ERR PFX
2001                                "%s: tag= %lld xfer_data failed\n",
2002                                __func__, (unsigned long long)tag);
2003                         goto out;
2004                 }
2005         }
2006
2007         if (srpt_post_send(ch, ioctx,
2008                            sizeof *srp_rsp +
2009                            be32_to_cpu(srp_rsp->sense_data_len))) {
2010                 printk(KERN_ERR PFX "%s: ch->state= %d tag= %lld\n",
2011                        __func__, ch->state,
2012                        (unsigned long long)tag);
2013                 ret = SCST_TGT_RES_FATAL_ERROR;
2014         }
2015
2016 out:
2017         return ret;
2018
2019 out_aborted:
2020         ret = SCST_TGT_RES_SUCCESS;
2021         scst_set_delivery_status(scmnd, SCST_CMD_DELIVERY_ABORTED);
2022         scst_tgt_cmd_done(scmnd, SCST_CONTEXT_SAME);
2023         goto out;
2024 }
2025
2026 /*
2027  * Called by the SCST core to inform ib_srpt that a received task management
2028  * function has been completed. Must not block.
2029  */
2030 static void srpt_tsk_mgmt_done(struct scst_mgmt_cmd *mcmnd)
2031 {
2032         struct srpt_rdma_ch *ch;
2033         struct srpt_mgmt_ioctx *mgmt_ioctx;
2034         struct srpt_ioctx *ioctx;
2035
2036         mgmt_ioctx = scst_mgmt_cmd_get_tgt_priv(mcmnd);
2037         BUG_ON(!mgmt_ioctx);
2038
2039         ch = mgmt_ioctx->ch;
2040         BUG_ON(!ch);
2041
2042         ioctx = mgmt_ioctx->ioctx;
2043         BUG_ON(!ioctx);
2044
2045         printk(KERN_WARNING PFX
2046                "%s: tsk_mgmt_done for tag= %lld status=%d\n",
2047                __func__, (unsigned long long)mgmt_ioctx->tag,
2048                scst_mgmt_cmd_get_status(mcmnd));
2049
2050         srpt_build_tskmgmt_rsp(ch, ioctx,
2051                                (scst_mgmt_cmd_get_status(mcmnd) ==
2052                                 SCST_MGMT_STATUS_SUCCESS) ?
2053                                SRP_TSK_MGMT_SUCCESS : SRP_TSK_MGMT_FAILED,
2054                                mgmt_ioctx->tag);
2055         srpt_post_send(ch, ioctx, sizeof(struct srp_rsp) + 4);
2056
2057         scst_mgmt_cmd_set_tgt_priv(mcmnd, NULL);
2058
2059         kfree(mgmt_ioctx);
2060 }
2061
2062 /*
2063  * Called by the SCST core to inform ib_srpt that the command 'scmnd' is about
2064  * to be freed. May be called in IRQ context.
2065  */
2066 static void srpt_on_free_cmd(struct scst_cmd *scmnd)
2067 {
2068         struct srpt_rdma_ch *ch;
2069         struct srpt_ioctx *ioctx;
2070
2071         ioctx = scst_cmd_get_tgt_priv(scmnd);
2072         BUG_ON(!ioctx);
2073
2074         ch = scst_sess_get_tgt_priv(scst_cmd_get_session(scmnd));
2075         BUG_ON(!ch);
2076
2077         spin_lock_irq(&ch->spinlock);
2078         list_del(&ioctx->scmnd_list);
2079         ch->active_scmnd_cnt--;
2080         spin_unlock_irq(&ch->spinlock);
2081
2082         srpt_reset_ioctx(ch, ioctx);
2083         scst_cmd_set_tgt_priv(scmnd, NULL);
2084 }
2085
2086 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20)
2087 static void srpt_refresh_port_work(void *ctx)
2088 #else
2089 static void srpt_refresh_port_work(struct work_struct *work)
2090 #endif
2091 {
2092 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20)
2093         struct srpt_port *sport = (struct srpt_port *)ctx;
2094 #else
2095         struct srpt_port *sport = container_of(work, struct srpt_port, work);
2096 #endif
2097
2098         srpt_refresh_port(sport);
2099 }
2100
2101 /*
2102  * Called by the SCST core to detect target adapters. Returns the number of
2103  * detected target adapters.
2104  */
2105 static int srpt_detect(struct scst_tgt_template *tp)
2106 {
2107         struct srpt_device *sdev;
2108         struct srpt_port *sport;
2109         int i;
2110         int count = 0;
2111
2112         list_for_each_entry(sdev, &srpt_devices, list) {
2113
2114                 sdev->scst_tgt = scst_register(tp, NULL);
2115                 if (!sdev->scst_tgt)
2116                         goto out;
2117
2118                 scst_tgt_set_tgt_priv(sdev->scst_tgt, sdev);
2119
2120                 for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
2121                         sport = &sdev->port[i - 1];
2122                         sport->sdev = sdev;
2123                         sport->port = i;
2124 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20) && ! defined(BACKPORT_LINUX_WORKQUEUE_TO_2_6_19)
2125                         INIT_WORK(&sport->work, srpt_refresh_port_work, sport);
2126 #else
2127                         INIT_WORK(&sport->work, srpt_refresh_port_work);
2128 #endif
2129
2130                         if (srpt_refresh_port(sport)) {
2131                                 scst_unregister(sdev->scst_tgt);
2132                                 goto out;
2133                         }
2134                 }
2135
2136                 ++count;
2137         }
2138 out:
2139         return count;
2140 }
2141
2142 /*
2143  * Called by the SCST core to free up the resources associated with device
2144  * 'scst_tgt'.
2145  */
2146 static int srpt_release(struct scst_tgt *scst_tgt)
2147 {
2148         struct srpt_device *sdev = scst_tgt_get_tgt_priv(scst_tgt);
2149         struct srpt_port *sport;
2150         struct srpt_rdma_ch *ch, *tmp_ch;
2151         struct ib_port_modify port_modify = {
2152                 .clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP
2153         };
2154         int i;
2155
2156         list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list)
2157             srpt_release_channel(ch, 1);
2158
2159         for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
2160                 sport = &sdev->port[i - 1];
2161                 ib_modify_port(sdev->device, sport->port, 0, &port_modify);
2162                 ib_unregister_mad_agent(sport->mad_agent);
2163         }
2164
2165         scst_tgt_set_tgt_priv(scst_tgt, NULL);
2166
2167         complete(&sdev->scst_released);
2168
2169         return 0;
2170 }
2171
2172 /*
2173  * Entry point for ib_srpt's kernel thread. This kernel thread is only created
2174  * when the module parameter 'thread' is not zero (the default is zero).
2175  * This thread processes the ioctx list srpt_thread.thread_ioctx_list.
2176  *
2177  * @pre thread != 0
2178  */
2179 static int srpt_ioctx_thread(void *arg)
2180 {
2181         struct srpt_ioctx *ioctx;
2182
2183         /* Hibernation / freezing of the SRPT kernel thread is not supported. */
2184         current->flags |= PF_NOFREEZE;
2185
2186         spin_lock_irq(&srpt_thread.thread_lock);
2187         while (!kthread_should_stop()) {
2188                 wait_queue_t wait;
2189                 init_waitqueue_entry(&wait, current);
2190
2191                 if (!srpt_test_ioctx_list()) {
2192                         add_wait_queue_exclusive(&ioctx_list_waitQ, &wait);
2193
2194                         for (;;) {
2195                                 set_current_state(TASK_INTERRUPTIBLE);
2196                                 if (srpt_test_ioctx_list())
2197                                         break;
2198                                 spin_unlock_irq(&srpt_thread.thread_lock);
2199                                 schedule();
2200                                 spin_lock_irq(&srpt_thread.thread_lock);
2201                         }
2202                         set_current_state(TASK_RUNNING);
2203                         remove_wait_queue(&ioctx_list_waitQ, &wait);
2204                 }
2205
2206                 while (!list_empty(&srpt_thread.thread_ioctx_list)) {
2207                         ioctx = list_entry(srpt_thread.thread_ioctx_list.next,
2208                                            struct srpt_ioctx, comp_list);
2209
2210                         list_del(&ioctx->comp_list);
2211
2212                         spin_unlock_irq(&srpt_thread.thread_lock);
2213                         switch (ioctx->op) {
2214                         case IB_WC_SEND:
2215                                 srpt_handle_send_comp(ioctx->ch, ioctx,
2216                                         SCST_CONTEXT_DIRECT);
2217                                 break;
2218                         case IB_WC_RDMA_WRITE:
2219                         case IB_WC_RDMA_READ:
2220                                 srpt_handle_rdma_comp(ioctx->ch, ioctx);
2221                                 break;
2222                         case IB_WC_RECV:
2223                                 srpt_handle_new_iu(ioctx->ch, ioctx);
2224                                 break;
2225                         default:
2226                                 break;
2227                         }
2228                         spin_lock_irq(&srpt_thread.thread_lock);
2229                 }
2230         }
2231         spin_unlock_irq(&srpt_thread.thread_lock);
2232
2233         return 0;
2234 }
2235
2236 /* SCST target template for the SRP target implementation. */
2237 static struct scst_tgt_template srpt_template = {
2238         .name = DRV_NAME,
2239         .sg_tablesize = SRPT_DEF_SG_TABLESIZE,
2240         .xmit_response_atomic = 1,
2241         .rdy_to_xfer_atomic = 1,
2242         .no_proc_entry = 1,
2243         .detect = srpt_detect,
2244         .release = srpt_release,
2245         .xmit_response = srpt_xmit_response,
2246         .rdy_to_xfer = srpt_rdy_to_xfer,
2247         .on_free_cmd = srpt_on_free_cmd,
2248         .task_mgmt_fn_done = srpt_tsk_mgmt_done
2249 };
2250
2251 /*
2252  * The callback function srpt_release_class_dev() is called whenever a
2253  * device is removed from the /sys/class/infiniband_srpt device class.
2254  */
2255 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2256 static void srpt_release_class_dev(struct class_device *class_dev)
2257 #else
2258 static void srpt_release_class_dev(struct device *dev)
2259 #endif
2260 {
2261 }
2262
2263 static struct class srpt_class = {
2264         .name = "infiniband_srpt",
2265 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2266         .release = srpt_release_class_dev
2267 #else
2268         .dev_release = srpt_release_class_dev
2269 #endif
2270 };
2271
2272 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2273 static ssize_t show_login_info(struct class_device *class_dev, char *buf)
2274 #else
2275 static ssize_t show_login_info(struct device *dev,
2276                                struct device_attribute *attr, char *buf)
2277 #endif
2278 {
2279         struct srpt_device *sdev =
2280 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2281                 container_of(class_dev, struct srpt_device, class_dev);
2282 #else
2283                 container_of(dev, struct srpt_device, dev);
2284 #endif
2285         struct srpt_port *sport;
2286         int i;
2287         int len = 0;
2288
2289         for (i = 0; i < sdev->device->phys_port_cnt; i++) {
2290                 sport = &sdev->port[i];
2291
2292                 len += sprintf(buf + len,
2293                                "tid_ext=%016llx,ioc_guid=%016llx,pkey=ffff,"
2294                                "dgid=%04x%04x%04x%04x%04x%04x%04x%04x,"
2295                                "service_id=%016llx\n",
2296                                (unsigned long long) mellanox_ioc_guid,
2297                                (unsigned long long) mellanox_ioc_guid,
2298                                be16_to_cpu(((__be16 *) sport->gid.raw)[0]),
2299                                be16_to_cpu(((__be16 *) sport->gid.raw)[1]),
2300                                be16_to_cpu(((__be16 *) sport->gid.raw)[2]),
2301                                be16_to_cpu(((__be16 *) sport->gid.raw)[3]),
2302                                be16_to_cpu(((__be16 *) sport->gid.raw)[4]),
2303                                be16_to_cpu(((__be16 *) sport->gid.raw)[5]),
2304                                be16_to_cpu(((__be16 *) sport->gid.raw)[6]),
2305                                be16_to_cpu(((__be16 *) sport->gid.raw)[7]),
2306                                (unsigned long long) mellanox_ioc_guid);
2307         }
2308
2309         return len;
2310 }
2311
2312 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2313 static CLASS_DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2314 #else
2315 static DEVICE_ATTR(login_info, S_IRUGO, show_login_info, NULL);
2316 #endif
2317
2318 /*
2319  * Callback function called by the InfiniBand core when either an InfiniBand
2320  * device has been added or during the ib_register_client() call for each
2321  * registered InfiniBand device.
2322  */
2323 static void srpt_add_one(struct ib_device *device)
2324 {
2325         struct srpt_device *sdev;
2326         struct ib_srq_init_attr srq_attr;
2327         int i;
2328
2329         sdev = kzalloc(sizeof *sdev, GFP_KERNEL);
2330         if (!sdev)
2331                 return;
2332
2333         sdev->device = device;
2334         init_completion(&sdev->scst_released);
2335
2336 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2337         sdev->class_dev.class = &srpt_class;
2338         sdev->class_dev.dev = device->dma_device;
2339         snprintf(sdev->class_dev.class_id, BUS_ID_SIZE,
2340                  "srpt-%s", device->name);
2341 #else
2342         sdev->dev.class = &srpt_class;
2343         sdev->dev.parent = device->dma_device;
2344         snprintf(sdev->dev.bus_id, BUS_ID_SIZE, "srpt-%s", device->name);
2345 #endif
2346
2347 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2348         if (class_device_register(&sdev->class_dev))
2349                 goto free_dev;
2350         if (class_device_create_file(&sdev->class_dev,
2351                                      &class_device_attr_login_info))
2352                 goto err_dev;
2353 #else
2354         if (device_register(&sdev->dev))
2355                 goto free_dev;
2356         if (device_create_file(&sdev->dev, &dev_attr_login_info))
2357                 goto err_dev;
2358 #endif
2359
2360         if (ib_query_device(device, &sdev->dev_attr))
2361                 goto err_dev;
2362
2363         sdev->pd = ib_alloc_pd(device);
2364         if (IS_ERR(sdev->pd))
2365                 goto err_dev;
2366
2367         sdev->mr = ib_get_dma_mr(sdev->pd, IB_ACCESS_LOCAL_WRITE);
2368         if (IS_ERR(sdev->mr))
2369                 goto err_pd;
2370
2371         srq_attr.event_handler = srpt_srq_event;
2372         srq_attr.srq_context = (void *)sdev;
2373         srq_attr.attr.max_wr = min(SRPT_SRQ_SIZE, sdev->dev_attr.max_srq_wr);
2374         srq_attr.attr.max_sge = 1;
2375         srq_attr.attr.srq_limit = 0;
2376
2377         sdev->srq = ib_create_srq(sdev->pd, &srq_attr);
2378         if (IS_ERR(sdev->srq))
2379                 goto err_mr;
2380
2381         printk(KERN_DEBUG PFX "%s: create SRQ #wr= %d max_allow=%d dev= %s\n",
2382                __func__, srq_attr.attr.max_wr,
2383               sdev->dev_attr.max_srq_wr, device->name);
2384
2385         if (!mellanox_ioc_guid)
2386                 mellanox_ioc_guid = be64_to_cpu(device->node_guid);
2387
2388         sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);
2389         if (IS_ERR(sdev->cm_id))
2390                 goto err_srq;
2391
2392         /* print out target login information */
2393         printk(KERN_DEBUG PFX "Target login info: id_ext=%016llx,"
2394                 "ioc_guid=%016llx,pkey=ffff,service_id=%016llx\n",
2395                 (unsigned long long) mellanox_ioc_guid,
2396                 (unsigned long long) mellanox_ioc_guid,
2397                 (unsigned long long) mellanox_ioc_guid);
2398
2399         /*
2400          * We do not have a consistent service_id (ie. also id_ext of target_id)
2401          * to identify this target. We currently use the guid of the first HCA
2402          * in the system as service_id; therefore, the target_id will change
2403          * if this HCA is gone bad and replaced by different HCA
2404          */
2405         if (ib_cm_listen(sdev->cm_id, cpu_to_be64(mellanox_ioc_guid), 0, NULL))
2406                 goto err_cm;
2407
2408         INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
2409                               srpt_event_handler);
2410         if (ib_register_event_handler(&sdev->event_handler))
2411                 goto err_cm;
2412
2413         if (srpt_alloc_ioctx_ring(sdev))
2414                 goto err_event;
2415
2416         INIT_LIST_HEAD(&sdev->rch_list);
2417         spin_lock_init(&sdev->spinlock);
2418
2419         for (i = 0; i < SRPT_SRQ_SIZE; ++i)
2420                 srpt_post_recv(sdev, sdev->ioctx_ring[i]);
2421
2422         list_add_tail(&sdev->list, &srpt_devices);
2423
2424         ib_set_client_data(device, &srpt_client, sdev);
2425
2426         return;
2427
2428 err_event:
2429         ib_unregister_event_handler(&sdev->event_handler);
2430 err_cm:
2431         ib_destroy_cm_id(sdev->cm_id);
2432 err_srq:
2433         ib_destroy_srq(sdev->srq);
2434 err_mr:
2435         ib_dereg_mr(sdev->mr);
2436 err_pd:
2437         ib_dealloc_pd(sdev->pd);
2438 err_dev:
2439 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2440         class_device_unregister(&sdev->class_dev);
2441 #else
2442         device_unregister(&sdev->dev);
2443 #endif
2444 free_dev:
2445         kfree(sdev);
2446 }
2447
2448 /*
2449  * Callback function called by the InfiniBand core when either an InfiniBand
2450  * device has been removed or during the ib_unregister_client() call for each
2451  * registered InfiniBand device.
2452  */
2453 static void srpt_remove_one(struct ib_device *device)
2454 {
2455         struct srpt_device *sdev;
2456         int i;
2457
2458         sdev = ib_get_client_data(device, &srpt_client);
2459         if (!sdev)
2460                 return;
2461
2462         wait_for_completion(&sdev->scst_released);
2463
2464         ib_unregister_event_handler(&sdev->event_handler);
2465         ib_destroy_cm_id(sdev->cm_id);
2466         ib_destroy_srq(sdev->srq);
2467         ib_dereg_mr(sdev->mr);
2468         ib_dealloc_pd(sdev->pd);
2469 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
2470         class_device_unregister(&sdev->class_dev);
2471 #else
2472         device_unregister(&sdev->dev);
2473 #endif
2474
2475         for (i = 0; i < SRPT_SRQ_SIZE; ++i)
2476                 srpt_free_ioctx(sdev, sdev->ioctx_ring[i]);
2477
2478         list_del(&sdev->list);
2479         kfree(sdev);
2480 }
2481
2482 static int __init srpt_init_module(void)
2483 {
2484         int ret;
2485
2486         INIT_LIST_HEAD(&srpt_devices);
2487
2488         ret = class_register(&srpt_class);
2489         if (ret) {
2490                 printk(KERN_ERR PFX "couldn't register class ib_srpt\n");
2491                 return ret;
2492         }
2493
2494         ret = ib_register_client(&srpt_client);
2495         if (ret) {
2496                 printk(KERN_ERR PFX "couldn't register IB client\n");
2497                 goto mem_out;
2498         }
2499
2500         ret = scst_register_target_template(&srpt_template);
2501         if (ret < 0) {
2502                 printk(KERN_ERR PFX "couldn't register with scst\n");
2503                 ret = -ENODEV;
2504                 goto ib_out;
2505         }
2506
2507         if (thread) {
2508                 spin_lock_init(&srpt_thread.thread_lock);
2509                 INIT_LIST_HEAD(&srpt_thread.thread_ioctx_list);
2510                 srpt_thread.thread = kthread_run(srpt_ioctx_thread,
2511                                                  NULL, "srpt_thread");
2512                 if (IS_ERR(srpt_thread.thread)) {
2513                         srpt_thread.thread = NULL;
2514                         thread = 0;
2515                 }
2516         }
2517
2518         return 0;
2519
2520 ib_out:
2521         ib_unregister_client(&srpt_client);
2522 mem_out:
2523         class_unregister(&srpt_class);
2524         return ret;
2525 }
2526
2527 static void __exit srpt_cleanup_module(void)
2528 {
2529         if (srpt_thread.thread)
2530                 kthread_stop(srpt_thread.thread);
2531         scst_unregister_target_template(&srpt_template);
2532         ib_unregister_client(&srpt_client);
2533         class_unregister(&srpt_class);
2534 }
2535
2536 module_init(srpt_init_module);
2537 module_exit(srpt_cleanup_module);