3fed97c612755fea3df5188f74904554895b1113
[mirror/scst/.git] / iscsi-scst / kernel / iscsi.c
1 /*
2  *  Copyright (C) 2002 - 2003 Ardis Technolgies <roman@ardistech.com>
3  *  Copyright (C) 2007 - 2008 Vladislav Bolkhovitin
4  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
5  *
6  *  This program is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU General Public License
8  *  as published by the Free Software Foundation, version 2
9  *  of the License.
10  *
11  *  This program is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  *  GNU General Public License for more details.
15  */
16
17 #include <linux/module.h>
18 #include <linux/hash.h>
19 #include <linux/kthread.h>
20 #include <net/tcp.h>
21 #include <scsi/scsi.h>
22
23 #include "iscsi.h"
24 #include "digest.h"
25
26 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
27 #warning "Patch put_page_callback-<kernel-version>.patch not applied on your \
28         kernel. ISCSI-SCST will run in the performance degraded mode. Refer \
29         README file for details."
30 #endif
31
32 #define ISCSI_INIT_WRITE_WAKE           0x1
33 #define ISCSI_INIT_WRITE_REMOVE_HASH    0x2
34
35 static int ctr_major;
36 static char ctr_name[] = "iscsi-scst-ctl";
37 static int iscsi_template_registered;
38
39 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
40 unsigned long iscsi_trace_flag = ISCSI_DEFAULT_LOG_FLAGS;
41 #endif
42
43 static struct kmem_cache *iscsi_cmnd_cache;
44
45 DEFINE_SPINLOCK(iscsi_rd_lock);
46 LIST_HEAD(iscsi_rd_list);
47 DECLARE_WAIT_QUEUE_HEAD(iscsi_rd_waitQ);
48
49 DEFINE_SPINLOCK(iscsi_wr_lock);
50 LIST_HEAD(iscsi_wr_list);
51 DECLARE_WAIT_QUEUE_HEAD(iscsi_wr_waitQ);
52
53 static struct page *dummy_page;
54 static struct scatterlist dummy_sg;
55
56 struct iscsi_thread_t {
57         struct task_struct *thr;
58         struct list_head threads_list_entry;
59 };
60
61 static LIST_HEAD(iscsi_threads_list);
62
63 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd);
64 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status);
65 static void cmnd_prepare_get_rejected_cmd_data(struct iscsi_cmnd *cmnd);
66 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess);
67 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd);
68
69 static inline u32 cmnd_write_size(struct iscsi_cmnd *cmnd)
70 {
71         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
72
73         if (hdr->flags & ISCSI_CMD_WRITE)
74                 return be32_to_cpu(hdr->data_length);
75         return 0;
76 }
77
78 static inline u32 cmnd_read_size(struct iscsi_cmnd *cmnd)
79 {
80         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
81
82         if (hdr->flags & ISCSI_CMD_READ) {
83                 struct iscsi_rlength_ahdr *ahdr =
84                         (struct iscsi_rlength_ahdr *)cmnd->pdu.ahs;
85
86                 if (!(hdr->flags & ISCSI_CMD_WRITE))
87                         return be32_to_cpu(hdr->data_length);
88                 if (ahdr && ahdr->ahstype == ISCSI_AHSTYPE_RLENGTH)
89                         return be32_to_cpu(ahdr->read_length);
90         }
91         return 0;
92 }
93
94 static inline void iscsi_restart_cmnd(struct iscsi_cmnd *cmnd)
95 {
96         EXTRACHECKS_BUG_ON(cmnd->data_waiting);
97
98         cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
99         scst_restart_cmd(cmnd->scst_cmd, SCST_PREPROCESS_STATUS_SUCCESS,
100                 SCST_CONTEXT_THREAD);
101 }
102
103 static inline void iscsi_restart_waiting_cmnd(struct iscsi_cmnd *cmnd)
104 {
105         /*
106          * There is no race with conn_abort(), since all functions
107          * called from single read thread
108          */
109         iscsi_extracheck_is_rd_thread(cmnd->conn);
110         cmnd->data_waiting = 0;
111
112         iscsi_restart_cmnd(cmnd);
113 }
114
115 static inline void iscsi_fail_waiting_cmnd(struct iscsi_cmnd *cmnd)
116 {
117         TRACE_MGMT_DBG("Failing data waiting cmd %p", cmnd);
118
119         /*
120          * There is no race with conn_abort(), since all functions
121          * called from single read thread
122          */
123         iscsi_extracheck_is_rd_thread(cmnd->conn);
124         cmnd->data_waiting = 0;
125
126         req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
127 }
128
129 struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn, struct iscsi_cmnd *parent)
130 {
131         struct iscsi_cmnd *cmnd;
132
133         /* ToDo: __GFP_NOFAIL?? */
134 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
135         cmnd = kmem_cache_alloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
136         memset(cmnd, 0, sizeof(*cmnd));
137 #else
138         cmnd = kmem_cache_zalloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
139 #endif
140
141         atomic_set(&cmnd->ref_cnt, 1);
142         cmnd->scst_state = ISCSI_CMD_STATE_NEW;
143         cmnd->conn = conn;
144         cmnd->parent_req = parent;
145         init_waitqueue_head(&cmnd->scst_waitQ);
146
147         if (parent == NULL) {
148                 conn_get(conn);
149
150 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
151                 atomic_set(&cmnd->net_ref_cnt, 0);
152 #endif
153                 spin_lock_init(&cmnd->rsp_cmd_lock);
154                 INIT_LIST_HEAD(&cmnd->rsp_cmd_list);
155                 INIT_LIST_HEAD(&cmnd->rx_ddigest_cmd_list);
156
157                 spin_lock_bh(&conn->cmd_list_lock);
158                 list_add_tail(&cmnd->cmd_list_entry, &conn->cmd_list);
159                 spin_unlock_bh(&conn->cmd_list_lock);
160         }
161
162         TRACE_DBG("conn %p, parent %p, cmnd %p", conn, parent, cmnd);
163         return cmnd;
164 }
165
166 /* Frees a command. Also frees the additional header. */
167 static void cmnd_free(struct iscsi_cmnd *cmnd)
168 {
169         TRACE_DBG("%p", cmnd);
170
171         if (unlikely(cmnd->tm_aborted)) {
172                 TRACE_MGMT_DBG("Free aborted cmd %p (scst cmd %p, state %d, "
173                         "parent_req %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
174                         cmnd->parent_req);
175         }
176
177         /* Catch users from cmd_list or rsp_cmd_list */
178         EXTRACHECKS_BUG_ON(atomic_read(&cmnd->ref_cnt) != 0);
179
180         kfree(cmnd->pdu.ahs);
181
182         if (unlikely(cmnd->on_write_list || cmnd->on_written_list)) {
183                 struct iscsi_scsi_cmd_hdr *req = cmnd_hdr(cmnd);
184
185                 PRINT_CRIT_ERROR("cmnd %p still on some list?, %x, %x, %x, %x, %x, %x, %x",
186                         cmnd, req->opcode, req->scb[0], req->flags, req->itt,
187                         be32_to_cpu(req->data_length),
188                         req->cmd_sn, be32_to_cpu(cmnd->pdu.datasize));
189
190                 if (unlikely(cmnd->parent_req)) {
191                         struct iscsi_scsi_cmd_hdr *preq =
192                                         cmnd_hdr(cmnd->parent_req);
193                         PRINT_CRIT_ERROR("%p %x %u", preq, preq->opcode, preq->scb[0]);
194                 }
195                 sBUG();
196         }
197
198         kmem_cache_free(iscsi_cmnd_cache, cmnd);
199         return;
200 }
201
202 void cmnd_done(struct iscsi_cmnd *cmnd)
203 {
204         TRACE_DBG("%p", cmnd);
205
206         if (unlikely(cmnd->tm_aborted)) {
207                 TRACE_MGMT_DBG("Done aborted cmd %p (scst cmd %p, state %d, "
208                         "parent_req %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
209                         cmnd->parent_req);
210         }
211
212         EXTRACHECKS_BUG_ON(cmnd->on_rx_digest_list);
213
214         if (cmnd->on_written_list) {
215                 struct iscsi_conn *conn = cmnd->conn;
216                 TRACE_DBG("Deleting cmd %p from conn %p written_list", cmnd,
217                         conn);
218                 spin_lock_bh(&conn->write_list_lock);
219                 list_del(&cmnd->write_list_entry);
220                 cmnd->on_written_list = 0;
221                 spin_unlock_bh(&conn->write_list_lock);
222         }
223
224         if (cmnd->parent_req == NULL) {
225                 struct iscsi_conn *conn = cmnd->conn;
226                 TRACE_DBG("Deleting req %p from conn %p", cmnd, conn);
227
228                 spin_lock_bh(&conn->cmd_list_lock);
229                 list_del(&cmnd->cmd_list_entry);
230                 spin_unlock_bh(&conn->cmd_list_lock);
231
232                 conn_put(conn);
233
234                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rsp_cmd_list));
235                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rx_ddigest_cmd_list));
236
237                 /* Order between above and below code is important! */
238
239                 if (cmnd->scst_cmd) {
240                         switch (cmnd->scst_state) {
241                         case ISCSI_CMD_STATE_PROCESSED:
242                                 TRACE_DBG("cmd %p PROCESSED", cmnd);
243                                 scst_tgt_cmd_done(cmnd->scst_cmd,
244                                         SCST_CONTEXT_DIRECT);
245                                 break;
246                         case ISCSI_CMD_STATE_AFTER_PREPROC:
247                         {
248                                 struct scst_cmd *scst_cmd = cmnd->scst_cmd;
249                                 TRACE_DBG("cmd %p AFTER_PREPROC", cmnd);
250                                 cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
251                                 cmnd->scst_cmd = NULL;
252                                 scst_restart_cmd(scst_cmd,
253                                         SCST_PREPROCESS_STATUS_ERROR_FATAL,
254                                         SCST_CONTEXT_THREAD);
255                                 break;
256                         }
257                         default:
258                                 PRINT_CRIT_ERROR("Unexpected cmnd scst state %d",
259                                                  cmnd->scst_state);
260                                 sBUG();
261                                 break;
262                         }
263                 }
264         } else {
265                 EXTRACHECKS_BUG_ON(cmnd->scst_cmd != NULL);
266                 TRACE_DBG("Deleting rsp %p from parent %p", cmnd,
267                         cmnd->parent_req);
268
269                 spin_lock_bh(&cmnd->parent_req->rsp_cmd_lock);
270                 list_del(&cmnd->rsp_cmd_list_entry);
271                 spin_unlock_bh(&cmnd->parent_req->rsp_cmd_lock);
272
273                 cmnd_put(cmnd->parent_req);
274         }
275
276         /* Order between above and below code is important! */
277
278         if (cmnd->own_sg) {
279                 TRACE_DBG("%s", "own_sg");
280                 if (cmnd->sg != &dummy_sg)
281                         scst_free(cmnd->sg, cmnd->sg_cnt);
282 #ifdef CONFIG_SCST_DEBUG
283                 cmnd->own_sg = 0;
284                 cmnd->sg = NULL;
285                 cmnd->sg_cnt = -1;
286 #endif
287         }
288
289         if (cmnd->dec_active_cmnds) {
290                 struct iscsi_session *sess = cmnd->conn->session;
291                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
292                         "new value %d)", cmnd, sess,
293                         atomic_read(&sess->active_cmds)-1);
294                 atomic_dec(&sess->active_cmds);
295 #ifdef CONFIG_SCST_EXTRACHECKS
296                 if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
297                         PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
298                                 atomic_read(&sess->active_cmds));
299                         sBUG();
300                 }
301 #endif
302         }
303
304         cmnd_free(cmnd);
305         return;
306 }
307
308 /*
309  * Corresponding conn may also gets destroyed atfer this function, except only
310  * if it's called from the read thread!
311  *
312  * It can't be called in parallel with iscsi_cmnds_init_write()!
313  */
314 void req_cmnd_release_force(struct iscsi_cmnd *req, int flags)
315 {
316         struct iscsi_cmnd *rsp, *t;
317         struct iscsi_conn *conn = req->conn;
318         LIST_HEAD(cmds_list);
319
320         TRACE_ENTRY();
321
322         TRACE_MGMT_DBG("%p", req);
323
324         sBUG_ON(req == conn->read_cmnd);
325
326         if (flags & ISCSI_FORCE_RELEASE_WRITE) {
327                 spin_lock_bh(&conn->write_list_lock);
328                 list_for_each_entry_safe(rsp, t, &conn->write_list,
329                                                 write_list_entry) {
330                         if (rsp->parent_req != req)
331                                 continue;
332
333                         cmd_del_from_write_list(rsp);
334
335                         list_add_tail(&rsp->write_list_entry, &cmds_list);
336                 }
337                 spin_unlock_bh(&conn->write_list_lock);
338
339                 list_for_each_entry_safe(rsp, t, &cmds_list, write_list_entry) {
340                         TRACE_MGMT_DBG("Putting write rsp %p", rsp);
341                         list_del(&rsp->write_list_entry);
342                         cmnd_put(rsp);
343                 }
344         }
345
346 again_rsp:
347         spin_lock_bh(&req->rsp_cmd_lock);
348         list_for_each_entry_reverse(rsp, &req->rsp_cmd_list, rsp_cmd_list_entry) {
349                 bool r;
350
351                 if (rsp->force_cleanup_done)
352                         continue;
353
354                 rsp->force_cleanup_done = 1;
355
356                 if (cmnd_get_check(rsp))
357                         continue;
358
359                 spin_unlock_bh(&req->rsp_cmd_lock);
360
361                 spin_lock_bh(&conn->write_list_lock);
362                 r = rsp->on_write_list || rsp->write_processing_started;
363                 spin_unlock_bh(&conn->write_list_lock);
364
365                 cmnd_put(rsp);
366
367                 if (r)
368                         goto again_rsp;
369
370                 /*
371                  * If both on_write_list and write_processing_started not set,
372                  * we can safely put() rsp.
373                  */
374                 TRACE_MGMT_DBG("Putting rsp %p", rsp);
375                 cmnd_put(rsp);
376                 goto again_rsp;
377         }
378         spin_unlock_bh(&req->rsp_cmd_lock);
379
380         req_cmnd_release(req);
381
382         TRACE_EXIT();
383         return;
384 }
385
386 /*
387  * Corresponding conn may also gets destroyed atfer this function, except only
388  * if it's called from the read thread!
389  */
390 void req_cmnd_release(struct iscsi_cmnd *req)
391 {
392         struct iscsi_cmnd *c, *t;
393
394         TRACE_ENTRY();
395
396         TRACE_DBG("%p", req);
397
398 #ifdef CONFIG_SCST_EXTRACHECKS
399         sBUG_ON(req->release_called);
400         req->release_called = 1;
401 #endif
402
403         if (unlikely(req->tm_aborted)) {
404                 TRACE_MGMT_DBG("Release aborted req cmd %p (scst cmd %p, "
405                         "state %d)", req, req->scst_cmd, req->scst_state);
406         }
407
408         sBUG_ON(req->parent_req != NULL);
409
410         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
411                                 rx_ddigest_cmd_list_entry) {
412                 cmd_del_from_rx_ddigest_list(c);
413                 cmnd_put(c);
414         }
415
416         if (req->hashed)
417                 cmnd_remove_hash(req);
418
419         if (req->dec_active_cmnds) {
420                 struct iscsi_session *sess = req->conn->session;
421                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
422                         "new value %d)", req, sess,
423                         atomic_read(&sess->active_cmds)-1);
424                 atomic_dec(&sess->active_cmds);
425                 req->dec_active_cmnds = 0;
426 #ifdef CONFIG_SCST_EXTRACHECKS
427                 if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
428                         PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
429                                 atomic_read(&sess->active_cmds));
430                         sBUG();
431                 }
432 #endif
433         }
434
435         cmnd_put(req);
436
437         TRACE_EXIT();
438         return;
439 }
440
441 /*
442  * Corresponding conn may also gets destroyed atfer this function, except only
443  * if it's called from the read thread!
444  */
445 void rsp_cmnd_release(struct iscsi_cmnd *cmnd)
446 {
447         TRACE_DBG("%p", cmnd);
448
449 #ifdef CONFIG_SCST_EXTRACHECKS
450         sBUG_ON(cmnd->release_called);
451         cmnd->release_called = 1;
452 #endif
453
454         sBUG_ON(cmnd->hashed);
455         sBUG_ON(cmnd->parent_req == NULL);
456
457         cmnd_put(cmnd);
458         return;
459 }
460
461 /**
462  * create a new command used as response.
463  *
464  * iscsi_cmnd_create_rsp_cmnd -
465  * @cmnd: ptr to request command
466  *
467  * @return    ptr to response command or NULL
468  */
469 static struct iscsi_cmnd *iscsi_cmnd_create_rsp_cmnd(struct iscsi_cmnd *parent)
470 {
471         struct iscsi_cmnd *rsp;
472
473         rsp = cmnd_alloc(parent->conn, parent);
474
475         spin_lock_bh(&parent->rsp_cmd_lock);
476         TRACE_DBG("Adding rsp %p to parent %p", rsp, parent);
477         list_add_tail(&rsp->rsp_cmd_list_entry, &parent->rsp_cmd_list);
478         spin_unlock_bh(&parent->rsp_cmd_lock);
479         cmnd_get(parent);
480         return rsp;
481 }
482
483 static inline struct iscsi_cmnd *get_rsp_cmnd(struct iscsi_cmnd *req)
484 {
485         struct iscsi_cmnd *res = NULL;
486
487         /* Currently this lock isn't needed, but just in case.. */
488         spin_lock_bh(&req->rsp_cmd_lock);
489         if (!list_empty(&req->rsp_cmd_list)) {
490                 res = list_entry(req->rsp_cmd_list.prev, struct iscsi_cmnd,
491                         rsp_cmd_list_entry);
492         }
493         spin_unlock_bh(&req->rsp_cmd_lock);
494
495         return res;
496 }
497
498 static void iscsi_cmnds_init_write(struct list_head *send, int flags)
499 {
500         struct iscsi_cmnd *rsp = list_entry(send->next, struct iscsi_cmnd,
501                                                 write_list_entry);
502         struct iscsi_conn *conn = rsp->conn;
503         struct list_head *pos, *next;
504
505         sBUG_ON(list_empty(send));
506
507         /*
508          * If we don't remove hashed req cmd from the hash list here, before
509          * submitting it for transmittion, we will have a race, when for
510          * some reason cmd's release is delayed after transmittion and
511          * initiator sends cmd with the same ITT => this command will be
512          * erroneously rejected as a duplicate.
513          */
514         if ((flags & ISCSI_INIT_WRITE_REMOVE_HASH) && rsp->parent_req->hashed &&
515             (rsp->parent_req->r2t_length == 0) &&
516             (rsp->parent_req->outstanding_r2t == 0))
517                 cmnd_remove_hash(rsp->parent_req);
518
519         if (!(conn->ddigest_type & DIGEST_NONE)) {
520                 list_for_each(pos, send) {
521                         rsp = list_entry(pos, struct iscsi_cmnd,
522                                                 write_list_entry);
523
524                         if (rsp->pdu.datasize != 0) {
525                                 TRACE_DBG("Doing data digest (%p:%x)", rsp,
526                                         cmnd_opcode(rsp));
527                                 digest_tx_data(rsp);
528                         }
529                 }
530         }
531
532         spin_lock_bh(&conn->write_list_lock);
533         list_for_each_safe(pos, next, send) {
534                 rsp = list_entry(pos, struct iscsi_cmnd, write_list_entry);
535
536                 TRACE_DBG("%p:%x", rsp, cmnd_opcode(rsp));
537
538                 sBUG_ON(conn != rsp->conn);
539
540                 list_del(&rsp->write_list_entry);
541                 cmd_add_on_write_list(conn, rsp);
542         }
543         spin_unlock_bh(&conn->write_list_lock);
544
545         if (flags & ISCSI_INIT_WRITE_WAKE)
546                 iscsi_make_conn_wr_active(conn);
547 }
548
549 static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags)
550 {
551         LIST_HEAD(head);
552
553         if (unlikely(rsp->on_write_list)) {
554                 PRINT_CRIT_ERROR("cmd already on write list (%x %x %x %x %u %u "
555                         "%u %u %u %u %u %d %d",
556                         cmnd_itt(rsp), cmnd_ttt(rsp), cmnd_opcode(rsp),
557                         cmnd_scsicode(rsp), rsp->r2t_sn,
558                         rsp->r2t_length, rsp->is_unsolicited_data,
559                         rsp->target_task_tag, rsp->outstanding_r2t,
560                         rsp->hdigest, rsp->ddigest,
561                         list_empty(&rsp->rsp_cmd_list), rsp->hashed);
562                 sBUG();
563         }
564         list_add(&rsp->write_list_entry, &head);
565         iscsi_cmnds_init_write(&head, flags);
566 }
567
568 static void iscsi_set_datasize(struct iscsi_cmnd *cmnd, u32 offset, u32 size)
569 {
570         cmnd->pdu.datasize = size;
571
572         if (size & 3) {
573                 u32 last_off = offset + size;
574                 int idx = last_off >> PAGE_SHIFT;
575                 u8 *p = (u8 *)page_address(sg_page(&cmnd->sg[idx])) +
576                         (last_off & ~PAGE_MASK);
577                 int i = 4 - (size & 3);
578                 while (i--)
579                         *p++ = 0;
580         }
581 }
582
583 static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
584 {
585         struct iscsi_cmnd *rsp;
586         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
587         struct iscsi_data_in_hdr *rsp_hdr;
588         u32 pdusize, expsize, scsisize, size, offset, sn;
589         LIST_HEAD(send);
590
591         TRACE_DBG("req %p", req);
592
593         pdusize = req->conn->session->sess_param.max_xmit_data_length;
594         expsize = cmnd_read_size(req);
595         size = min(expsize, (u32)req->bufflen);
596         offset = 0;
597         sn = 0;
598
599         while (1) {
600                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
601                 TRACE_DBG("rsp %p", rsp);
602                 rsp->sg = req->sg;
603                 rsp->sg_cnt = req->sg_cnt;
604                 rsp->bufflen = req->bufflen;
605                 rsp_hdr = (struct iscsi_data_in_hdr *)&rsp->pdu.bhs;
606
607                 rsp_hdr->opcode = ISCSI_OP_SCSI_DATA_IN;
608                 rsp_hdr->itt = req_hdr->itt;
609                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
610                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
611                 rsp_hdr->data_sn = cpu_to_be32(sn);
612
613                 if (size <= pdusize) {
614                         TRACE_DBG("offset %d, size %d", offset, size);
615                         iscsi_set_datasize(rsp, offset, size);
616                         if (send_status) {
617                                 TRACE_DBG("status %x", status);
618                                 rsp_hdr->flags =
619                                         ISCSI_FLG_FINAL | ISCSI_FLG_STATUS;
620                                 rsp_hdr->cmd_status = status;
621                         }
622                         scsisize = req->bufflen;
623                         if (scsisize < expsize) {
624                                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
625                                 size = expsize - scsisize;
626                         } else if (scsisize > expsize) {
627                                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
628                                 size = scsisize - expsize;
629                         } else
630                                 size = 0;
631                         rsp_hdr->residual_count = cpu_to_be32(size);
632                         list_add_tail(&rsp->write_list_entry, &send);
633                         break;
634                 }
635
636                 TRACE_DBG("pdusize %d, offset %d, size %d", pdusize, offset,
637                         size);
638
639                 iscsi_set_datasize(rsp, offset, pdusize);
640
641                 size -= pdusize;
642                 offset += pdusize;
643                 sn++;
644
645                 list_add_tail(&rsp->write_list_entry, &send);
646         }
647         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_REMOVE_HASH);
648 }
649
650 static struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req, int status,
651         const u8 *sense_buf, int sense_len)
652 {
653         struct iscsi_cmnd *rsp;
654         struct iscsi_scsi_rsp_hdr *rsp_hdr;
655         struct iscsi_sense_data *sense;
656         struct scatterlist *sg;
657
658         rsp = iscsi_cmnd_create_rsp_cmnd(req);
659         TRACE_DBG("%p", rsp);
660
661         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
662         rsp_hdr->opcode = ISCSI_OP_SCSI_RSP;
663         rsp_hdr->flags = ISCSI_FLG_FINAL;
664         rsp_hdr->response = ISCSI_RESPONSE_COMMAND_COMPLETED;
665         rsp_hdr->cmd_status = status;
666         rsp_hdr->itt = cmnd_hdr(req)->itt;
667
668         if (SCST_SENSE_VALID(sense_buf)) {
669                 TRACE_DBG("%s", "SENSE VALID");
670                 /* ToDo: __GFP_NOFAIL ?? */
671                 sg = rsp->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
672                                         &rsp->sg_cnt);
673                 if (sg == NULL) {
674                         ;/* ToDo */;
675                 }
676                 rsp->own_sg = 1;
677                 sense = (struct iscsi_sense_data *)page_address(sg_page(&sg[0]));
678                 sense->length = cpu_to_be16(sense_len);
679                 memcpy(sense->data, sense_buf, sense_len);
680                 rsp->pdu.datasize = sizeof(struct iscsi_sense_data) + sense_len;
681                 rsp->bufflen = (rsp->pdu.datasize + 3) & -4;
682                 if (rsp->bufflen - rsp->pdu.datasize) {
683                         int i = rsp->pdu.datasize;
684                         u8 *p = (u8 *)sense + i;
685
686                         while (i < rsp->bufflen) {
687                                 *p++ = 0;
688                                 i++;
689                         }
690                 }
691         } else {
692                 rsp->pdu.datasize = 0;
693                 rsp->bufflen = 0;
694         }
695
696         return rsp;
697 }
698
699 static struct iscsi_cmnd *create_sense_rsp(struct iscsi_cmnd *req,
700         u8 sense_key, u8 asc, u8 ascq)
701 {
702         u8 sense[14];
703         memset(sense, 0, sizeof(sense));
704         sense[0] = 0xf0;
705         sense[2] = sense_key;
706         sense[7] = 6;   /* Additional sense length */
707         sense[12] = asc;
708         sense[13] = ascq;
709         return create_status_rsp(req, SAM_STAT_CHECK_CONDITION, sense,
710                 sizeof(sense));
711 }
712
713 static void iscsi_cmnd_reject(struct iscsi_cmnd *req, int reason)
714 {
715         struct iscsi_cmnd *rsp;
716         struct iscsi_reject_hdr *rsp_hdr;
717         struct scatterlist *sg;
718         char *addr;
719
720         TRACE_MGMT_DBG("Reject: req %p, reason %x", req, reason);
721
722         sBUG_ON(req->rejected);
723         req->rejected = 1;
724         req->reject_reason = ISCSI_REJECT_CMD;
725
726         rsp = iscsi_cmnd_create_rsp_cmnd(req);
727         rsp_hdr = (struct iscsi_reject_hdr *)&rsp->pdu.bhs;
728
729         rsp_hdr->opcode = ISCSI_OP_REJECT;
730         rsp_hdr->ffffffff = ISCSI_RESERVED_TAG;
731         rsp_hdr->reason = reason;
732
733         /* ToDo: __GFP_NOFAIL ?? */
734         sg = rsp->sg = scst_alloc(PAGE_SIZE, GFP_KERNEL|__GFP_NOFAIL,
735                                 &rsp->sg_cnt);
736         if (sg == NULL) {
737                 ;/* ToDo */;
738         }
739         rsp->own_sg = 1;
740         addr = page_address(sg_page(&sg[0]));
741         clear_page(addr);
742         memcpy(addr, &req->pdu.bhs, sizeof(struct iscsi_hdr));
743         rsp->bufflen = rsp->pdu.datasize = sizeof(struct iscsi_hdr);
744
745         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
746                                          ISCSI_INIT_WRITE_WAKE);
747
748         cmnd_prepare_get_rejected_cmd_data(req);
749 }
750
751 static inline int iscsi_get_allowed_cmds(struct iscsi_session *sess)
752 {
753         int res = max(-1, (int)sess->max_queued_cmnds -
754                                 atomic_read(&sess->active_cmds)-1);
755         TRACE_DBG("allowed cmds %d (sess %p, active_cmds %d)", res,
756                 sess, atomic_read(&sess->active_cmds));
757         return res;
758 }
759
760 static u32 cmnd_set_sn(struct iscsi_cmnd *cmnd, int set_stat_sn)
761 {
762         struct iscsi_conn *conn = cmnd->conn;
763         struct iscsi_session *sess = conn->session;
764         u32 res;
765
766         spin_lock(&sess->sn_lock);
767
768         if (set_stat_sn)
769                 cmnd->pdu.bhs.sn = cpu_to_be32(conn->stat_sn++);
770         cmnd->pdu.bhs.exp_sn = cpu_to_be32(sess->exp_cmd_sn);
771         cmnd->pdu.bhs.max_sn = cpu_to_be32(sess->exp_cmd_sn +
772                                  iscsi_get_allowed_cmds(sess));
773
774         res = cpu_to_be32(conn->stat_sn);
775
776         spin_unlock(&sess->sn_lock);
777         return res;
778 }
779
780 /* Called under sn_lock */
781 static void __update_stat_sn(struct iscsi_cmnd *cmnd)
782 {
783         struct iscsi_conn *conn = cmnd->conn;
784         u32 exp_stat_sn;
785
786         cmnd->pdu.bhs.exp_sn = exp_stat_sn = be32_to_cpu(cmnd->pdu.bhs.exp_sn);
787         TRACE_DBG("%x,%x", cmnd_opcode(cmnd), exp_stat_sn);
788         if ((int)(exp_stat_sn - conn->exp_stat_sn) > 0 &&
789             (int)(exp_stat_sn - conn->stat_sn) <= 0) {
790                 /* free pdu resources */
791                 cmnd->conn->exp_stat_sn = exp_stat_sn;
792         }
793 }
794
795 static inline void update_stat_sn(struct iscsi_cmnd *cmnd)
796 {
797         spin_lock(&cmnd->conn->session->sn_lock);
798         __update_stat_sn(cmnd);
799         spin_unlock(&cmnd->conn->session->sn_lock);
800 }
801
802 /* Called under sn_lock */
803 static int check_cmd_sn(struct iscsi_cmnd *cmnd)
804 {
805         struct iscsi_session *session = cmnd->conn->session;
806         u32 cmd_sn;
807
808         cmnd->pdu.bhs.sn = cmd_sn = be32_to_cpu(cmnd->pdu.bhs.sn);
809         TRACE_DBG("%d(%d)", cmd_sn, session->exp_cmd_sn);
810         if (likely((s32)(cmd_sn - session->exp_cmd_sn) >= 0))
811                 return 0;
812         PRINT_ERROR("sequence error (%x,%x)", cmd_sn, session->exp_cmd_sn);
813         return -ISCSI_REASON_PROTOCOL_ERROR;
814 }
815
816 static inline struct iscsi_cmnd *__cmnd_find_hash(struct iscsi_session *session,
817         u32 itt, u32 ttt)
818 {
819         struct list_head *head;
820         struct iscsi_cmnd *cmnd;
821
822         head = &session->cmnd_hash[cmnd_hashfn(itt)];
823
824         list_for_each_entry(cmnd, head, hash_list_entry) {
825                 if (cmnd->pdu.bhs.itt == itt) {
826                         if ((ttt != ISCSI_RESERVED_TAG) && (ttt != cmnd->target_task_tag))
827                                 continue;
828                         return cmnd;
829                 }
830         }
831         return NULL;
832 }
833
834 static struct iscsi_cmnd *cmnd_find_hash(struct iscsi_session *session,
835         u32 itt, u32 ttt)
836 {
837         struct iscsi_cmnd *cmnd;
838
839         spin_lock(&session->cmnd_hash_lock);
840         cmnd = __cmnd_find_hash(session, itt, ttt);
841         spin_unlock(&session->cmnd_hash_lock);
842
843         return cmnd;
844 }
845
846 static struct iscsi_cmnd *cmnd_find_hash_get(struct iscsi_session *session,
847         u32 itt, u32 ttt)
848 {
849         struct iscsi_cmnd *cmnd;
850
851         spin_lock(&session->cmnd_hash_lock);
852         cmnd = __cmnd_find_hash(session, itt, ttt);
853         if (cmnd != NULL) {
854                 if (unlikely(cmnd_get_check(cmnd)))
855                         cmnd = NULL;
856         }
857         spin_unlock(&session->cmnd_hash_lock);
858
859         return cmnd;
860 }
861
862 static int cmnd_insert_hash(struct iscsi_cmnd *cmnd)
863 {
864         struct iscsi_session *session = cmnd->conn->session;
865         struct iscsi_cmnd *tmp;
866         struct list_head *head;
867         int err = 0;
868         u32 itt = cmnd->pdu.bhs.itt;
869
870         TRACE_DBG("%p:%x", cmnd, itt);
871         if (unlikely(itt == ISCSI_RESERVED_TAG)) {
872                 PRINT_ERROR("%s", "ITT is RESERVED_TAG");
873                 PRINT_BUFFER("Incorrect BHS", &cmnd->pdu.bhs,
874                         sizeof(cmnd->pdu.bhs));
875                 err = -ISCSI_REASON_PROTOCOL_ERROR;
876                 goto out;
877         }
878
879         spin_lock(&session->cmnd_hash_lock);
880
881         head = &session->cmnd_hash[cmnd_hashfn(cmnd->pdu.bhs.itt)];
882
883         tmp = __cmnd_find_hash(session, itt, ISCSI_RESERVED_TAG);
884         if (likely(!tmp)) {
885                 list_add_tail(&cmnd->hash_list_entry, head);
886                 cmnd->hashed = 1;
887         } else {
888                 PRINT_ERROR("Task %x in progress, cmnd %p", itt, cmnd);
889                 err = -ISCSI_REASON_TASK_IN_PROGRESS;
890         }
891
892         spin_unlock(&session->cmnd_hash_lock);
893
894         if (likely(!err)) {
895                 spin_lock(&session->sn_lock);
896                 __update_stat_sn(cmnd);
897                 err = check_cmd_sn(cmnd);
898                 spin_unlock(&session->sn_lock);
899         }
900
901 out:
902         return err;
903 }
904
905 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd)
906 {
907         struct iscsi_session *session = cmnd->conn->session;
908         struct iscsi_cmnd *tmp;
909
910         spin_lock(&session->cmnd_hash_lock);
911
912         tmp = __cmnd_find_hash(session, cmnd->pdu.bhs.itt, ISCSI_RESERVED_TAG);
913
914         if (likely(tmp && tmp == cmnd)) {
915                 list_del(&cmnd->hash_list_entry);
916                 cmnd->hashed = 0;
917         } else {
918                 PRINT_ERROR("%p:%x not found", cmnd, cmnd_itt(cmnd));
919         }
920
921         spin_unlock(&session->cmnd_hash_lock);
922 }
923
924 static void cmnd_prepare_get_rejected_cmd_data(struct iscsi_cmnd *cmnd)
925 {
926         struct iscsi_conn *conn = cmnd->conn;
927         struct scatterlist *sg = cmnd->sg;
928         char *addr;
929         u32 size;
930         int i;
931
932         TRACE_MGMT_DBG("Skipping (%p, %x %x %x %u, %p, scst state %d)", cmnd,
933                 cmnd_itt(cmnd), cmnd_opcode(cmnd), cmnd_hdr(cmnd)->scb[0],
934                 cmnd->pdu.datasize, cmnd->scst_cmd, cmnd->scst_state);
935
936         iscsi_extracheck_is_rd_thread(conn);
937
938         size = cmnd->pdu.datasize;
939         if (!size)
940                 return;
941
942         if (sg == NULL) {
943                 /*
944                  * There are no problems with the safety from concurrent
945                  * accesses to dummy_page in dummy_sg, since data only
946                  * will be read and then discarded.
947                  */
948                 sg = cmnd->sg = &dummy_sg;
949                 cmnd->bufflen = PAGE_SIZE;
950                 cmnd->own_sg = 1;
951         }
952
953         addr = page_address(sg_page(&sg[0]));
954         sBUG_ON(addr == NULL);
955         size = (size + 3) & -4;
956         conn->read_size = size;
957         for (i = 0; size > PAGE_SIZE; i++, size -= cmnd->bufflen) {
958                 sBUG_ON(i >= ISCSI_CONN_IOV_MAX);
959                 conn->read_iov[i].iov_base = addr;
960                 conn->read_iov[i].iov_len = cmnd->bufflen;
961         }
962         conn->read_iov[i].iov_base = addr;
963         conn->read_iov[i].iov_len = size;
964         conn->read_msg.msg_iov = conn->read_iov;
965         conn->read_msg.msg_iovlen = ++i;
966
967         return;
968 }
969
970 static void cmnd_reject_scsi_cmd(struct iscsi_cmnd *req)
971 {
972         struct iscsi_cmnd *rsp;
973         struct iscsi_scsi_rsp_hdr *rsp_hdr;
974         u32 size;
975
976         TRACE_DBG("%p", req);
977
978         sBUG_ON(req->rejected);
979         req->rejected = 1;
980         req->reject_reason = ISCSI_REJECT_SCSI_CMD;
981
982         rsp = get_rsp_cmnd(req);
983         if (rsp == NULL) {
984                 /* That can be true for aborted commands */
985                 goto out_reject;
986         }
987
988         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
989
990         sBUG_ON(cmnd_opcode(rsp) != ISCSI_OP_SCSI_RSP);
991
992         size = cmnd_write_size(req);
993         if (size) {
994                 rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
995                 rsp_hdr->residual_count = cpu_to_be32(size);
996         }
997         size = cmnd_read_size(req);
998         if (size) {
999                 if (cmnd_hdr(req)->flags & ISCSI_CMD_WRITE) {
1000                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_UNDERFLOW;
1001                         rsp_hdr->bi_residual_count = cpu_to_be32(size);
1002                 } else {
1003                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
1004                         rsp_hdr->residual_count = cpu_to_be32(size);
1005                 }
1006         }
1007
1008         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
1009                                          ISCSI_INIT_WRITE_WAKE);
1010
1011 out_reject:
1012         cmnd_prepare_get_rejected_cmd_data(req);
1013         return;
1014 }
1015
1016 static int cmnd_prepare_recv_pdu(struct iscsi_conn *conn,
1017         struct iscsi_cmnd *cmd, u32 offset, u32 size)
1018 {
1019         struct scatterlist *sg = cmd->sg;
1020         int bufflen = cmd->bufflen;
1021         int idx, i;
1022         char *addr;
1023         int res = 0;
1024
1025         TRACE_DBG("%p %u,%u", cmd->sg, offset, size);
1026
1027         iscsi_extracheck_is_rd_thread(conn);
1028
1029         if (unlikely((offset >= bufflen) ||
1030                      (offset + size > bufflen))) {
1031                 PRINT_ERROR("Wrong ltn (%u %u %u)", offset, size, bufflen);
1032                 mark_conn_closed(conn);
1033                 res = -EIO;
1034                 goto out;
1035         }
1036
1037         offset += sg[0].offset;
1038         idx = offset >> PAGE_SHIFT;
1039         offset &= ~PAGE_MASK;
1040
1041         conn->read_msg.msg_iov = conn->read_iov;
1042         conn->read_size = size = (size + 3) & -4;
1043
1044         i = 0;
1045         while (1) {
1046                 addr = page_address(sg_page(&sg[idx]));
1047                 sBUG_ON(addr == NULL);
1048                 conn->read_iov[i].iov_base = addr + offset;
1049                 if (offset + size <= PAGE_SIZE) {
1050                         TRACE_DBG("idx=%d, offset=%u, size=%d, addr=%p",
1051                                 idx, offset, size, addr);
1052                         conn->read_iov[i].iov_len = size;
1053                         conn->read_msg.msg_iovlen = ++i;
1054                         break;
1055                 }
1056                 conn->read_iov[i].iov_len = PAGE_SIZE - offset;
1057                 TRACE_DBG("idx=%d, offset=%u, size=%d, iov_len=%zd, addr=%p",
1058                         idx, offset, size, conn->read_iov[i].iov_len, addr);
1059                 size -= conn->read_iov[i].iov_len;
1060                 offset = 0;
1061                 if (unlikely(++i >= ISCSI_CONN_IOV_MAX)) {
1062                         PRINT_ERROR("Initiator %s violated negotiated "
1063                                 "parameters by sending too much data (size "
1064                                 "left %d)", conn->session->initiator_name, size);
1065                         mark_conn_closed(conn);
1066                         res = -EINVAL;
1067                         break;
1068                 }
1069                 idx++;
1070         }
1071         TRACE_DBG("msg_iov=%p, msg_iovlen=%zd",
1072                 conn->read_msg.msg_iov, conn->read_msg.msg_iovlen);
1073
1074 out:
1075         return res;
1076 }
1077
1078 static void send_r2t(struct iscsi_cmnd *req)
1079 {
1080         struct iscsi_session *session = req->conn->session;
1081         struct iscsi_cmnd *rsp;
1082         struct iscsi_r2t_hdr *rsp_hdr;
1083         u32 offset, burst;
1084         LIST_HEAD(send);
1085
1086         if (unlikely(req->tm_aborted)) {
1087                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted on R2T "
1088                         "(r2t_length %d, outstanding_r2t %d)", req,
1089                         req->scst_cmd, req->r2t_length, req->outstanding_r2t);
1090                 if (req->outstanding_r2t == 0)
1091                         iscsi_fail_waiting_cmnd(req);
1092                 goto out;
1093         }
1094
1095         /*
1096          * There is no race with data_out_start() and conn_abort(), since
1097          * all functions called from single read thread
1098          */
1099         iscsi_extracheck_is_rd_thread(req->conn);
1100
1101         burst = session->sess_param.max_burst_length;
1102         offset = be32_to_cpu(cmnd_hdr(req)->data_length) - req->r2t_length;
1103
1104         do {
1105                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
1106                 rsp->pdu.bhs.ttt = req->target_task_tag;
1107                 rsp_hdr = (struct iscsi_r2t_hdr *)&rsp->pdu.bhs;
1108                 rsp_hdr->opcode = ISCSI_OP_R2T;
1109                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1110                 rsp_hdr->lun = cmnd_hdr(req)->lun;
1111                 rsp_hdr->itt = cmnd_hdr(req)->itt;
1112                 rsp_hdr->r2t_sn = cpu_to_be32(req->r2t_sn++);
1113                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
1114                 if (req->r2t_length > burst) {
1115                         rsp_hdr->data_length = cpu_to_be32(burst);
1116                         req->r2t_length -= burst;
1117                         offset += burst;
1118                 } else {
1119                         rsp_hdr->data_length = cpu_to_be32(req->r2t_length);
1120                         req->r2t_length = 0;
1121                 }
1122
1123                 TRACE_WRITE("%x %u %u %u %u", cmnd_itt(req),
1124                         be32_to_cpu(rsp_hdr->data_length),
1125                         be32_to_cpu(rsp_hdr->buffer_offset),
1126                         be32_to_cpu(rsp_hdr->r2t_sn), req->outstanding_r2t);
1127
1128                 list_add_tail(&rsp->write_list_entry, &send);
1129
1130                 if (++req->outstanding_r2t >= session->sess_param.max_outstanding_r2t)
1131                         break;
1132
1133         } while (req->r2t_length != 0);
1134
1135         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_WAKE);
1136
1137 out:
1138         return;
1139 }
1140
1141 static int iscsi_pre_exec(struct scst_cmd *scst_cmd)
1142 {
1143         int res = SCST_PREPROCESS_STATUS_SUCCESS;
1144         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
1145                 scst_cmd_get_tgt_priv(scst_cmd);
1146         struct iscsi_cmnd *c, *t;
1147
1148         TRACE_ENTRY();
1149
1150         EXTRACHECKS_BUG_ON(scst_cmd_atomic(scst_cmd));
1151
1152         if (scst_cmd_get_data_direction(scst_cmd) == SCST_DATA_READ) {
1153                 EXTRACHECKS_BUG_ON(!list_empty(&req->rx_ddigest_cmd_list));
1154                 goto out;
1155         }
1156
1157         /* If data digest isn't used this list will be empty */
1158         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
1159                                 rx_ddigest_cmd_list_entry) {
1160                 TRACE_DBG("Checking digest of RX ddigest cmd %p", c);
1161                 if (digest_rx_data(c) != 0) {
1162                         scst_set_cmd_error(scst_cmd,
1163                                 SCST_LOAD_SENSE(iscsi_sense_crc_error));
1164                         res = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
1165                         /*
1166                          * The rest of rx_ddigest_cmd_list will be freed
1167                          * in req_cmnd_release()
1168                          */
1169                         goto out;
1170                 }
1171                 cmd_del_from_rx_ddigest_list(c);
1172                 cmnd_put(c);
1173         }
1174
1175 out:
1176         TRACE_EXIT_RES(res);
1177         return res;
1178 }
1179
1180 static int noop_out_start(struct iscsi_cmnd *cmnd)
1181 {
1182         struct iscsi_conn *conn = cmnd->conn;
1183         u32 size, tmp;
1184         int i, err = 0;
1185
1186         TRACE_DBG("%p", cmnd);
1187
1188         iscsi_extracheck_is_rd_thread(conn);
1189
1190         if (unlikely(cmnd_ttt(cmnd) != cpu_to_be32(ISCSI_RESERVED_TAG))) {
1191                 /*
1192                  * We don't request a NOP-Out by sending a NOP-In.
1193                  * See 10.18.2 in the draft 20.
1194                  */
1195                 PRINT_ERROR("Initiator sent command with not RESERVED tag and "
1196                         "TTT %x", cmnd_itt(cmnd));
1197                 err = -ISCSI_REASON_PROTOCOL_ERROR;
1198                 goto out;
1199         }
1200
1201         if (cmnd_itt(cmnd) == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1202                 if (unlikely(!(cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE)))
1203                         PRINT_ERROR("%s", "Initiator sent RESERVED tag for "
1204                                 "non-immediate command");
1205                 spin_lock(&conn->session->sn_lock);
1206                 __update_stat_sn(cmnd);
1207                 err = check_cmd_sn(cmnd);
1208                 spin_unlock(&conn->session->sn_lock);
1209                 if (unlikely(err))
1210                         goto out;
1211         } else {
1212                 err = cmnd_insert_hash(cmnd);
1213                 if (unlikely(err < 0)) {
1214                         PRINT_ERROR("Can't insert in hash: ignore this request %x",
1215                                     cmnd_itt(cmnd));
1216                         goto out;
1217                 }
1218         }
1219
1220         size = cmnd->pdu.datasize;
1221
1222         if (size) {
1223                 size = (size + 3) & -4;
1224                 conn->read_msg.msg_iov = conn->read_iov;
1225                 if (cmnd->pdu.bhs.itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1226                         struct scatterlist *sg;
1227
1228                         cmnd->sg = sg = scst_alloc(size, GFP_KERNEL,
1229                                                 &cmnd->sg_cnt);
1230                         if (sg == NULL) {
1231                                 TRACE(TRACE_OUT_OF_MEM, "Allocating buffer for "
1232                                         "%d NOP-Out payload failed", size);
1233                                 err = -ISCSI_REASON_OUT_OF_RESOURCES;
1234                                 goto out;
1235                         }
1236
1237                         /* We already checked it in check_segment_length() */
1238                         sBUG_ON(cmnd->sg_cnt > ISCSI_CONN_IOV_MAX);
1239
1240                         cmnd->own_sg = 1;
1241                         cmnd->bufflen = size;
1242
1243                         for (i = 0; i < cmnd->sg_cnt; i++) {
1244                                 conn->read_iov[i].iov_base =
1245                                         page_address(sg_page(&sg[i]));
1246                                 tmp = min_t(u32, size, PAGE_SIZE);
1247                                 conn->read_iov[i].iov_len = tmp;
1248                                 conn->read_size += tmp;
1249                                 size -= tmp;
1250                         }
1251                         sBUG_ON(size != 0);
1252                 } else {
1253                         /*
1254                          * There are no problems with the safety from concurrent
1255                          * accesses to dummy_page, since for ISCSI_RESERVED_TAG
1256                          * the data only read and then discarded.
1257                          */
1258                         for (i = 0; i < ISCSI_CONN_IOV_MAX; i++) {
1259                                 conn->read_iov[i].iov_base =
1260                                         page_address(dummy_page);
1261                                 tmp = min_t(u32, size, PAGE_SIZE);
1262                                 conn->read_iov[i].iov_len = tmp;
1263                                 conn->read_size += tmp;
1264                                 size -= tmp;
1265                         }
1266
1267                         /* We already checked size in check_segment_length() */
1268                         sBUG_ON(size != 0);
1269                 }
1270
1271                 conn->read_msg.msg_iovlen = i;
1272                 TRACE_DBG("msg_iov=%p, msg_iovlen=%zd", conn->read_msg.msg_iov,
1273                         conn->read_msg.msg_iovlen);
1274         }
1275
1276 out:
1277         return err;
1278 }
1279
1280 static inline u32 get_next_ttt(struct iscsi_conn *conn)
1281 {
1282         u32 ttt;
1283         struct iscsi_session *session = conn->session;
1284
1285         iscsi_extracheck_is_rd_thread(conn);
1286
1287         if (session->next_ttt == ISCSI_RESERVED_TAG)
1288                 session->next_ttt++;
1289         ttt = session->next_ttt++;
1290
1291         return cpu_to_be32(ttt);
1292 }
1293
1294 static int scsi_cmnd_start(struct iscsi_cmnd *req)
1295 {
1296         struct iscsi_conn *conn = req->conn;
1297         struct iscsi_session *session = conn->session;
1298         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1299         struct scst_cmd *scst_cmd;
1300         scst_data_direction dir;
1301         int res = 0;
1302
1303         TRACE_ENTRY();
1304
1305         TRACE_DBG("scsi command: %02x", req_hdr->scb[0]);
1306
1307         TRACE_DBG("Incrementing active_cmds (cmd %p, sess %p, "
1308                 "new value %d)", req, session,
1309                 atomic_read(&session->active_cmds)+1);
1310         atomic_inc(&session->active_cmds);
1311         req->dec_active_cmnds = 1;
1312
1313         scst_cmd = scst_rx_cmd(session->scst_sess,
1314                 (uint8_t *)&req_hdr->lun, sizeof(req_hdr->lun),
1315                 req_hdr->scb, sizeof(req_hdr->scb), SCST_NON_ATOMIC);
1316         if (scst_cmd == NULL) {
1317                 create_status_rsp(req, SAM_STAT_BUSY, NULL, 0);
1318                 cmnd_reject_scsi_cmd(req);
1319                 goto out;
1320         }
1321
1322         req->scst_cmd = scst_cmd;
1323         scst_cmd_set_tag(scst_cmd, req_hdr->itt);
1324         scst_cmd_set_tgt_priv(scst_cmd, req);
1325 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
1326         scst_cmd_set_data_buf_tgt_alloc(scst_cmd);
1327 #endif
1328
1329         if (req_hdr->flags & ISCSI_CMD_READ)
1330                 dir = SCST_DATA_READ;
1331         else if (req_hdr->flags & ISCSI_CMD_WRITE)
1332                 dir = SCST_DATA_WRITE;
1333         else
1334                 dir = SCST_DATA_NONE;
1335         scst_cmd_set_expected(scst_cmd, dir, be32_to_cpu(req_hdr->data_length));
1336
1337         switch (req_hdr->flags & ISCSI_CMD_ATTR_MASK) {
1338         case ISCSI_CMD_SIMPLE:
1339                 scst_cmd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1340                 break;
1341         case ISCSI_CMD_HEAD_OF_QUEUE:
1342                 scst_cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1343                 break;
1344         case ISCSI_CMD_ORDERED:
1345                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1346                 break;
1347         case ISCSI_CMD_ACA:
1348                 scst_cmd->queue_type = SCST_CMD_QUEUE_ACA;
1349                 break;
1350         case ISCSI_CMD_UNTAGGED:
1351                 scst_cmd->queue_type = SCST_CMD_QUEUE_UNTAGGED;
1352                 break;
1353         default:
1354                 PRINT_ERROR("Unknown task code %x, use ORDERED instead",
1355                         req_hdr->flags & ISCSI_CMD_ATTR_MASK);
1356                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1357                 break;
1358         }
1359
1360         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1361         scst_cmd_set_tgt_sn(scst_cmd, req_hdr->cmd_sn);
1362
1363         TRACE_DBG("START Command (tag %d, queue_type %d)",
1364                 req_hdr->itt, scst_cmd->queue_type);
1365         req->scst_state = ISCSI_CMD_STATE_RX_CMD;
1366         scst_cmd_init_stage1_done(scst_cmd, SCST_CONTEXT_DIRECT, 0);
1367
1368         wait_event(req->scst_waitQ, (req->scst_state != ISCSI_CMD_STATE_RX_CMD));
1369
1370         if (unlikely(req->scst_state != ISCSI_CMD_STATE_AFTER_PREPROC)) {
1371                 TRACE_DBG("req %p is in %x state", req, req->scst_state);
1372                 if (req->scst_state == ISCSI_CMD_STATE_PROCESSED) {
1373                         cmnd_reject_scsi_cmd(req);
1374                         goto out;
1375                 }
1376                 if (unlikely(req->tm_aborted)) {
1377                         TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
1378                                 req->scst_cmd);
1379                         cmnd_prepare_get_rejected_cmd_data(req);
1380                         goto out;
1381                 }
1382                 sBUG();
1383         }
1384
1385         dir = scst_cmd_get_data_direction(scst_cmd);
1386         if (dir != SCST_DATA_WRITE) {
1387                 if (unlikely(!(req_hdr->flags & ISCSI_CMD_FINAL) ||
1388                              req->pdu.datasize)) {
1389                         PRINT_ERROR("Unexpected unsolicited data (ITT %x "
1390                                 "CDB %x", cmnd_itt(req), req_hdr->scb[0]);
1391                         create_sense_rsp(req, ABORTED_COMMAND, 0xc, 0xc);
1392                         cmnd_reject_scsi_cmd(req);
1393                         goto out;
1394                 }
1395         }
1396
1397         if (dir == SCST_DATA_WRITE) {
1398                 req->is_unsolicited_data = !(req_hdr->flags & ISCSI_CMD_FINAL);
1399                 req->r2t_length = be32_to_cpu(req_hdr->data_length) - req->pdu.datasize;
1400                 if (req->r2t_length > 0)
1401                         req->data_waiting = 1;
1402         }
1403         req->target_task_tag = get_next_ttt(conn);
1404         req->sg = scst_cmd_get_sg(scst_cmd);
1405         req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
1406         req->bufflen = scst_cmd_get_bufflen(scst_cmd);
1407         if (unlikely(req->r2t_length > req->bufflen)) {
1408                 PRINT_ERROR("req->r2t_length %d > req->bufflen %d",
1409                         req->r2t_length, req->bufflen);
1410                 req->r2t_length = req->bufflen;
1411         }
1412
1413         TRACE_DBG("req=%p, dir=%d, is_unsolicited_data=%d, "
1414                 "r2t_length=%d, bufflen=%d", req, dir,
1415                 req->is_unsolicited_data, req->r2t_length, req->bufflen);
1416
1417         if (unlikely(!session->sess_param.immediate_data &&
1418                      req->pdu.datasize)) {
1419                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1420                         "forbidden immediate data sent (ITT %x, op  %x)",
1421                         session->initiator_name, cmnd_itt(req), req_hdr->scb[0]);
1422                 res = -EINVAL;
1423                 goto out;
1424         }
1425
1426         if (unlikely(session->sess_param.initial_r2t &&
1427                      !(req_hdr->flags & ISCSI_CMD_FINAL))) {
1428                 PRINT_ERROR("Initiator %s violated negotiated paremeters: "
1429                         "initial R2T is required (ITT %x, op  %x)",
1430                         session->initiator_name, cmnd_itt(req), req_hdr->scb[0]);
1431                 res = -EINVAL;
1432                 goto out;
1433         }
1434
1435         if (req->pdu.datasize) {
1436                 if (unlikely(dir != SCST_DATA_WRITE)) {
1437                         PRINT_ERROR("pdu.datasize(%d) >0, but dir(%x) isn't WRITE",
1438                                 req->pdu.datasize, dir);
1439                         create_sense_rsp(req, ABORTED_COMMAND, 0xc, 0xc);
1440                         cmnd_reject_scsi_cmd(req);
1441                 } else
1442                         res = cmnd_prepare_recv_pdu(conn, req, 0, req->pdu.datasize);
1443         }
1444 out:
1445         /* Aborted commands will be freed in cmnd_rx_end() */
1446         TRACE_EXIT_RES(res);
1447         return res;
1448 }
1449
1450 static int data_out_start(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
1451 {
1452         struct iscsi_data_out_hdr *req_hdr = (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1453         struct iscsi_cmnd *orig_req = NULL;
1454         u32 offset = be32_to_cpu(req_hdr->buffer_offset);
1455         int res = 0;
1456
1457         TRACE_ENTRY();
1458
1459         /*
1460          * There is no race with send_r2t() and conn_abort(), since
1461          * all functions called from single read thread
1462          */
1463         iscsi_extracheck_is_rd_thread(cmnd->conn);
1464
1465         update_stat_sn(cmnd);
1466
1467         cmnd->cmd_req = orig_req = cmnd_find_hash(conn->session, req_hdr->itt,
1468                                         req_hdr->ttt);
1469         if (unlikely(orig_req == NULL)) {
1470                 /* It might happen if req was aborted and then freed */
1471                 TRACE(TRACE_MGMT_MINOR, "Unable to find scsi task %x %x",
1472                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1473                 goto out_reject;
1474         }
1475
1476         if (orig_req->is_unsolicited_data) {
1477                 if (unlikely(orig_req->r2t_length < cmnd->pdu.datasize)) {
1478                         PRINT_ERROR("Data size (%d) > R2T length (%d)",
1479                                 cmnd->pdu.datasize, orig_req->r2t_length);
1480                         mark_conn_closed(conn);
1481                         res = -EINVAL;
1482                         goto out;
1483                 }
1484                 orig_req->r2t_length -= cmnd->pdu.datasize;
1485         }
1486
1487         /* Check unsolicited burst data */
1488         if (unlikely((req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) &&
1489                      (orig_req->pdu.bhs.flags & ISCSI_FLG_FINAL))) {
1490                 PRINT_ERROR("Unexpected data from %x %x",
1491                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1492                 mark_conn_closed(conn);
1493                 res = -EINVAL;
1494                 goto out;
1495         }
1496
1497         TRACE_WRITE("%u %p %p %u %u", req_hdr->ttt, cmnd, orig_req,
1498                 offset, cmnd->pdu.datasize);
1499
1500         res = cmnd_prepare_recv_pdu(conn, orig_req, offset, cmnd->pdu.datasize);
1501
1502 out:
1503         TRACE_EXIT_RES(res);
1504         return res;
1505
1506 out_reject:
1507         sBUG_ON(cmnd->rejected);
1508         cmnd->rejected = 1;
1509         cmnd->reject_reason = ISCSI_REJECT_DATA;
1510         cmnd_prepare_get_rejected_cmd_data(cmnd);
1511         goto out;
1512 }
1513
1514 static void data_out_end(struct iscsi_cmnd *cmnd)
1515 {
1516         struct iscsi_data_out_hdr *req_hdr = (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1517         struct iscsi_cmnd *req;
1518
1519         sBUG_ON(cmnd == NULL);
1520         req = cmnd->cmd_req;
1521         sBUG_ON(req == NULL);
1522
1523         TRACE_DBG("cmnd %p, req %p", cmnd, req);
1524
1525         iscsi_extracheck_is_rd_thread(cmnd->conn);
1526
1527         if (!(cmnd->conn->ddigest_type & DIGEST_NONE) &&
1528             !cmnd->ddigest_checked) {
1529                 cmd_add_on_rx_ddigest_list(req, cmnd);
1530                 cmnd_get(cmnd);
1531         }
1532
1533         if (req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1534                 TRACE_DBG("ISCSI_RESERVED_TAG, FINAL %x",
1535                         req_hdr->flags & ISCSI_FLG_FINAL);
1536
1537                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1538                         req->is_unsolicited_data = 0;
1539                         if (req->pending)
1540                                 goto out_put;
1541                 } else
1542                         goto out_put;
1543         } else {
1544                 TRACE_DBG("FINAL %x, outstanding_r2t %d, r2t_length %d",
1545                         req_hdr->flags & ISCSI_FLG_FINAL,
1546                         req->outstanding_r2t, req->r2t_length);
1547
1548                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1549                         if (unlikely(req->is_unsolicited_data)) {
1550                                 PRINT_ERROR("Unexpected unsolicited data "
1551                                         "(r2t_length %u, outstanding_r2t %d)",
1552                                         req->r2t_length, req->is_unsolicited_data);
1553                                 mark_conn_closed(req->conn);
1554                                 goto out_put;
1555                         }
1556                         req->outstanding_r2t--;
1557                 } else
1558                         goto out_put;
1559         }
1560
1561         if (req->r2t_length != 0) {
1562                 if (!req->is_unsolicited_data)
1563                         send_r2t(req);
1564         } else
1565                 iscsi_restart_waiting_cmnd(req);
1566
1567 out_put:
1568         cmnd_put(cmnd);
1569         return;
1570 }
1571
1572 static void __cmnd_abort(struct iscsi_cmnd *cmnd)
1573 {
1574         /*
1575          * Here, if cmnd is data_waiting, we should iscsi_fail_waiting_cmnd()
1576          * it. But, since this function can be called from any thread, not only
1577          * from the read one, we at the moment can't do that, because of
1578          * absence of appropriate locking protection. But this isn't a stuff
1579          * for 1.0.0. So, currently a misbehaving initiator, not sending
1580          * data in R2T state for a sharing between targets device, for which
1581          * for some reason an aborting TM command, e.g. TARGET RESET, from
1582          * another initiator is issued, can block response for this TM command
1583          * virtually forever and by this make the issuing initiator eventually
1584          * put the device offline.
1585          *
1586          * ToDo in the next version, possibly a simple connection mutex, taken
1587          * by the read thread before starting any processing and by this
1588          * function, should be sufficient.
1589          */
1590
1591         TRACE_MGMT_DBG("Aborting cmd %p, scst_cmd %p (scst state %x, "
1592                 "ref_cnt %d, itt %x, sn %u, op %x, r2t_len %x, CDB op %x, "
1593                 "size to write %u, is_unsolicited_data %d, "
1594                 "outstanding_r2t %d, data_waiting %d, sess->exp_cmd_sn %u, "
1595                 "conn %p, rd_task %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
1596                 atomic_read(&cmnd->ref_cnt), cmnd_itt(cmnd), cmnd->pdu.bhs.sn,
1597                 cmnd_opcode(cmnd), cmnd->r2t_length, cmnd_scsicode(cmnd),
1598                 cmnd_write_size(cmnd), cmnd->is_unsolicited_data,
1599                 cmnd->outstanding_r2t, cmnd->data_waiting,
1600                 cmnd->conn->session->exp_cmd_sn, cmnd->conn,
1601                 cmnd->conn->rd_task);
1602
1603 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
1604         TRACE_MGMT_DBG("net_ref_cnt %d", atomic_read(&cmnd->net_ref_cnt));
1605 #endif
1606
1607         cmnd->tm_aborted = 1;
1608
1609         return;
1610 }
1611
1612 /* Must be called from the read thread */
1613 static int cmnd_abort(struct iscsi_cmnd *req)
1614 {
1615         struct iscsi_session *session = req->conn->session;
1616         struct iscsi_task_mgt_hdr *req_hdr =
1617                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1618         struct iscsi_cmnd *cmnd;
1619         int err;
1620
1621         req_hdr->ref_cmd_sn = be32_to_cpu(req_hdr->ref_cmd_sn);
1622
1623         if (after(req_hdr->ref_cmd_sn, req_hdr->cmd_sn)) {
1624                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) > CmdSN(%u)",
1625                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn);
1626                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1627                 goto out;
1628         }
1629
1630         cmnd = cmnd_find_hash_get(session, req_hdr->rtt, ISCSI_RESERVED_TAG);
1631         if (cmnd) {
1632                 struct iscsi_conn *conn = cmnd->conn;
1633                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1634
1635                 if (req_hdr->lun != hdr->lun) {
1636                         PRINT_ERROR("ABORT TASK: LUN mismatch: req LUN "
1637                                     "%llx, cmd LUN %llx, rtt %u",
1638                                     (long long unsigned int)req_hdr->lun,
1639                                     (long long unsigned int)hdr->lun,
1640                                     req_hdr->rtt);
1641                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1642                         goto out_put;
1643                 }
1644
1645                 if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
1646                         if (req_hdr->ref_cmd_sn != req_hdr->cmd_sn) {
1647                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != TM cmd "
1648                                         "CmdSN(%u) for immediate command %p",
1649                                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
1650                                         cmnd);
1651                                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1652                                 goto out_put;
1653                         }
1654                 } else {
1655                         if (req_hdr->ref_cmd_sn != hdr->cmd_sn) {
1656                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != "
1657                                         "CmdSN(%u) for command %p",
1658                                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
1659                                         cmnd);
1660                                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1661                                 goto out_put;
1662                         }
1663                 }
1664
1665                 if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1666                     (req_hdr->cmd_sn == hdr->cmd_sn)) {
1667                         PRINT_ERROR("ABORT TASK: SN mismatch: req SN %x, "
1668                                 "cmd SN %x, rtt %u", req_hdr->cmd_sn,
1669                                 hdr->cmd_sn, req_hdr->rtt);
1670                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1671                         goto out_put;
1672                 }
1673
1674                 spin_lock_bh(&conn->cmd_list_lock);
1675                 __cmnd_abort(cmnd);
1676                 spin_unlock_bh(&conn->cmd_list_lock);
1677
1678                 cmnd_put(cmnd);
1679                 err = 0;
1680         } else {
1681                 TRACE_MGMT_DBG("cmd RTT %x not found", req_hdr->rtt);
1682                 err = ISCSI_RESPONSE_UNKNOWN_TASK;
1683         }
1684
1685 out:
1686         return err;
1687
1688 out_put:
1689         cmnd_put(cmnd);
1690         goto out;
1691 }
1692
1693 /* Must be called from the read thread */
1694 static int target_abort(struct iscsi_cmnd *req, int all)
1695 {
1696         struct iscsi_target *target = req->conn->session->target;
1697         struct iscsi_task_mgt_hdr *req_hdr =
1698                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1699         struct iscsi_session *session;
1700         struct iscsi_conn *conn;
1701         struct iscsi_cmnd *cmnd;
1702
1703         mutex_lock(&target->target_mutex);
1704
1705         list_for_each_entry(session, &target->session_list, session_list_entry) {
1706                 list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
1707                         spin_lock_bh(&conn->cmd_list_lock);
1708                         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1709                                 if (cmnd == req)
1710                                         continue;
1711                                 if (all)
1712                                         __cmnd_abort(cmnd);
1713                                 else if (req_hdr->lun == cmnd_hdr(cmnd)->lun)
1714                                         __cmnd_abort(cmnd);
1715                         }
1716                         spin_unlock_bh(&conn->cmd_list_lock);
1717                 }
1718         }
1719
1720         mutex_unlock(&target->target_mutex);
1721         return 0;
1722 }
1723
1724 /* Must be called from the read thread */
1725 static void task_set_abort(struct iscsi_cmnd *req)
1726 {
1727         struct iscsi_session *session = req->conn->session;
1728         struct iscsi_task_mgt_hdr *req_hdr =
1729                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1730         struct iscsi_target *target = session->target;
1731         struct iscsi_conn *conn;
1732         struct iscsi_cmnd *cmnd;
1733
1734         mutex_lock(&target->target_mutex);
1735
1736         list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
1737                 spin_lock_bh(&conn->cmd_list_lock);
1738                 list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1739                         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1740                         if (cmnd == req)
1741                                 continue;
1742                         if (req_hdr->lun != hdr->lun)
1743                                 continue;
1744                         if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1745                             req_hdr->cmd_sn == hdr->cmd_sn)
1746                                 continue;
1747                         __cmnd_abort(cmnd);
1748                 }
1749                 spin_unlock_bh(&conn->cmd_list_lock);
1750         }
1751
1752         mutex_unlock(&target->target_mutex);
1753         return;
1754 }
1755
1756 /* Must be called from the read thread */
1757 void conn_abort(struct iscsi_conn *conn)
1758 {
1759         struct iscsi_cmnd *cmnd;
1760
1761         TRACE_MGMT_DBG("Aborting conn %p", conn);
1762
1763         iscsi_extracheck_is_rd_thread(conn);
1764
1765         spin_lock_bh(&conn->cmd_list_lock);
1766 again:
1767         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1768                 __cmnd_abort(cmnd);
1769                 if (cmnd->data_waiting) {
1770                         if (!cmnd_get_check(cmnd)) {
1771                                 spin_unlock_bh(&conn->cmd_list_lock);
1772
1773                                 /* ToDo: this is racy for MC/S */
1774                                 TRACE_MGMT_DBG("Restarting data waiting cmd %p",
1775                                         cmnd);
1776                                 iscsi_fail_waiting_cmnd(cmnd);
1777
1778                                 cmnd_put(cmnd);
1779
1780                                 /*
1781                                  * We are in the read thread, so we may not
1782                                  * worry that after cmnd release conn gets
1783                                  * released as well.
1784                                  */
1785                                 spin_lock_bh(&conn->cmd_list_lock);
1786                                 goto again;
1787                         }
1788                 }
1789         }
1790         spin_unlock_bh(&conn->cmd_list_lock);
1791
1792         return;
1793 }
1794
1795 static void execute_task_management(struct iscsi_cmnd *req)
1796 {
1797         struct iscsi_conn *conn = req->conn;
1798         struct iscsi_session *sess = conn->session;
1799         struct iscsi_task_mgt_hdr *req_hdr =
1800                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1801         int rc, status, function = req_hdr->function & ISCSI_FUNCTION_MASK;
1802         struct scst_rx_mgmt_params params;
1803
1804         TRACE((function == ISCSI_FUNCTION_ABORT_TASK) ? TRACE_MGMT_MINOR : TRACE_MGMT,
1805                 "TM fn %d", function);
1806
1807         TRACE_MGMT_DBG("TM req %p, itt %x, rtt %x, sn %u, con %p", req,
1808                 cmnd_itt(req), req_hdr->rtt, req_hdr->cmd_sn, conn);
1809
1810         iscsi_extracheck_is_rd_thread(conn);
1811
1812         spin_lock(&sess->sn_lock);
1813         sess->tm_active++;
1814         sess->tm_sn = req_hdr->cmd_sn;
1815         if (sess->tm_rsp != NULL) {
1816                 struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
1817
1818                 TRACE(TRACE_MGMT_MINOR, "Dropping delayed TM rsp %p", tm_rsp);
1819
1820                 sess->tm_rsp = NULL;
1821                 sess->tm_active--;
1822
1823                 spin_unlock(&sess->sn_lock);
1824
1825                 sBUG_ON(sess->tm_active < 0);
1826
1827                 rsp_cmnd_release(tm_rsp);
1828         } else
1829                 spin_unlock(&sess->sn_lock);
1830
1831         memset(&params, 0, sizeof(params));
1832         params.atomic = SCST_NON_ATOMIC;
1833         params.tgt_priv = req;
1834
1835         if ((function != ISCSI_FUNCTION_ABORT_TASK) &&
1836             (req_hdr->rtt != ISCSI_RESERVED_TAG)) {
1837                 PRINT_ERROR("Invalid RTT %x (TM fn %x)", req_hdr->rtt,
1838                         function);
1839                 rc = -1;
1840                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1841                 goto reject;
1842         }
1843
1844         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1845
1846         switch (function) {
1847         case ISCSI_FUNCTION_ABORT_TASK:
1848                 rc = -1;
1849                 status = cmnd_abort(req);
1850                 if (status == 0) {
1851                         params.fn = SCST_ABORT_TASK;
1852                         params.tag = req_hdr->rtt;
1853                         params.tag_set = 1;
1854                         params.lun = (uint8_t *)&req_hdr->lun;
1855                         params.lun_len = sizeof(req_hdr->lun);
1856                         params.lun_set = 1;
1857                         params.cmd_sn = req_hdr->cmd_sn;
1858                         params.cmd_sn_set = 1;
1859                         rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1860                                 &params);
1861                         status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1862                 }
1863                 break;
1864         case ISCSI_FUNCTION_ABORT_TASK_SET:
1865                 task_set_abort(req);
1866                 params.fn = SCST_ABORT_TASK_SET;
1867                 params.lun = (uint8_t *)&req_hdr->lun;
1868                 params.lun_len = sizeof(req_hdr->lun);
1869                 params.lun_set = 1;
1870                 params.cmd_sn = req_hdr->cmd_sn;
1871                 params.cmd_sn_set = 1;
1872                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1873                         &params);
1874                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1875                 break;
1876         case ISCSI_FUNCTION_CLEAR_TASK_SET:
1877                 task_set_abort(req);
1878                 params.fn = SCST_CLEAR_TASK_SET;
1879                 params.lun = (uint8_t *)&req_hdr->lun;
1880                 params.lun_len = sizeof(req_hdr->lun);
1881                 params.lun_set = 1;
1882                 params.cmd_sn = req_hdr->cmd_sn;
1883                 params.cmd_sn_set = 1;
1884                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1885                         &params);
1886                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1887                 break;
1888         case ISCSI_FUNCTION_CLEAR_ACA:
1889                 params.fn = SCST_CLEAR_ACA;
1890                 params.lun = (uint8_t *)&req_hdr->lun;
1891                 params.lun_len = sizeof(req_hdr->lun);
1892                 params.lun_set = 1;
1893                 params.cmd_sn = req_hdr->cmd_sn;
1894                 params.cmd_sn_set = 1;
1895                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1896                         &params);
1897                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1898                 break;
1899         case ISCSI_FUNCTION_TARGET_COLD_RESET:
1900         case ISCSI_FUNCTION_TARGET_WARM_RESET:
1901                 target_abort(req, 1);
1902                 params.fn = SCST_TARGET_RESET;
1903                 params.cmd_sn = req_hdr->cmd_sn;
1904                 params.cmd_sn_set = 1;
1905                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1906                         &params);
1907                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1908                 break;
1909         case ISCSI_FUNCTION_LOGICAL_UNIT_RESET:
1910                 target_abort(req, 0);
1911                 params.fn = SCST_LUN_RESET;
1912                 params.lun = (uint8_t *)&req_hdr->lun;
1913                 params.lun_len = sizeof(req_hdr->lun);
1914                 params.lun_set = 1;
1915                 params.cmd_sn = req_hdr->cmd_sn;
1916                 params.cmd_sn_set = 1;
1917                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
1918                         &params);
1919                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1920                 break;
1921         case ISCSI_FUNCTION_TASK_REASSIGN:
1922                 rc = -1;
1923                 status = ISCSI_RESPONSE_ALLEGIANCE_REASSIGNMENT_UNSUPPORTED;
1924                 break;
1925         default:
1926                 PRINT_ERROR("Unknown TM function %d", function);
1927                 rc = -1;
1928                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
1929                 break;
1930         }
1931
1932 reject:
1933         if (rc != 0)
1934                 iscsi_send_task_mgmt_resp(req, status);
1935
1936         return;
1937 }
1938
1939 static void noop_out_exec(struct iscsi_cmnd *req)
1940 {
1941         struct iscsi_cmnd *rsp;
1942         struct iscsi_nop_in_hdr *rsp_hdr;
1943
1944         TRACE_DBG("%p", req);
1945
1946         if (cmnd_itt(req) != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1947                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
1948
1949                 rsp_hdr = (struct iscsi_nop_in_hdr *)&rsp->pdu.bhs;
1950                 rsp_hdr->opcode = ISCSI_OP_NOOP_IN;
1951                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1952                 rsp_hdr->itt = req->pdu.bhs.itt;
1953                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
1954
1955                 if (req->pdu.datasize)
1956                         sBUG_ON(req->sg == NULL);
1957                 else
1958                         sBUG_ON(req->sg != NULL);
1959
1960                 if (req->sg) {
1961                         rsp->sg = req->sg;
1962                         rsp->sg_cnt = req->sg_cnt;
1963                         rsp->bufflen = req->bufflen;
1964                 }
1965
1966                 sBUG_ON(get_pgcnt(req->pdu.datasize, 0) > ISCSI_CONN_IOV_MAX);
1967
1968                 rsp->pdu.datasize = req->pdu.datasize;
1969                 iscsi_cmnd_init_write(rsp,
1970                         ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
1971                 req_cmnd_release(req);
1972         } else
1973                 cmnd_put(req);
1974 }
1975
1976 static void logout_exec(struct iscsi_cmnd *req)
1977 {
1978         struct iscsi_logout_req_hdr *req_hdr;
1979         struct iscsi_cmnd *rsp;
1980         struct iscsi_logout_rsp_hdr *rsp_hdr;
1981
1982         PRINT_INFO("Logout received from initiator %s",
1983                 req->conn->session->initiator_name);
1984         TRACE_DBG("%p", req);
1985
1986         req_hdr = (struct iscsi_logout_req_hdr *)&req->pdu.bhs;
1987         rsp = iscsi_cmnd_create_rsp_cmnd(req);
1988         rsp_hdr = (struct iscsi_logout_rsp_hdr *)&rsp->pdu.bhs;
1989         rsp_hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1990         rsp_hdr->flags = ISCSI_FLG_FINAL;
1991         rsp_hdr->itt = req_hdr->itt;
1992         rsp->should_close_conn = 1;
1993         iscsi_cmnd_init_write(rsp,
1994                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
1995         req_cmnd_release(req);
1996 }
1997
1998 static void iscsi_cmnd_exec(struct iscsi_cmnd *cmnd)
1999 {
2000         TRACE_ENTRY();
2001
2002         TRACE_DBG("%p,%x,%u", cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn);
2003
2004         iscsi_extracheck_is_rd_thread(cmnd->conn);
2005
2006         if (unlikely(cmnd->tm_aborted)) {
2007                 TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted", cmnd,
2008                         cmnd->scst_cmd);
2009                 req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
2010                 goto out;
2011         }
2012
2013         if (unlikely(cmnd->rejected))
2014                 goto out_rejected;
2015
2016         switch (cmnd_opcode(cmnd)) {
2017         case ISCSI_OP_SCSI_CMD:
2018                 if (cmnd->r2t_length != 0) {
2019                         if (!cmnd->is_unsolicited_data) {
2020                                 send_r2t(cmnd);
2021                                 break;
2022                         }
2023                 } else
2024                         iscsi_restart_cmnd(cmnd);
2025                 break;
2026         case ISCSI_OP_NOOP_OUT:
2027                 noop_out_exec(cmnd);
2028                 break;
2029         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2030                 execute_task_management(cmnd);
2031                 break;
2032         case ISCSI_OP_LOGOUT_CMD:
2033                 logout_exec(cmnd);
2034                 break;
2035         default:
2036                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2037                 req_cmnd_release(cmnd);
2038                 break;
2039         }
2040 out:
2041         TRACE_EXIT();
2042         return;
2043
2044 out_rejected:
2045         TRACE_MGMT_DBG("Rejected cmd %p (reason %d)", cmnd,
2046                 cmnd->reject_reason);
2047         switch (cmnd->reject_reason) {
2048         default:
2049                 PRINT_ERROR("Unexpected reject reason %d", cmnd->reject_reason);
2050                 /* go through */
2051         case ISCSI_REJECT_SCSI_CMD:
2052                 req_cmnd_release(cmnd);
2053                 break;
2054         }
2055         goto out;
2056 }
2057
2058 static void __cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd,
2059         u32 offset, u32 size)
2060 {
2061         TRACE_DBG("%p %u,%u,%u", cmnd, offset, size, cmnd->bufflen);
2062
2063         iscsi_extracheck_is_wr_thread(conn);
2064
2065         sBUG_ON(offset > cmnd->bufflen);
2066         sBUG_ON(offset + size > cmnd->bufflen);
2067
2068         conn->write_offset = offset;
2069         conn->write_size += size;
2070 }
2071
2072 static void cmnd_send_pdu(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
2073 {
2074         u32 size;
2075
2076         if (!cmnd->pdu.datasize)
2077                 return;
2078
2079         size = (cmnd->pdu.datasize + 3) & -4;
2080         sBUG_ON(cmnd->sg == NULL);
2081         sBUG_ON(cmnd->bufflen != size);
2082         __cmnd_send_pdu(conn, cmnd, 0, size);
2083 }
2084
2085 static void set_cork(struct socket *sock, int on)
2086 {
2087         int opt = on;
2088         mm_segment_t oldfs;
2089
2090         oldfs = get_fs();
2091         set_fs(get_ds());
2092         sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK, (void *)&opt, sizeof(opt));
2093         set_fs(oldfs);
2094 }
2095
2096 void cmnd_tx_start(struct iscsi_cmnd *cmnd)
2097 {
2098         struct iscsi_conn *conn = cmnd->conn;
2099
2100         TRACE_DBG("%p:%p:%x", conn, cmnd, cmnd_opcode(cmnd));
2101         iscsi_cmnd_set_length(&cmnd->pdu);
2102
2103         iscsi_extracheck_is_wr_thread(conn);
2104
2105         set_cork(conn->sock, 1);
2106
2107         conn->write_iop = conn->write_iov;
2108         conn->write_iop->iov_base = &cmnd->pdu.bhs;
2109         conn->write_iop->iov_len = sizeof(cmnd->pdu.bhs);
2110         conn->write_iop_used = 1;
2111         conn->write_size = sizeof(cmnd->pdu.bhs);
2112
2113         switch (cmnd_opcode(cmnd)) {
2114         case ISCSI_OP_NOOP_IN:
2115                 cmnd_set_sn(cmnd, 1);
2116                 cmnd_send_pdu(conn, cmnd);
2117                 break;
2118         case ISCSI_OP_SCSI_RSP:
2119                 cmnd_set_sn(cmnd, 1);
2120                 cmnd_send_pdu(conn, cmnd);
2121                 break;
2122         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2123                 cmnd_set_sn(cmnd, 1);
2124                 break;
2125         case ISCSI_OP_TEXT_RSP:
2126                 cmnd_set_sn(cmnd, 1);
2127                 break;
2128         case ISCSI_OP_SCSI_DATA_IN:
2129         {
2130                 struct iscsi_data_in_hdr *rsp = (struct iscsi_data_in_hdr *)&cmnd->pdu.bhs;
2131                 u32 offset = cpu_to_be32(rsp->buffer_offset);
2132
2133                 cmnd_set_sn(cmnd, (rsp->flags & ISCSI_FLG_FINAL) ? 1 : 0);
2134                 __cmnd_send_pdu(conn, cmnd, offset, cmnd->pdu.datasize);
2135                 break;
2136         }
2137         case ISCSI_OP_LOGOUT_RSP:
2138                 cmnd_set_sn(cmnd, 1);
2139                 break;
2140         case ISCSI_OP_R2T:
2141                 cmnd->pdu.bhs.sn = cmnd_set_sn(cmnd, 0);
2142                 break;
2143         case ISCSI_OP_ASYNC_MSG:
2144                 cmnd_set_sn(cmnd, 1);
2145                 break;
2146         case ISCSI_OP_REJECT:
2147                 cmnd_set_sn(cmnd, 1);
2148                 cmnd_send_pdu(conn, cmnd);
2149                 break;
2150         default:
2151                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2152                 break;
2153         }
2154
2155         /* move this? */
2156         conn->write_size = (conn->write_size + 3) & -4;
2157         iscsi_dump_pdu(&cmnd->pdu);
2158 }
2159
2160 void cmnd_tx_end(struct iscsi_cmnd *cmnd)
2161 {
2162         struct iscsi_conn *conn = cmnd->conn;
2163
2164         TRACE_DBG("%p:%x (should_close_conn %d)", cmnd, cmnd_opcode(cmnd),
2165                 cmnd->should_close_conn);
2166
2167         switch (cmnd_opcode(cmnd)) {
2168         case ISCSI_OP_NOOP_IN:
2169         case ISCSI_OP_SCSI_RSP:
2170         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2171         case ISCSI_OP_TEXT_RSP:
2172         case ISCSI_OP_R2T:
2173         case ISCSI_OP_ASYNC_MSG:
2174         case ISCSI_OP_REJECT:
2175         case ISCSI_OP_SCSI_DATA_IN:
2176         case ISCSI_OP_LOGOUT_RSP:
2177                 break;
2178         default:
2179                 PRINT_CRIT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2180                 sBUG();
2181                 break;
2182         }
2183
2184         if (cmnd->should_close_conn) {
2185                 PRINT_INFO("Closing connection at initiator %s request",
2186                         conn->session->initiator_name);
2187                 mark_conn_closed(conn);
2188         }
2189
2190         set_cork(cmnd->conn->sock, 0);
2191 }
2192
2193 /*
2194  * Push the command for execution. This functions reorders the commands.
2195  * Called from the read thread.
2196  */
2197 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd)
2198 {
2199         struct iscsi_session *session = cmnd->conn->session;
2200         struct list_head *entry;
2201         u32 cmd_sn;
2202
2203         TRACE_DBG("%p:%x %u,%u",
2204                 cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn, session->exp_cmd_sn);
2205
2206         iscsi_extracheck_is_rd_thread(cmnd->conn);
2207
2208         sBUG_ON(cmnd->parent_req != NULL);
2209
2210         if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
2211                 TRACE_DBG("Immediate cmd %p (cmd_sn %u)", cmnd,
2212                         cmnd->pdu.bhs.sn);
2213                 iscsi_cmnd_exec(cmnd);
2214                 goto out;
2215         }
2216
2217         spin_lock(&session->sn_lock);
2218
2219         cmd_sn = cmnd->pdu.bhs.sn;
2220         if (cmd_sn == session->exp_cmd_sn) {
2221                 while (1) {
2222                         session->exp_cmd_sn = ++cmd_sn;
2223
2224                         if (unlikely(session->tm_active > 0)) {
2225                                 if (before(cmd_sn, session->tm_sn)) {
2226                                         struct iscsi_conn *conn = cmnd->conn;
2227
2228                                         spin_unlock(&session->sn_lock);
2229
2230                                         spin_lock_bh(&conn->cmd_list_lock);
2231                                         __cmnd_abort(cmnd);
2232                                         spin_unlock_bh(&conn->cmd_list_lock);
2233
2234                                         spin_lock(&session->sn_lock);
2235                                 }
2236                                 iscsi_check_send_delayed_tm_resp(session);
2237                         }
2238
2239                         spin_unlock(&session->sn_lock);
2240
2241                         iscsi_cmnd_exec(cmnd);
2242
2243                         if (list_empty(&session->pending_list))
2244                                 break;
2245                         cmnd = list_entry(session->pending_list.next, struct iscsi_cmnd,
2246                                                 pending_list_entry);
2247                         if (cmnd->pdu.bhs.sn != cmd_sn)
2248                                 break;
2249
2250                         list_del(&cmnd->pending_list_entry);
2251                         cmnd->pending = 0;
2252
2253                         TRACE_DBG("Processing pending cmd %p (cmd_sn %u)",
2254                                 cmnd, cmd_sn);
2255
2256                         spin_lock(&session->sn_lock);
2257                 }
2258         } else {
2259                 int drop = 0;
2260
2261                 TRACE_DBG("Pending cmd %p (cmd_sn %u, exp_cmd_sn %u)",
2262                         cmnd, cmd_sn, session->exp_cmd_sn);
2263
2264                 /*
2265                  * iSCSI RFC 3720: "The target MUST silently ignore any
2266                  * non-immediate command outside of [from ExpCmdSN to MaxCmdSN
2267                  * inclusive] range". But we won't honor the MaxCmdSN
2268                  * requirement, because, since we adjust MaxCmdSN from the
2269                  * separate write thread, rarery it is possible that initiator
2270                  * can legally send command with CmdSN>MaxSN. But it won't
2271                  * hurt anything, in the worst case it will lead to
2272                  * additional QUEUE FULL status.
2273                  */
2274
2275                 if (unlikely(before(cmd_sn, session->exp_cmd_sn))) {
2276                         PRINT_ERROR("Unexpected cmd_sn (%u,%u)", cmd_sn,
2277                                 session->exp_cmd_sn);
2278                         drop = 1;
2279                 }
2280
2281 #if 0
2282                 if (unlikely(after(cmd_sn, session->exp_cmd_sn +
2283                                         iscsi_get_allowed_cmds(session)))) {
2284                         TRACE_MGMT_DBG("Too large cmd_sn %u (exp_cmd_sn %u, "
2285                                 "max_sn %u)", cmd_sn, session->exp_cmd_sn,
2286                                 iscsi_get_allowed_cmds(session));
2287                 }
2288 #endif
2289
2290                 spin_unlock(&session->sn_lock);
2291
2292                 if (unlikely(drop)) {
2293                         req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
2294                         goto out;
2295                 }
2296
2297                 if (unlikely(cmnd->tm_aborted)) {
2298                         struct iscsi_cmnd *tm_clone;
2299
2300                         TRACE_MGMT_DBG("Pending aborted cmnd %p, creating TM "
2301                                 "clone (scst cmd %p, state %d)", cmnd,
2302                                 cmnd->scst_cmd, cmnd->scst_state);
2303
2304                         tm_clone = cmnd_alloc(cmnd->conn, NULL);
2305                         if (tm_clone != NULL) {
2306                                 tm_clone->tm_aborted = 1;
2307                                 tm_clone->pdu = cmnd->pdu;
2308
2309                                 TRACE_MGMT_DBG("TM clone %p created", tm_clone);
2310
2311                                 iscsi_cmnd_exec(cmnd);
2312                                 cmnd = tm_clone;
2313                         } else
2314                                 PRINT_ERROR("%s", "Unable to create TM clone");
2315                 }
2316
2317                 list_for_each(entry, &session->pending_list) {
2318                         struct iscsi_cmnd *tmp = list_entry(entry, struct iscsi_cmnd,
2319                                                         pending_list_entry);
2320                         if (before(cmd_sn, tmp->pdu.bhs.sn))
2321                                 break;
2322                 }
2323
2324                 list_add_tail(&cmnd->pending_list_entry, entry);
2325                 cmnd->pending = 1;
2326         }
2327 out:
2328         return;
2329 }
2330
2331 static int check_segment_length(struct iscsi_cmnd *cmnd)
2332 {
2333         struct iscsi_conn *conn = cmnd->conn;
2334         struct iscsi_session *session = conn->session;
2335
2336         if (unlikely(cmnd->pdu.datasize > session->sess_param.max_recv_data_length)) {
2337                 PRINT_ERROR("Initiator %s violated negotiated parameters: "
2338                         "data too long (ITT %x, datasize %u, "
2339                         "max_recv_data_length %u", session->initiator_name,
2340                         cmnd_itt(cmnd), cmnd->pdu.datasize,
2341                         session->sess_param.max_recv_data_length);
2342                 mark_conn_closed(conn);
2343                 return -EINVAL;
2344         }
2345         return 0;
2346 }
2347
2348 int cmnd_rx_start(struct iscsi_cmnd *cmnd)
2349 {
2350         struct iscsi_conn *conn = cmnd->conn;
2351         int res, rc;
2352
2353         iscsi_dump_pdu(&cmnd->pdu);
2354
2355         res = check_segment_length(cmnd);
2356         if (res != 0)
2357                 goto out;
2358
2359         switch (cmnd_opcode(cmnd)) {
2360         case ISCSI_OP_NOOP_OUT:
2361                 rc = noop_out_start(cmnd);
2362                 break;
2363         case ISCSI_OP_SCSI_CMD:
2364                 rc = cmnd_insert_hash(cmnd);
2365                 if (likely(rc == 0)) {
2366                         res = scsi_cmnd_start(cmnd);
2367                         if (unlikely(res != 0))
2368                                 goto out;
2369                 }
2370                 break;
2371         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2372                 rc = cmnd_insert_hash(cmnd);
2373                 break;
2374         case ISCSI_OP_SCSI_DATA_OUT:
2375                 res = data_out_start(conn, cmnd);
2376                 rc = 0; /* to avoid compiler warning */
2377                 if (unlikely(res != 0))
2378                         goto out;
2379                 break;
2380         case ISCSI_OP_LOGOUT_CMD:
2381                 rc = cmnd_insert_hash(cmnd);
2382                 break;
2383         case ISCSI_OP_TEXT_CMD:
2384         case ISCSI_OP_SNACK_CMD:
2385                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2386                 break;
2387         default:
2388                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2389                 break;
2390         }
2391
2392         if (unlikely(rc < 0)) {
2393                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
2394                 PRINT_ERROR("Error %d (iSCSI opcode %x, ITT %x, op %x)", rc,
2395                         cmnd_opcode(cmnd), cmnd_itt(cmnd),
2396                         (cmnd_opcode(cmnd) == ISCSI_OP_SCSI_CMD ?
2397                                 hdr->scb[0] : -1));
2398                 iscsi_cmnd_reject(cmnd, -rc);
2399         }
2400
2401 out:
2402         TRACE_EXIT_RES(res);
2403         return res;
2404 }
2405
2406 void cmnd_rx_end(struct iscsi_cmnd *cmnd)
2407 {
2408         TRACE_ENTRY();
2409
2410         TRACE_DBG("%p:%x", cmnd, cmnd_opcode(cmnd));
2411
2412         if (unlikely(cmnd->rejected))
2413                 goto out_rejected;
2414
2415 cont:
2416         switch (cmnd_opcode(cmnd)) {
2417         case ISCSI_OP_SCSI_CMD:
2418         case ISCSI_OP_NOOP_OUT:
2419         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2420         case ISCSI_OP_LOGOUT_CMD:
2421                 iscsi_session_push_cmnd(cmnd);
2422                 break;
2423         case ISCSI_OP_SCSI_DATA_OUT:
2424                 data_out_end(cmnd);
2425                 break;
2426         default:
2427                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2428                 req_cmnd_release(cmnd);
2429                 break;
2430         }
2431
2432 out:
2433         TRACE_EXIT();
2434         return;
2435
2436 out_rejected:
2437         switch (cmnd->reject_reason) {
2438         default:
2439                 PRINT_ERROR("Unexpected reject reason %d", cmnd->reject_reason);
2440                 /* go through */
2441         case ISCSI_REJECT_CMD:
2442         case ISCSI_REJECT_DATA:
2443                 req_cmnd_release(cmnd);
2444                 break;
2445         case ISCSI_REJECT_SCSI_CMD:
2446                 goto cont;
2447         }
2448         goto out;
2449 }
2450
2451 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
2452 static int iscsi_alloc_data_buf(struct scst_cmd *cmd)
2453 {
2454         if (scst_cmd_get_data_direction(cmd) == SCST_DATA_READ) {
2455                 /*
2456                  * sock->ops->sendpage() is async zero copy operation,
2457                  * so we must be sure not to free and reuse
2458                  * the command's buffer before the sending was completed
2459                  * by the network layers. It is possible only if we
2460                  * don't use SGV cache.
2461                  */
2462                 scst_cmd_set_no_sgv(cmd);
2463         }
2464         return 1;
2465 }
2466 #endif
2467
2468 static inline void iscsi_set_state_wake_up(struct iscsi_cmnd *req,
2469         int new_state)
2470 {
2471         /*
2472          * We use wait_event() to wait for the state change, but it checks its
2473          * condition without any protection, so without cmnd_get() it is
2474          * possible that req will die "immediately" after the state assignment
2475          * and wake_up() will operate on dead data.
2476          */
2477         cmnd_get_ordered(req);
2478         req->scst_state = new_state;
2479         wake_up(&req->scst_waitQ);
2480         cmnd_put(req);
2481         return;
2482 }
2483
2484 static void iscsi_preprocessing_done(struct scst_cmd *scst_cmd)
2485 {
2486         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
2487                                 scst_cmd_get_tgt_priv(scst_cmd);
2488
2489         TRACE_DBG("req %p", req);
2490
2491         EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_RX_CMD);
2492
2493         iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_AFTER_PREPROC);
2494         return;
2495 }
2496
2497 /*
2498  * No locks.
2499  *
2500  * IMPORTANT! Connection conn must be protected by additional conn_get()
2501  * upon entrance in this function, because otherwise it could be destroyed
2502  * inside as a result of iscsi_send(), which releases sent commands.
2503  */
2504 static void iscsi_try_local_processing(struct iscsi_conn *conn)
2505 {
2506         int local;
2507
2508         TRACE_ENTRY();
2509
2510         spin_lock_bh(&iscsi_wr_lock);
2511         switch (conn->wr_state) {
2512         case ISCSI_CONN_WR_STATE_IN_LIST:
2513                 list_del(&conn->wr_list_entry);
2514                 /* go through */
2515         case ISCSI_CONN_WR_STATE_IDLE:
2516 #ifdef CONFIG_SCST_EXTRACHECKS
2517                 conn->wr_task = current;
2518 #endif
2519                 conn->wr_state = ISCSI_CONN_WR_STATE_PROCESSING;
2520                 conn->wr_space_ready = 0;
2521                 local = 1;
2522                 break;
2523         default:
2524                 local = 0;
2525                 break;
2526         }
2527         spin_unlock_bh(&iscsi_wr_lock);
2528
2529         if (local) {
2530                 int rc = 1;
2531
2532                 if (test_write_ready(conn))
2533                         rc = iscsi_send(conn);
2534
2535                 spin_lock_bh(&iscsi_wr_lock);
2536 #ifdef CONFIG_SCST_EXTRACHECKS
2537                 conn->wr_task = NULL;
2538 #endif
2539                 if ((rc <= 0) || test_write_ready(conn)) {
2540                         list_add_tail(&conn->wr_list_entry, &iscsi_wr_list);
2541                         conn->wr_state = ISCSI_CONN_WR_STATE_IN_LIST;
2542                         wake_up(&iscsi_wr_waitQ);
2543                 } else
2544                         conn->wr_state = ISCSI_CONN_WR_STATE_IDLE;
2545                 spin_unlock_bh(&iscsi_wr_lock);
2546         }
2547
2548         TRACE_EXIT();
2549         return;
2550 }
2551
2552 static int iscsi_xmit_response(struct scst_cmd *scst_cmd)
2553 {
2554         int is_send_status = scst_cmd_get_is_send_status(scst_cmd);
2555         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
2556                                         scst_cmd_get_tgt_priv(scst_cmd);
2557         struct iscsi_conn *conn = req->conn;
2558         int status = scst_cmd_get_status(scst_cmd);
2559         u8 *sense = scst_cmd_get_sense_buffer(scst_cmd);
2560         int sense_len = scst_cmd_get_sense_buffer_len(scst_cmd);
2561         int old_state = req->scst_state;
2562
2563         if (scst_cmd_atomic(scst_cmd))
2564                 return SCST_TGT_RES_NEED_THREAD_CTX;
2565
2566         scst_cmd_set_tgt_priv(scst_cmd, NULL);
2567
2568         req->tm_aborted |= scst_cmd_aborted(scst_cmd) ? 1 : 0;
2569         if (unlikely(req->tm_aborted)) {
2570                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
2571                         req->scst_cmd);
2572
2573                 scst_set_delivery_status(req->scst_cmd,
2574                         SCST_CMD_DELIVERY_ABORTED);
2575
2576                 if (old_state == ISCSI_CMD_STATE_RESTARTED) {
2577                         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2578                         req_cmnd_release_force(req, ISCSI_FORCE_RELEASE_WRITE);
2579                 } else
2580                         iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
2581
2582                 goto out;
2583         }
2584
2585         if (unlikely(old_state != ISCSI_CMD_STATE_RESTARTED)) {
2586                 TRACE_DBG("req %p on %d state", req, old_state);
2587
2588                 create_status_rsp(req, status, sense, sense_len);
2589
2590                 switch (old_state) {
2591                 case ISCSI_CMD_STATE_RX_CMD:
2592                 case ISCSI_CMD_STATE_AFTER_PREPROC:
2593                         break;
2594                 default:
2595                         sBUG();
2596                 }
2597
2598                 iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
2599                 goto out;
2600         }
2601
2602         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2603
2604         req->bufflen = scst_cmd_get_resp_data_len(scst_cmd);
2605         req->sg = scst_cmd_get_sg(scst_cmd);
2606         req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
2607
2608         TRACE_DBG("req %p, is_send_status=%x, req->bufflen=%d, req->sg=%p, "
2609                 "req->sg_cnt %d", req, is_send_status, req->bufflen, req->sg,
2610                 req->sg_cnt);
2611
2612         if (unlikely((req->bufflen != 0) && !is_send_status)) {
2613                 PRINT_CRIT_ERROR("%s", "Sending DATA without STATUS is unsupported");
2614                 scst_set_cmd_error(scst_cmd,
2615                         SCST_LOAD_SENSE(scst_sense_hardw_error));
2616                 sBUG();
2617         }
2618
2619         if (req->bufflen != 0) {
2620                 /*
2621                  * Check above makes sure that is_send_status is set,
2622                  * so status is valid here, but in future that could change.
2623                  * ToDo
2624                  */
2625                 if (status != SAM_STAT_CHECK_CONDITION) {
2626                         send_data_rsp(req, status, is_send_status);
2627                 } else {
2628                         struct iscsi_cmnd *rsp;
2629                         struct iscsi_scsi_rsp_hdr *rsp_hdr;
2630                         int resid;
2631                         send_data_rsp(req, 0, 0);
2632                         if (is_send_status) {
2633                                 rsp = create_status_rsp(req, status, sense,
2634                                         sense_len);
2635                                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
2636                                 resid = cmnd_read_size(req) - req->bufflen;
2637                                 if (resid > 0) {
2638                                         rsp_hdr->flags |=
2639                                                 ISCSI_FLG_RESIDUAL_UNDERFLOW;
2640                                         rsp_hdr->residual_count = cpu_to_be32(resid);
2641                                 } else if (resid < 0) {
2642                                         rsp_hdr->flags |=
2643                                                 ISCSI_FLG_RESIDUAL_OVERFLOW;
2644                                         rsp_hdr->residual_count = cpu_to_be32(-resid);
2645                                 }
2646                                 iscsi_cmnd_init_write(rsp,
2647                                         ISCSI_INIT_WRITE_REMOVE_HASH);
2648                         }
2649                 }
2650         } else if (is_send_status) {
2651                 struct iscsi_cmnd *rsp;
2652                 struct iscsi_scsi_rsp_hdr *rsp_hdr;
2653                 u32 resid;
2654                 rsp = create_status_rsp(req, status, sense, sense_len);
2655                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *) &rsp->pdu.bhs;
2656                 resid = cmnd_read_size(req);
2657                 if (resid != 0) {
2658                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
2659                         rsp_hdr->residual_count = cpu_to_be32(resid);
2660                 }
2661                 iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH);
2662         }
2663 #ifdef CONFIG_SCST_EXTRACHECKS
2664         else
2665                 sBUG();
2666 #endif
2667
2668         conn_get_ordered(conn);
2669         req_cmnd_release(req);
2670         iscsi_try_local_processing(conn);
2671         conn_put(conn);
2672
2673 out:
2674         return SCST_TGT_RES_SUCCESS;
2675 }
2676
2677 /* Called under sn_lock */
2678 static bool iscsi_is_delay_tm_resp(struct iscsi_cmnd *rsp)
2679 {
2680         bool res = 0;
2681         struct iscsi_task_mgt_hdr *req_hdr =
2682                 (struct iscsi_task_mgt_hdr *)&rsp->parent_req->pdu.bhs;
2683         int function = req_hdr->function & ISCSI_FUNCTION_MASK;
2684         struct iscsi_session *sess = rsp->conn->session;
2685
2686         TRACE_ENTRY();
2687
2688         /* This should be checked for immediate TM commands as well */
2689
2690         switch (function) {
2691         default:
2692                 if (before(sess->exp_cmd_sn, req_hdr->cmd_sn))
2693                         res = 1;
2694                 break;
2695         }
2696
2697         TRACE_EXIT_RES(res);
2698         return res;
2699 }
2700
2701 /* Called under sn_lock, but might drop it inside, then reaquire */
2702 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess)
2703 {
2704         struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
2705
2706         TRACE_ENTRY();
2707
2708         if (tm_rsp == NULL)
2709                 goto out;
2710
2711         if (iscsi_is_delay_tm_resp(tm_rsp))
2712                 goto out;
2713
2714         TRACE(TRACE_MGMT_MINOR, "Sending delayed rsp %p", tm_rsp);
2715
2716         sess->tm_rsp = NULL;
2717         sess->tm_active--;
2718
2719         spin_unlock(&sess->sn_lock);
2720
2721         sBUG_ON(sess->tm_active < 0);
2722
2723         iscsi_cmnd_init_write(tm_rsp,
2724                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2725
2726         spin_lock(&sess->sn_lock);
2727
2728 out:
2729         TRACE_EXIT();
2730         return;
2731 }
2732
2733 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status)
2734 {
2735         struct iscsi_cmnd *rsp;
2736         struct iscsi_task_mgt_hdr *req_hdr =
2737                                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2738         struct iscsi_task_rsp_hdr *rsp_hdr;
2739         struct iscsi_session *sess = req->conn->session;
2740         int fn = req_hdr->function & ISCSI_FUNCTION_MASK;
2741
2742         TRACE_ENTRY();
2743
2744         TRACE_MGMT_DBG("TM req %p finished", req);
2745         TRACE((req_hdr->function == ISCSI_FUNCTION_ABORT_TASK) ?
2746                          TRACE_MGMT_MINOR : TRACE_MGMT,
2747                 "TM fn %d finished, status %d", fn, status);
2748
2749         rsp = iscsi_cmnd_create_rsp_cmnd(req);
2750         rsp_hdr = (struct iscsi_task_rsp_hdr *)&rsp->pdu.bhs;
2751
2752         rsp_hdr->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
2753         rsp_hdr->flags = ISCSI_FLG_FINAL;
2754         rsp_hdr->itt = req_hdr->itt;
2755         rsp_hdr->response = status;
2756
2757         if (fn == ISCSI_FUNCTION_TARGET_COLD_RESET)
2758                 rsp->should_close_conn = 1;
2759
2760         sBUG_ON(sess->tm_rsp != NULL);
2761
2762         spin_lock(&sess->sn_lock);
2763         if (iscsi_is_delay_tm_resp(rsp)) {
2764                 TRACE(TRACE_MGMT_MINOR, "Delaying TM fn %x response %p "
2765                         "(req %p), because not all affected commands received "
2766                         "(TM cmd sn %u, exp sn %u)",
2767                         req_hdr->function & ISCSI_FUNCTION_MASK, rsp, req,
2768                         req_hdr->cmd_sn, sess->exp_cmd_sn);
2769                 sess->tm_rsp = rsp;
2770                 spin_unlock(&sess->sn_lock);
2771                 goto out_release;
2772         }
2773         sess->tm_active--;
2774         spin_unlock(&sess->sn_lock);
2775
2776         sBUG_ON(sess->tm_active < 0);
2777
2778         iscsi_cmnd_init_write(rsp,
2779                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2780
2781 out_release:
2782         req_cmnd_release(req);
2783
2784         TRACE_EXIT();
2785         return;
2786 }
2787
2788 static inline int iscsi_get_mgmt_response(int status)
2789 {
2790         switch (status) {
2791         case SCST_MGMT_STATUS_SUCCESS:
2792                 return ISCSI_RESPONSE_FUNCTION_COMPLETE;
2793
2794         case SCST_MGMT_STATUS_TASK_NOT_EXIST:
2795                 return ISCSI_RESPONSE_UNKNOWN_TASK;
2796
2797         case SCST_MGMT_STATUS_LUN_NOT_EXIST:
2798                 return ISCSI_RESPONSE_UNKNOWN_LUN;
2799
2800         case SCST_MGMT_STATUS_FN_NOT_SUPPORTED:
2801                 return ISCSI_RESPONSE_FUNCTION_UNSUPPORTED;
2802
2803         case SCST_MGMT_STATUS_REJECTED:
2804         case SCST_MGMT_STATUS_FAILED:
2805         default:
2806                 return ISCSI_RESPONSE_FUNCTION_REJECTED;
2807         }
2808 }
2809
2810 static void iscsi_task_mgmt_fn_done(struct scst_mgmt_cmd *scst_mcmd)
2811 {
2812         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
2813                                 scst_mgmt_cmd_get_tgt_priv(scst_mcmd);
2814         int status = iscsi_get_mgmt_response(scst_mgmt_cmd_get_status(scst_mcmd));
2815
2816         TRACE_MGMT_DBG("req %p, scst_mcmd %p, fn %d, scst status %d",
2817                 req, scst_mcmd, scst_mgmt_cmd_get_fn(scst_mcmd),
2818                 scst_mgmt_cmd_get_status(scst_mcmd));
2819
2820         iscsi_send_task_mgmt_resp(req, status);
2821
2822         scst_mgmt_cmd_set_tgt_priv(scst_mcmd, NULL);
2823
2824         return;
2825 }
2826
2827 static int iscsi_target_detect(struct scst_tgt_template *templ)
2828 {
2829         /* Nothing to do */
2830         return 0;
2831 }
2832
2833 static int iscsi_target_release(struct scst_tgt *scst_tgt)
2834 {
2835         /* Nothing to do */
2836         return 0;
2837 }
2838
2839 struct scst_tgt_template iscsi_template = {
2840         .name = "iscsi",
2841         .sg_tablesize = ISCSI_CONN_IOV_MAX,
2842         .threads_num = 0,
2843         .no_clustering = 1,
2844         .xmit_response_atomic = 0,
2845         .detect = iscsi_target_detect,
2846         .release = iscsi_target_release,
2847         .xmit_response = iscsi_xmit_response,
2848 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
2849         .alloc_data_buf = iscsi_alloc_data_buf,
2850 #endif
2851         .preprocessing_done = iscsi_preprocessing_done,
2852         .pre_exec = iscsi_pre_exec,
2853         .task_mgmt_fn_done = iscsi_task_mgmt_fn_done,
2854 };
2855
2856 static __init int iscsi_run_threads(int count, char *name, int (*fn)(void *))
2857 {
2858         int res = 0;
2859         int i;
2860         struct iscsi_thread_t *thr;
2861
2862         for (i = 0; i < count; i++) {
2863                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
2864                 if (!thr) {
2865                         res = -ENOMEM;
2866                         PRINT_ERROR("Failed to allocate thr %d", res);
2867                         goto out;
2868                 }
2869                 thr->thr = kthread_run(fn, NULL, "%s%d", name, i);
2870                 if (IS_ERR(thr->thr)) {
2871                         res = PTR_ERR(thr->thr);
2872                         PRINT_ERROR("kthread_create() failed: %d", res);
2873                         kfree(thr);
2874                         goto out;
2875                 }
2876                 list_add(&thr->threads_list_entry, &iscsi_threads_list);
2877         }
2878
2879 out:
2880         return res;
2881 }
2882
2883 static void iscsi_stop_threads(void)
2884 {
2885         struct iscsi_thread_t *t, *tmp;
2886
2887         list_for_each_entry_safe(t, tmp, &iscsi_threads_list,
2888                                 threads_list_entry) {
2889                 int rc = kthread_stop(t->thr);
2890                 if (rc < 0)
2891                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
2892                 list_del(&t->threads_list_entry);
2893                 kfree(t);
2894         }
2895 }
2896
2897 static int __init iscsi_init(void)
2898 {
2899         int err = 0;
2900         int num;
2901
2902         PRINT_INFO("iSCSI SCST Target - version %s", ISCSI_VERSION_STRING);
2903
2904         dummy_page = alloc_pages(GFP_KERNEL, 0);
2905         if (dummy_page == NULL) {
2906                 PRINT_ERROR("%s", "Dummy page allocation failed");
2907                 goto out;
2908         }
2909
2910         sg_init_table(&dummy_sg, 1);
2911         sg_set_page(&dummy_sg, dummy_page, PAGE_SIZE, 0);
2912
2913 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
2914         err = net_set_get_put_page_callbacks(iscsi_get_page_callback,
2915                         iscsi_put_page_callback);
2916         if (err != 0) {
2917                 PRINT_INFO("Unable to set page callbackes: %d", err);
2918                 goto out_free_dummy;
2919         }
2920 #else
2921         PRINT_INFO("%s", "Patch put_page_callback-<kernel-version>.patch "
2922                 "not applied on your kernel. Running in the performance "
2923                 "degraded mode. Refer README file for details");
2924 #endif
2925
2926         ctr_major = register_chrdev(0, ctr_name, &ctr_fops);
2927         if (ctr_major < 0) {
2928                 PRINT_ERROR("failed to register the control device %d", ctr_major);
2929                 err = ctr_major;
2930                 goto out_callb;
2931         }
2932
2933         err = event_init();
2934         if (err < 0)
2935                 goto out_reg;
2936
2937         iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS);
2938         if (!iscsi_cmnd_cache) {
2939                 err = -ENOMEM;
2940                 goto out_event;
2941         }
2942
2943         err = scst_register_target_template(&iscsi_template);
2944         if (err < 0)
2945                 goto out_kmem;
2946
2947         iscsi_template_registered = 1;
2948
2949         err = iscsi_procfs_init();
2950         if (err < 0)
2951                 goto out_reg_tmpl;
2952
2953         num = max(num_online_cpus(), 2);
2954
2955         err = iscsi_run_threads(num, "iscsird", istrd);
2956         if (err != 0)
2957                 goto out_thr;
2958
2959         err = iscsi_run_threads(num, "iscsiwr", istwr);
2960         if (err != 0)
2961                 goto out_thr;
2962
2963 out:
2964         return err;
2965
2966 out_thr:
2967         iscsi_procfs_exit();
2968         iscsi_stop_threads();
2969
2970 out_reg_tmpl:
2971         scst_unregister_target_template(&iscsi_template);
2972
2973 out_kmem:
2974         kmem_cache_destroy(iscsi_cmnd_cache);
2975
2976 out_event:
2977         event_exit();
2978
2979 out_reg:
2980         unregister_chrdev(ctr_major, ctr_name);
2981
2982 out_callb:
2983 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
2984         net_set_get_put_page_callbacks(NULL, NULL);
2985 #endif
2986
2987 out_free_dummy:
2988         __free_pages(dummy_page, 0);
2989         goto out;
2990 }
2991
2992 static void __exit iscsi_exit(void)
2993 {
2994         iscsi_stop_threads();
2995
2996         unregister_chrdev(ctr_major, ctr_name);
2997
2998         iscsi_procfs_exit();
2999         event_exit();
3000
3001         kmem_cache_destroy(iscsi_cmnd_cache);
3002
3003         scst_unregister_target_template(&iscsi_template);
3004
3005 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3006         net_set_get_put_page_callbacks(NULL, NULL);
3007 #endif
3008
3009         __free_pages(dummy_page, 0);
3010         return;
3011 }
3012
3013 module_init(iscsi_init);
3014 module_exit(iscsi_exit);
3015
3016 MODULE_LICENSE("GPL");