A huge chunk of related to each other changes, which had to be tested together.
[mirror/scst/.git] / iscsi-scst / kernel / iscsi.c
1 /*
2  *  Copyright (C) 2002 - 2003 Ardis Technolgies <roman@ardistech.com>
3  *  Copyright (C) 2007 - 2009 Vladislav Bolkhovitin
4  *  Copyright (C) 2007 - 2009 ID7 Ltd.
5  *
6  *  This program is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU General Public License
8  *  as published by the Free Software Foundation, version 2
9  *  of the License.
10  *
11  *  This program is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  *  GNU General Public License for more details.
15  */
16
17 #include <linux/module.h>
18 #include <linux/hash.h>
19 #include <linux/kthread.h>
20 #include <linux/scatterlist.h>
21 #include <net/tcp.h>
22 #include <scsi/scsi.h>
23
24 #include "iscsi.h"
25 #include "digest.h"
26
27 #ifndef GENERATING_UPSTREAM_PATCH
28 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
29 #warning "Patch put_page_callback-<kernel-version>.patch not applied on your\
30  kernel or CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION\
31  config option not set. ISCSI-SCST will be working with not the best\
32  performance. Refer README file for details."
33 #endif
34 #endif
35
36 #define ISCSI_INIT_WRITE_WAKE           0x1
37
38 static int ctr_major;
39 static char ctr_name[] = "iscsi-scst-ctl";
40 static int iscsi_template_registered;
41
42 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
43 unsigned long iscsi_trace_flag = ISCSI_DEFAULT_LOG_FLAGS;
44 #endif
45
46 static struct kmem_cache *iscsi_cmnd_cache;
47
48 DEFINE_SPINLOCK(iscsi_rd_lock);
49 LIST_HEAD(iscsi_rd_list);
50 DECLARE_WAIT_QUEUE_HEAD(iscsi_rd_waitQ);
51
52 DEFINE_SPINLOCK(iscsi_wr_lock);
53 LIST_HEAD(iscsi_wr_list);
54 DECLARE_WAIT_QUEUE_HEAD(iscsi_wr_waitQ);
55
56 static struct page *dummy_page;
57 static struct scatterlist dummy_sg;
58
59 struct iscsi_thread_t {
60         struct task_struct *thr;
61         struct list_head threads_list_entry;
62 };
63
64 static LIST_HEAD(iscsi_threads_list);
65
66 static void cmnd_remove_data_wait_hash(struct iscsi_cmnd *cmnd);
67 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status);
68 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess);
69 static void req_cmnd_release(struct iscsi_cmnd *req);
70 static int iscsi_preliminary_complete(struct iscsi_cmnd *req,
71         struct iscsi_cmnd *orig_req, bool get_data);
72 static int cmnd_insert_data_wait_hash(struct iscsi_cmnd *cmnd);
73 static void __cmnd_abort(struct iscsi_cmnd *cmnd);
74 static void iscsi_set_resid(struct iscsi_cmnd *rsp, bool bufflen_set);
75 static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags);
76
77 static void req_del_from_write_timeout_list(struct iscsi_cmnd *req)
78 {
79         struct iscsi_conn *conn;
80
81         TRACE_ENTRY();
82
83         if (!req->on_write_timeout_list)
84                 goto out;
85
86         conn = req->conn;
87
88         TRACE_DBG("Deleting cmd %p from conn %p write_timeout_list",
89                 req, conn);
90
91         spin_lock_bh(&conn->write_list_lock);
92         list_del(&req->write_timeout_list_entry);
93         req->on_write_timeout_list = 0;
94         spin_unlock_bh(&conn->write_list_lock);
95
96 out:
97         TRACE_EXIT();
98         return;
99 }
100
101 static inline u32 cmnd_write_size(struct iscsi_cmnd *cmnd)
102 {
103         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
104
105         if (hdr->flags & ISCSI_CMD_WRITE)
106                 return be32_to_cpu(hdr->data_length);
107         return 0;
108 }
109
110 static inline int cmnd_read_size(struct iscsi_cmnd *cmnd)
111 {
112         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
113
114         if (hdr->flags & ISCSI_CMD_READ) {
115                 struct iscsi_ahs_hdr *ahdr;
116
117                 if (!(hdr->flags & ISCSI_CMD_WRITE))
118                         return be32_to_cpu(hdr->data_length);
119
120                 ahdr = (struct iscsi_ahs_hdr *)cmnd->pdu.ahs;
121                 if (ahdr != NULL) {
122                         uint8_t *p = (uint8_t *)ahdr;
123                         unsigned int size = 0;
124                         do {
125                                 int s;
126
127                                 ahdr = (struct iscsi_ahs_hdr *)p;
128
129                                 if (ahdr->ahstype == ISCSI_AHSTYPE_RLENGTH) {
130                                         struct iscsi_rlength_ahdr *rh =
131                                               (struct iscsi_rlength_ahdr *)ahdr;
132                                         return be32_to_cpu(rh->read_length);
133                                 }
134
135                                 s = 3 + be16_to_cpu(ahdr->ahslength);
136                                 s = (s + 3) & -4;
137                                 size += s;
138                                 p += s;
139                         } while (size < cmnd->pdu.ahssize);
140                 }
141                 return -1;
142         }
143         return 0;
144 }
145
146 void iscsi_restart_cmnd(struct iscsi_cmnd *cmnd)
147 {
148         int status;
149
150         TRACE_ENTRY();
151
152         EXTRACHECKS_BUG_ON(cmnd->r2t_len_to_receive != 0);
153         EXTRACHECKS_BUG_ON(cmnd->r2t_len_to_send != 0);
154
155         iscsi_extracheck_is_rd_thread(cmnd->conn);
156
157         req_del_from_write_timeout_list(cmnd);
158
159         /*
160          * Let's remove cmnd from the hash earlier to keep it smaller.
161          * See also corresponding comment in req_cmnd_release().
162          */
163         if (cmnd->hashed)
164                 cmnd_remove_data_wait_hash(cmnd);
165
166         if (unlikely(test_bit(ISCSI_CONN_REINSTATING,
167                         &cmnd->conn->conn_aflags))) {
168                 struct iscsi_target *target = cmnd->conn->session->target;
169                 bool get_out;
170
171                 mutex_lock(&target->target_mutex);
172
173                 get_out = test_bit(ISCSI_CONN_REINSTATING,
174                                 &cmnd->conn->conn_aflags);
175                 /* Let's don't look dead */
176                 if (scst_cmd_get_cdb(cmnd->scst_cmd)[0] == TEST_UNIT_READY)
177                         get_out = false;
178
179                 if (!get_out)
180                         goto unlock_cont;
181
182                 TRACE_MGMT_DBG("Pending cmnd %p, because conn %p is "
183                         "reinstated", cmnd, cmnd->conn);
184
185                 cmnd->scst_state = ISCSI_CMD_STATE_REINST_PENDING;
186                 list_add_tail(&cmnd->reinst_pending_cmd_list_entry,
187                         &cmnd->conn->reinst_pending_cmd_list);
188
189 unlock_cont:
190                 mutex_unlock(&target->target_mutex);
191
192                 if (get_out)
193                         goto out;
194         }
195
196         if (unlikely(cmnd->prelim_compl_flags != 0)) {
197                 if (test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags)) {
198                         TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted", cmnd,
199                                 cmnd->scst_cmd);
200                         req_cmnd_release_force(cmnd);
201                         goto out;
202                 }
203
204                 if (cmnd->scst_cmd == NULL) {
205                         TRACE_MGMT_DBG("Finishing preliminary completed cmd %p "
206                                 "with NULL scst_cmd", cmnd);
207                         req_cmnd_release(cmnd);
208                         goto out;
209                 }
210                 EXTRACHECKS_BUG_ON(scst_cmd_get_status(cmnd->scst_cmd) == 0);
211
212                 status = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
213         } else
214                 status = SCST_PREPROCESS_STATUS_SUCCESS;
215
216         cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
217
218         scst_restart_cmd(cmnd->scst_cmd, status, SCST_CONTEXT_THREAD);
219
220 out:
221         TRACE_EXIT();
222         return;
223 }
224
225 void iscsi_fail_data_waiting_cmnd(struct iscsi_cmnd *cmnd)
226 {
227         TRACE_ENTRY();
228
229         TRACE_MGMT_DBG("Failing data waiting cmnd %p", cmnd);
230
231         /*
232          * There is no race with conn_abort(), since all functions
233          * called from single read thread
234          */
235         iscsi_extracheck_is_rd_thread(cmnd->conn);
236         cmnd->r2t_len_to_receive = 0;
237         cmnd->r2t_len_to_send = 0;
238
239         req_cmnd_release_force(cmnd);
240
241         TRACE_EXIT();
242         return;
243 }
244
245 struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn,
246                               struct iscsi_cmnd *parent)
247 {
248         struct iscsi_cmnd *cmnd;
249
250         /* ToDo: __GFP_NOFAIL?? */
251 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
252         cmnd = kmem_cache_alloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
253         memset(cmnd, 0, sizeof(*cmnd));
254 #else
255         cmnd = kmem_cache_zalloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
256 #endif
257
258         atomic_set(&cmnd->ref_cnt, 1);
259         cmnd->scst_state = ISCSI_CMD_STATE_NEW;
260         cmnd->conn = conn;
261         cmnd->parent_req = parent;
262
263         if (parent == NULL) {
264                 conn_get(conn);
265
266 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
267                 atomic_set(&cmnd->net_ref_cnt, 0);
268 #endif
269                 INIT_LIST_HEAD(&cmnd->rsp_cmd_list);
270                 INIT_LIST_HEAD(&cmnd->rx_ddigest_cmd_list);
271                 cmnd->target_task_tag = cpu_to_be32(ISCSI_RESERVED_TAG);
272
273                 spin_lock_bh(&conn->cmd_list_lock);
274                 list_add_tail(&cmnd->cmd_list_entry, &conn->cmd_list);
275                 spin_unlock_bh(&conn->cmd_list_lock);
276         }
277
278         TRACE_DBG("conn %p, parent %p, cmnd %p", conn, parent, cmnd);
279         return cmnd;
280 }
281
282 /* Frees a command. Also frees the additional header. */
283 static void cmnd_free(struct iscsi_cmnd *cmnd)
284 {
285         TRACE_ENTRY();
286
287         TRACE_DBG("cmnd %p", cmnd);
288
289         if (unlikely(test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags))) {
290                 TRACE_MGMT_DBG("Free aborted cmd %p (scst cmd %p, state %d, "
291                         "parent_req %p)", cmnd, cmnd->scst_cmd,
292                         cmnd->scst_state, cmnd->parent_req);
293         }
294
295         /* Catch users from cmd_list or rsp_cmd_list */
296         EXTRACHECKS_BUG_ON(atomic_read(&cmnd->ref_cnt) != 0);
297
298         kfree(cmnd->pdu.ahs);
299
300 #ifdef CONFIG_SCST_EXTRACHECKS
301         if (unlikely(cmnd->on_write_list || cmnd->on_write_timeout_list)) {
302                 struct iscsi_scsi_cmd_hdr *req = cmnd_hdr(cmnd);
303
304                 PRINT_CRIT_ERROR("cmnd %p still on some list?, %x, %x, %x, "
305                         "%x, %x, %x, %x", cmnd, req->opcode, req->scb[0],
306                         req->flags, req->itt, be32_to_cpu(req->data_length),
307                         req->cmd_sn, be32_to_cpu(cmnd->pdu.datasize));
308
309                 if (unlikely(cmnd->parent_req)) {
310                         struct iscsi_scsi_cmd_hdr *preq =
311                                         cmnd_hdr(cmnd->parent_req);
312                         PRINT_CRIT_ERROR("%p %x %u", preq, preq->opcode,
313                                 preq->scb[0]);
314                 }
315                 sBUG();
316         }
317 #endif
318
319         kmem_cache_free(iscsi_cmnd_cache, cmnd);
320
321         TRACE_EXIT();
322         return;
323 }
324
325 /* Might be called under some lock and on SIRQ */
326 void cmnd_done(struct iscsi_cmnd *cmnd)
327 {
328         TRACE_ENTRY();
329
330         TRACE_DBG("cmnd %p", cmnd);
331
332         if (unlikely(test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags))) {
333                 TRACE_MGMT_DBG("Done aborted cmd %p (scst cmd %p, state %d, "
334                         "parent_req %p)", cmnd, cmnd->scst_cmd,
335                         cmnd->scst_state, cmnd->parent_req);
336         }
337
338         EXTRACHECKS_BUG_ON(cmnd->on_rx_digest_list);
339         EXTRACHECKS_BUG_ON(cmnd->hashed);
340
341         req_del_from_write_timeout_list(cmnd);
342
343         if (cmnd->parent_req == NULL) {
344                 struct iscsi_conn *conn = cmnd->conn;
345                 struct iscsi_cmnd *rsp, *t;
346
347                 TRACE_DBG("Deleting req %p from conn %p", cmnd, conn);
348
349                 spin_lock_bh(&conn->cmd_list_lock);
350                 list_del(&cmnd->cmd_list_entry);
351                 spin_unlock_bh(&conn->cmd_list_lock);
352
353                 conn_put(conn);
354
355                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rx_ddigest_cmd_list));
356
357                 /* Order between above and below code is important! */
358
359                 if ((cmnd->scst_cmd != NULL) || (cmnd->scst_aen != NULL)) {
360                         switch (cmnd->scst_state) {
361                         case ISCSI_CMD_STATE_PROCESSED:
362                                 TRACE_DBG("cmd %p PROCESSED", cmnd);
363                                 scst_tgt_cmd_done(cmnd->scst_cmd,
364                                         SCST_CONTEXT_DIRECT_ATOMIC);
365                                 break;
366
367                         case ISCSI_CMD_STATE_AFTER_PREPROC:
368                         {
369                                 /* It can be for some aborted commands */
370                                 struct scst_cmd *scst_cmd = cmnd->scst_cmd;
371                                 TRACE_DBG("cmd %p AFTER_PREPROC", cmnd);
372                                 cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
373                                 cmnd->scst_cmd = NULL;
374                                 scst_restart_cmd(scst_cmd,
375                                         SCST_PREPROCESS_STATUS_ERROR_FATAL,
376                                         SCST_CONTEXT_THREAD);
377                                 break;
378                         }
379
380                         case ISCSI_CMD_STATE_AEN:
381                                 TRACE_DBG("cmd %p AEN PROCESSED", cmnd);
382                                 scst_aen_done(cmnd->scst_aen);
383                                 break;
384
385                         case ISCSI_CMD_STATE_OUT_OF_SCST_PRELIM_COMPL:
386                                 break;
387
388                         default:
389                                 PRINT_CRIT_ERROR("Unexpected cmnd scst state "
390                                         "%d", cmnd->scst_state);
391                                 sBUG();
392                                 break;
393                         }
394                 }
395
396                 if (cmnd->own_sg) {
397                         TRACE_DBG("own_sg for req %p", cmnd);
398                         if (cmnd->sg != &dummy_sg)
399                                 scst_free(cmnd->sg, cmnd->sg_cnt);
400 #ifdef CONFIG_SCST_DEBUG
401                         cmnd->own_sg = 0;
402                         cmnd->sg = NULL;
403                         cmnd->sg_cnt = -1;
404 #endif
405                 }
406
407                 if (cmnd->dec_active_cmnds) {
408                         struct iscsi_session *sess = cmnd->conn->session;
409                         TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
410                                 "new value %d)", cmnd, sess,
411                                 atomic_read(&sess->active_cmds)-1);
412                         atomic_dec(&sess->active_cmds);
413 #ifdef CONFIG_SCST_EXTRACHECKS
414                         if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
415                                 PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
416                                         atomic_read(&sess->active_cmds));
417                                 sBUG();
418                         }
419 #endif
420                 }
421
422                 list_for_each_entry_safe(rsp, t, &cmnd->rsp_cmd_list,
423                                         rsp_cmd_list_entry) {
424                         cmnd_free(rsp);
425                 }
426
427                 cmnd_free(cmnd);
428         } else {
429                 if (cmnd->own_sg) {
430                         TRACE_DBG("own_sg for rsp %p", cmnd);
431                         if ((cmnd->sg != &dummy_sg) && (cmnd->sg != cmnd->rsp_sg))
432                                 scst_free(cmnd->sg, cmnd->sg_cnt);
433 #ifdef CONFIG_SCST_DEBUG
434                         cmnd->own_sg = 0;
435                         cmnd->sg = NULL;
436                         cmnd->sg_cnt = -1;
437 #endif
438                 }
439
440                 EXTRACHECKS_BUG_ON(cmnd->dec_active_cmnds);
441
442                 if (cmnd == cmnd->parent_req->main_rsp) {
443                         TRACE_DBG("Finishing main rsp %p (req %p)", cmnd,
444                                 cmnd->parent_req);
445                         cmnd->parent_req->main_rsp = NULL;
446                 }
447
448                 cmnd_put(cmnd->parent_req);
449                 /*
450                  * rsp will be freed on the last parent's put and can already
451                  * be freed!!
452                  */
453         }
454
455         TRACE_EXIT();
456         return;
457 }
458
459 /*
460  * Corresponding conn may also get destroyed after this function, except only
461  * if it's called from the read thread!
462  *
463  * It can't be called in parallel with iscsi_cmnds_init_write()!
464  */
465 void req_cmnd_release_force(struct iscsi_cmnd *req)
466 {
467         struct iscsi_cmnd *rsp, *t;
468         struct iscsi_conn *conn = req->conn;
469         LIST_HEAD(cmds_list);
470
471         TRACE_ENTRY();
472
473         TRACE_MGMT_DBG("req %p", req);
474
475         sBUG_ON(req == conn->read_cmnd);
476
477         spin_lock_bh(&conn->write_list_lock);
478         list_for_each_entry_safe(rsp, t, &conn->write_list, write_list_entry) {
479                 if (rsp->parent_req != req)
480                         continue;
481
482                 cmd_del_from_write_list(rsp);
483
484                 list_add_tail(&rsp->write_list_entry, &cmds_list);
485         }
486         spin_unlock_bh(&conn->write_list_lock);
487
488         list_for_each_entry_safe(rsp, t, &cmds_list, write_list_entry) {
489                 TRACE_MGMT_DBG("Putting write rsp %p", rsp);
490                 list_del(&rsp->write_list_entry);
491                 cmnd_put(rsp);
492         }
493
494         /* Supposed nobody can add responses in the list anymore */
495         list_for_each_entry_reverse(rsp, &req->rsp_cmd_list,
496                         rsp_cmd_list_entry) {
497                 bool r;
498
499                 if (rsp->force_cleanup_done)
500                         continue;
501
502                 rsp->force_cleanup_done = 1;
503
504                 if (cmnd_get_check(rsp))
505                         continue;
506
507                 spin_lock_bh(&conn->write_list_lock);
508                 r = rsp->on_write_list || rsp->write_processing_started;
509                 spin_unlock_bh(&conn->write_list_lock);
510
511                 cmnd_put(rsp);
512
513                 if (r)
514                         continue;
515
516                 /*
517                  * If both on_write_list and write_processing_started not set,
518                  * we can safely put() rsp.
519                  */
520                 TRACE_MGMT_DBG("Putting rsp %p", rsp);
521                 cmnd_put(rsp);
522         }
523
524         if (req->main_rsp != NULL) {
525                 TRACE_MGMT_DBG("Putting main rsp %p", req->main_rsp);
526                 cmnd_put(req->main_rsp);
527                 req->main_rsp = NULL;
528         }
529
530         req_cmnd_release(req);
531
532         TRACE_EXIT();
533         return;
534 }
535
536 /*
537  * Corresponding conn may also get destroyed after this function, except only
538  * if it's called from the read thread!
539  */
540 static void req_cmnd_release(struct iscsi_cmnd *req)
541 {
542         struct iscsi_cmnd *c, *t;
543
544         TRACE_ENTRY();
545
546         TRACE_DBG("req %p", req);
547
548 #ifdef CONFIG_SCST_EXTRACHECKS
549         sBUG_ON(req->release_called);
550         req->release_called = 1;
551 #endif
552
553         if (unlikely(test_bit(ISCSI_CMD_ABORTED, &req->prelim_compl_flags))) {
554                 TRACE_MGMT_DBG("Release aborted req cmd %p (scst cmd %p, "
555                         "state %d)", req, req->scst_cmd, req->scst_state);
556         }
557
558         sBUG_ON(req->parent_req != NULL);
559
560         /*
561          * We have to remove hashed req from the hash list before sending
562          * response. Otherwise we can have a race, when for some reason cmd's
563          * release (and, hence, removal from the hash) is delayed after the
564          * transmission and initiator sends cmd with the same ITT, hence
565          * the new command will be erroneously rejected as a duplicate.
566          */
567         if (unlikely(req->hashed)) {
568                 /* It sometimes can happen during errors recovery */
569                 cmnd_remove_data_wait_hash(req);
570         }
571
572         if (unlikely(req->main_rsp != NULL)) {
573                 TRACE_DBG("Sending main rsp %p", req->main_rsp);
574                 iscsi_cmnd_init_write(req->main_rsp, ISCSI_INIT_WRITE_WAKE);
575                 req->main_rsp = NULL;
576         }
577
578         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
579                                 rx_ddigest_cmd_list_entry) {
580                 cmd_del_from_rx_ddigest_list(c);
581                 cmnd_put(c);
582         }
583
584         EXTRACHECKS_BUG_ON(req->pending);
585
586         if (req->dec_active_cmnds) {
587                 struct iscsi_session *sess = req->conn->session;
588                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
589                         "new value %d)", req, sess,
590                         atomic_read(&sess->active_cmds)-1);
591                 atomic_dec(&sess->active_cmds);
592                 req->dec_active_cmnds = 0;
593 #ifdef CONFIG_SCST_EXTRACHECKS
594                 if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
595                         PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
596                                 atomic_read(&sess->active_cmds));
597                         sBUG();
598                 }
599 #endif
600         }
601
602         cmnd_put(req);
603
604         TRACE_EXIT();
605         return;
606 }
607
608 /*
609  * Corresponding conn may also get destroyed after this function, except only
610  * if it's called from the read thread!
611  */
612 void rsp_cmnd_release(struct iscsi_cmnd *cmnd)
613 {
614         TRACE_DBG("%p", cmnd);
615
616 #ifdef CONFIG_SCST_EXTRACHECKS
617         sBUG_ON(cmnd->release_called);
618         cmnd->release_called = 1;
619 #endif
620
621         EXTRACHECKS_BUG_ON(cmnd->parent_req == NULL);
622
623         cmnd_put(cmnd);
624         return;
625 }
626
627 static struct iscsi_cmnd *iscsi_alloc_rsp(struct iscsi_cmnd *parent)
628 {
629         struct iscsi_cmnd *rsp;
630
631         TRACE_ENTRY();
632
633         rsp = cmnd_alloc(parent->conn, parent);
634
635         TRACE_DBG("Adding rsp %p to parent %p", rsp, parent);
636         list_add_tail(&rsp->rsp_cmd_list_entry, &parent->rsp_cmd_list);
637
638         cmnd_get(parent);
639
640         TRACE_EXIT_HRES((unsigned long)rsp);
641         return rsp;
642 }
643
644 static inline struct iscsi_cmnd *iscsi_alloc_main_rsp(struct iscsi_cmnd *parent)
645 {
646         struct iscsi_cmnd *rsp;
647
648         TRACE_ENTRY();
649
650         EXTRACHECKS_BUG_ON(parent->main_rsp != 0);
651
652         rsp = iscsi_alloc_rsp(parent);
653         parent->main_rsp = rsp;
654
655         TRACE_EXIT_HRES((unsigned long)rsp);
656         return rsp;
657 }
658
659 static void iscsi_cmnds_init_write(struct list_head *send, int flags)
660 {
661         struct iscsi_cmnd *rsp = list_entry(send->next, struct iscsi_cmnd,
662                                                 write_list_entry);
663         struct iscsi_conn *conn = rsp->conn;
664         struct list_head *pos, *next;
665
666         sBUG_ON(list_empty(send));
667
668         if (!(conn->ddigest_type & DIGEST_NONE)) {
669                 list_for_each(pos, send) {
670                         rsp = list_entry(pos, struct iscsi_cmnd,
671                                                 write_list_entry);
672
673                         if (rsp->pdu.datasize != 0) {
674                                 TRACE_DBG("Doing data digest (%p:%x)", rsp,
675                                         cmnd_opcode(rsp));
676                                 digest_tx_data(rsp);
677                         }
678                 }
679         }
680
681         spin_lock_bh(&conn->write_list_lock);
682         list_for_each_safe(pos, next, send) {
683                 rsp = list_entry(pos, struct iscsi_cmnd, write_list_entry);
684
685                 TRACE_DBG("%p:%x", rsp, cmnd_opcode(rsp));
686
687                 sBUG_ON(conn != rsp->conn);
688
689                 list_del(&rsp->write_list_entry);
690                 cmd_add_on_write_list(conn, rsp);
691         }
692         spin_unlock_bh(&conn->write_list_lock);
693
694         if (flags & ISCSI_INIT_WRITE_WAKE)
695                 iscsi_make_conn_wr_active(conn);
696
697         return;
698 }
699
700 static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags)
701 {
702         LIST_HEAD(head);
703
704 #ifdef CONFIG_SCST_EXTRACHECKS
705         if (unlikely(rsp->on_write_list)) {
706                 PRINT_CRIT_ERROR("cmd already on write list (%x %x %x "
707                         "%u %u %d %d", cmnd_itt(rsp),
708                         cmnd_opcode(rsp), cmnd_scsicode(rsp),
709                         rsp->hdigest, rsp->ddigest,
710                         list_empty(&rsp->rsp_cmd_list), rsp->hashed);
711                 sBUG();
712         }
713 #endif
714         list_add_tail(&rsp->write_list_entry, &head);
715         iscsi_cmnds_init_write(&head, flags);
716         return;
717 }
718
719 static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
720 {
721         struct iscsi_cmnd *rsp;
722         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
723         struct iscsi_data_in_hdr *rsp_hdr;
724         u32 pdusize, expsize, size, offset, sn;
725         LIST_HEAD(send);
726
727         TRACE_DBG("req %p", req);
728
729         pdusize = req->conn->session->sess_param.max_xmit_data_length;
730         expsize = req->read_size;
731         size = min(expsize, (u32)req->bufflen);
732         offset = 0;
733         sn = 0;
734
735         while (1) {
736                 rsp = iscsi_alloc_rsp(req);
737                 TRACE_DBG("rsp %p", rsp);
738                 rsp->sg = req->sg;
739                 rsp->sg_cnt = req->sg_cnt;
740                 rsp->bufflen = req->bufflen;
741                 rsp_hdr = (struct iscsi_data_in_hdr *)&rsp->pdu.bhs;
742
743                 rsp_hdr->opcode = ISCSI_OP_SCSI_DATA_IN;
744                 rsp_hdr->itt = req_hdr->itt;
745                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
746                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
747                 rsp_hdr->data_sn = cpu_to_be32(sn);
748
749                 if (size <= pdusize) {
750                         TRACE_DBG("offset %d, size %d", offset, size);
751                         rsp->pdu.datasize = size;
752                         if (send_status) {
753                                 unsigned int scsisize;
754
755                                 TRACE_DBG("status %x", status);
756
757                                 EXTRACHECKS_BUG_ON((cmnd_hdr(req)->flags & ISCSI_CMD_WRITE) != 0);
758
759                                 rsp_hdr->flags = ISCSI_FLG_FINAL | ISCSI_FLG_STATUS;
760                                 rsp_hdr->cmd_status = status;
761
762                                 scsisize = req->bufflen;
763                                 if (scsisize < expsize) {
764                                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
765                                         size = expsize - scsisize;
766                                 } else if (scsisize > expsize) {
767                                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
768                                         size = scsisize - expsize;
769                                 } else
770                                         size = 0;
771                                 rsp_hdr->residual_count = cpu_to_be32(size);
772                         }
773                         list_add_tail(&rsp->write_list_entry, &send);
774                         break;
775                 }
776
777                 TRACE_DBG("pdusize %d, offset %d, size %d", pdusize, offset,
778                         size);
779
780                 rsp->pdu.datasize = pdusize;
781
782                 size -= pdusize;
783                 offset += pdusize;
784                 sn++;
785
786                 list_add_tail(&rsp->write_list_entry, &send);
787         }
788         iscsi_cmnds_init_write(&send, 0);
789         return;
790 }
791
792 static void iscsi_init_status_rsp(struct iscsi_cmnd *rsp,
793         int status, const u8 *sense_buf, int sense_len, bool bufflen_set)
794 {
795         struct iscsi_cmnd *req = rsp->parent_req;
796         struct iscsi_scsi_rsp_hdr *rsp_hdr;
797         struct scatterlist *sg;
798
799         TRACE_ENTRY();
800
801         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
802         rsp_hdr->opcode = ISCSI_OP_SCSI_RSP;
803         rsp_hdr->flags = ISCSI_FLG_FINAL;
804         rsp_hdr->response = ISCSI_RESPONSE_COMMAND_COMPLETED;
805         rsp_hdr->cmd_status = status;
806         rsp_hdr->itt = cmnd_hdr(req)->itt;
807
808         if (SCST_SENSE_VALID(sense_buf)) {
809                 TRACE_DBG("%s", "SENSE VALID");
810
811                 sg = rsp->sg = rsp->rsp_sg;
812                 rsp->sg_cnt = 2;
813                 rsp->own_sg = 1;
814
815                 sg_init_table(sg, 2);
816                 sg_set_buf(&sg[0], &rsp->sense_hdr, sizeof(rsp->sense_hdr));
817                 sg_set_buf(&sg[1], sense_buf, sense_len);
818
819                 rsp->sense_hdr.length = cpu_to_be16(sense_len);
820
821                 rsp->pdu.datasize = sizeof(rsp->sense_hdr) + sense_len;
822                 rsp->bufflen = rsp->pdu.datasize;
823         } else {
824                 rsp->pdu.datasize = 0;
825                 rsp->bufflen = 0;
826         }
827
828         iscsi_set_resid(rsp, bufflen_set);
829
830         TRACE_EXIT();
831         return;
832 }
833
834 static inline struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req,
835         int status, const u8 *sense_buf, int sense_len, bool bufflen_set)
836 {
837         struct iscsi_cmnd *rsp;
838
839         TRACE_ENTRY();
840
841         rsp = iscsi_alloc_rsp(req);
842         TRACE_DBG("rsp %p", rsp);
843
844         iscsi_init_status_rsp(rsp, status, sense_buf, sense_len, bufflen_set);
845
846         TRACE_EXIT_HRES((unsigned long)rsp);
847         return rsp;
848 }
849
850 static struct iscsi_cmnd *create_prelim_status_rsp(struct iscsi_cmnd *req,
851         int status, const u8 *sense_buf, int sense_len)
852 {
853         struct iscsi_cmnd *rsp;
854
855         TRACE_ENTRY();
856
857         rsp = iscsi_alloc_main_rsp(req);
858         TRACE_DBG("main rsp %p", rsp);
859
860         iscsi_init_status_rsp(rsp, status, sense_buf, sense_len, false);
861
862         TRACE_EXIT_HRES((unsigned long)rsp);
863         return rsp;
864 }
865
866 static int iscsi_set_prelim_r2t_len_to_receive(struct iscsi_cmnd *req)
867 {
868         struct iscsi_hdr *req_hdr = &req->pdu.bhs;
869         int res = 0;
870
871         TRACE_ENTRY();
872
873         if (req_hdr->flags & ISCSI_CMD_FINAL)
874                 goto out;
875
876         res = cmnd_insert_data_wait_hash(req);
877         if (res != 0) {
878                 /*
879                  * We have to close connection, because otherwise a data
880                  * corruption is possible if we allow to receive data
881                  * for this request in another request with dublicated ITT.
882                  */
883                 mark_conn_closed(req->conn);
884                 goto out;
885         }
886
887         /*
888          * We need to wait for one or more PDUs. Let's simplify
889          * other code and pretend we need to receive 1 byte.
890          * In data_out_start() we will correct it.
891          */
892         if (req->outstanding_r2t == 0) {
893                 req->outstanding_r2t = 1;
894                 req_add_to_write_timeout_list(req);
895         }
896         req->r2t_len_to_receive = 1;
897         req->r2t_len_to_send = 0;
898
899         TRACE_DBG("req %p, op %x, outstanding_r2t %d, r2t_len_to_receive %d, "
900                 "r2t_len_to_send %d", req, cmnd_opcode(req),
901                 req->outstanding_r2t, req->r2t_len_to_receive,
902                 req->r2t_len_to_send);
903
904 out:
905         TRACE_EXIT_RES(res);
906         return res;
907 }
908
909 static int create_preliminary_status_rsp(struct iscsi_cmnd *req,
910         int status, const u8 *sense_buf, int sense_len)
911 {
912         int res = 0;
913         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
914
915         TRACE_ENTRY();
916
917         if (req->prelim_compl_flags != 0) {
918                 TRACE_MGMT_DBG("req %p already prelim completed", req);
919                 goto out;
920         }
921
922         req->scst_state = ISCSI_CMD_STATE_OUT_OF_SCST_PRELIM_COMPL;
923
924         if ((req_hdr->flags & ISCSI_CMD_READ) &&
925             (req_hdr->flags & ISCSI_CMD_WRITE)) {
926                 int sz = cmnd_read_size(req);
927                 if (sz > 0)
928                         req->read_size = sz;
929         } else if (req_hdr->flags & ISCSI_CMD_READ)
930                 req->read_size = be32_to_cpu(req_hdr->data_length);
931
932         create_prelim_status_rsp(req, status, sense_buf, sense_len);
933         res = iscsi_preliminary_complete(req, req, true);
934
935 out:
936         TRACE_EXIT_RES(res);
937         return res;
938 }
939
940 static int set_scst_preliminary_status_rsp(struct iscsi_cmnd *req,
941         bool get_data, int key, int asc, int ascq)
942 {
943         int res = 0;
944
945         TRACE_ENTRY();
946
947         if (req->scst_cmd == NULL) {
948                 /* There must be already error set */
949                 goto complete;
950         }
951
952         scst_set_cmd_error(req->scst_cmd, key, asc, ascq);
953
954 complete:
955         res = iscsi_preliminary_complete(req, req, get_data);
956
957         TRACE_EXIT_RES(res);
958         return res;
959 }
960
961 static int create_reject_rsp(struct iscsi_cmnd *req, int reason, bool get_data)
962 {
963         int res = 0;
964         struct iscsi_cmnd *rsp;
965         struct iscsi_reject_hdr *rsp_hdr;
966         struct scatterlist *sg;
967
968         TRACE_ENTRY();
969
970         TRACE_MGMT_DBG("Reject: req %p, reason %x", req, reason);
971
972         if (cmnd_opcode(req) == ISCSI_OP_SCSI_CMD) {
973                 if (req->scst_cmd == NULL) {
974                         /* BUSY status must be already set */
975                         struct iscsi_scsi_rsp_hdr *rsp_hdr;
976                         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&req->main_rsp->pdu.bhs;
977                         sBUG_ON(rsp_hdr->cmd_status == 0);
978                         /*
979                          * Let's not send REJECT here. The initiator will retry
980                          * and, hopefully, next time we will not fail allocating
981                          * scst_cmd, so we will then send the REJECT.
982                          */
983                         goto out;
984                 } else
985                         set_scst_preliminary_status_rsp(req, get_data,
986                                 SCST_LOAD_SENSE(scst_sense_invalid_message));
987         }
988
989         rsp = iscsi_alloc_main_rsp(req);
990         rsp_hdr = (struct iscsi_reject_hdr *)&rsp->pdu.bhs;
991
992         rsp_hdr->opcode = ISCSI_OP_REJECT;
993         rsp_hdr->ffffffff = ISCSI_RESERVED_TAG;
994         rsp_hdr->reason = reason;
995
996         sg = rsp->sg = rsp->rsp_sg;
997         rsp->sg_cnt = 1;
998         rsp->own_sg = 1;
999         sg_init_one(sg, &req->pdu.bhs, sizeof(struct iscsi_hdr));
1000         rsp->bufflen = rsp->pdu.datasize = sizeof(struct iscsi_hdr);
1001
1002         res = iscsi_preliminary_complete(req, req, true);
1003
1004 out:
1005         TRACE_EXIT_RES(res);
1006         return res;
1007 }
1008
1009 static inline int iscsi_get_allowed_cmds(struct iscsi_session *sess)
1010 {
1011         int res = max(-1, (int)sess->max_queued_cmnds -
1012                                 atomic_read(&sess->active_cmds)-1);
1013         TRACE_DBG("allowed cmds %d (sess %p, active_cmds %d)", res,
1014                 sess, atomic_read(&sess->active_cmds));
1015         return res;
1016 }
1017
1018 static u32 cmnd_set_sn(struct iscsi_cmnd *cmnd, int set_stat_sn)
1019 {
1020         struct iscsi_conn *conn = cmnd->conn;
1021         struct iscsi_session *sess = conn->session;
1022         u32 res;
1023
1024         spin_lock(&sess->sn_lock);
1025
1026         if (set_stat_sn)
1027                 cmnd->pdu.bhs.sn = cpu_to_be32(conn->stat_sn++);
1028         cmnd->pdu.bhs.exp_sn = cpu_to_be32(sess->exp_cmd_sn);
1029         cmnd->pdu.bhs.max_sn = cpu_to_be32(sess->exp_cmd_sn +
1030                                  iscsi_get_allowed_cmds(sess));
1031
1032         res = cpu_to_be32(conn->stat_sn);
1033
1034         spin_unlock(&sess->sn_lock);
1035         return res;
1036 }
1037
1038 /* Called under sn_lock */
1039 static void __update_stat_sn(struct iscsi_cmnd *cmnd)
1040 {
1041         struct iscsi_conn *conn = cmnd->conn;
1042         u32 exp_stat_sn;
1043
1044         cmnd->pdu.bhs.exp_sn = exp_stat_sn = be32_to_cpu(cmnd->pdu.bhs.exp_sn);
1045         TRACE_DBG("%x,%x", cmnd_opcode(cmnd), exp_stat_sn);
1046         if ((int)(exp_stat_sn - conn->exp_stat_sn) > 0 &&
1047             (int)(exp_stat_sn - conn->stat_sn) <= 0) {
1048                 /* free pdu resources */
1049                 cmnd->conn->exp_stat_sn = exp_stat_sn;
1050         }
1051         return;
1052 }
1053
1054 static inline void update_stat_sn(struct iscsi_cmnd *cmnd)
1055 {
1056         spin_lock(&cmnd->conn->session->sn_lock);
1057         __update_stat_sn(cmnd);
1058         spin_unlock(&cmnd->conn->session->sn_lock);
1059         return;
1060 }
1061
1062 /* Called under sn_lock */
1063 static int check_cmd_sn(struct iscsi_cmnd *cmnd)
1064 {
1065         struct iscsi_session *session = cmnd->conn->session;
1066         u32 cmd_sn;
1067
1068         cmnd->pdu.bhs.sn = cmd_sn = be32_to_cpu(cmnd->pdu.bhs.sn);
1069         TRACE_DBG("%d(%d)", cmd_sn, session->exp_cmd_sn);
1070         if (likely((s32)(cmd_sn - session->exp_cmd_sn) >= 0))
1071                 return 0;
1072         PRINT_ERROR("sequence error (%x,%x)", cmd_sn, session->exp_cmd_sn);
1073         return -ISCSI_REASON_PROTOCOL_ERROR;
1074 }
1075
1076 static struct iscsi_cmnd *cmnd_find_itt_get(struct iscsi_conn *conn, u32 itt)
1077 {
1078         struct iscsi_cmnd *cmnd, *found_cmnd = NULL;
1079
1080         spin_lock_bh(&conn->cmd_list_lock);
1081         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1082                 if ((cmnd->pdu.bhs.itt == itt) && !cmnd_get_check(cmnd)) {
1083                         found_cmnd = cmnd;
1084                         break;
1085                 }
1086         }
1087         spin_unlock_bh(&conn->cmd_list_lock);
1088
1089         return found_cmnd;
1090 }
1091
1092 /**
1093  ** We use the ITT hash only to find original request PDU for subsequent
1094  ** Data-Out PDUs.
1095  **/
1096
1097 /* Must be called under cmnd_data_wait_hash_lock */
1098 static struct iscsi_cmnd *__cmnd_find_data_wait_hash(struct iscsi_conn *conn,
1099         u32 itt)
1100 {
1101         struct list_head *head;
1102         struct iscsi_cmnd *cmnd;
1103
1104         head = &conn->session->cmnd_data_wait_hash[cmnd_hashfn(itt)];
1105
1106         list_for_each_entry(cmnd, head, hash_list_entry) {
1107                 if (cmnd->pdu.bhs.itt == itt)
1108                         return cmnd;
1109         }
1110         return NULL;
1111 }
1112
1113 static struct iscsi_cmnd *cmnd_find_data_wait_hash(struct iscsi_conn *conn,
1114         u32 itt)
1115 {
1116         struct iscsi_cmnd *res;
1117         struct iscsi_session *session = conn->session;
1118
1119         spin_lock(&session->cmnd_data_wait_hash_lock);
1120         res = __cmnd_find_data_wait_hash(conn, itt);
1121         spin_unlock(&session->cmnd_data_wait_hash_lock);
1122
1123         return res;
1124 }
1125
1126 static inline u32 get_next_ttt(struct iscsi_conn *conn)
1127 {
1128         u32 ttt;
1129         struct iscsi_session *session = conn->session;
1130
1131         /* Not compatible with MC/S! */
1132
1133         iscsi_extracheck_is_rd_thread(conn);
1134
1135         if (unlikely(session->next_ttt == ISCSI_RESERVED_TAG))
1136                 session->next_ttt++;
1137         ttt = session->next_ttt++;
1138
1139         return ttt;
1140 }
1141
1142 static int cmnd_insert_data_wait_hash(struct iscsi_cmnd *cmnd)
1143 {
1144         struct iscsi_session *session = cmnd->conn->session;
1145         struct iscsi_cmnd *tmp;
1146         struct list_head *head;
1147         int err = 0;
1148         u32 itt = cmnd->pdu.bhs.itt;
1149
1150         if (unlikely(cmnd->hashed)) {
1151                 /* It can be for preliminary completed commands */
1152                 goto out;
1153         }
1154
1155         /*
1156          * We don't need TTT, because ITT/buffer_offset pair is sufficient
1157          * to find out the original request and buffer for Data-Out PDUs, but
1158          * crazy iSCSI spec requires us to send this superfluous field in
1159          * R2T PDUs and some initiators may rely on it.
1160          */
1161         cmnd->target_task_tag = get_next_ttt(cmnd->conn);
1162
1163         TRACE_DBG("%p:%x", cmnd, itt);
1164         if (unlikely(itt == ISCSI_RESERVED_TAG)) {
1165                 PRINT_ERROR("%s", "ITT is RESERVED_TAG");
1166                 PRINT_BUFFER("Incorrect BHS", &cmnd->pdu.bhs,
1167                         sizeof(cmnd->pdu.bhs));
1168                 err = -ISCSI_REASON_PROTOCOL_ERROR;
1169                 goto out;
1170         }
1171
1172         spin_lock(&session->cmnd_data_wait_hash_lock);
1173
1174         head = &session->cmnd_data_wait_hash[cmnd_hashfn(itt)];
1175
1176         tmp = __cmnd_find_data_wait_hash(cmnd->conn, itt);
1177         if (likely(!tmp)) {
1178                 TRACE_DBG("Adding cmnd %p to the hash (ITT %x)", cmnd,
1179                         cmnd_itt(cmnd));
1180                 list_add_tail(&cmnd->hash_list_entry, head);
1181                 cmnd->hashed = 1;
1182         } else {
1183                 PRINT_ERROR("Task %x in progress, cmnd %p", itt, cmnd);
1184                 err = -ISCSI_REASON_TASK_IN_PROGRESS;
1185         }
1186
1187         spin_unlock(&session->cmnd_data_wait_hash_lock);
1188
1189 out:
1190         return err;
1191 }
1192
1193 static void cmnd_remove_data_wait_hash(struct iscsi_cmnd *cmnd)
1194 {
1195         struct iscsi_session *session = cmnd->conn->session;
1196         struct iscsi_cmnd *tmp;
1197
1198         spin_lock(&session->cmnd_data_wait_hash_lock);
1199
1200         tmp = __cmnd_find_data_wait_hash(cmnd->conn, cmnd->pdu.bhs.itt);
1201
1202         if (likely(tmp && tmp == cmnd)) {
1203                 TRACE_DBG("Deleting cmnd %p from the hash (ITT %x)", cmnd,
1204                         cmnd_itt(cmnd));
1205                 list_del(&cmnd->hash_list_entry);
1206                 cmnd->hashed = 0;
1207         } else
1208                 PRINT_ERROR("%p:%x not found", cmnd, cmnd_itt(cmnd));
1209
1210         spin_unlock(&session->cmnd_data_wait_hash_lock);
1211
1212         return;
1213 }
1214
1215 static void cmnd_prepare_get_rejected_immed_data(struct iscsi_cmnd *cmnd)
1216 {
1217         struct iscsi_conn *conn = cmnd->conn;
1218         struct scatterlist *sg = cmnd->sg;
1219         char __user *addr;
1220         u32 size;
1221         unsigned int i;
1222
1223         TRACE_ENTRY();
1224
1225         TRACE_DBG_FLAG(iscsi_get_flow_ctrl_or_mgmt_dbg_log_flag(cmnd),
1226                 "Skipping (cmnd %p, ITT %x, op %x, cmd op %x, "
1227                 "datasize %u, scst_cmd %p, scst state %d)", cmnd,
1228                 cmnd_itt(cmnd), cmnd_opcode(cmnd), cmnd_hdr(cmnd)->scb[0],
1229                 cmnd->pdu.datasize, cmnd->scst_cmd, cmnd->scst_state);
1230
1231         iscsi_extracheck_is_rd_thread(conn);
1232
1233         size = cmnd->pdu.datasize;
1234         if (!size)
1235                 goto out;
1236
1237         /* We already checked pdu.datasize in check_segment_length() */
1238
1239         if (sg == NULL) {
1240                 /*
1241                  * There are no problems with the safety from concurrent
1242                  * accesses to dummy_page in dummy_sg, since data only
1243                  * will be read and then discarded.
1244                  */
1245                 sg = cmnd->sg = &dummy_sg;
1246                 cmnd->bufflen = PAGE_SIZE;
1247                 cmnd->own_sg = 1;
1248         }
1249
1250         addr = (char __force __user *)(page_address(sg_page(&sg[0])));
1251         sBUG_ON(addr == NULL);
1252         conn->read_size = size;
1253         for (i = 0; size > PAGE_SIZE; i++, size -= cmnd->bufflen) {
1254                 /* We already checked pdu.datasize in check_segment_length() */
1255                 sBUG_ON(i >= ISCSI_CONN_IOV_MAX);
1256                 conn->read_iov[i].iov_base = addr;
1257                 conn->read_iov[i].iov_len = cmnd->bufflen;
1258         }
1259         conn->read_iov[i].iov_base = addr;
1260         conn->read_iov[i].iov_len = size;
1261         conn->read_msg.msg_iov = conn->read_iov;
1262         conn->read_msg.msg_iovlen = ++i;
1263
1264 out:
1265         TRACE_EXIT();
1266         return;
1267 }
1268
1269 static void iscsi_set_resid(struct iscsi_cmnd *rsp, bool bufflen_set)
1270 {
1271         struct iscsi_cmnd *req = rsp->parent_req;
1272         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1273         struct iscsi_scsi_rsp_hdr *rsp_hdr;
1274         int resid, resp_len, in_resp_len;
1275
1276         if ((req_hdr->flags & ISCSI_CMD_READ) &&
1277             (req_hdr->flags & ISCSI_CMD_WRITE)) {
1278                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
1279
1280                 if (bufflen_set) {
1281                         resp_len = req->bufflen;
1282                         if (req->scst_cmd != NULL)
1283                                 in_resp_len = scst_cmd_get_in_bufflen(req->scst_cmd);
1284                         else
1285                                 in_resp_len = 0;
1286                 } else {
1287                         resp_len = 0;
1288                         in_resp_len = 0;
1289                 }
1290
1291                 resid = be32_to_cpu(req_hdr->data_length) - in_resp_len;
1292                 if (resid > 0) {
1293                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
1294                         rsp_hdr->residual_count = cpu_to_be32(resid);
1295                 } else if (resid < 0) {
1296                         resid = -resid;
1297                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
1298                         rsp_hdr->residual_count = cpu_to_be32(resid);
1299                 }
1300
1301                 resid = req->read_size - resp_len;
1302                 if (resid > 0) {
1303                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_UNDERFLOW;
1304                         rsp_hdr->bi_residual_count = cpu_to_be32(resid);
1305                 } else if (resid < 0) {
1306                         resid = -resid;
1307                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_OVERFLOW;
1308                         rsp_hdr->bi_residual_count = cpu_to_be32(resid);
1309                 }
1310         } else {
1311                 if (bufflen_set)
1312                         resp_len = req->bufflen;
1313                 else
1314                         resp_len = 0;
1315
1316                 resid = req->read_size - resp_len;
1317                 if (resid > 0) {
1318                         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
1319                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
1320                         rsp_hdr->residual_count = cpu_to_be32(resid);
1321                 } else if (resid < 0) {
1322                         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
1323                         resid = -resid;
1324                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
1325                         rsp_hdr->residual_count = cpu_to_be32(resid);
1326                 }
1327         }
1328         return;
1329 }
1330
1331 static int iscsi_preliminary_complete(struct iscsi_cmnd *req,
1332         struct iscsi_cmnd *orig_req, bool get_data)
1333 {
1334         struct iscsi_hdr *req_hdr = &req->pdu.bhs;
1335         int res = 0;
1336         bool set_r2t_len;
1337
1338         TRACE_ENTRY();
1339
1340         TRACE_DBG_FLAG(iscsi_get_flow_ctrl_or_mgmt_dbg_log_flag(orig_req),
1341                 "Prelim completed req %p, orig_req %p (FINAL %x, "
1342                 "outstanding_r2t %d)", req, orig_req,
1343                 (req_hdr->flags & ISCSI_CMD_FINAL), orig_req->outstanding_r2t);
1344
1345         iscsi_extracheck_is_rd_thread(req->conn);
1346         sBUG_ON(req->parent_req != NULL);
1347
1348         if (test_bit(ISCSI_CMD_PRELIM_COMPLETED, &req->prelim_compl_flags)) {
1349                 TRACE_MGMT_DBG("req %p already prelim completed", req);
1350                 /* To not try to get data twice */
1351                 get_data = false;
1352         }
1353
1354         set_r2t_len = !req->hashed &&
1355                       (cmnd_opcode(req) == ISCSI_OP_SCSI_CMD) &&
1356                       !test_bit(ISCSI_CMD_PRELIM_COMPLETED,
1357                                 &orig_req->prelim_compl_flags);
1358         set_bit(ISCSI_CMD_PRELIM_COMPLETED, &orig_req->prelim_compl_flags);
1359
1360         TRACE_DBG("get_data %d, set_r2t_len %d", get_data, set_r2t_len);
1361
1362         if (get_data)
1363                 cmnd_prepare_get_rejected_immed_data(req);
1364
1365         if (set_r2t_len)
1366                 res = iscsi_set_prelim_r2t_len_to_receive(orig_req);
1367
1368         TRACE_EXIT_RES(res);
1369         return res;
1370 }
1371
1372 static int cmnd_prepare_recv_pdu(struct iscsi_conn *conn,
1373         struct iscsi_cmnd *cmd, u32 offset, u32 size)
1374 {
1375         struct scatterlist *sg = cmd->sg;
1376         unsigned int bufflen = cmd->bufflen;
1377         unsigned int idx, i;
1378         char __user *addr;
1379         int res = 0;
1380
1381         TRACE_DBG("%p %u,%u", cmd->sg, offset, size);
1382
1383         iscsi_extracheck_is_rd_thread(conn);
1384
1385         if (unlikely((offset >= bufflen) ||
1386                      (offset + size > bufflen))) {
1387                 PRINT_ERROR("Wrong ltn (%u %u %u)", offset, size, bufflen);
1388                 mark_conn_closed(conn);
1389                 res = -EIO;
1390                 goto out;
1391         }
1392
1393         offset += sg[0].offset;
1394         idx = offset >> PAGE_SHIFT;
1395         offset &= ~PAGE_MASK;
1396
1397         conn->read_msg.msg_iov = conn->read_iov;
1398         conn->read_size = size;
1399
1400         i = 0;
1401         while (1) {
1402                 addr = (char __force __user *)(page_address(sg_page(&sg[idx])));
1403                 sBUG_ON(addr == NULL);
1404                 conn->read_iov[i].iov_base = addr + offset;
1405                 if (offset + size <= PAGE_SIZE) {
1406                         TRACE_DBG("idx=%d, offset=%u, size=%d, addr=%p",
1407                                 idx, offset, size, addr);
1408                         conn->read_iov[i].iov_len = size;
1409                         conn->read_msg.msg_iovlen = ++i;
1410                         break;
1411                 }
1412                 conn->read_iov[i].iov_len = PAGE_SIZE - offset;
1413                 TRACE_DBG("idx=%d, offset=%u, size=%d, iov_len=%zd, addr=%p",
1414                         idx, offset, size, conn->read_iov[i].iov_len, addr);
1415                 size -= conn->read_iov[i].iov_len;
1416                 if (unlikely(++i >= ISCSI_CONN_IOV_MAX)) {
1417                         PRINT_ERROR("Initiator %s violated negotiated "
1418                                 "parameters by sending too much data (size "
1419                                 "left %d)", conn->session->initiator_name,
1420                                 size);
1421                         mark_conn_closed(conn);
1422                         res = -EINVAL;
1423                         break;
1424                 }
1425                 idx++;
1426                 offset = sg[idx].offset;
1427         }
1428         TRACE_DBG("msg_iov=%p, msg_iovlen=%zd",
1429                 conn->read_msg.msg_iov, conn->read_msg.msg_iovlen);
1430
1431 out:
1432         return res;
1433 }
1434
1435 static void send_r2t(struct iscsi_cmnd *req)
1436 {
1437         struct iscsi_session *sess = req->conn->session;
1438         struct iscsi_cmnd *rsp;
1439         struct iscsi_r2t_hdr *rsp_hdr;
1440         u32 offset, burst;
1441         LIST_HEAD(send);
1442
1443         TRACE_ENTRY();
1444
1445         EXTRACHECKS_BUG_ON(req->r2t_len_to_send == 0);
1446
1447         /*
1448          * There is no race with data_out_start() and conn_abort(), since
1449          * all functions called from single read thread
1450          */
1451         iscsi_extracheck_is_rd_thread(req->conn);
1452
1453         /*
1454          * We don't need to check for PRELIM_COMPLETED here, because for such
1455          * commands we set r2t_len_to_send = 0, hence made sure we won't
1456          * called here.
1457          */
1458
1459         EXTRACHECKS_BUG_ON(req->outstanding_r2t >
1460                            sess->sess_param.max_outstanding_r2t);
1461
1462         if (req->outstanding_r2t == sess->sess_param.max_outstanding_r2t)
1463                 goto out;
1464
1465         burst = sess->sess_param.max_burst_length;
1466         offset = be32_to_cpu(cmnd_hdr(req)->data_length) -
1467                         req->r2t_len_to_send;
1468
1469         do {
1470                 rsp = iscsi_alloc_rsp(req);
1471                 rsp->pdu.bhs.ttt = req->target_task_tag;
1472                 rsp_hdr = (struct iscsi_r2t_hdr *)&rsp->pdu.bhs;
1473                 rsp_hdr->opcode = ISCSI_OP_R2T;
1474                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1475                 rsp_hdr->lun = cmnd_hdr(req)->lun;
1476                 rsp_hdr->itt = cmnd_hdr(req)->itt;
1477                 rsp_hdr->r2t_sn = cpu_to_be32(req->r2t_sn++);
1478                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
1479                 if (req->r2t_len_to_send > burst) {
1480                         rsp_hdr->data_length = cpu_to_be32(burst);
1481                         req->r2t_len_to_send -= burst;
1482                         offset += burst;
1483                 } else {
1484                         rsp_hdr->data_length = cpu_to_be32(req->r2t_len_to_send);
1485                         req->r2t_len_to_send = 0;
1486                 }
1487
1488                 TRACE_WRITE("req %p, data_length %u, buffer_offset %u, "
1489                         "r2t_sn %u, outstanding_r2t %u", req,
1490                         be32_to_cpu(rsp_hdr->data_length),
1491                         be32_to_cpu(rsp_hdr->buffer_offset),
1492                         be32_to_cpu(rsp_hdr->r2t_sn), req->outstanding_r2t);
1493
1494                 list_add_tail(&rsp->write_list_entry, &send);
1495                 req->outstanding_r2t++;
1496
1497         } while ((req->outstanding_r2t < sess->sess_param.max_outstanding_r2t) &&
1498                  (req->r2t_len_to_send != 0));
1499
1500         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_WAKE);
1501
1502 out:
1503         TRACE_EXIT();
1504         return;
1505 }
1506
1507 static int iscsi_pre_exec(struct scst_cmd *scst_cmd)
1508 {
1509         int res = SCST_PREPROCESS_STATUS_SUCCESS;
1510         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
1511                 scst_cmd_get_tgt_priv(scst_cmd);
1512         struct iscsi_cmnd *c, *t;
1513
1514         TRACE_ENTRY();
1515
1516         EXTRACHECKS_BUG_ON(scst_cmd_atomic(scst_cmd));
1517
1518         /* If data digest isn't used this list will be empty */
1519         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
1520                                 rx_ddigest_cmd_list_entry) {
1521                 TRACE_DBG("Checking digest of RX ddigest cmd %p", c);
1522                 if (digest_rx_data(c) != 0) {
1523                         scst_set_cmd_error(scst_cmd,
1524                                 SCST_LOAD_SENSE(iscsi_sense_crc_error));
1525                         res = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
1526                         /*
1527                          * The rest of rx_ddigest_cmd_list will be freed
1528                          * in req_cmnd_release()
1529                          */
1530                         goto out;
1531                 }
1532                 cmd_del_from_rx_ddigest_list(c);
1533                 cmnd_put(c);
1534         }
1535
1536 out:
1537         TRACE_EXIT_RES(res);
1538         return res;
1539 }
1540
1541 static int noop_out_start(struct iscsi_cmnd *cmnd)
1542 {
1543         struct iscsi_conn *conn = cmnd->conn;
1544         struct iscsi_hdr *req_hdr = &cmnd->pdu.bhs;
1545         u32 size, tmp;
1546         int i, err = 0;
1547
1548         TRACE_DBG("%p", cmnd);
1549
1550         iscsi_extracheck_is_rd_thread(conn);
1551
1552         if (!(req_hdr->flags & ISCSI_FLG_FINAL)) {
1553                 PRINT_ERROR("%s", "Initiator sent NOP-Out with not a single "
1554                         "PDU");
1555                 err = -ISCSI_REASON_PROTOCOL_ERROR;
1556                 goto out;
1557         }
1558
1559         if (cmnd_itt(cmnd) == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1560                 if (unlikely(!(cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE)))
1561                         PRINT_ERROR("%s", "Initiator sent RESERVED tag for "
1562                                 "non-immediate NOP-Out command");
1563         }
1564
1565         spin_lock(&conn->session->sn_lock);
1566         __update_stat_sn(cmnd);
1567         err = check_cmd_sn(cmnd);
1568         spin_unlock(&conn->session->sn_lock);
1569         if (unlikely(err))
1570                 goto out;
1571
1572         size = cmnd->pdu.datasize;
1573
1574         if (size) {
1575                 conn->read_msg.msg_iov = conn->read_iov;
1576                 if (cmnd->pdu.bhs.itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1577                         struct scatterlist *sg;
1578
1579                         cmnd->sg = sg = scst_alloc(size, GFP_KERNEL,
1580                                                 &cmnd->sg_cnt);
1581                         if (sg == NULL) {
1582                                 TRACE(TRACE_OUT_OF_MEM, "Allocating buffer for"
1583                                       " %d NOP-Out payload failed", size);
1584                                 err = -ISCSI_REASON_OUT_OF_RESOURCES;
1585                                 goto out;
1586                         }
1587
1588                         /* We already checked it in check_segment_length() */
1589                         sBUG_ON(cmnd->sg_cnt > (signed)ISCSI_CONN_IOV_MAX);
1590
1591                         cmnd->own_sg = 1;
1592                         cmnd->bufflen = size;
1593
1594                         for (i = 0; i < cmnd->sg_cnt; i++) {
1595                                 conn->read_iov[i].iov_base =
1596                                         (void __force __user *)(page_address(sg_page(&sg[i])));
1597                                 tmp = min_t(u32, size, PAGE_SIZE);
1598                                 conn->read_iov[i].iov_len = tmp;
1599                                 conn->read_size += tmp;
1600                                 size -= tmp;
1601                         }
1602                         sBUG_ON(size != 0);
1603                 } else {
1604                         /*
1605                          * There are no problems with the safety from concurrent
1606                          * accesses to dummy_page, since for ISCSI_RESERVED_TAG
1607                          * the data only read and then discarded.
1608                          */
1609                         for (i = 0; i < (signed)ISCSI_CONN_IOV_MAX; i++) {
1610                                 conn->read_iov[i].iov_base =
1611                                         (void __force __user *)(page_address(dummy_page));
1612                                 tmp = min_t(u32, size, PAGE_SIZE);
1613                                 conn->read_iov[i].iov_len = tmp;
1614                                 conn->read_size += tmp;
1615                                 size -= tmp;
1616                         }
1617
1618                         /* We already checked size in check_segment_length() */
1619                         sBUG_ON(size != 0);
1620                 }
1621
1622                 conn->read_msg.msg_iovlen = i;
1623                 TRACE_DBG("msg_iov=%p, msg_iovlen=%zd", conn->read_msg.msg_iov,
1624                         conn->read_msg.msg_iovlen);
1625         }
1626
1627 out:
1628         return err;
1629 }
1630
1631 int cmnd_rx_continue(struct iscsi_cmnd *req)
1632 {
1633         struct iscsi_conn *conn = req->conn;
1634         struct iscsi_session *session = conn->session;
1635         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1636         struct scst_cmd *scst_cmd = req->scst_cmd;
1637         scst_data_direction dir;
1638         bool unsolicited_data_expected = false;
1639         int res = 0;
1640
1641         TRACE_ENTRY();
1642
1643         TRACE_DBG("scsi command: %x", req_hdr->scb[0]);
1644
1645         EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_AFTER_PREPROC);
1646
1647         dir = scst_cmd_get_data_direction(scst_cmd);
1648
1649         /* Check prelim_compl_flags here to save R2Ts */
1650         if (unlikely(scst_cmd_completed(scst_cmd) ||
1651             unlikely(req->prelim_compl_flags != 0))) {
1652                 res = iscsi_preliminary_complete(req, req, true);
1653                 goto trace;
1654         }
1655
1656         /* For prelim completed commands sg&K can be already set! */
1657
1658         if (dir != SCST_DATA_BIDI) {
1659                 req->sg = scst_cmd_get_sg(scst_cmd);
1660                 req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
1661                 req->bufflen = scst_cmd_get_bufflen(scst_cmd);
1662         } else {
1663                 req->sg = scst_cmd_get_in_sg(scst_cmd);
1664                 req->sg_cnt = scst_cmd_get_in_sg_cnt(scst_cmd);
1665                 req->bufflen = scst_cmd_get_in_bufflen(scst_cmd);
1666         }
1667
1668         if (dir & SCST_DATA_WRITE) {
1669                 unsolicited_data_expected = !(req_hdr->flags & ISCSI_CMD_FINAL);
1670
1671                 if (unlikely(session->sess_param.initial_r2t &&
1672                     unsolicited_data_expected)) {
1673                         PRINT_ERROR("Initiator %s violated negotiated "
1674                                 "parameters: initial R2T is required (ITT %x, "
1675                                 "op  %x)", session->initiator_name,
1676                                 cmnd_itt(req), req_hdr->scb[0]);
1677                         goto out_close;
1678                 }
1679
1680                 if (unlikely(!session->sess_param.immediate_data &&
1681                     req->pdu.datasize)) {
1682                         PRINT_ERROR("Initiator %s violated negotiated "
1683                                 "parameters: forbidden immediate data sent "
1684                                 "(ITT %x, op  %x)", session->initiator_name,
1685                                 cmnd_itt(req), req_hdr->scb[0]);
1686                         goto out_close;
1687                 }
1688
1689                 if (unlikely(session->sess_param.first_burst_length < req->pdu.datasize)) {
1690                         PRINT_ERROR("Initiator %s violated negotiated "
1691                                 "parameters: immediate data len (%d) > "
1692                                 "first_burst_length (%d) (ITT %x, op  %x)",
1693                                 session->initiator_name,
1694                                 req->pdu.datasize,
1695                                 session->sess_param.first_burst_length,
1696                                 cmnd_itt(req), req_hdr->scb[0]);
1697                         goto out_close;
1698                 }
1699
1700                 req->r2t_len_to_receive = be32_to_cpu(req_hdr->data_length) -
1701                                           req->pdu.datasize;
1702
1703                 if (unlikely(req->r2t_len_to_receive > req->bufflen)) {
1704                         PRINT_ERROR("req->r2t_len_to_receive %d > req->bufflen "
1705                                 "%d", req->r2t_len_to_receive, req->bufflen);
1706                         goto out_close;
1707                 }
1708
1709                 res = cmnd_insert_data_wait_hash(req);
1710                 if (unlikely(res != 0)) {
1711                         /*
1712                          * We have to close connection, because otherwise a data
1713                          * corruption is possible if we allow to receive data
1714                          * for this request in another request with dublicated
1715                          * ITT.
1716                          */
1717                         goto out_close;
1718                 }
1719
1720                 if (unsolicited_data_expected) {
1721                         req->outstanding_r2t = 1;
1722                         req->r2t_len_to_send = req->r2t_len_to_receive -
1723                                 min_t(unsigned int,
1724                                       session->sess_param.first_burst_length -
1725                                                 req->pdu.datasize,
1726                                       req->r2t_len_to_receive);
1727                 } else
1728                         req->r2t_len_to_send = req->r2t_len_to_receive;
1729
1730                 req_add_to_write_timeout_list(req);
1731
1732                 if (req->pdu.datasize) {
1733                         res = cmnd_prepare_recv_pdu(conn, req, 0, req->pdu.datasize);
1734                         /* For performance better to send R2Ts ASAP */
1735                         if (likely(res == 0) && (req->r2t_len_to_send != 0))
1736                                 send_r2t(req);
1737                 }
1738         } else {
1739                 if (unlikely(!(req_hdr->flags & ISCSI_CMD_FINAL) ||
1740                              req->pdu.datasize)) {
1741                         PRINT_ERROR("Unexpected unsolicited data (ITT %x "
1742                                 "CDB %x", cmnd_itt(req), req_hdr->scb[0]);
1743                         set_scst_preliminary_status_rsp(req, true,
1744                                 SCST_LOAD_SENSE(iscsi_sense_unexpected_unsolicited_data));
1745                 }
1746         }
1747
1748 trace:  
1749         TRACE_DBG("req=%p, dir=%d, unsolicited_data_expected=%d, "
1750                 "r2t_len_to_receive=%d, r2t_len_to_send=%d, bufflen=%d, "
1751                 "own_sg %d", req, dir, unsolicited_data_expected,
1752                 req->r2t_len_to_receive, req->r2t_len_to_send, req->bufflen,
1753                 req->own_sg);
1754
1755 out:
1756         TRACE_EXIT_RES(res);
1757         return res;
1758
1759 out_close:
1760         mark_conn_closed(conn);
1761         res = -EINVAL;
1762         goto out;
1763 }
1764
1765 static int scsi_cmnd_start(struct iscsi_cmnd *req)
1766 {
1767         struct iscsi_conn *conn = req->conn;
1768         struct iscsi_session *session = conn->session;
1769         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1770         struct scst_cmd *scst_cmd;
1771         scst_data_direction dir;
1772         struct iscsi_ahs_hdr *ahdr;
1773         int res = 0;
1774
1775         TRACE_ENTRY();
1776
1777         TRACE_DBG("scsi command: %x", req_hdr->scb[0]);
1778
1779         TRACE_DBG("Incrementing active_cmds (cmd %p, sess %p, "
1780                 "new value %d)", req, session,
1781                 atomic_read(&session->active_cmds)+1);
1782         atomic_inc(&session->active_cmds);
1783         req->dec_active_cmnds = 1;
1784
1785         scst_cmd = scst_rx_cmd(session->scst_sess,
1786                 (uint8_t *)&req_hdr->lun, sizeof(req_hdr->lun),
1787                 req_hdr->scb, sizeof(req_hdr->scb), SCST_NON_ATOMIC);
1788         if (scst_cmd == NULL) {
1789                 res = create_preliminary_status_rsp(req, SAM_STAT_BUSY,
1790                         NULL, 0);
1791                 goto out;
1792         }
1793
1794         req->scst_cmd = scst_cmd;
1795         scst_cmd_set_tag(scst_cmd, req_hdr->itt);
1796         scst_cmd_set_tgt_priv(scst_cmd, req);
1797
1798         if ((req_hdr->flags & ISCSI_CMD_READ) &&
1799             (req_hdr->flags & ISCSI_CMD_WRITE)) {
1800                 int sz = cmnd_read_size(req);
1801                 if (unlikely(sz < 0)) {
1802                         PRINT_ERROR("%s", "BIDI data transfer, but initiator "
1803                                 "not supplied Bidirectional Read Expected Data "
1804                                 "Transfer Length AHS");
1805                         set_scst_preliminary_status_rsp(req, true,
1806                            SCST_LOAD_SENSE(scst_sense_parameter_value_invalid));
1807                 } else {
1808                         req->read_size = sz;
1809                         dir = SCST_DATA_BIDI;
1810                         scst_cmd_set_expected(scst_cmd, dir, sz);
1811                         scst_cmd_set_expected_in_transfer_len(scst_cmd,
1812                                 be32_to_cpu(req_hdr->data_length));
1813 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
1814                         scst_cmd_set_tgt_need_alloc_data_buf(scst_cmd);
1815 #endif
1816                 }
1817         } else if (req_hdr->flags & ISCSI_CMD_READ) {
1818                 req->read_size = be32_to_cpu(req_hdr->data_length);
1819                 dir = SCST_DATA_READ;
1820                 scst_cmd_set_expected(scst_cmd, dir, req->read_size);
1821 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
1822                 scst_cmd_set_tgt_need_alloc_data_buf(scst_cmd);
1823 #endif
1824         } else if (req_hdr->flags & ISCSI_CMD_WRITE) {
1825                 dir = SCST_DATA_WRITE;
1826                 scst_cmd_set_expected(scst_cmd, dir,
1827                         be32_to_cpu(req_hdr->data_length));
1828         } else {
1829                 dir = SCST_DATA_NONE;
1830                 scst_cmd_set_expected(scst_cmd, dir, 0);
1831         }
1832
1833         switch (req_hdr->flags & ISCSI_CMD_ATTR_MASK) {
1834         case ISCSI_CMD_SIMPLE:
1835                 scst_cmd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1836                 break;
1837         case ISCSI_CMD_HEAD_OF_QUEUE:
1838                 scst_cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1839                 break;
1840         case ISCSI_CMD_ORDERED:
1841                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1842                 break;
1843         case ISCSI_CMD_ACA:
1844                 scst_cmd->queue_type = SCST_CMD_QUEUE_ACA;
1845                 break;
1846         case ISCSI_CMD_UNTAGGED:
1847                 scst_cmd->queue_type = SCST_CMD_QUEUE_UNTAGGED;
1848                 break;
1849         default:
1850                 PRINT_ERROR("Unknown task code %x, use ORDERED instead",
1851                         req_hdr->flags & ISCSI_CMD_ATTR_MASK);
1852                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1853                 break;
1854         }
1855
1856         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1857         scst_cmd_set_tgt_sn(scst_cmd, req_hdr->cmd_sn);
1858
1859         ahdr = (struct iscsi_ahs_hdr *)req->pdu.ahs;
1860         if (ahdr != NULL) {
1861                 uint8_t *p = (uint8_t *)ahdr;
1862                 unsigned int size = 0;
1863                 do {
1864                         int s;
1865
1866                         ahdr = (struct iscsi_ahs_hdr *)p;
1867
1868                         if (ahdr->ahstype == ISCSI_AHSTYPE_CDB) {
1869                                 struct iscsi_cdb_ahdr *eca =
1870                                         (struct iscsi_cdb_ahdr *)ahdr;
1871                                 scst_cmd_set_ext_cdb(scst_cmd, eca->cdb,
1872                                         be16_to_cpu(ahdr->ahslength) - 1);
1873                                 break;
1874                         }
1875                         s = 3 + be16_to_cpu(ahdr->ahslength);
1876                         s = (s + 3) & -4;
1877                         size += s;
1878                         p += s;
1879                 } while (size < req->pdu.ahssize);
1880         }
1881
1882         TRACE_DBG("START Command (itt %x, queue_type %d)",
1883                 req_hdr->itt, scst_cmd->queue_type);
1884         req->scst_state = ISCSI_CMD_STATE_RX_CMD;
1885         conn->rx_task = current;
1886         scst_cmd_init_stage1_done(scst_cmd, SCST_CONTEXT_DIRECT, 0);
1887
1888         if (req->scst_state != ISCSI_CMD_STATE_RX_CMD)
1889                 res = cmnd_rx_continue(req);
1890         else {
1891                 TRACE_DBG("Delaying req %p post processing (scst_state %d)",
1892                         req, req->scst_state);
1893                 res = 1;
1894         }
1895
1896 out:
1897         TRACE_EXIT_RES(res);
1898         return res;
1899 }
1900
1901 static int data_out_start(struct iscsi_cmnd *cmnd)
1902 {
1903         struct iscsi_conn *conn = cmnd->conn;
1904         struct iscsi_data_out_hdr *req_hdr =
1905                 (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1906         struct iscsi_cmnd *orig_req;
1907         struct iscsi_hdr *orig_req_hdr;
1908         u32 offset = be32_to_cpu(req_hdr->buffer_offset);
1909         int res = 0;
1910
1911         TRACE_ENTRY();
1912
1913         /*
1914          * There is no race with send_r2t() and conn_abort(), since
1915          * all functions called from single read thread
1916          */
1917         iscsi_extracheck_is_rd_thread(cmnd->conn);
1918
1919         update_stat_sn(cmnd);
1920
1921         orig_req = cmnd_find_data_wait_hash(conn, req_hdr->itt);
1922         cmnd->cmd_req = orig_req;
1923         if (unlikely(orig_req == NULL)) {
1924                 /*
1925                  * It shouldn't happen, since we don't abort any request until
1926                  * we received all related PDUs from the initiator or timeout
1927                  * them. Let's quietly drop such PDUs.
1928                  */
1929                 TRACE(TRACE_MGMT_MINOR, "Unable to find scsi task ITT %x",
1930                         cmnd_itt(cmnd));
1931                 res = iscsi_preliminary_complete(cmnd, cmnd, true);
1932                 goto out;
1933         }
1934
1935         if (unlikely(orig_req->r2t_len_to_receive < cmnd->pdu.datasize)) {
1936                 if (orig_req->prelim_compl_flags != 0) {
1937                         /* We can have fake r2t_len_to_receive */
1938                         goto go;
1939                 }
1940                 PRINT_ERROR("Data size (%d) > R2T length to receive (%d)",
1941                         cmnd->pdu.datasize, orig_req->r2t_len_to_receive);
1942                 set_scst_preliminary_status_rsp(orig_req, false,
1943                         SCST_LOAD_SENSE(iscsi_sense_incorrect_amount_of_data));
1944                 goto go;
1945         }
1946
1947         /* Crazy iSCSI spec requires us to make this unneeded check */
1948         orig_req_hdr = &orig_req->pdu.bhs;
1949         if (unlikely(orig_req_hdr->lun != req_hdr->lun)) {
1950                 PRINT_ERROR("Wrong LUN (%lld) in Data-Out PDU (expected %lld), "
1951                         "orig_req %p, cmnd %p", (unsigned long long)req_hdr->lun,
1952                         (unsigned long long)orig_req_hdr->lun, orig_req, cmnd);
1953                 create_reject_rsp(orig_req, ISCSI_REASON_PROTOCOL_ERROR, false);
1954                 goto go;
1955         }
1956
1957 go:
1958         if (req_hdr->flags & ISCSI_FLG_FINAL)
1959                 orig_req->outstanding_r2t--;
1960
1961         if (unlikely(orig_req->prelim_compl_flags != 0)) {
1962                 res = iscsi_preliminary_complete(cmnd, orig_req, true);
1963                 goto out;
1964         }
1965
1966         TRACE_WRITE("cmnd %p, orig_req %p, offset %u, datasize %u", cmnd,
1967                 orig_req, offset, cmnd->pdu.datasize);
1968
1969         res = cmnd_prepare_recv_pdu(conn, orig_req, offset, cmnd->pdu.datasize);
1970
1971 out:
1972         TRACE_EXIT_RES(res);
1973         return res;
1974 }
1975
1976 static void data_out_end(struct iscsi_cmnd *cmnd)
1977 {
1978         struct iscsi_data_out_hdr *req_hdr =
1979                 (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1980         struct iscsi_cmnd *req;
1981
1982         TRACE_ENTRY();
1983
1984         EXTRACHECKS_BUG_ON(cmnd == NULL);
1985         req = cmnd->cmd_req;
1986         if (unlikely(req == NULL))
1987                 goto out;
1988
1989         TRACE_DBG("cmnd %p, req %p", cmnd, req);
1990
1991         iscsi_extracheck_is_rd_thread(cmnd->conn);
1992
1993         if (!(cmnd->conn->ddigest_type & DIGEST_NONE) &&
1994             !cmnd->ddigest_checked) {
1995                 cmd_add_on_rx_ddigest_list(req, cmnd);
1996                 cmnd_get(cmnd);
1997         }
1998
1999         /*
2000          * Now we received the data and can adjust r2t_len_to_receive of the
2001          * orig req. We couldn't do it earlier, because it will break data
2002          * receiving errors recovery (calls of iscsi_fail_data_waiting_cmnd()).
2003          */
2004         req->r2t_len_to_receive -= cmnd->pdu.datasize;
2005
2006         if (unlikely(req->prelim_compl_flags != 0)) {
2007                 /*
2008                  * We might need to wait for one or more PDUs. Let's simplify
2009                  * other code.
2010                  */
2011                 req->r2t_len_to_receive = req->outstanding_r2t;
2012                 req->r2t_len_to_send = 0;
2013         }
2014
2015         TRACE_DBG("req %p, FINAL %x, outstanding_r2t %d, r2t_len_to_receive %d,"
2016                 " r2t_len_to_send %d", req, req_hdr->flags & ISCSI_FLG_FINAL,
2017                 req->outstanding_r2t, req->r2t_len_to_receive,
2018                 req->r2t_len_to_send);
2019
2020         if (!(req_hdr->flags & ISCSI_FLG_FINAL))
2021                 goto out;
2022
2023         if (req->r2t_len_to_receive == 0) {
2024                 if (!req->pending)
2025                         iscsi_restart_cmnd(req);
2026         } else if (req->r2t_len_to_send != 0)
2027                 send_r2t(req);
2028
2029 out:
2030         TRACE_EXIT();
2031         return;
2032 }
2033
2034 /* Might be called under target_mutex and cmd_list_lock */
2035 static void __cmnd_abort(struct iscsi_cmnd *cmnd)
2036 {
2037         unsigned long timeout_time = jiffies + ISCSI_TM_DATA_WAIT_SCHED_TIMEOUT;
2038         struct iscsi_conn *conn = cmnd->conn;
2039
2040         TRACE_MGMT_DBG("Aborting cmd %p, scst_cmd %p (scst state %x, "
2041                 "ref_cnt %d, on_write_timeout_list %d, write_start %ld, ITT %x, "
2042                 "sn %u, op %x, r2t_len_to_receive %d, r2t_len_to_send %d, "
2043                 "CDB op %x, size to write %u, outstanding_r2t %d, "
2044                 "sess->exp_cmd_sn %u, conn %p, rd_task %p)",
2045                 cmnd, cmnd->scst_cmd, cmnd->scst_state,
2046                 atomic_read(&cmnd->ref_cnt), cmnd->on_write_timeout_list,
2047                 cmnd->write_start, cmnd_itt(cmnd), cmnd->pdu.bhs.sn,
2048                 cmnd_opcode(cmnd), cmnd->r2t_len_to_receive,
2049                 cmnd->r2t_len_to_send, cmnd_scsicode(cmnd),
2050                 cmnd_write_size(cmnd), cmnd->outstanding_r2t,
2051                 cmnd->conn->session->exp_cmd_sn, cmnd->conn,
2052                 cmnd->conn->rd_task);
2053
2054 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
2055         TRACE_MGMT_DBG("net_ref_cnt %d", atomic_read(&cmnd->net_ref_cnt));
2056 #endif
2057
2058         /*
2059          * Lock to sync with iscsi_check_tm_data_wait_timeouts(), including
2060          * CMD_ABORTED bit set.
2061          */
2062         spin_lock_bh(&iscsi_rd_lock);
2063
2064         /*
2065          * We suppose that preliminary commands completion is tested by
2066          * comparing prelim_compl_flags with 0. Otherwise a race is possible,
2067          * like sending command in SCST core as PRELIM_COMPLETED, while it
2068          * wasn't aborted in it yet and have as the result a wrong success
2069          * status sent to the initiator.
2070          */
2071         set_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags);
2072
2073         TRACE_MGMT_DBG("Setting conn_tm_active for conn %p", conn);
2074         conn->conn_tm_active = 1;
2075
2076         spin_unlock_bh(&iscsi_rd_lock);
2077
2078         /*
2079          * We need the lock to sync with req_add_to_write_timeout_list() and
2080          * close races for rsp_timer.expires.
2081          */
2082         spin_lock_bh(&conn->write_list_lock);
2083         if (!timer_pending(&conn->rsp_timer) ||
2084             time_after(conn->rsp_timer.expires, timeout_time)) {
2085                 TRACE_MGMT_DBG("Mod timer on %ld (conn %p)", timeout_time,
2086                         conn);
2087                 mod_timer(&conn->rsp_timer, timeout_time);
2088         } else
2089                 TRACE_MGMT_DBG("Timer for conn %p is going to fire on %ld "
2090                         "(timeout time %ld)", conn, conn->rsp_timer.expires,
2091                         timeout_time);
2092         spin_unlock_bh(&conn->write_list_lock);
2093
2094         return;
2095 }
2096
2097 /* Must be called from the read or conn close thread */
2098 static int cmnd_abort(struct iscsi_cmnd *req, int *status)
2099 {
2100         struct iscsi_task_mgt_hdr *req_hdr =
2101                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2102         struct iscsi_cmnd *cmnd;
2103         int res = -1;
2104
2105         req_hdr->ref_cmd_sn = be32_to_cpu(req_hdr->ref_cmd_sn);
2106
2107         if (!before(req_hdr->ref_cmd_sn, req_hdr->cmd_sn)) {
2108                 TRACE(TRACE_MGMT, "ABORT TASK: RefCmdSN(%u) > CmdSN(%u)",
2109                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn);
2110                 *status = ISCSI_RESPONSE_UNKNOWN_TASK;
2111                 goto out;
2112         }
2113
2114         cmnd = cmnd_find_itt_get(req->conn, req_hdr->rtt);
2115         if (cmnd) {
2116                 struct iscsi_conn *conn = cmnd->conn;
2117                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
2118
2119                 if (req_hdr->lun != hdr->lun) {
2120                         PRINT_ERROR("ABORT TASK: LUN mismatch: req LUN "
2121                                     "%llx, cmd LUN %llx, rtt %u",
2122                                     (long long unsigned int)req_hdr->lun,
2123                                     (long long unsigned int)hdr->lun,
2124                                     req_hdr->rtt);
2125                         *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2126                         goto out_put;
2127                 }
2128
2129                 if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
2130                         if (req_hdr->ref_cmd_sn != req_hdr->cmd_sn) {
2131                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != TM "
2132                                         "cmd CmdSN(%u) for immediate command "
2133                                         "%p", req_hdr->ref_cmd_sn,
2134                                         req_hdr->cmd_sn, cmnd);
2135                                 *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2136                                 goto out_put;
2137                         }
2138                 } else {
2139                         if (req_hdr->ref_cmd_sn != hdr->cmd_sn) {
2140                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != "
2141                                         "CmdSN(%u) for command %p",
2142                                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
2143                                         cmnd);
2144                                 *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2145                                 goto out_put;
2146                         }
2147                 }
2148
2149                 if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
2150                     (req_hdr->cmd_sn == hdr->cmd_sn)) {
2151                         PRINT_ERROR("ABORT TASK: SN mismatch: req SN %x, "
2152                                 "cmd SN %x, rtt %u", req_hdr->cmd_sn,
2153                                 hdr->cmd_sn, req_hdr->rtt);
2154                         *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2155                         goto out_put;
2156                 }
2157
2158                 spin_lock_bh(&conn->cmd_list_lock);
2159                 __cmnd_abort(cmnd);
2160                 spin_unlock_bh(&conn->cmd_list_lock);
2161
2162                 cmnd_put(cmnd);
2163                 res = 0;
2164         } else {
2165                 TRACE_MGMT_DBG("cmd RTT %x not found", req_hdr->rtt);
2166                 /*
2167                  * iSCSI RFC:
2168                  *
2169                  * b)  If the Referenced Task Tag does not identify an existing task,
2170                  * but if the CmdSN indicated by the RefCmdSN field in the Task
2171                  * Management function request is within the valid CmdSN window
2172                  * and less than the CmdSN of the Task Management function
2173                  * request itself, then targets must consider the CmdSN received
2174                  * and return the "Function complete" response.
2175                  *
2176                  * c)  If the Referenced Task Tag does not identify an existing task
2177                  * and if the CmdSN indicated by the RefCmdSN field in the Task
2178                  * Management function request is outside the valid CmdSN window,
2179                  * then targets must return the "Task does not exist" response.
2180                  *
2181                  * 128 seems to be a good "window".
2182                  */
2183                 if (between(req_hdr->ref_cmd_sn, req_hdr->cmd_sn - 128,
2184                             req_hdr->cmd_sn)) {
2185                         *status = ISCSI_RESPONSE_FUNCTION_COMPLETE;
2186                         res = 0;
2187                 } else
2188                         *status = ISCSI_RESPONSE_UNKNOWN_TASK;
2189         }
2190
2191 out:
2192         return res;
2193
2194 out_put:
2195         cmnd_put(cmnd);
2196         goto out;
2197 }
2198
2199 /* Must be called from the read or conn close thread */
2200 static int target_abort(struct iscsi_cmnd *req, int all)
2201 {
2202         struct iscsi_target *target = req->conn->session->target;
2203         struct iscsi_task_mgt_hdr *req_hdr =
2204                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2205         struct iscsi_session *session;
2206         struct iscsi_conn *conn;
2207         struct iscsi_cmnd *cmnd;
2208
2209         mutex_lock(&target->target_mutex);
2210
2211         list_for_each_entry(session, &target->session_list,
2212                             session_list_entry) {
2213                 list_for_each_entry(conn, &session->conn_list,
2214                                     conn_list_entry) {
2215                         spin_lock_bh(&conn->cmd_list_lock);
2216                         list_for_each_entry(cmnd, &conn->cmd_list,
2217                                             cmd_list_entry) {
2218                                 if (cmnd == req)
2219                                         continue;
2220                                 if (all)
2221                                         __cmnd_abort(cmnd);
2222                                 else if (req_hdr->lun == cmnd_hdr(cmnd)->lun)
2223                                         __cmnd_abort(cmnd);
2224                         }
2225                         spin_unlock_bh(&conn->cmd_list_lock);
2226                 }
2227         }
2228
2229         mutex_unlock(&target->target_mutex);
2230         return 0;
2231 }
2232
2233 /* Must be called from the read or conn close thread */
2234 static void task_set_abort(struct iscsi_cmnd *req)
2235 {
2236         struct iscsi_session *session = req->conn->session;
2237         struct iscsi_task_mgt_hdr *req_hdr =
2238                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2239         struct iscsi_target *target = session->target;
2240         struct iscsi_conn *conn;
2241         struct iscsi_cmnd *cmnd;
2242
2243         mutex_lock(&target->target_mutex);
2244
2245         list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
2246                 spin_lock_bh(&conn->cmd_list_lock);
2247                 list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
2248                         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
2249                         if (cmnd == req)
2250                                 continue;
2251                         if (req_hdr->lun != hdr->lun)
2252                                 continue;
2253                         if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
2254                             req_hdr->cmd_sn == hdr->cmd_sn)
2255                                 continue;
2256                         __cmnd_abort(cmnd);
2257                 }
2258                 spin_unlock_bh(&conn->cmd_list_lock);
2259         }
2260
2261         mutex_unlock(&target->target_mutex);
2262         return;
2263 }
2264
2265 /* Must be called from the read or conn close thread */
2266 void conn_abort(struct iscsi_conn *conn)
2267 {
2268         struct iscsi_cmnd *cmnd;
2269
2270         TRACE_MGMT_DBG("Aborting conn %p", conn);
2271
2272         iscsi_extracheck_is_rd_thread(conn);
2273
2274         spin_lock_bh(&conn->cmd_list_lock);
2275 again:
2276         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
2277                 __cmnd_abort(cmnd);
2278                 if (cmnd->r2t_len_to_receive != 0) {
2279                         if (!cmnd_get_check(cmnd)) {
2280                                 spin_unlock_bh(&conn->cmd_list_lock);
2281
2282                                 /* ToDo: this is racy for MC/S */
2283                                 iscsi_fail_data_waiting_cmnd(cmnd);
2284
2285                                 cmnd_put(cmnd);
2286
2287                                 /*
2288                                  * We are in the read thread, so we may not
2289                                  * worry that after cmnd release conn gets
2290                                  * released as well.
2291                                  */
2292                                 spin_lock_bh(&conn->cmd_list_lock);
2293                                 goto again;
2294                         }
2295                 }
2296         }
2297         spin_unlock_bh(&conn->cmd_list_lock);
2298
2299         return;
2300 }
2301
2302 static void execute_task_management(struct iscsi_cmnd *req)
2303 {
2304         struct iscsi_conn *conn = req->conn;
2305         struct iscsi_session *sess = conn->session;
2306         struct iscsi_task_mgt_hdr *req_hdr =
2307                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2308         int rc, status, function = req_hdr->function & ISCSI_FUNCTION_MASK;
2309         struct scst_rx_mgmt_params params;
2310
2311         TRACE((function == ISCSI_FUNCTION_ABORT_TASK) ?
2312                         TRACE_MGMT_MINOR : TRACE_MGMT,
2313                 "TM fn %d", function);
2314
2315         TRACE_MGMT_DBG("TM req %p, ITT %x, RTT %x, sn %u, con %p", req,
2316                 cmnd_itt(req), req_hdr->rtt, req_hdr->cmd_sn, conn);
2317
2318         iscsi_extracheck_is_rd_thread(conn);
2319
2320         spin_lock(&sess->sn_lock);
2321         sess->tm_active++;
2322         sess->tm_sn = req_hdr->cmd_sn;
2323         if (sess->tm_rsp != NULL) {
2324                 struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
2325
2326                 TRACE(TRACE_MGMT_MINOR, "Dropping delayed TM rsp %p", tm_rsp);
2327
2328                 sess->tm_rsp = NULL;
2329                 sess->tm_active--;
2330
2331                 spin_unlock(&sess->sn_lock);
2332
2333                 sBUG_ON(sess->tm_active < 0);
2334
2335                 rsp_cmnd_release(tm_rsp);
2336         } else
2337                 spin_unlock(&sess->sn_lock);
2338
2339         memset(&params, 0, sizeof(params));
2340         params.atomic = SCST_NON_ATOMIC;
2341         params.tgt_priv = req;
2342
2343         if ((function != ISCSI_FUNCTION_ABORT_TASK) &&
2344             (req_hdr->rtt != ISCSI_RESERVED_TAG)) {
2345                 PRINT_ERROR("Invalid RTT %x (TM fn %d)", req_hdr->rtt,
2346                         function);
2347                 rc = -1;
2348                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2349                 goto reject;
2350         }
2351
2352         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
2353
2354         switch (function) {
2355         case ISCSI_FUNCTION_ABORT_TASK:
2356                 rc = cmnd_abort(req, &status);
2357                 if (rc == 0) {
2358                         params.fn = SCST_ABORT_TASK;
2359                         params.tag = req_hdr->rtt;
2360                         params.tag_set = 1;
2361                         params.lun = (uint8_t *)&req_hdr->lun;
2362                         params.lun_len = sizeof(req_hdr->lun);
2363                         params.lun_set = 1;
2364                         params.cmd_sn = req_hdr->cmd_sn;
2365                         params.cmd_sn_set = 1;
2366                         rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2367                                 &params);
2368                         status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2369                 }
2370                 break;
2371         case ISCSI_FUNCTION_ABORT_TASK_SET:
2372                 task_set_abort(req);
2373                 params.fn = SCST_ABORT_TASK_SET;
2374                 params.lun = (uint8_t *)&req_hdr->lun;
2375                 params.lun_len = sizeof(req_hdr->lun);
2376                 params.lun_set = 1;
2377                 params.cmd_sn = req_hdr->cmd_sn;
2378                 params.cmd_sn_set = 1;
2379                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2380                         &params);
2381                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2382                 break;
2383         case ISCSI_FUNCTION_CLEAR_TASK_SET:
2384                 task_set_abort(req);
2385                 params.fn = SCST_CLEAR_TASK_SET;
2386                 params.lun = (uint8_t *)&req_hdr->lun;
2387                 params.lun_len = sizeof(req_hdr->lun);
2388                 params.lun_set = 1;
2389                 params.cmd_sn = req_hdr->cmd_sn;
2390                 params.cmd_sn_set = 1;
2391                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2392                         &params);
2393                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2394                 break;
2395         case ISCSI_FUNCTION_CLEAR_ACA:
2396                 params.fn = SCST_CLEAR_ACA;
2397                 params.lun = (uint8_t *)&req_hdr->lun;
2398                 params.lun_len = sizeof(req_hdr->lun);
2399                 params.lun_set = 1;
2400                 params.cmd_sn = req_hdr->cmd_sn;
2401                 params.cmd_sn_set = 1;
2402                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2403                         &params);
2404                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2405                 break;
2406         case ISCSI_FUNCTION_TARGET_COLD_RESET:
2407         case ISCSI_FUNCTION_TARGET_WARM_RESET:
2408                 target_abort(req, 1);
2409                 params.fn = SCST_TARGET_RESET;
2410                 params.cmd_sn = req_hdr->cmd_sn;
2411                 params.cmd_sn_set = 1;
2412                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2413                         &params);
2414                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2415                 break;
2416         case ISCSI_FUNCTION_LOGICAL_UNIT_RESET:
2417                 target_abort(req, 0);
2418                 params.fn = SCST_LUN_RESET;
2419                 params.lun = (uint8_t *)&req_hdr->lun;
2420                 params.lun_len = sizeof(req_hdr->lun);
2421                 params.lun_set = 1;
2422                 params.cmd_sn = req_hdr->cmd_sn;
2423                 params.cmd_sn_set = 1;
2424                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2425                         &params);
2426                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2427                 break;
2428         case ISCSI_FUNCTION_TASK_REASSIGN:
2429                 rc = -1;
2430                 status = ISCSI_RESPONSE_ALLEGIANCE_REASSIGNMENT_UNSUPPORTED;
2431                 break;
2432         default:
2433                 PRINT_ERROR("Unknown TM function %d", function);
2434                 rc = -1;
2435                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2436                 break;
2437         }
2438
2439 reject:
2440         if (rc != 0)
2441                 iscsi_send_task_mgmt_resp(req, status);
2442
2443         return;
2444 }
2445
2446 static void noop_out_exec(struct iscsi_cmnd *req)
2447 {
2448         struct iscsi_cmnd *rsp;
2449         struct iscsi_nop_in_hdr *rsp_hdr;
2450
2451         TRACE_ENTRY();
2452
2453         TRACE_DBG("%p", req);
2454
2455         if (cmnd_itt(req) != cpu_to_be32(ISCSI_RESERVED_TAG)) {
2456                 rsp = iscsi_alloc_main_rsp(req);
2457
2458                 rsp_hdr = (struct iscsi_nop_in_hdr *)&rsp->pdu.bhs;
2459                 rsp_hdr->opcode = ISCSI_OP_NOOP_IN;
2460                 rsp_hdr->flags = ISCSI_FLG_FINAL;
2461                 rsp_hdr->itt = req->pdu.bhs.itt;
2462                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
2463
2464                 if (req->pdu.datasize)
2465                         sBUG_ON(req->sg == NULL);
2466                 else
2467                         sBUG_ON(req->sg != NULL);
2468
2469                 if (req->sg) {
2470                         rsp->sg = req->sg;
2471                         rsp->sg_cnt = req->sg_cnt;
2472                         rsp->bufflen = req->bufflen;
2473                 }
2474
2475                 /* We already checked it in check_segment_length() */
2476                 sBUG_ON(get_pgcnt(req->pdu.datasize, 0) > ISCSI_CONN_IOV_MAX);
2477
2478                 rsp->pdu.datasize = req->pdu.datasize;
2479         }
2480
2481         req_cmnd_release(req);
2482
2483         TRACE_EXIT();
2484         return;
2485 }
2486
2487 static void logout_exec(struct iscsi_cmnd *req)
2488 {
2489         struct iscsi_logout_req_hdr *req_hdr;
2490         struct iscsi_cmnd *rsp;
2491         struct iscsi_logout_rsp_hdr *rsp_hdr;
2492
2493         PRINT_INFO("Logout received from initiator %s",
2494                 req->conn->session->initiator_name);
2495         TRACE_DBG("%p", req);
2496
2497         req_hdr = (struct iscsi_logout_req_hdr *)&req->pdu.bhs;
2498         rsp = iscsi_alloc_main_rsp(req);
2499         rsp_hdr = (struct iscsi_logout_rsp_hdr *)&rsp->pdu.bhs;
2500         rsp_hdr->opcode = ISCSI_OP_LOGOUT_RSP;
2501         rsp_hdr->flags = ISCSI_FLG_FINAL;
2502         rsp_hdr->itt = req_hdr->itt;
2503         rsp->should_close_conn = 1;
2504
2505         req_cmnd_release(req);
2506
2507         return;
2508 }
2509
2510 static void iscsi_cmnd_exec(struct iscsi_cmnd *cmnd)
2511 {
2512         TRACE_ENTRY();
2513
2514         TRACE_DBG("cmnd %p, op %x, SN %u", cmnd, cmnd_opcode(cmnd),
2515                 cmnd->pdu.bhs.sn);
2516
2517         iscsi_extracheck_is_rd_thread(cmnd->conn);
2518
2519         if (cmnd_opcode(cmnd) == ISCSI_OP_SCSI_CMD) {
2520                 if (cmnd->r2t_len_to_receive == 0)
2521                         iscsi_restart_cmnd(cmnd);
2522                 else if (cmnd->r2t_len_to_send != 0)
2523                         send_r2t(cmnd);
2524                 goto out;
2525         }
2526
2527         if (cmnd->prelim_compl_flags != 0) {
2528                 TRACE_MGMT_DBG("Terminating prelim completed non-SCSI cmnd %p "
2529                         "(op %x)", cmnd, cmnd_opcode(cmnd));
2530                 req_cmnd_release(cmnd);
2531                 goto out;
2532         }
2533
2534         switch (cmnd_opcode(cmnd)) {
2535         case ISCSI_OP_NOOP_OUT:
2536                 noop_out_exec(cmnd);
2537                 break;
2538         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2539                 execute_task_management(cmnd);
2540                 break;
2541         case ISCSI_OP_LOGOUT_CMD:
2542                 logout_exec(cmnd);
2543                 break;
2544         default:
2545                 PRINT_CRIT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
2546                 sBUG();
2547                 break;
2548         }
2549
2550 out:
2551         TRACE_EXIT();
2552         return;
2553 }
2554
2555 static void set_cork(struct socket *sock, int on)
2556 {
2557         int opt = on;
2558         mm_segment_t oldfs;
2559
2560         oldfs = get_fs();
2561         set_fs(get_ds());
2562         sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK,
2563                               (void __force __user *)&opt, sizeof(opt));
2564         set_fs(oldfs);
2565         return;
2566 }
2567
2568 void cmnd_tx_start(struct iscsi_cmnd *cmnd)
2569 {
2570         struct iscsi_conn *conn = cmnd->conn;
2571
2572         TRACE_DBG("conn %p, cmnd %p, opcode %x", conn, cmnd, cmnd_opcode(cmnd));
2573         iscsi_cmnd_set_length(&cmnd->pdu);
2574
2575         iscsi_extracheck_is_wr_thread(conn);
2576
2577         set_cork(conn->sock, 1);
2578
2579         conn->write_iop = conn->write_iov;
2580         conn->write_iop->iov_base = (void __force __user *)(&cmnd->pdu.bhs);
2581         conn->write_iop->iov_len = sizeof(cmnd->pdu.bhs);
2582         conn->write_iop_used = 1;
2583         conn->write_size = sizeof(cmnd->pdu.bhs) + cmnd->pdu.datasize;
2584         conn->write_offset = 0;
2585
2586         switch (cmnd_opcode(cmnd)) {
2587         case ISCSI_OP_NOOP_IN:
2588                 cmnd_set_sn(cmnd, 1);
2589                 break;
2590         case ISCSI_OP_SCSI_RSP:
2591                 cmnd_set_sn(cmnd, 1);
2592                 break;
2593         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2594                 cmnd_set_sn(cmnd, 1);
2595                 break;
2596         case ISCSI_OP_TEXT_RSP:
2597                 cmnd_set_sn(cmnd, 1);
2598                 break;
2599         case ISCSI_OP_SCSI_DATA_IN:
2600         {
2601                 struct iscsi_data_in_hdr *rsp =
2602                         (struct iscsi_data_in_hdr *)&cmnd->pdu.bhs;
2603                 u32 offset = cpu_to_be32(rsp->buffer_offset);
2604
2605                 TRACE_DBG("cmnd %p, offset %u, datasize %u, bufflen %u", cmnd,
2606                         offset, cmnd->pdu.datasize, cmnd->bufflen);
2607
2608                 sBUG_ON(offset > cmnd->bufflen);
2609                 sBUG_ON(offset + cmnd->pdu.datasize > cmnd->bufflen);
2610
2611                 conn->write_offset = offset;
2612
2613                 cmnd_set_sn(cmnd, (rsp->flags & ISCSI_FLG_FINAL) ? 1 : 0);
2614                 break;
2615         }
2616         case ISCSI_OP_LOGOUT_RSP:
2617                 cmnd_set_sn(cmnd, 1);
2618                 break;
2619         case ISCSI_OP_R2T:
2620                 cmnd->pdu.bhs.sn = cmnd_set_sn(cmnd, 0);
2621                 break;
2622         case ISCSI_OP_ASYNC_MSG:
2623                 cmnd_set_sn(cmnd, 1);
2624                 break;
2625         case ISCSI_OP_REJECT:
2626                 cmnd_set_sn(cmnd, 1);
2627                 break;
2628         default:
2629                 PRINT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
2630                 break;
2631         }
2632
2633         iscsi_dump_pdu(&cmnd->pdu);
2634         return;
2635 }
2636
2637 void cmnd_tx_end(struct iscsi_cmnd *cmnd)
2638 {
2639         struct iscsi_conn *conn = cmnd->conn;
2640
2641         TRACE_DBG("%p:%x (should_close_conn %d, should_close_all_conn %d)",
2642                 cmnd, cmnd_opcode(cmnd), cmnd->should_close_conn,
2643                 cmnd->should_close_all_conn);
2644
2645 #ifdef CONFIG_SCST_EXTRACHECKS
2646         switch (cmnd_opcode(cmnd)) {
2647         case ISCSI_OP_NOOP_IN:
2648         case ISCSI_OP_SCSI_RSP:
2649         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2650         case ISCSI_OP_TEXT_RSP:
2651         case ISCSI_OP_R2T:
2652         case ISCSI_OP_ASYNC_MSG:
2653         case ISCSI_OP_REJECT:
2654         case ISCSI_OP_SCSI_DATA_IN:
2655         case ISCSI_OP_LOGOUT_RSP:
2656                 break;
2657         default:
2658                 PRINT_CRIT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2659                 sBUG();
2660                 break;
2661         }
2662 #endif
2663
2664         if (unlikely(cmnd->should_close_conn)) {
2665                 if (cmnd->should_close_all_conn) {
2666                         PRINT_INFO("Closing all connections for target %x at "
2667                                 "initiator's %s request",
2668                                 cmnd->conn->session->target->tid,
2669                                 conn->session->initiator_name);
2670                         target_del_all_sess(cmnd->conn->session->target, 0);
2671                 } else {
2672                         PRINT_INFO("Closing connection at initiator's %s "
2673                                 "request", conn->session->initiator_name);
2674                         mark_conn_closed(conn);
2675                 }
2676         }
2677
2678         set_cork(cmnd->conn->sock, 0);
2679         return;
2680 }
2681
2682 /*
2683  * Push the command for execution. This functions reorders the commands.
2684  * Called from the read thread.
2685  *
2686  * Basically, since we don't support MC/S and TCP guarantees data delivery
2687  * order, all that SN's stuff isn't needed at all (commands delivery order is
2688  * a natural commands execution order), but insane iSCSI spec requires
2689  * us to check it and we have to, because some crazy initiators can rely
2690  * on the SN's based order and reorder requests during sending. For all other
2691  * normal initiators all that code is a NOP.
2692  */
2693 static void iscsi_push_cmnd(struct iscsi_cmnd *cmnd)
2694 {
2695         struct iscsi_session *session = cmnd->conn->session;
2696         struct list_head *entry;
2697         u32 cmd_sn;
2698
2699         TRACE_DBG("cmnd %p, iSCSI opcode %x, sn %u, exp sn %u", cmnd,
2700                 cmnd_opcode(cmnd), cmnd->pdu.bhs.sn, session->exp_cmd_sn);
2701
2702         iscsi_extracheck_is_rd_thread(cmnd->conn);
2703
2704         sBUG_ON(cmnd->parent_req != NULL);
2705
2706         if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
2707                 TRACE_DBG("Immediate cmd %p (cmd_sn %u)", cmnd,
2708                         cmnd->pdu.bhs.sn);
2709                 iscsi_cmnd_exec(cmnd);
2710                 goto out;
2711         }
2712
2713         spin_lock(&session->sn_lock);
2714
2715         cmd_sn = cmnd->pdu.bhs.sn;
2716         if (cmd_sn == session->exp_cmd_sn) {
2717                 while (1) {
2718                         session->exp_cmd_sn = ++cmd_sn;
2719
2720                         if (unlikely(session->tm_active > 0)) {
2721                                 if (before(cmd_sn, session->tm_sn)) {
2722                                         struct iscsi_conn *conn = cmnd->conn;
2723
2724                                         spin_unlock(&session->sn_lock);
2725
2726                                         spin_lock_bh(&conn->cmd_list_lock);
2727                                         __cmnd_abort(cmnd);
2728                                         spin_unlock_bh(&conn->cmd_list_lock);
2729
2730                                         spin_lock(&session->sn_lock);
2731                                 }
2732                                 iscsi_check_send_delayed_tm_resp(session);
2733                         }
2734
2735                         spin_unlock(&session->sn_lock);
2736
2737                         iscsi_cmnd_exec(cmnd);
2738
2739                         spin_lock(&session->sn_lock);
2740
2741                         if (list_empty(&session->pending_list))
2742                                 break;
2743                         cmnd = list_entry(session->pending_list.next,
2744                                           struct iscsi_cmnd,
2745                                           pending_list_entry);
2746                         if (cmnd->pdu.bhs.sn != cmd_sn)
2747                                 break;
2748
2749                         list_del(&cmnd->pending_list_entry);
2750                         cmnd->pending = 0;
2751
2752                         TRACE_MGMT_DBG("Processing pending cmd %p (cmd_sn %u)",
2753                                 cmnd, cmd_sn);
2754                 }
2755         } else {
2756                 int drop = 0;
2757
2758                 TRACE_DBG("Pending cmd %p (cmd_sn %u, exp_cmd_sn %u)",
2759                         cmnd, cmd_sn, session->exp_cmd_sn);
2760
2761                 /*
2762                  * iSCSI RFC 3720: "The target MUST silently ignore any
2763                  * non-immediate command outside of [from ExpCmdSN to MaxCmdSN
2764                  * inclusive] range". But we won't honor the MaxCmdSN
2765                  * requirement, because, since we adjust MaxCmdSN from the
2766                  * separate write thread, rarely it is possible that initiator
2767                  * can legally send command with CmdSN>MaxSN. But it won't
2768                  * hurt anything, in the worst case it will lead to
2769                  * additional QUEUE FULL status.
2770                  */
2771
2772                 if (unlikely(before(cmd_sn, session->exp_cmd_sn))) {
2773                         PRINT_ERROR("Unexpected cmd_sn (%u,%u)", cmd_sn,
2774                                 session->exp_cmd_sn);
2775                         drop = 1;
2776                 }
2777
2778 #if 0
2779                 if (unlikely(after(cmd_sn, session->exp_cmd_sn +
2780                                         iscsi_get_allowed_cmds(session)))) {
2781                         TRACE_MGMT_DBG("Too large cmd_sn %u (exp_cmd_sn %u, "
2782                                 "max_sn %u)", cmd_sn, session->exp_cmd_sn,
2783                                 iscsi_get_allowed_cmds(session));
2784                 }
2785 #endif
2786
2787                 spin_unlock(&session->sn_lock);
2788
2789                 if (unlikely(drop)) {
2790                         req_cmnd_release_force(cmnd);
2791                         goto out;
2792                 }
2793
2794                 if (unlikely(test_bit(ISCSI_CMD_ABORTED,
2795                                         &cmnd->prelim_compl_flags))) {
2796                         struct iscsi_cmnd *tm_clone;
2797
2798                         TRACE_MGMT_DBG("Pending aborted cmnd %p, creating TM "
2799                                 "clone (scst cmd %p, state %d)", cmnd,
2800                                 cmnd->scst_cmd, cmnd->scst_state);
2801
2802                         tm_clone = cmnd_alloc(cmnd->conn, NULL);
2803                         if (tm_clone != NULL) {
2804                                 set_bit(ISCSI_CMD_ABORTED,
2805                                         &tm_clone->prelim_compl_flags);
2806                                 tm_clone->pdu = cmnd->pdu;
2807
2808                                 TRACE_MGMT_DBG("TM clone %p created",
2809                                                tm_clone);
2810
2811                                 iscsi_cmnd_exec(cmnd);
2812                                 cmnd = tm_clone;
2813                         } else
2814                                 PRINT_ERROR("%s", "Unable to create TM clone");
2815                 }
2816
2817                 TRACE_MGMT_DBG("Pending cmnd %p (op %x, sn %u, exp sn %u)",
2818                         cmnd, cmnd_opcode(cmnd), cmd_sn, session->exp_cmd_sn);
2819
2820                 spin_lock(&session->sn_lock);
2821                 list_for_each(entry, &session->pending_list) {
2822                         struct iscsi_cmnd *tmp =
2823                                 list_entry(entry, struct iscsi_cmnd,
2824                                            pending_list_entry);
2825                         if (before(cmd_sn, tmp->pdu.bhs.sn))
2826                                 break;
2827                 }
2828                 list_add_tail(&cmnd->pending_list_entry, entry);
2829                 cmnd->pending = 1;
2830         }
2831
2832         spin_unlock(&session->sn_lock);
2833 out:
2834         return;
2835 }
2836
2837 static int check_segment_length(struct iscsi_cmnd *cmnd)
2838 {
2839         struct iscsi_conn *conn = cmnd->conn;
2840         struct iscsi_session *session = conn->session;
2841
2842         if (unlikely(cmnd->pdu.datasize > session->sess_param.max_recv_data_length)) {
2843                 PRINT_ERROR("Initiator %s violated negotiated parameters: "
2844                         "data too long (ITT %x, datasize %u, "
2845                         "max_recv_data_length %u", session->initiator_name,
2846                         cmnd_itt(cmnd), cmnd->pdu.datasize,
2847                         session->sess_param.max_recv_data_length);
2848                 mark_conn_closed(conn);
2849                 return -EINVAL;
2850         }
2851         return 0;
2852 }
2853
2854 int cmnd_rx_start(struct iscsi_cmnd *cmnd)
2855 {
2856         int res, rc;
2857
2858         iscsi_dump_pdu(&cmnd->pdu);
2859
2860         res = check_segment_length(cmnd);
2861         if (res != 0)
2862                 goto out;
2863
2864         switch (cmnd_opcode(cmnd)) {
2865         case ISCSI_OP_SCSI_CMD:
2866                 res = scsi_cmnd_start(cmnd);
2867                 if (unlikely(res < 0))
2868                         goto out;
2869                 spin_lock(&cmnd->conn->session->sn_lock);
2870                 __update_stat_sn(cmnd);
2871                 rc = check_cmd_sn(cmnd);
2872                 spin_unlock(&cmnd->conn->session->sn_lock);
2873                 break;
2874         case ISCSI_OP_SCSI_DATA_OUT:
2875                 res = data_out_start(cmnd);
2876                 goto out;
2877         case ISCSI_OP_NOOP_OUT:
2878                 rc = noop_out_start(cmnd);
2879                 break;
2880         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2881         case ISCSI_OP_LOGOUT_CMD:
2882                 spin_lock(&cmnd->conn->session->sn_lock);
2883                 __update_stat_sn(cmnd);
2884                 rc = check_cmd_sn(cmnd);
2885                 spin_unlock(&cmnd->conn->session->sn_lock);
2886                 break;
2887         case ISCSI_OP_TEXT_CMD:
2888         case ISCSI_OP_SNACK_CMD:
2889         default:
2890                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2891                 break;
2892         }
2893
2894         if (unlikely(rc < 0)) {
2895                 PRINT_ERROR("Error %d (iSCSI opcode %x, ITT %x)", rc,
2896                         cmnd_opcode(cmnd), cmnd_itt(cmnd));
2897                 res = create_reject_rsp(cmnd, -rc, true);
2898         }
2899
2900 out:
2901         TRACE_EXIT_RES(res);
2902         return res;
2903 }
2904
2905 void cmnd_rx_end(struct iscsi_cmnd *cmnd)
2906 {
2907         TRACE_ENTRY();
2908
2909         TRACE_DBG("cmnd %p, opcode %x", cmnd, cmnd_opcode(cmnd));
2910
2911         switch (cmnd_opcode(cmnd)) {
2912         case ISCSI_OP_SCSI_CMD:
2913         case ISCSI_OP_NOOP_OUT:
2914         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2915         case ISCSI_OP_LOGOUT_CMD:
2916                 iscsi_push_cmnd(cmnd);
2917                 goto out;
2918         case ISCSI_OP_SCSI_DATA_OUT:
2919                 data_out_end(cmnd);
2920                 break;
2921         default:
2922                 PRINT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
2923                 break;
2924         }
2925
2926         req_cmnd_release(cmnd);
2927
2928 out:
2929         TRACE_EXIT();
2930         return;
2931 }
2932
2933 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
2934 static int iscsi_alloc_data_buf(struct scst_cmd *cmd)
2935 {
2936         /*
2937          * sock->ops->sendpage() is async zero copy operation,
2938          * so we must be sure not to free and reuse
2939          * the command's buffer before the sending was completed
2940          * by the network layers. It is possible only if we
2941          * don't use SGV cache.
2942          */
2943         EXTRACHECKS_BUG_ON(!(scst_cmd_get_data_direction(cmd) & SCST_DATA_READ));
2944         scst_cmd_set_no_sgv(cmd);
2945         return 1;
2946 }
2947 #endif
2948
2949 static void iscsi_preprocessing_done(struct scst_cmd *scst_cmd)
2950 {
2951         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
2952                                 scst_cmd_get_tgt_priv(scst_cmd);
2953
2954         TRACE_DBG("req %p", req);
2955
2956         if (req->conn->rx_task == current)
2957                 req->scst_state = ISCSI_CMD_STATE_AFTER_PREPROC;
2958         else {
2959                 /*
2960                  * We wait for the state change without any protection, so
2961                  * without cmnd_get() it is possible that req will die
2962                  * "immediately" after the state assignment and
2963                  * iscsi_make_conn_rd_active() will operate on dead data.
2964                  * We use the ordered version of cmnd_get(), because "get"
2965                  * must be done before the state assignment.
2966                  *
2967                  * We protected from the race on calling cmnd_rx_continue(),
2968                  * because there can be only one read thread processing
2969                  * connection.
2970                  */
2971                 cmnd_get_ordered(req);
2972                 req->scst_state = ISCSI_CMD_STATE_AFTER_PREPROC;
2973                 iscsi_make_conn_rd_active(req->conn);
2974                 if (unlikely(req->conn->closing)) {
2975                         TRACE_DBG("Waking up closing conn %p", req->conn);
2976                         wake_up(&req->conn->read_state_waitQ);
2977                 }
2978                 cmnd_put(req);
2979         }
2980
2981         return;
2982 }
2983
2984 /*
2985  * No locks.
2986  *
2987  * IMPORTANT! Connection conn must be protected by additional conn_get()
2988  * upon entrance in this function, because otherwise it could be destroyed
2989  * inside as a result of iscsi_send(), which releases sent commands.
2990  */
2991 static void iscsi_try_local_processing(struct iscsi_conn *conn)
2992 {
2993         int local;
2994
2995         TRACE_ENTRY();
2996
2997         spin_lock_bh(&iscsi_wr_lock);
2998         switch (conn->wr_state) {
2999         case ISCSI_CONN_WR_STATE_IN_LIST:
3000                 list_del(&conn->wr_list_entry);
3001                 /* go through */
3002         case ISCSI_CONN_WR_STATE_IDLE:
3003 #ifdef CONFIG_SCST_EXTRACHECKS
3004                 conn->wr_task = current;
3005 #endif
3006                 conn->wr_state = ISCSI_CONN_WR_STATE_PROCESSING;
3007                 conn->wr_space_ready = 0;
3008                 local = 1;
3009                 break;
3010         default:
3011                 local = 0;
3012                 break;
3013         }
3014         spin_unlock_bh(&iscsi_wr_lock);
3015
3016         if (local) {
3017                 int rc = 1;
3018
3019                 if (test_write_ready(conn))
3020                         rc = iscsi_send(conn);
3021
3022                 spin_lock_bh(&iscsi_wr_lock);
3023 #ifdef CONFIG_SCST_EXTRACHECKS
3024                 conn->wr_task = NULL;
3025 #endif
3026                 if ((rc <= 0) || test_write_ready(conn)) {
3027                         list_add_tail(&conn->wr_list_entry, &iscsi_wr_list);
3028                         conn->wr_state = ISCSI_CONN_WR_STATE_IN_LIST;
3029                         wake_up(&iscsi_wr_waitQ);
3030                 } else
3031                         conn->wr_state = ISCSI_CONN_WR_STATE_IDLE;
3032                 spin_unlock_bh(&iscsi_wr_lock);
3033         }
3034
3035         TRACE_EXIT();
3036         return;
3037 }
3038
3039 static int iscsi_xmit_response(struct scst_cmd *scst_cmd)
3040 {
3041         int is_send_status = scst_cmd_get_is_send_status(scst_cmd);
3042         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
3043                                         scst_cmd_get_tgt_priv(scst_cmd);
3044         struct iscsi_conn *conn = req->conn;
3045         int status = scst_cmd_get_status(scst_cmd);
3046         u8 *sense = scst_cmd_get_sense_buffer(scst_cmd);
3047         int sense_len = scst_cmd_get_sense_buffer_len(scst_cmd);
3048
3049         if (unlikely(scst_cmd_atomic(scst_cmd)))
3050                 return SCST_TGT_RES_NEED_THREAD_CTX;
3051
3052         scst_cmd_set_tgt_priv(scst_cmd, NULL);
3053
3054         EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_RESTARTED);
3055
3056         if (unlikely(scst_cmd_aborted(scst_cmd)))
3057                 set_bit(ISCSI_CMD_ABORTED, &req->prelim_compl_flags);
3058
3059         if (unlikely(req->prelim_compl_flags != 0)) {
3060                 if (test_bit(ISCSI_CMD_ABORTED, &req->prelim_compl_flags)) {
3061                         TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
3062                                 req->scst_cmd);
3063                         scst_set_delivery_status(req->scst_cmd,
3064                                 SCST_CMD_DELIVERY_ABORTED);
3065                         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
3066                         req_cmnd_release_force(req);
3067                         goto out;
3068                 }
3069
3070                 TRACE_DBG("Prelim completed req %p", req);
3071
3072                 /* To make sure we didn't miss anything above */
3073                 sBUG_ON(status == 0);
3074
3075                 /*
3076                  * We could preliminary have finished req before we
3077                  * knew its device, so check if we return correct sense
3078                  * format.
3079                  */
3080                 scst_check_convert_sense(scst_cmd);
3081
3082                 if (!req->own_sg) {
3083                         req->sg = scst_cmd_get_sg(scst_cmd);
3084                         req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
3085                 }
3086         } else {
3087                 EXTRACHECKS_BUG_ON(req->own_sg);
3088                 req->sg = scst_cmd_get_sg(scst_cmd);
3089                 req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
3090         }
3091
3092         req->bufflen = scst_cmd_get_resp_data_len(scst_cmd);
3093
3094         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
3095
3096         TRACE_DBG("req %p, is_send_status=%x, req->bufflen=%d, req->sg=%p, "
3097                 "req->sg_cnt %d", req, is_send_status, req->bufflen, req->sg,
3098                 req->sg_cnt);
3099
3100         EXTRACHECKS_BUG_ON(req->hashed);
3101         if (req->main_rsp != NULL)
3102                 EXTRACHECKS_BUG_ON(cmnd_opcode(req->main_rsp) != ISCSI_OP_REJECT);
3103
3104         if (unlikely((req->bufflen != 0) && !is_send_status)) {
3105                 PRINT_CRIT_ERROR("%s", "Sending DATA without STATUS is "
3106                         "unsupported");
3107                 scst_set_cmd_error(scst_cmd,
3108                         SCST_LOAD_SENSE(scst_sense_hardw_error));
3109                 sBUG(); /* ToDo */
3110         }
3111
3112         if (req->bufflen != 0) {
3113                 /*
3114                  * Check above makes sure that is_send_status is set,
3115                  * so status is valid here, but in future that could change.
3116                  * ToDo
3117                  */
3118                 if ((status != SAM_STAT_CHECK_CONDITION) &&
3119                     ((cmnd_hdr(req)->flags & (ISCSI_CMD_WRITE|ISCSI_CMD_READ)) !=
3120                                 (ISCSI_CMD_WRITE|ISCSI_CMD_READ))) {
3121                         send_data_rsp(req, status, is_send_status);
3122                 } else {
3123                         struct iscsi_cmnd *rsp;
3124                         send_data_rsp(req, 0, 0);
3125                         if (is_send_status) {
3126                                 rsp = create_status_rsp(req, status, sense,
3127                                         sense_len, true);
3128                                 iscsi_cmnd_init_write(rsp, 0);
3129                         }
3130                 }
3131         } else if (is_send_status) {
3132                 struct iscsi_cmnd *rsp;
3133                 rsp = create_status_rsp(req, status, sense, sense_len, false);
3134                 iscsi_cmnd_init_write(rsp, 0);
3135         }
3136 #ifdef CONFIG_SCST_EXTRACHECKS
3137         else
3138                 sBUG();
3139 #endif
3140
3141         /*
3142          * "_ordered" here to protect from reorder, which can lead to
3143          * preliminary connection destroy in req_cmnd_release(). Just in
3144          * case, actually, because reordering shouldn't go so far, but who
3145          * knows..
3146          */
3147         conn_get_ordered(conn);
3148         req_cmnd_release(req);
3149         iscsi_try_local_processing(conn);
3150         conn_put(conn);
3151
3152 out:
3153         return SCST_TGT_RES_SUCCESS;
3154 }
3155
3156 /* Called under sn_lock */
3157 static bool iscsi_is_delay_tm_resp(struct iscsi_cmnd *rsp)
3158 {
3159         bool res = 0;
3160         struct iscsi_task_mgt_hdr *req_hdr =
3161                 (struct iscsi_task_mgt_hdr *)&rsp->parent_req->pdu.bhs;
3162         int function = req_hdr->function & ISCSI_FUNCTION_MASK;
3163         struct iscsi_session *sess = rsp->conn->session;
3164
3165         TRACE_ENTRY();
3166
3167         /* This should be checked for immediate TM commands as well */
3168
3169         switch (function) {
3170         default:
3171                 if (before(sess->exp_cmd_sn, req_hdr->cmd_sn))
3172                         res = 1;
3173                 break;
3174         }
3175
3176         TRACE_EXIT_RES(res);
3177         return res;
3178 }
3179
3180 /* Called under sn_lock, but might drop it inside, then reaquire */
3181 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess)
3182         __acquires(&sn_lock)
3183         __releases(&sn_lock)
3184 {
3185         struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
3186
3187         TRACE_ENTRY();
3188
3189         if (tm_rsp == NULL)
3190                 goto out;
3191
3192         if (iscsi_is_delay_tm_resp(tm_rsp))
3193                 goto out;
3194
3195         TRACE(TRACE_MGMT_MINOR, "Sending delayed rsp %p", tm_rsp);
3196
3197         sess->tm_rsp = NULL;
3198         sess->tm_active--;
3199
3200         spin_unlock(&sess->sn_lock);
3201
3202         sBUG_ON(sess->tm_active < 0);
3203
3204         iscsi_cmnd_init_write(tm_rsp, ISCSI_INIT_WRITE_WAKE);
3205
3206         spin_lock(&sess->sn_lock);
3207
3208 out:
3209         TRACE_EXIT();
3210         return;
3211 }
3212
3213 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status)
3214 {
3215         struct iscsi_cmnd *rsp;
3216         struct iscsi_task_mgt_hdr *req_hdr =
3217                                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
3218         struct iscsi_task_rsp_hdr *rsp_hdr;
3219         struct iscsi_session *sess = req->conn->session;
3220         int fn = req_hdr->function & ISCSI_FUNCTION_MASK;
3221
3222         TRACE_ENTRY();
3223
3224         TRACE_MGMT_DBG("TM req %p finished", req);
3225         TRACE((req_hdr->function == ISCSI_FUNCTION_ABORT_TASK) ?
3226                          TRACE_MGMT_MINOR : TRACE_MGMT,
3227                 "TM fn %d finished, status %d", fn, status);
3228
3229         rsp = iscsi_alloc_rsp(req);
3230         rsp_hdr = (struct iscsi_task_rsp_hdr *)&rsp->pdu.bhs;
3231
3232         rsp_hdr->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
3233         rsp_hdr->flags = ISCSI_FLG_FINAL;
3234         rsp_hdr->itt = req_hdr->itt;
3235         rsp_hdr->response = status;
3236
3237         if (fn == ISCSI_FUNCTION_TARGET_COLD_RESET) {
3238                 rsp->should_close_conn = 1;
3239                 rsp->should_close_all_conn = 1;
3240         }
3241
3242         sBUG_ON(sess->tm_rsp != NULL);
3243
3244         spin_lock(&sess->sn_lock);
3245         if (iscsi_is_delay_tm_resp(rsp)) {
3246                 TRACE(TRACE_MGMT_MINOR, "Delaying TM fn %d response %p "
3247                         "(req %p), because not all affected commands received "
3248                         "(TM cmd sn %u, exp sn %u)",
3249                         req_hdr->function & ISCSI_FUNCTION_MASK, rsp, req,
3250                         req_hdr->cmd_sn, sess->exp_cmd_sn);
3251                 sess->tm_rsp = rsp;
3252                 spin_unlock(&sess->sn_lock);
3253                 goto out_release;
3254         }
3255         sess->tm_active--;
3256         spin_unlock(&sess->sn_lock);
3257
3258         sBUG_ON(sess->tm_active < 0);
3259
3260         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_WAKE);
3261
3262 out_release:
3263         req_cmnd_release(req);
3264
3265         TRACE_EXIT();
3266         return;
3267 }
3268
3269 static inline int iscsi_get_mgmt_response(int status)
3270 {
3271         switch (status) {
3272         case SCST_MGMT_STATUS_SUCCESS:
3273                 return ISCSI_RESPONSE_FUNCTION_COMPLETE;
3274
3275         case SCST_MGMT_STATUS_TASK_NOT_EXIST:
3276                 return ISCSI_RESPONSE_UNKNOWN_TASK;
3277
3278         case SCST_MGMT_STATUS_LUN_NOT_EXIST:
3279                 return ISCSI_RESPONSE_UNKNOWN_LUN;
3280
3281         case SCST_MGMT_STATUS_FN_NOT_SUPPORTED:
3282                 return ISCSI_RESPONSE_FUNCTION_UNSUPPORTED;
3283
3284         case SCST_MGMT_STATUS_REJECTED:
3285         case SCST_MGMT_STATUS_FAILED:
3286         default:
3287                 return ISCSI_RESPONSE_FUNCTION_REJECTED;
3288         }
3289 }
3290
3291 static void iscsi_task_mgmt_fn_done(struct scst_mgmt_cmd *scst_mcmd)
3292 {
3293         int fn = scst_mgmt_cmd_get_fn(scst_mcmd);
3294         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
3295                                 scst_mgmt_cmd_get_tgt_priv(scst_mcmd);
3296         int status =
3297                 iscsi_get_mgmt_response(scst_mgmt_cmd_get_status(scst_mcmd));
3298
3299         if ((status == ISCSI_RESPONSE_UNKNOWN_TASK) &&
3300             (fn == SCST_ABORT_TASK)) {
3301                 /* If we are here, we found the task, so must succeed */
3302                 status = ISCSI_RESPONSE_FUNCTION_COMPLETE;
3303         }
3304
3305         TRACE_MGMT_DBG("req %p, scst_mcmd %p, fn %d, scst status %d, status %d",
3306                 req, scst_mcmd, fn, scst_mgmt_cmd_get_status(scst_mcmd),
3307                 status);
3308
3309         switch (fn) {
3310         case SCST_NEXUS_LOSS_SESS:
3311         case SCST_ABORT_ALL_TASKS_SESS:
3312                 /* They are internal */
3313                 break;
3314         default:
3315                 iscsi_send_task_mgmt_resp(req, status);
3316                 scst_mgmt_cmd_set_tgt_priv(scst_mcmd, NULL);
3317                 break;
3318         }
3319         return;
3320 }
3321
3322 static int iscsi_scsi_aen(struct scst_aen *aen)
3323 {
3324         int res = SCST_AEN_RES_SUCCESS;
3325         uint64_t lun = scst_aen_get_lun(aen);
3326         const uint8_t *sense = scst_aen_get_sense(aen);
3327         int sense_len = scst_aen_get_sense_len(aen);
3328         struct iscsi_session *sess = scst_sess_get_tgt_priv(
3329                                         scst_aen_get_sess(aen));
3330         struct iscsi_conn *conn;
3331         bool found;
3332         struct iscsi_cmnd *fake_req, *rsp;
3333         struct iscsi_async_msg_hdr *rsp_hdr;
3334         struct scatterlist *sg;
3335
3336         TRACE_ENTRY();
3337
3338         TRACE_MGMT_DBG("SCSI AEN to sess %p (initiator %s)", sess,
3339                 sess->initiator_name);
3340
3341         mutex_lock(&sess->target->target_mutex);
3342
3343         found = false;
3344         list_for_each_entry_reverse(conn, &sess->conn_list, conn_list_entry) {
3345                 if (!test_bit(ISCSI_CONN_SHUTTINGDOWN, &conn->conn_aflags) &&
3346                     (conn->conn_reinst_successor == NULL)) {
3347                         found = true;
3348                         break;
3349                 }
3350         }
3351         if (!found) {
3352                 TRACE_MGMT_DBG("Unable to find alive conn for sess %p", sess);
3353                 goto out_err;
3354         }
3355
3356         /* Create a fake request */
3357         fake_req = cmnd_alloc(conn, NULL);
3358         if (fake_req == NULL) {
3359                 PRINT_ERROR("%s", "Unable to alloc fake AEN request");
3360                 goto out_err;
3361         }
3362
3363         mutex_unlock(&sess->target->target_mutex);
3364
3365         rsp = iscsi_alloc_main_rsp(fake_req);
3366         if (rsp == NULL) {
3367                 PRINT_ERROR("%s", "Unable to alloc AEN rsp");
3368                 goto out_err_free_req;
3369         }
3370
3371         fake_req->scst_state = ISCSI_CMD_STATE_AEN;
3372         fake_req->scst_aen = aen;
3373
3374         rsp_hdr = (struct iscsi_async_msg_hdr *)&rsp->pdu.bhs;
3375
3376         rsp_hdr->opcode = ISCSI_OP_ASYNC_MSG;
3377         rsp_hdr->flags = ISCSI_FLG_FINAL;
3378         rsp_hdr->lun = lun; /* it's already in SCSI form */
3379         rsp_hdr->ffffffff = 0xffffffff;
3380         rsp_hdr->async_event = ISCSI_ASYNC_SCSI;
3381
3382         sg = rsp->sg = rsp->rsp_sg;
3383         rsp->sg_cnt = 2;
3384         rsp->own_sg = 1;
3385
3386         sg_init_table(sg, 2);
3387         sg_set_buf(&sg[0], &rsp->sense_hdr, sizeof(rsp->sense_hdr));
3388         sg_set_buf(&sg[1], sense, sense_len);
3389
3390         rsp->sense_hdr.length = cpu_to_be16(sense_len);
3391         rsp->pdu.datasize = sizeof(rsp->sense_hdr) + sense_len;
3392         rsp->bufflen = rsp->pdu.datasize;
3393
3394         req_cmnd_release(fake_req);
3395
3396 out:
3397         TRACE_EXIT_RES(res);
3398         return res;
3399
3400 out_err_free_req:
3401         req_cmnd_release(fake_req);
3402
3403 out_err:
3404         mutex_unlock(&sess->target->target_mutex);
3405         res = SCST_AEN_RES_FAILED;
3406         goto out;
3407 }
3408
3409 static int iscsi_report_aen(struct scst_aen *aen)
3410 {
3411         int res;
3412         int event_fn = scst_aen_get_event_fn(aen);
3413
3414         TRACE_ENTRY();
3415
3416         switch (event_fn) {
3417         case SCST_AEN_SCSI:
3418                 res = iscsi_scsi_aen(aen);
3419                 break;
3420         default:
3421                 TRACE_MGMT_DBG("Unsupported AEN %d", event_fn);
3422                 res = SCST_AEN_RES_NOT_SUPPORTED;
3423                 break;
3424         }
3425
3426         TRACE_EXIT_RES(res);
3427         return res;
3428 }
3429
3430 static int iscsi_target_detect(struct scst_tgt_template *templ)
3431 {
3432         /* Nothing to do */
3433         return 0;
3434 }
3435
3436 static int iscsi_target_release(struct scst_tgt *scst_tgt)
3437 {
3438         /* Nothing to do */
3439         return 0;
3440 }
3441
3442 #ifndef CONFIG_SCST_PROC
3443 static struct scst_trace_log iscsi_local_trace_tbl[] = {
3444     { TRACE_D_WRITE,            "d_write" },
3445     { TRACE_CONN_OC,            "conn" },
3446     { TRACE_CONN_OC_DBG,        "conn_dbg" },
3447     { TRACE_D_IOV,              "iov" },
3448     { TRACE_D_DUMP_PDU,         "pdu" },
3449     { TRACE_NET_PG,             "net_page" },
3450     { 0,                        NULL }
3451 };
3452
3453 #define ISCSI_TRACE_TLB_HELP    ", d_read, d_write, conn, conn_dbg, iov, pdu, net_page"
3454 #endif
3455
3456 struct scst_tgt_template iscsi_template = {
3457         .name = "iscsi",
3458         .sg_tablesize = 0xFFFF /* no limit */,
3459         .threads_num = 0,
3460         .no_clustering = 1,
3461         .xmit_response_atomic = 0,
3462 #ifndef CONFIG_SCST_PROC
3463         .tgtt_attrs = iscsi_attrs,
3464         .tgt_attrs = iscsi_tgt_attrs,
3465         .sess_attrs = iscsi_sess_attrs,
3466         .enable_tgt = iscsi_enable_target,
3467         .is_tgt_enabled = iscsi_is_target_enabled,
3468 #endif
3469 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
3470         .default_trace_flags = ISCSI_DEFAULT_LOG_FLAGS,
3471         .trace_flags = &trace_flag,
3472 #ifndef CONFIG_SCST_PROC
3473         .trace_tbl = iscsi_local_trace_tbl,
3474         .trace_tbl_help = ISCSI_TRACE_TLB_HELP,
3475 #endif
3476 #endif
3477         .detect = iscsi_target_detect,
3478         .release = iscsi_target_release,
3479         .xmit_response = iscsi_xmit_response,
3480 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3481         .alloc_data_buf = iscsi_alloc_data_buf,
3482 #endif
3483         .preprocessing_done = iscsi_preprocessing_done,
3484         .pre_exec = iscsi_pre_exec,
3485         .task_mgmt_affected_cmds_done = iscsi_task_mgmt_affected_cmds_done,
3486         .task_mgmt_fn_done = iscsi_task_mgmt_fn_done,
3487         .report_aen = iscsi_report_aen,
3488 };
3489
3490 static __init int iscsi_run_threads(int count, char *name, int (*fn)(void *))
3491 {
3492         int res = 0;
3493         int i;
3494         struct iscsi_thread_t *thr;
3495
3496         for (i = 0; i < count; i++) {
3497                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
3498                 if (!thr) {
3499                         res = -ENOMEM;
3500                         PRINT_ERROR("Failed to allocate thr %d", res);
3501                         goto out;
3502                 }
3503                 thr->thr = kthread_run(fn, NULL, "%s%d", name, i);
3504                 if (IS_ERR(thr->thr)) {
3505                         res = PTR_ERR(thr->thr);
3506                         PRINT_ERROR("kthread_create() failed: %d", res);
3507                         kfree(thr);
3508                         goto out;
3509                 }
3510                 list_add_tail(&thr->threads_list_entry, &iscsi_threads_list);
3511         }
3512
3513 out:
3514         return res;
3515 }
3516
3517 static void iscsi_stop_threads(void)
3518 {
3519         struct iscsi_thread_t *t, *tmp;
3520
3521         list_for_each_entry_safe(t, tmp, &iscsi_threads_list,
3522                                 threads_list_entry) {
3523                 int rc = kthread_stop(t->thr);
3524                 if (rc < 0)
3525                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
3526                 list_del(&t->threads_list_entry);
3527                 kfree(t);
3528         }
3529         return;
3530 }
3531
3532 static int __init iscsi_init(void)
3533 {
3534         int err = 0;
3535         int num;
3536
3537         PRINT_INFO("iSCSI SCST Target - version %s", ISCSI_VERSION_STRING);
3538
3539         dummy_page = alloc_pages(GFP_KERNEL, 0);
3540         if (dummy_page == NULL) {
3541                 PRINT_ERROR("%s", "Dummy page allocation failed");
3542                 goto out;
3543         }
3544
3545         sg_init_table(&dummy_sg, 1);
3546         sg_set_page(&dummy_sg, dummy_page, PAGE_SIZE, 0);
3547
3548 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3549         err = net_set_get_put_page_callbacks(iscsi_get_page_callback,
3550                         iscsi_put_page_callback);
3551         if (err != 0) {
3552                 PRINT_INFO("Unable to set page callbackes: %d", err);
3553                 goto out_free_dummy;
3554         }
3555 #else
3556 #ifndef GENERATING_UPSTREAM_PATCH
3557         PRINT_WARNING("%s",
3558                 "CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION "
3559                 "not enabled in your kernel. ISCSI-SCST will be working with "
3560                 "not the best performance. Refer README file for details.");
3561 #endif
3562 #endif
3563
3564         ctr_major = register_chrdev(0, ctr_name, &ctr_fops);
3565         if (ctr_major < 0) {
3566                 PRINT_ERROR("failed to register the control device %d",
3567                             ctr_major);
3568                 err = ctr_major;
3569                 goto out_callb;
3570         }
3571
3572         err = event_init();
3573         if (err < 0)
3574                 goto out_reg;
3575
3576         iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS);
3577         if (!iscsi_cmnd_cache) {
3578                 err = -ENOMEM;
3579                 goto out_event;
3580         }
3581
3582         err = scst_register_target_template(&iscsi_template);
3583         if (err < 0)
3584                 goto out_kmem;
3585
3586         iscsi_template_registered = 1;
3587
3588 #ifdef CONFIG_SCST_PROC
3589         err = iscsi_procfs_init();
3590         if (err < 0)
3591                 goto out_reg_tmpl;
3592 #endif
3593
3594         num = max((int)num_online_cpus(), 2);
3595
3596         err = iscsi_run_threads(num, "iscsird", istrd);
3597         if (err != 0)
3598                 goto out_thr;
3599
3600         err = iscsi_run_threads(num, "iscsiwr", istwr);
3601         if (err != 0)
3602                 goto out_thr;
3603
3604 out:
3605         return err;
3606
3607 out_thr:
3608 #ifdef CONFIG_SCST_PROC
3609         iscsi_procfs_exit();
3610 #endif
3611         iscsi_stop_threads();
3612
3613 #ifdef CONFIG_SCST_PROC
3614 out_reg_tmpl:
3615 #endif
3616         scst_unregister_target_template(&iscsi_template);
3617
3618 out_kmem:
3619         kmem_cache_destroy(iscsi_cmnd_cache);
3620
3621 out_event:
3622         event_exit();
3623
3624 out_reg:
3625         unregister_chrdev(ctr_major, ctr_name);
3626
3627 out_callb:
3628 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3629         net_set_get_put_page_callbacks(NULL, NULL);