826c0e311797f33f8908981b0d454023eac454ba
[mirror/scst/.git] / iscsi-scst / kernel / iscsi.c
1 /*
2  *  Copyright (C) 2002 - 2003 Ardis Technolgies <roman@ardistech.com>
3  *  Copyright (C) 2007 - 2009 Vladislav Bolkhovitin
4  *  Copyright (C) 2007 - 2009 ID7 Ltd.
5  *
6  *  This program is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU General Public License
8  *  as published by the Free Software Foundation, version 2
9  *  of the License.
10  *
11  *  This program is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  *  GNU General Public License for more details.
15  */
16
17 #include <linux/module.h>
18 #include <linux/hash.h>
19 #include <linux/kthread.h>
20 #include <linux/scatterlist.h>
21 #include <net/tcp.h>
22 #include <scsi/scsi.h>
23
24 #include "iscsi.h"
25 #include "digest.h"
26
27 #ifndef GENERATING_UPSTREAM_PATCH
28 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
29 #warning "Patch put_page_callback-<kernel-version>.patch not applied on your\
30  kernel or CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION\
31  config option not set. ISCSI-SCST will be working with not the best\
32  performance. Refer README file for details."
33 #endif
34 #endif
35
36 #define ISCSI_INIT_WRITE_WAKE           0x1
37
38 static int ctr_major;
39 static char ctr_name[] = "iscsi-scst-ctl";
40 static int iscsi_template_registered;
41
42 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
43 unsigned long iscsi_trace_flag = ISCSI_DEFAULT_LOG_FLAGS;
44 #endif
45
46 static struct kmem_cache *iscsi_cmnd_cache;
47
48 DEFINE_SPINLOCK(iscsi_rd_lock);
49 LIST_HEAD(iscsi_rd_list);
50 DECLARE_WAIT_QUEUE_HEAD(iscsi_rd_waitQ);
51
52 DEFINE_SPINLOCK(iscsi_wr_lock);
53 LIST_HEAD(iscsi_wr_list);
54 DECLARE_WAIT_QUEUE_HEAD(iscsi_wr_waitQ);
55
56 static struct page *dummy_page;
57 static struct scatterlist dummy_sg;
58
59 struct iscsi_thread_t {
60         struct task_struct *thr;
61         struct list_head threads_list_entry;
62 };
63
64 static LIST_HEAD(iscsi_threads_list);
65
66 static void cmnd_remove_data_wait_hash(struct iscsi_cmnd *cmnd);
67 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status);
68 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess);
69 static void req_cmnd_release(struct iscsi_cmnd *req);
70 static int iscsi_preliminary_complete(struct iscsi_cmnd *req,
71         struct iscsi_cmnd *orig_req, bool get_data);
72 static int cmnd_insert_data_wait_hash(struct iscsi_cmnd *cmnd);
73 static void __cmnd_abort(struct iscsi_cmnd *cmnd);
74 static void iscsi_set_resid(struct iscsi_cmnd *rsp, bool bufflen_set);
75 static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags);
76
77 static void req_del_from_write_timeout_list(struct iscsi_cmnd *req)
78 {
79         struct iscsi_conn *conn;
80
81         TRACE_ENTRY();
82
83         if (!req->on_write_timeout_list)
84                 goto out;
85
86         conn = req->conn;
87
88         TRACE_DBG("Deleting cmd %p from conn %p write_timeout_list",
89                 req, conn);
90
91         spin_lock_bh(&conn->write_list_lock);
92         list_del(&req->write_timeout_list_entry);
93         req->on_write_timeout_list = 0;
94         spin_unlock_bh(&conn->write_list_lock);
95
96 out:
97         TRACE_EXIT();
98         return;
99 }
100
101 static inline u32 cmnd_write_size(struct iscsi_cmnd *cmnd)
102 {
103         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
104
105         if (hdr->flags & ISCSI_CMD_WRITE)
106                 return be32_to_cpu(hdr->data_length);
107         return 0;
108 }
109
110 static inline int cmnd_read_size(struct iscsi_cmnd *cmnd)
111 {
112         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
113
114         if (hdr->flags & ISCSI_CMD_READ) {
115                 struct iscsi_ahs_hdr *ahdr;
116
117                 if (!(hdr->flags & ISCSI_CMD_WRITE))
118                         return be32_to_cpu(hdr->data_length);
119
120                 ahdr = (struct iscsi_ahs_hdr *)cmnd->pdu.ahs;
121                 if (ahdr != NULL) {
122                         uint8_t *p = (uint8_t *)ahdr;
123                         unsigned int size = 0;
124                         do {
125                                 int s;
126
127                                 ahdr = (struct iscsi_ahs_hdr *)p;
128
129                                 if (ahdr->ahstype == ISCSI_AHSTYPE_RLENGTH) {
130                                         struct iscsi_rlength_ahdr *rh =
131                                               (struct iscsi_rlength_ahdr *)ahdr;
132                                         return be32_to_cpu(rh->read_length);
133                                 }
134
135                                 s = 3 + be16_to_cpu(ahdr->ahslength);
136                                 s = (s + 3) & -4;
137                                 size += s;
138                                 p += s;
139                         } while (size < cmnd->pdu.ahssize);
140                 }
141                 return -1;
142         }
143         return 0;
144 }
145
146 void iscsi_restart_cmnd(struct iscsi_cmnd *cmnd)
147 {
148         int status;
149
150         TRACE_ENTRY();
151
152         EXTRACHECKS_BUG_ON(cmnd->r2t_len_to_receive != 0);
153         EXTRACHECKS_BUG_ON(cmnd->r2t_len_to_send != 0);
154
155         iscsi_extracheck_is_rd_thread(cmnd->conn);
156
157         req_del_from_write_timeout_list(cmnd);
158
159         /*
160          * Let's remove cmnd from the hash earlier to keep it smaller.
161          * See also corresponding comment in req_cmnd_release().
162          */
163         if (cmnd->hashed)
164                 cmnd_remove_data_wait_hash(cmnd);
165
166         if (unlikely(test_bit(ISCSI_CONN_REINSTATING,
167                         &cmnd->conn->conn_aflags))) {
168                 struct iscsi_target *target = cmnd->conn->session->target;
169                 bool get_out;
170
171                 mutex_lock(&target->target_mutex);
172
173                 get_out = test_bit(ISCSI_CONN_REINSTATING,
174                                 &cmnd->conn->conn_aflags);
175                 /* Let's don't look dead */
176                 if (scst_cmd_get_cdb(cmnd->scst_cmd)[0] == TEST_UNIT_READY)
177                         get_out = false;
178
179                 if (!get_out)
180                         goto unlock_cont;
181
182                 TRACE_MGMT_DBG("Pending cmnd %p, because conn %p is "
183                         "reinstated", cmnd, cmnd->conn);
184
185                 cmnd->scst_state = ISCSI_CMD_STATE_REINST_PENDING;
186                 list_add_tail(&cmnd->reinst_pending_cmd_list_entry,
187                         &cmnd->conn->reinst_pending_cmd_list);
188
189 unlock_cont:
190                 mutex_unlock(&target->target_mutex);
191
192                 if (get_out)
193                         goto out;
194         }
195
196         if (unlikely(cmnd->prelim_compl_flags != 0)) {
197                 if (test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags)) {
198                         TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted", cmnd,
199                                 cmnd->scst_cmd);
200                         req_cmnd_release_force(cmnd);
201                         goto out;
202                 }
203
204                 if (cmnd->scst_cmd == NULL) {
205                         TRACE_MGMT_DBG("Finishing preliminary completed cmd %p "
206                                 "with NULL scst_cmd", cmnd);
207                         req_cmnd_release(cmnd);
208                         goto out;
209                 }
210
211                 status = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
212         } else
213                 status = SCST_PREPROCESS_STATUS_SUCCESS;
214
215         cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
216
217         scst_restart_cmd(cmnd->scst_cmd, status, SCST_CONTEXT_THREAD);
218
219 out:
220         TRACE_EXIT();
221         return;
222 }
223
224 void iscsi_fail_data_waiting_cmnd(struct iscsi_cmnd *cmnd)
225 {
226         TRACE_ENTRY();
227
228         TRACE_MGMT_DBG("Failing data waiting cmnd %p", cmnd);
229
230         /*
231          * There is no race with conn_abort(), since all functions
232          * called from single read thread
233          */
234         iscsi_extracheck_is_rd_thread(cmnd->conn);
235         cmnd->r2t_len_to_receive = 0;
236         cmnd->r2t_len_to_send = 0;
237
238         req_cmnd_release_force(cmnd);
239
240         TRACE_EXIT();
241         return;
242 }
243
244 struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn,
245                               struct iscsi_cmnd *parent)
246 {
247         struct iscsi_cmnd *cmnd;
248
249         /* ToDo: __GFP_NOFAIL?? */
250 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
251         cmnd = kmem_cache_alloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
252         memset(cmnd, 0, sizeof(*cmnd));
253 #else
254         cmnd = kmem_cache_zalloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
255 #endif
256
257         atomic_set(&cmnd->ref_cnt, 1);
258         cmnd->scst_state = ISCSI_CMD_STATE_NEW;
259         cmnd->conn = conn;
260         cmnd->parent_req = parent;
261
262         if (parent == NULL) {
263                 conn_get(conn);
264
265 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
266                 atomic_set(&cmnd->net_ref_cnt, 0);
267 #endif
268                 INIT_LIST_HEAD(&cmnd->rsp_cmd_list);
269                 INIT_LIST_HEAD(&cmnd->rx_ddigest_cmd_list);
270                 cmnd->target_task_tag = cpu_to_be32(ISCSI_RESERVED_TAG);
271
272                 spin_lock_bh(&conn->cmd_list_lock);
273                 list_add_tail(&cmnd->cmd_list_entry, &conn->cmd_list);
274                 spin_unlock_bh(&conn->cmd_list_lock);
275         }
276
277         TRACE_DBG("conn %p, parent %p, cmnd %p", conn, parent, cmnd);
278         return cmnd;
279 }
280
281 /* Frees a command. Also frees the additional header. */
282 static void cmnd_free(struct iscsi_cmnd *cmnd)
283 {
284         TRACE_ENTRY();
285
286         TRACE_DBG("cmnd %p", cmnd);
287
288         if (unlikely(test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags))) {
289                 TRACE_MGMT_DBG("Free aborted cmd %p (scst cmd %p, state %d, "
290                         "parent_req %p)", cmnd, cmnd->scst_cmd,
291                         cmnd->scst_state, cmnd->parent_req);
292         }
293
294         /* Catch users from cmd_list or rsp_cmd_list */
295         EXTRACHECKS_BUG_ON(atomic_read(&cmnd->ref_cnt) != 0);
296
297         kfree(cmnd->pdu.ahs);
298
299 #ifdef CONFIG_SCST_EXTRACHECKS
300         if (unlikely(cmnd->on_write_list || cmnd->on_write_timeout_list)) {
301                 struct iscsi_scsi_cmd_hdr *req = cmnd_hdr(cmnd);
302
303                 PRINT_CRIT_ERROR("cmnd %p still on some list?, %x, %x, %x, "
304                         "%x, %x, %x, %x", cmnd, req->opcode, req->scb[0],
305                         req->flags, req->itt, be32_to_cpu(req->data_length),
306                         req->cmd_sn, be32_to_cpu(cmnd->pdu.datasize));
307
308                 if (unlikely(cmnd->parent_req)) {
309                         struct iscsi_scsi_cmd_hdr *preq =
310                                         cmnd_hdr(cmnd->parent_req);
311                         PRINT_CRIT_ERROR("%p %x %u", preq, preq->opcode,
312                                 preq->scb[0]);
313                 }
314                 sBUG();
315         }
316 #endif
317
318         kmem_cache_free(iscsi_cmnd_cache, cmnd);
319
320         TRACE_EXIT();
321         return;
322 }
323
324 /* Might be called under some lock and on SIRQ */
325 void cmnd_done(struct iscsi_cmnd *cmnd)
326 {
327         TRACE_ENTRY();
328
329         TRACE_DBG("cmnd %p", cmnd);
330
331         if (unlikely(test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags))) {
332                 TRACE_MGMT_DBG("Done aborted cmd %p (scst cmd %p, state %d, "
333                         "parent_req %p)", cmnd, cmnd->scst_cmd,
334                         cmnd->scst_state, cmnd->parent_req);
335         }
336
337         EXTRACHECKS_BUG_ON(cmnd->on_rx_digest_list);
338         EXTRACHECKS_BUG_ON(cmnd->hashed);
339
340         req_del_from_write_timeout_list(cmnd);
341
342         if (cmnd->parent_req == NULL) {
343                 struct iscsi_conn *conn = cmnd->conn;
344                 struct iscsi_cmnd *rsp, *t;
345
346                 TRACE_DBG("Deleting req %p from conn %p", cmnd, conn);
347
348                 spin_lock_bh(&conn->cmd_list_lock);
349                 list_del(&cmnd->cmd_list_entry);
350                 spin_unlock_bh(&conn->cmd_list_lock);
351
352                 conn_put(conn);
353
354                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rx_ddigest_cmd_list));
355
356                 /* Order between above and below code is important! */
357
358                 if ((cmnd->scst_cmd != NULL) || (cmnd->scst_aen != NULL)) {
359                         switch (cmnd->scst_state) {
360                         case ISCSI_CMD_STATE_PROCESSED:
361                                 TRACE_DBG("cmd %p PROCESSED", cmnd);
362                                 scst_tgt_cmd_done(cmnd->scst_cmd,
363                                         SCST_CONTEXT_DIRECT_ATOMIC);
364                                 break;
365
366                         case ISCSI_CMD_STATE_AFTER_PREPROC:
367                         {
368                                 /* It can be for some aborted commands */
369                                 struct scst_cmd *scst_cmd = cmnd->scst_cmd;
370                                 TRACE_DBG("cmd %p AFTER_PREPROC", cmnd);
371                                 cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
372                                 cmnd->scst_cmd = NULL;
373                                 scst_restart_cmd(scst_cmd,
374                                         SCST_PREPROCESS_STATUS_ERROR_FATAL,
375                                         SCST_CONTEXT_THREAD);
376                                 break;
377                         }
378
379                         case ISCSI_CMD_STATE_AEN:
380                                 TRACE_DBG("cmd %p AEN PROCESSED", cmnd);
381                                 scst_aen_done(cmnd->scst_aen);
382                                 break;
383
384                         case ISCSI_CMD_STATE_OUT_OF_SCST_PRELIM_COMPL:
385                                 break;
386
387                         default:
388                                 PRINT_CRIT_ERROR("Unexpected cmnd scst state "
389                                         "%d", cmnd->scst_state);
390                                 sBUG();
391                                 break;
392                         }
393                 }
394
395                 if (cmnd->own_sg) {
396                         TRACE_DBG("own_sg for req %p", cmnd);
397                         if (cmnd->sg != &dummy_sg)
398                                 scst_free(cmnd->sg, cmnd->sg_cnt);
399 #ifdef CONFIG_SCST_DEBUG
400                         cmnd->own_sg = 0;
401                         cmnd->sg = NULL;
402                         cmnd->sg_cnt = -1;
403 #endif
404                 }
405
406                 if (cmnd->dec_active_cmnds) {
407                         struct iscsi_session *sess = cmnd->conn->session;
408                         TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
409                                 "new value %d)", cmnd, sess,
410                                 atomic_read(&sess->active_cmds)-1);
411                         atomic_dec(&sess->active_cmds);
412 #ifdef CONFIG_SCST_EXTRACHECKS
413                         if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
414                                 PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
415                                         atomic_read(&sess->active_cmds));
416                                 sBUG();
417                         }
418 #endif
419                 }
420
421                 list_for_each_entry_safe(rsp, t, &cmnd->rsp_cmd_list,
422                                         rsp_cmd_list_entry) {
423                         cmnd_free(rsp);
424                 }
425
426                 cmnd_free(cmnd);
427         } else {
428                 if (cmnd->own_sg) {
429                         TRACE_DBG("own_sg for rsp %p", cmnd);
430                         if ((cmnd->sg != &dummy_sg) && (cmnd->sg != cmnd->rsp_sg))
431                                 scst_free(cmnd->sg, cmnd->sg_cnt);
432 #ifdef CONFIG_SCST_DEBUG
433                         cmnd->own_sg = 0;
434                         cmnd->sg = NULL;
435                         cmnd->sg_cnt = -1;
436 #endif
437                 }
438
439                 EXTRACHECKS_BUG_ON(cmnd->dec_active_cmnds);
440
441                 if (cmnd == cmnd->parent_req->main_rsp) {
442                         TRACE_DBG("Finishing main rsp %p (req %p)", cmnd,
443                                 cmnd->parent_req);
444                         cmnd->parent_req->main_rsp = NULL;
445                 }
446
447                 cmnd_put(cmnd->parent_req);
448                 /*
449                  * rsp will be freed on the last parent's put and can already
450                  * be freed!!
451                  */
452         }
453
454         TRACE_EXIT();
455         return;
456 }
457
458 /*
459  * Corresponding conn may also get destroyed after this function, except only
460  * if it's called from the read thread!
461  *
462  * It can't be called in parallel with iscsi_cmnds_init_write()!
463  */
464 void req_cmnd_release_force(struct iscsi_cmnd *req)
465 {
466         struct iscsi_cmnd *rsp, *t;
467         struct iscsi_conn *conn = req->conn;
468         LIST_HEAD(cmds_list);
469
470         TRACE_ENTRY();
471
472         TRACE_MGMT_DBG("req %p", req);
473
474         sBUG_ON(req == conn->read_cmnd);
475
476         spin_lock_bh(&conn->write_list_lock);
477         list_for_each_entry_safe(rsp, t, &conn->write_list, write_list_entry) {
478                 if (rsp->parent_req != req)
479                         continue;
480
481                 cmd_del_from_write_list(rsp);
482
483                 list_add_tail(&rsp->write_list_entry, &cmds_list);
484         }
485         spin_unlock_bh(&conn->write_list_lock);
486
487         list_for_each_entry_safe(rsp, t, &cmds_list, write_list_entry) {
488                 TRACE_MGMT_DBG("Putting write rsp %p", rsp);
489                 list_del(&rsp->write_list_entry);
490                 cmnd_put(rsp);
491         }
492
493         /* Supposed nobody can add responses in the list anymore */
494         list_for_each_entry_reverse(rsp, &req->rsp_cmd_list,
495                         rsp_cmd_list_entry) {
496                 bool r;
497
498                 if (rsp->force_cleanup_done)
499                         continue;
500
501                 rsp->force_cleanup_done = 1;
502
503                 if (cmnd_get_check(rsp))
504                         continue;
505
506                 spin_lock_bh(&conn->write_list_lock);
507                 r = rsp->on_write_list || rsp->write_processing_started;
508                 spin_unlock_bh(&conn->write_list_lock);
509
510                 cmnd_put(rsp);
511
512                 if (r)
513                         continue;
514
515                 /*
516                  * If both on_write_list and write_processing_started not set,
517                  * we can safely put() rsp.
518                  */
519                 TRACE_MGMT_DBG("Putting rsp %p", rsp);
520                 cmnd_put(rsp);
521         }
522
523         if (req->main_rsp != NULL) {
524                 TRACE_MGMT_DBG("Putting main rsp %p", req->main_rsp);
525                 cmnd_put(req->main_rsp);
526                 req->main_rsp = NULL;
527         }
528
529         req_cmnd_release(req);
530
531         TRACE_EXIT();
532         return;
533 }
534
535 /*
536  * Corresponding conn may also get destroyed after this function, except only
537  * if it's called from the read thread!
538  */
539 static void req_cmnd_release(struct iscsi_cmnd *req)
540 {
541         struct iscsi_cmnd *c, *t;
542
543         TRACE_ENTRY();
544
545         TRACE_DBG("req %p", req);
546
547 #ifdef CONFIG_SCST_EXTRACHECKS
548         sBUG_ON(req->release_called);
549         req->release_called = 1;
550 #endif
551
552         if (unlikely(test_bit(ISCSI_CMD_ABORTED, &req->prelim_compl_flags))) {
553                 TRACE_MGMT_DBG("Release aborted req cmd %p (scst cmd %p, "
554                         "state %d)", req, req->scst_cmd, req->scst_state);
555         }
556
557         sBUG_ON(req->parent_req != NULL);
558
559         /*
560          * We have to remove hashed req from the hash list before sending
561          * response. Otherwise we can have a race, when for some reason cmd's
562          * release (and, hence, removal from the hash) is delayed after the
563          * transmission and initiator sends cmd with the same ITT, hence
564          * the new command will be erroneously rejected as a duplicate.
565          */
566         if (unlikely(req->hashed)) {
567                 /* It sometimes can happen during errors recovery */
568                 cmnd_remove_data_wait_hash(req);
569         }
570
571         if (unlikely(req->main_rsp != NULL)) {
572                 TRACE_DBG("Sending main rsp %p", req->main_rsp);
573                 iscsi_cmnd_init_write(req->main_rsp, ISCSI_INIT_WRITE_WAKE);
574                 req->main_rsp = NULL;
575         }
576
577         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
578                                 rx_ddigest_cmd_list_entry) {
579                 cmd_del_from_rx_ddigest_list(c);
580                 cmnd_put(c);
581         }
582
583         EXTRACHECKS_BUG_ON(req->pending);
584
585         if (req->dec_active_cmnds) {
586                 struct iscsi_session *sess = req->conn->session;
587                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
588                         "new value %d)", req, sess,
589                         atomic_read(&sess->active_cmds)-1);
590                 atomic_dec(&sess->active_cmds);
591                 req->dec_active_cmnds = 0;
592 #ifdef CONFIG_SCST_EXTRACHECKS
593                 if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
594                         PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
595                                 atomic_read(&sess->active_cmds));
596                         sBUG();
597                 }
598 #endif
599         }
600
601         cmnd_put(req);
602
603         TRACE_EXIT();
604         return;
605 }
606
607 /*
608  * Corresponding conn may also get destroyed after this function, except only
609  * if it's called from the read thread!
610  */
611 void rsp_cmnd_release(struct iscsi_cmnd *cmnd)
612 {
613         TRACE_DBG("%p", cmnd);
614
615 #ifdef CONFIG_SCST_EXTRACHECKS
616         sBUG_ON(cmnd->release_called);
617         cmnd->release_called = 1;
618 #endif
619
620         EXTRACHECKS_BUG_ON(cmnd->parent_req == NULL);
621
622         cmnd_put(cmnd);
623         return;
624 }
625
626 static struct iscsi_cmnd *iscsi_alloc_rsp(struct iscsi_cmnd *parent)
627 {
628         struct iscsi_cmnd *rsp;
629
630         TRACE_ENTRY();
631
632         rsp = cmnd_alloc(parent->conn, parent);
633
634         TRACE_DBG("Adding rsp %p to parent %p", rsp, parent);
635         list_add_tail(&rsp->rsp_cmd_list_entry, &parent->rsp_cmd_list);
636
637         cmnd_get(parent);
638
639         TRACE_EXIT_HRES((unsigned long)rsp);
640         return rsp;
641 }
642
643 static inline struct iscsi_cmnd *iscsi_alloc_main_rsp(struct iscsi_cmnd *parent)
644 {
645         struct iscsi_cmnd *rsp;
646
647         TRACE_ENTRY();
648
649         EXTRACHECKS_BUG_ON(parent->main_rsp != NULL);
650
651         rsp = iscsi_alloc_rsp(parent);
652         parent->main_rsp = rsp;
653
654         TRACE_EXIT_HRES((unsigned long)rsp);
655         return rsp;
656 }
657
658 static void iscsi_cmnds_init_write(struct list_head *send, int flags)
659 {
660         struct iscsi_cmnd *rsp = list_entry(send->next, struct iscsi_cmnd,
661                                                 write_list_entry);
662         struct iscsi_conn *conn = rsp->conn;
663         struct list_head *pos, *next;
664
665         sBUG_ON(list_empty(send));
666
667         if (!(conn->ddigest_type & DIGEST_NONE)) {
668                 list_for_each(pos, send) {
669                         rsp = list_entry(pos, struct iscsi_cmnd,
670                                                 write_list_entry);
671
672                         if (rsp->pdu.datasize != 0) {
673                                 TRACE_DBG("Doing data digest (%p:%x)", rsp,
674                                         cmnd_opcode(rsp));
675                                 digest_tx_data(rsp);
676                         }
677                 }
678         }
679
680         spin_lock_bh(&conn->write_list_lock);
681         list_for_each_safe(pos, next, send) {
682                 rsp = list_entry(pos, struct iscsi_cmnd, write_list_entry);
683
684                 TRACE_DBG("%p:%x", rsp, cmnd_opcode(rsp));
685
686                 sBUG_ON(conn != rsp->conn);
687
688                 list_del(&rsp->write_list_entry);
689                 cmd_add_on_write_list(conn, rsp);
690         }
691         spin_unlock_bh(&conn->write_list_lock);
692
693         if (flags & ISCSI_INIT_WRITE_WAKE)
694                 iscsi_make_conn_wr_active(conn);
695
696         return;
697 }
698
699 static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags)
700 {
701         LIST_HEAD(head);
702
703 #ifdef CONFIG_SCST_EXTRACHECKS
704         if (unlikely(rsp->on_write_list)) {
705                 PRINT_CRIT_ERROR("cmd already on write list (%x %x %x "
706                         "%u %u %d %d", cmnd_itt(rsp),
707                         cmnd_opcode(rsp), cmnd_scsicode(rsp),
708                         rsp->hdigest, rsp->ddigest,
709                         list_empty(&rsp->rsp_cmd_list), rsp->hashed);
710                 sBUG();
711         }
712 #endif
713         list_add_tail(&rsp->write_list_entry, &head);
714         iscsi_cmnds_init_write(&head, flags);
715         return;
716 }
717
718 static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
719 {
720         struct iscsi_cmnd *rsp;
721         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
722         struct iscsi_data_in_hdr *rsp_hdr;
723         u32 pdusize, expsize, size, offset, sn;
724         LIST_HEAD(send);
725
726         TRACE_DBG("req %p", req);
727
728         pdusize = req->conn->session->sess_param.max_xmit_data_length;
729         expsize = req->read_size;
730         size = min(expsize, (u32)req->bufflen);
731         offset = 0;
732         sn = 0;
733
734         while (1) {
735                 rsp = iscsi_alloc_rsp(req);
736                 TRACE_DBG("rsp %p", rsp);
737                 rsp->sg = req->sg;
738                 rsp->sg_cnt = req->sg_cnt;
739                 rsp->bufflen = req->bufflen;
740                 rsp_hdr = (struct iscsi_data_in_hdr *)&rsp->pdu.bhs;
741
742                 rsp_hdr->opcode = ISCSI_OP_SCSI_DATA_IN;
743                 rsp_hdr->itt = req_hdr->itt;
744                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
745                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
746                 rsp_hdr->data_sn = cpu_to_be32(sn);
747
748                 if (size <= pdusize) {
749                         TRACE_DBG("offset %d, size %d", offset, size);
750                         rsp->pdu.datasize = size;
751                         if (send_status) {
752                                 unsigned int scsisize;
753
754                                 TRACE_DBG("status %x", status);
755
756                                 EXTRACHECKS_BUG_ON((cmnd_hdr(req)->flags & ISCSI_CMD_WRITE) != 0);
757
758                                 rsp_hdr->flags = ISCSI_FLG_FINAL | ISCSI_FLG_STATUS;
759                                 rsp_hdr->cmd_status = status;
760
761                                 scsisize = req->bufflen;
762                                 if (scsisize < expsize) {
763                                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
764                                         size = expsize - scsisize;
765                                 } else if (scsisize > expsize) {
766                                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
767                                         size = scsisize - expsize;
768                                 } else
769                                         size = 0;
770                                 rsp_hdr->residual_count = cpu_to_be32(size);
771                         }
772                         list_add_tail(&rsp->write_list_entry, &send);
773                         break;
774                 }
775
776                 TRACE_DBG("pdusize %d, offset %d, size %d", pdusize, offset,
777                         size);
778
779                 rsp->pdu.datasize = pdusize;
780
781                 size -= pdusize;
782                 offset += pdusize;
783                 sn++;
784
785                 list_add_tail(&rsp->write_list_entry, &send);
786         }
787         iscsi_cmnds_init_write(&send, 0);
788         return;
789 }
790
791 static void iscsi_init_status_rsp(struct iscsi_cmnd *rsp,
792         int status, const u8 *sense_buf, int sense_len, bool bufflen_set)
793 {
794         struct iscsi_cmnd *req = rsp->parent_req;
795         struct iscsi_scsi_rsp_hdr *rsp_hdr;
796         struct scatterlist *sg;
797
798         TRACE_ENTRY();
799
800         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
801         rsp_hdr->opcode = ISCSI_OP_SCSI_RSP;
802         rsp_hdr->flags = ISCSI_FLG_FINAL;
803         rsp_hdr->response = ISCSI_RESPONSE_COMMAND_COMPLETED;
804         rsp_hdr->cmd_status = status;
805         rsp_hdr->itt = cmnd_hdr(req)->itt;
806
807         if (SCST_SENSE_VALID(sense_buf)) {
808                 TRACE_DBG("%s", "SENSE VALID");
809
810                 sg = rsp->sg = rsp->rsp_sg;
811                 rsp->sg_cnt = 2;
812                 rsp->own_sg = 1;
813
814                 sg_init_table(sg, 2);
815                 sg_set_buf(&sg[0], &rsp->sense_hdr, sizeof(rsp->sense_hdr));
816                 sg_set_buf(&sg[1], sense_buf, sense_len);
817
818                 rsp->sense_hdr.length = cpu_to_be16(sense_len);
819
820                 rsp->pdu.datasize = sizeof(rsp->sense_hdr) + sense_len;
821                 rsp->bufflen = rsp->pdu.datasize;
822         } else {
823                 rsp->pdu.datasize = 0;
824                 rsp->bufflen = 0;
825         }
826
827         iscsi_set_resid(rsp, bufflen_set);
828
829         TRACE_EXIT();
830         return;
831 }
832
833 static inline struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req,
834         int status, const u8 *sense_buf, int sense_len, bool bufflen_set)
835 {
836         struct iscsi_cmnd *rsp;
837
838         TRACE_ENTRY();
839
840         rsp = iscsi_alloc_rsp(req);
841         TRACE_DBG("rsp %p", rsp);
842
843         iscsi_init_status_rsp(rsp, status, sense_buf, sense_len, bufflen_set);
844
845         TRACE_EXIT_HRES((unsigned long)rsp);
846         return rsp;
847 }
848
849 static struct iscsi_cmnd *create_prelim_status_rsp(struct iscsi_cmnd *req,
850         int status, const u8 *sense_buf, int sense_len)
851 {
852         struct iscsi_cmnd *rsp;
853
854         TRACE_ENTRY();
855
856         rsp = iscsi_alloc_main_rsp(req);
857         TRACE_DBG("main rsp %p", rsp);
858
859         iscsi_init_status_rsp(rsp, status, sense_buf, sense_len, false);
860
861         TRACE_EXIT_HRES((unsigned long)rsp);
862         return rsp;
863 }
864
865 static int iscsi_set_prelim_r2t_len_to_receive(struct iscsi_cmnd *req)
866 {
867         struct iscsi_hdr *req_hdr = &req->pdu.bhs;
868         int res = 0;
869
870         TRACE_ENTRY();
871
872         if (req_hdr->flags & ISCSI_CMD_FINAL)
873                 goto out;
874
875         res = cmnd_insert_data_wait_hash(req);
876         if (res != 0) {
877                 /*
878                  * We have to close connection, because otherwise a data
879                  * corruption is possible if we allow to receive data
880                  * for this request in another request with dublicated ITT.
881                  */
882                 mark_conn_closed(req->conn);
883                 goto out;
884         }
885
886         /*
887          * We need to wait for one or more PDUs. Let's simplify
888          * other code and pretend we need to receive 1 byte.
889          * In data_out_start() we will correct it.
890          */
891         if (req->outstanding_r2t == 0) {
892                 req->outstanding_r2t = 1;
893                 req_add_to_write_timeout_list(req);
894         }
895         req->r2t_len_to_receive = 1;
896         req->r2t_len_to_send = 0;
897
898         TRACE_DBG("req %p, op %x, outstanding_r2t %d, r2t_len_to_receive %d, "
899                 "r2t_len_to_send %d", req, cmnd_opcode(req),
900                 req->outstanding_r2t, req->r2t_len_to_receive,
901                 req->r2t_len_to_send);
902
903 out:
904         TRACE_EXIT_RES(res);
905         return res;
906 }
907
908 static int create_preliminary_status_rsp(struct iscsi_cmnd *req,
909         int status, const u8 *sense_buf, int sense_len)
910 {
911         int res = 0;
912         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
913
914         TRACE_ENTRY();
915
916         if (req->prelim_compl_flags != 0) {
917                 TRACE_MGMT_DBG("req %p already prelim completed", req);
918                 goto out;
919         }
920
921         req->scst_state = ISCSI_CMD_STATE_OUT_OF_SCST_PRELIM_COMPL;
922
923         if ((req_hdr->flags & ISCSI_CMD_READ) &&
924             (req_hdr->flags & ISCSI_CMD_WRITE)) {
925                 int sz = cmnd_read_size(req);
926                 if (sz > 0)
927                         req->read_size = sz;
928         } else if (req_hdr->flags & ISCSI_CMD_READ)
929                 req->read_size = be32_to_cpu(req_hdr->data_length);
930
931         create_prelim_status_rsp(req, status, sense_buf, sense_len);
932         res = iscsi_preliminary_complete(req, req, true);
933
934 out:
935         TRACE_EXIT_RES(res);
936         return res;
937 }
938
939 static int set_scst_preliminary_status_rsp(struct iscsi_cmnd *req,
940         bool get_data, int key, int asc, int ascq)
941 {
942         int res = 0;
943
944         TRACE_ENTRY();
945
946         if (req->scst_cmd == NULL) {
947                 /* There must be already error set */
948                 goto complete;
949         }
950
951         scst_set_cmd_error(req->scst_cmd, key, asc, ascq);
952
953 complete:
954         res = iscsi_preliminary_complete(req, req, get_data);
955
956         TRACE_EXIT_RES(res);
957         return res;
958 }
959
960 static int create_reject_rsp(struct iscsi_cmnd *req, int reason, bool get_data)
961 {
962         int res = 0;
963         struct iscsi_cmnd *rsp;
964         struct iscsi_reject_hdr *rsp_hdr;
965         struct scatterlist *sg;
966
967         TRACE_ENTRY();
968
969         TRACE_MGMT_DBG("Reject: req %p, reason %x", req, reason);
970
971         if (cmnd_opcode(req) == ISCSI_OP_SCSI_CMD) {
972                 if (req->scst_cmd == NULL) {
973                         /* BUSY status must be already set */
974                         struct iscsi_scsi_rsp_hdr *rsp_hdr;
975                         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&req->main_rsp->pdu.bhs;
976                         sBUG_ON(rsp_hdr->cmd_status == 0);
977                         /*
978                          * Let's not send REJECT here. The initiator will retry
979                          * and, hopefully, next time we will not fail allocating
980                          * scst_cmd, so we will then send the REJECT.
981                          */
982                         goto out;
983                 } else
984                         set_scst_preliminary_status_rsp(req, get_data,
985                                 SCST_LOAD_SENSE(scst_sense_invalid_message));
986         }
987
988         rsp = iscsi_alloc_main_rsp(req);
989         rsp_hdr = (struct iscsi_reject_hdr *)&rsp->pdu.bhs;
990
991         rsp_hdr->opcode = ISCSI_OP_REJECT;
992         rsp_hdr->ffffffff = ISCSI_RESERVED_TAG;
993         rsp_hdr->reason = reason;
994
995         sg = rsp->sg = rsp->rsp_sg;
996         rsp->sg_cnt = 1;
997         rsp->own_sg = 1;
998         sg_init_one(sg, &req->pdu.bhs, sizeof(struct iscsi_hdr));
999         rsp->bufflen = rsp->pdu.datasize = sizeof(struct iscsi_hdr);
1000
1001         res = iscsi_preliminary_complete(req, req, true);
1002
1003 out:
1004         TRACE_EXIT_RES(res);
1005         return res;
1006 }
1007
1008 static inline int iscsi_get_allowed_cmds(struct iscsi_session *sess)
1009 {
1010         int res = max(-1, (int)sess->max_queued_cmnds -
1011                                 atomic_read(&sess->active_cmds)-1);
1012         TRACE_DBG("allowed cmds %d (sess %p, active_cmds %d)", res,
1013                 sess, atomic_read(&sess->active_cmds));
1014         return res;
1015 }
1016
1017 static u32 cmnd_set_sn(struct iscsi_cmnd *cmnd, int set_stat_sn)
1018 {
1019         struct iscsi_conn *conn = cmnd->conn;
1020         struct iscsi_session *sess = conn->session;
1021         u32 res;
1022
1023         spin_lock(&sess->sn_lock);
1024
1025         if (set_stat_sn)
1026                 cmnd->pdu.bhs.sn = cpu_to_be32(conn->stat_sn++);
1027         cmnd->pdu.bhs.exp_sn = cpu_to_be32(sess->exp_cmd_sn);
1028         cmnd->pdu.bhs.max_sn = cpu_to_be32(sess->exp_cmd_sn +
1029                                  iscsi_get_allowed_cmds(sess));
1030
1031         res = cpu_to_be32(conn->stat_sn);
1032
1033         spin_unlock(&sess->sn_lock);
1034         return res;
1035 }
1036
1037 /* Called under sn_lock */
1038 static void __update_stat_sn(struct iscsi_cmnd *cmnd)
1039 {
1040         struct iscsi_conn *conn = cmnd->conn;
1041         u32 exp_stat_sn;
1042
1043         cmnd->pdu.bhs.exp_sn = exp_stat_sn = be32_to_cpu(cmnd->pdu.bhs.exp_sn);
1044         TRACE_DBG("%x,%x", cmnd_opcode(cmnd), exp_stat_sn);
1045         if ((int)(exp_stat_sn - conn->exp_stat_sn) > 0 &&
1046             (int)(exp_stat_sn - conn->stat_sn) <= 0) {
1047                 /* free pdu resources */
1048                 cmnd->conn->exp_stat_sn = exp_stat_sn;
1049         }
1050         return;
1051 }
1052
1053 static inline void update_stat_sn(struct iscsi_cmnd *cmnd)
1054 {
1055         spin_lock(&cmnd->conn->session->sn_lock);
1056         __update_stat_sn(cmnd);
1057         spin_unlock(&cmnd->conn->session->sn_lock);
1058         return;
1059 }
1060
1061 /* Called under sn_lock */
1062 static int check_cmd_sn(struct iscsi_cmnd *cmnd)
1063 {
1064         struct iscsi_session *session = cmnd->conn->session;
1065         u32 cmd_sn;
1066
1067         cmnd->pdu.bhs.sn = cmd_sn = be32_to_cpu(cmnd->pdu.bhs.sn);
1068         TRACE_DBG("%d(%d)", cmd_sn, session->exp_cmd_sn);
1069         if (likely((s32)(cmd_sn - session->exp_cmd_sn) >= 0))
1070                 return 0;
1071         PRINT_ERROR("sequence error (%x,%x)", cmd_sn, session->exp_cmd_sn);
1072         return -ISCSI_REASON_PROTOCOL_ERROR;
1073 }
1074
1075 static struct iscsi_cmnd *cmnd_find_itt_get(struct iscsi_conn *conn, u32 itt)
1076 {
1077         struct iscsi_cmnd *cmnd, *found_cmnd = NULL;
1078
1079         spin_lock_bh(&conn->cmd_list_lock);
1080         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1081                 if ((cmnd->pdu.bhs.itt == itt) && !cmnd_get_check(cmnd)) {
1082                         found_cmnd = cmnd;
1083                         break;
1084                 }
1085         }
1086         spin_unlock_bh(&conn->cmd_list_lock);
1087
1088         return found_cmnd;
1089 }
1090
1091 /**
1092  ** We use the ITT hash only to find original request PDU for subsequent
1093  ** Data-Out PDUs.
1094  **/
1095
1096 /* Must be called under cmnd_data_wait_hash_lock */
1097 static struct iscsi_cmnd *__cmnd_find_data_wait_hash(struct iscsi_conn *conn,
1098         u32 itt)
1099 {
1100         struct list_head *head;
1101         struct iscsi_cmnd *cmnd;
1102
1103         head = &conn->session->cmnd_data_wait_hash[cmnd_hashfn(itt)];
1104
1105         list_for_each_entry(cmnd, head, hash_list_entry) {
1106                 if (cmnd->pdu.bhs.itt == itt)
1107                         return cmnd;
1108         }
1109         return NULL;
1110 }
1111
1112 static struct iscsi_cmnd *cmnd_find_data_wait_hash(struct iscsi_conn *conn,
1113         u32 itt)
1114 {
1115         struct iscsi_cmnd *res;
1116         struct iscsi_session *session = conn->session;
1117
1118         spin_lock(&session->cmnd_data_wait_hash_lock);
1119         res = __cmnd_find_data_wait_hash(conn, itt);
1120         spin_unlock(&session->cmnd_data_wait_hash_lock);
1121
1122         return res;
1123 }
1124
1125 static inline u32 get_next_ttt(struct iscsi_conn *conn)
1126 {
1127         u32 ttt;
1128         struct iscsi_session *session = conn->session;
1129
1130         /* Not compatible with MC/S! */
1131
1132         iscsi_extracheck_is_rd_thread(conn);
1133
1134         if (unlikely(session->next_ttt == ISCSI_RESERVED_TAG))
1135                 session->next_ttt++;
1136         ttt = session->next_ttt++;
1137
1138         return ttt;
1139 }
1140
1141 static int cmnd_insert_data_wait_hash(struct iscsi_cmnd *cmnd)
1142 {
1143         struct iscsi_session *session = cmnd->conn->session;
1144         struct iscsi_cmnd *tmp;
1145         struct list_head *head;
1146         int err = 0;
1147         u32 itt = cmnd->pdu.bhs.itt;
1148
1149         if (unlikely(cmnd->hashed)) {
1150                 /* It can be for preliminary completed commands */
1151                 goto out;
1152         }
1153
1154         /*
1155          * We don't need TTT, because ITT/buffer_offset pair is sufficient
1156          * to find out the original request and buffer for Data-Out PDUs, but
1157          * crazy iSCSI spec requires us to send this superfluous field in
1158          * R2T PDUs and some initiators may rely on it.
1159          */
1160         cmnd->target_task_tag = get_next_ttt(cmnd->conn);
1161
1162         TRACE_DBG("%p:%x", cmnd, itt);
1163         if (unlikely(itt == ISCSI_RESERVED_TAG)) {
1164                 PRINT_ERROR("%s", "ITT is RESERVED_TAG");
1165                 PRINT_BUFFER("Incorrect BHS", &cmnd->pdu.bhs,
1166                         sizeof(cmnd->pdu.bhs));
1167                 err = -ISCSI_REASON_PROTOCOL_ERROR;
1168                 goto out;
1169         }
1170
1171         spin_lock(&session->cmnd_data_wait_hash_lock);
1172
1173         head = &session->cmnd_data_wait_hash[cmnd_hashfn(itt)];
1174
1175         tmp = __cmnd_find_data_wait_hash(cmnd->conn, itt);
1176         if (likely(!tmp)) {
1177                 TRACE_DBG("Adding cmnd %p to the hash (ITT %x)", cmnd,
1178                         cmnd_itt(cmnd));
1179                 list_add_tail(&cmnd->hash_list_entry, head);
1180                 cmnd->hashed = 1;
1181         } else {
1182                 PRINT_ERROR("Task %x in progress, cmnd %p", itt, cmnd);
1183                 err = -ISCSI_REASON_TASK_IN_PROGRESS;
1184         }
1185
1186         spin_unlock(&session->cmnd_data_wait_hash_lock);
1187
1188 out:
1189         return err;
1190 }
1191
1192 static void cmnd_remove_data_wait_hash(struct iscsi_cmnd *cmnd)
1193 {
1194         struct iscsi_session *session = cmnd->conn->session;
1195         struct iscsi_cmnd *tmp;
1196
1197         spin_lock(&session->cmnd_data_wait_hash_lock);
1198
1199         tmp = __cmnd_find_data_wait_hash(cmnd->conn, cmnd->pdu.bhs.itt);
1200
1201         if (likely(tmp && tmp == cmnd)) {
1202                 TRACE_DBG("Deleting cmnd %p from the hash (ITT %x)", cmnd,
1203                         cmnd_itt(cmnd));
1204                 list_del(&cmnd->hash_list_entry);
1205                 cmnd->hashed = 0;
1206         } else
1207                 PRINT_ERROR("%p:%x not found", cmnd, cmnd_itt(cmnd));
1208
1209         spin_unlock(&session->cmnd_data_wait_hash_lock);
1210
1211         return;
1212 }
1213
1214 static void cmnd_prepare_get_rejected_immed_data(struct iscsi_cmnd *cmnd)
1215 {
1216         struct iscsi_conn *conn = cmnd->conn;
1217         struct scatterlist *sg = cmnd->sg;
1218         char __user *addr;
1219         u32 size;
1220         unsigned int i;
1221
1222         TRACE_ENTRY();
1223
1224         TRACE_DBG_FLAG(iscsi_get_flow_ctrl_or_mgmt_dbg_log_flag(cmnd),
1225                 "Skipping (cmnd %p, ITT %x, op %x, cmd op %x, "
1226                 "datasize %u, scst_cmd %p, scst state %d)", cmnd,
1227                 cmnd_itt(cmnd), cmnd_opcode(cmnd), cmnd_hdr(cmnd)->scb[0],
1228                 cmnd->pdu.datasize, cmnd->scst_cmd, cmnd->scst_state);
1229
1230         iscsi_extracheck_is_rd_thread(conn);
1231
1232         size = cmnd->pdu.datasize;
1233         if (!size)
1234                 goto out;
1235
1236         /* We already checked pdu.datasize in check_segment_length() */
1237
1238         if (sg == NULL) {
1239                 /*
1240                  * There are no problems with the safety from concurrent
1241                  * accesses to dummy_page in dummy_sg, since data only
1242                  * will be read and then discarded.
1243                  */
1244                 sg = cmnd->sg = &dummy_sg;
1245                 cmnd->bufflen = PAGE_SIZE;
1246                 cmnd->own_sg = 1;
1247         }
1248
1249         addr = (char __force __user *)(page_address(sg_page(&sg[0])));
1250         sBUG_ON(addr == NULL);
1251         conn->read_size = size;
1252         for (i = 0; size > PAGE_SIZE; i++, size -= cmnd->bufflen) {
1253                 /* We already checked pdu.datasize in check_segment_length() */
1254                 sBUG_ON(i >= ISCSI_CONN_IOV_MAX);
1255                 conn->read_iov[i].iov_base = addr;
1256                 conn->read_iov[i].iov_len = cmnd->bufflen;
1257         }
1258         conn->read_iov[i].iov_base = addr;
1259         conn->read_iov[i].iov_len = size;
1260         conn->read_msg.msg_iov = conn->read_iov;
1261         conn->read_msg.msg_iovlen = ++i;
1262
1263 out:
1264         TRACE_EXIT();
1265         return;
1266 }
1267
1268 static void iscsi_set_resid(struct iscsi_cmnd *rsp, bool bufflen_set)
1269 {
1270         struct iscsi_cmnd *req = rsp->parent_req;
1271         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1272         struct iscsi_scsi_rsp_hdr *rsp_hdr;
1273         int resid, resp_len, in_resp_len;
1274
1275         if ((req_hdr->flags & ISCSI_CMD_READ) &&
1276             (req_hdr->flags & ISCSI_CMD_WRITE)) {
1277                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
1278
1279                 if (bufflen_set) {
1280                         resp_len = req->bufflen;
1281                         if (req->scst_cmd != NULL)
1282                                 in_resp_len = scst_cmd_get_in_bufflen(req->scst_cmd);
1283                         else
1284                                 in_resp_len = 0;
1285                 } else {
1286                         resp_len = 0;
1287                         in_resp_len = 0;
1288                 }
1289
1290                 resid = be32_to_cpu(req_hdr->data_length) - in_resp_len;
1291                 if (resid > 0) {
1292                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
1293                         rsp_hdr->residual_count = cpu_to_be32(resid);
1294                 } else if (resid < 0) {
1295                         resid = -resid;
1296                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
1297                         rsp_hdr->residual_count = cpu_to_be32(resid);
1298                 }
1299
1300                 resid = req->read_size - resp_len;
1301                 if (resid > 0) {
1302                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_UNDERFLOW;
1303                         rsp_hdr->bi_residual_count = cpu_to_be32(resid);
1304                 } else if (resid < 0) {
1305                         resid = -resid;
1306                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_OVERFLOW;
1307                         rsp_hdr->bi_residual_count = cpu_to_be32(resid);
1308                 }
1309         } else {
1310                 if (bufflen_set)
1311                         resp_len = req->bufflen;
1312                 else
1313                         resp_len = 0;
1314
1315                 resid = req->read_size - resp_len;
1316                 if (resid > 0) {
1317                         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
1318                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
1319                         rsp_hdr->residual_count = cpu_to_be32(resid);
1320                 } else if (resid < 0) {
1321                         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
1322                         resid = -resid;
1323                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
1324                         rsp_hdr->residual_count = cpu_to_be32(resid);
1325                 }
1326         }
1327         return;
1328 }
1329
1330 static int iscsi_preliminary_complete(struct iscsi_cmnd *req,
1331         struct iscsi_cmnd *orig_req, bool get_data)
1332 {
1333         int res = 0;
1334         bool set_r2t_len;
1335
1336         TRACE_ENTRY();
1337
1338 #ifdef CONFIG_SCST_DEBUG
1339         {
1340                 struct iscsi_hdr *req_hdr = &req->pdu.bhs;
1341                 TRACE_DBG_FLAG(iscsi_get_flow_ctrl_or_mgmt_dbg_log_flag(orig_req),
1342                         "Prelim completed req %p, orig_req %p (FINAL %x, "
1343                         "outstanding_r2t %d)", req, orig_req,
1344                         (req_hdr->flags & ISCSI_CMD_FINAL),
1345                         orig_req->outstanding_r2t);
1346         }
1347 #endif
1348
1349         iscsi_extracheck_is_rd_thread(req->conn);
1350         sBUG_ON(req->parent_req != NULL);
1351
1352         if (test_bit(ISCSI_CMD_PRELIM_COMPLETED, &req->prelim_compl_flags)) {
1353                 TRACE_MGMT_DBG("req %p already prelim completed", req);
1354                 /* To not try to get data twice */
1355                 get_data = false;
1356         }
1357
1358         set_r2t_len = !req->hashed &&
1359                       (cmnd_opcode(req) == ISCSI_OP_SCSI_CMD) &&
1360                       !test_bit(ISCSI_CMD_PRELIM_COMPLETED,
1361                                 &orig_req->prelim_compl_flags);
1362         set_bit(ISCSI_CMD_PRELIM_COMPLETED, &orig_req->prelim_compl_flags);
1363
1364         TRACE_DBG("get_data %d, set_r2t_len %d", get_data, set_r2t_len);
1365
1366         if (get_data)
1367                 cmnd_prepare_get_rejected_immed_data(req);
1368
1369         if (set_r2t_len)
1370                 res = iscsi_set_prelim_r2t_len_to_receive(orig_req);
1371
1372         TRACE_EXIT_RES(res);
1373         return res;
1374 }
1375
1376 static int cmnd_prepare_recv_pdu(struct iscsi_conn *conn,
1377         struct iscsi_cmnd *cmd, u32 offset, u32 size)
1378 {
1379         struct scatterlist *sg = cmd->sg;
1380         unsigned int bufflen = cmd->bufflen;
1381         unsigned int idx, i;
1382         char __user *addr;
1383         int res = 0;
1384
1385         TRACE_DBG("%p %u,%u", cmd->sg, offset, size);
1386
1387         iscsi_extracheck_is_rd_thread(conn);
1388
1389         if (unlikely((offset >= bufflen) ||
1390                      (offset + size > bufflen))) {
1391                 PRINT_ERROR("Wrong ltn (%u %u %u)", offset, size, bufflen);
1392                 mark_conn_closed(conn);
1393                 res = -EIO;
1394                 goto out;
1395         }
1396
1397         offset += sg[0].offset;
1398         idx = offset >> PAGE_SHIFT;
1399         offset &= ~PAGE_MASK;
1400
1401         conn->read_msg.msg_iov = conn->read_iov;
1402         conn->read_size = size;
1403
1404         i = 0;
1405         while (1) {
1406                 addr = (char __force __user *)(page_address(sg_page(&sg[idx])));
1407                 sBUG_ON(addr == NULL);
1408                 conn->read_iov[i].iov_base = addr + offset;
1409                 if (offset + size <= PAGE_SIZE) {
1410                         TRACE_DBG("idx=%d, offset=%u, size=%d, addr=%p",
1411                                 idx, offset, size, addr);
1412                         conn->read_iov[i].iov_len = size;
1413                         conn->read_msg.msg_iovlen = ++i;
1414                         break;
1415                 }
1416                 conn->read_iov[i].iov_len = PAGE_SIZE - offset;
1417                 TRACE_DBG("idx=%d, offset=%u, size=%d, iov_len=%zd, addr=%p",
1418                         idx, offset, size, conn->read_iov[i].iov_len, addr);
1419                 size -= conn->read_iov[i].iov_len;
1420                 if (unlikely(++i >= ISCSI_CONN_IOV_MAX)) {
1421                         PRINT_ERROR("Initiator %s violated negotiated "
1422                                 "parameters by sending too much data (size "
1423                                 "left %d)", conn->session->initiator_name,
1424                                 size);
1425                         mark_conn_closed(conn);
1426                         res = -EINVAL;
1427                         break;
1428                 }
1429                 idx++;
1430                 offset = sg[idx].offset;
1431         }
1432         TRACE_DBG("msg_iov=%p, msg_iovlen=%zd",
1433                 conn->read_msg.msg_iov, conn->read_msg.msg_iovlen);
1434
1435 out:
1436         return res;
1437 }
1438
1439 static void send_r2t(struct iscsi_cmnd *req)
1440 {
1441         struct iscsi_session *sess = req->conn->session;
1442         struct iscsi_cmnd *rsp;
1443         struct iscsi_r2t_hdr *rsp_hdr;
1444         u32 offset, burst;
1445         LIST_HEAD(send);
1446
1447         TRACE_ENTRY();
1448
1449         EXTRACHECKS_BUG_ON(req->r2t_len_to_send == 0);
1450
1451         /*
1452          * There is no race with data_out_start() and conn_abort(), since
1453          * all functions called from single read thread
1454          */
1455         iscsi_extracheck_is_rd_thread(req->conn);
1456
1457         /*
1458          * We don't need to check for PRELIM_COMPLETED here, because for such
1459          * commands we set r2t_len_to_send = 0, hence made sure we won't
1460          * called here.
1461          */
1462
1463         EXTRACHECKS_BUG_ON(req->outstanding_r2t >
1464                            sess->sess_param.max_outstanding_r2t);
1465
1466         if (req->outstanding_r2t == sess->sess_param.max_outstanding_r2t)
1467                 goto out;
1468
1469         burst = sess->sess_param.max_burst_length;
1470         offset = be32_to_cpu(cmnd_hdr(req)->data_length) -
1471                         req->r2t_len_to_send;
1472
1473         do {
1474                 rsp = iscsi_alloc_rsp(req);
1475                 rsp->pdu.bhs.ttt = req->target_task_tag;
1476                 rsp_hdr = (struct iscsi_r2t_hdr *)&rsp->pdu.bhs;
1477                 rsp_hdr->opcode = ISCSI_OP_R2T;
1478                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1479                 rsp_hdr->lun = cmnd_hdr(req)->lun;
1480                 rsp_hdr->itt = cmnd_hdr(req)->itt;
1481                 rsp_hdr->r2t_sn = cpu_to_be32(req->r2t_sn++);
1482                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
1483                 if (req->r2t_len_to_send > burst) {
1484                         rsp_hdr->data_length = cpu_to_be32(burst);
1485                         req->r2t_len_to_send -= burst;
1486                         offset += burst;
1487                 } else {
1488                         rsp_hdr->data_length = cpu_to_be32(req->r2t_len_to_send);
1489                         req->r2t_len_to_send = 0;
1490                 }
1491
1492                 TRACE_WRITE("req %p, data_length %u, buffer_offset %u, "
1493                         "r2t_sn %u, outstanding_r2t %u", req,
1494                         be32_to_cpu(rsp_hdr->data_length),
1495                         be32_to_cpu(rsp_hdr->buffer_offset),
1496                         be32_to_cpu(rsp_hdr->r2t_sn), req->outstanding_r2t);
1497
1498                 list_add_tail(&rsp->write_list_entry, &send);
1499                 req->outstanding_r2t++;
1500
1501         } while ((req->outstanding_r2t < sess->sess_param.max_outstanding_r2t) &&
1502                  (req->r2t_len_to_send != 0));
1503
1504         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_WAKE);
1505
1506 out:
1507         TRACE_EXIT();
1508         return;
1509 }
1510
1511 static int iscsi_pre_exec(struct scst_cmd *scst_cmd)
1512 {
1513         int res = SCST_PREPROCESS_STATUS_SUCCESS;
1514         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
1515                 scst_cmd_get_tgt_priv(scst_cmd);
1516         struct iscsi_cmnd *c, *t;
1517
1518         TRACE_ENTRY();
1519
1520         EXTRACHECKS_BUG_ON(scst_cmd_atomic(scst_cmd));
1521
1522         /* If data digest isn't used this list will be empty */
1523         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
1524                                 rx_ddigest_cmd_list_entry) {
1525                 TRACE_DBG("Checking digest of RX ddigest cmd %p", c);
1526                 if (digest_rx_data(c) != 0) {
1527                         scst_set_cmd_error(scst_cmd,
1528                                 SCST_LOAD_SENSE(iscsi_sense_crc_error));
1529                         res = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
1530                         /*
1531                          * The rest of rx_ddigest_cmd_list will be freed
1532                          * in req_cmnd_release()
1533                          */
1534                         goto out;
1535                 }
1536                 cmd_del_from_rx_ddigest_list(c);
1537                 cmnd_put(c);
1538         }
1539
1540 out:
1541         TRACE_EXIT_RES(res);
1542         return res;
1543 }
1544
1545 static int noop_out_start(struct iscsi_cmnd *cmnd)
1546 {
1547         struct iscsi_conn *conn = cmnd->conn;
1548         struct iscsi_hdr *req_hdr = &cmnd->pdu.bhs;
1549         u32 size, tmp;
1550         int i, err = 0;
1551
1552         TRACE_DBG("%p", cmnd);
1553
1554         iscsi_extracheck_is_rd_thread(conn);
1555
1556         if (!(req_hdr->flags & ISCSI_FLG_FINAL)) {
1557                 PRINT_ERROR("%s", "Initiator sent NOP-Out with not a single "
1558                         "PDU");
1559                 err = -ISCSI_REASON_PROTOCOL_ERROR;
1560                 goto out;
1561         }
1562
1563         if (cmnd_itt(cmnd) == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1564                 if (unlikely(!(cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE)))
1565                         PRINT_ERROR("%s", "Initiator sent RESERVED tag for "
1566                                 "non-immediate NOP-Out command");
1567         }
1568
1569         spin_lock(&conn->session->sn_lock);
1570         __update_stat_sn(cmnd);
1571         err = check_cmd_sn(cmnd);
1572         spin_unlock(&conn->session->sn_lock);
1573         if (unlikely(err))
1574                 goto out;
1575
1576         size = cmnd->pdu.datasize;
1577
1578         if (size) {
1579                 conn->read_msg.msg_iov = conn->read_iov;
1580                 if (cmnd->pdu.bhs.itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1581                         struct scatterlist *sg;
1582
1583                         cmnd->sg = sg = scst_alloc(size, GFP_KERNEL,
1584                                                 &cmnd->sg_cnt);
1585                         if (sg == NULL) {
1586                                 TRACE(TRACE_OUT_OF_MEM, "Allocating buffer for"
1587                                       " %d NOP-Out payload failed", size);
1588                                 err = -ISCSI_REASON_OUT_OF_RESOURCES;
1589                                 goto out;
1590                         }
1591
1592                         /* We already checked it in check_segment_length() */
1593                         sBUG_ON(cmnd->sg_cnt > (signed)ISCSI_CONN_IOV_MAX);
1594
1595                         cmnd->own_sg = 1;
1596                         cmnd->bufflen = size;
1597
1598                         for (i = 0; i < cmnd->sg_cnt; i++) {
1599                                 conn->read_iov[i].iov_base =
1600                                         (void __force __user *)(page_address(sg_page(&sg[i])));
1601                                 tmp = min_t(u32, size, PAGE_SIZE);
1602                                 conn->read_iov[i].iov_len = tmp;
1603                                 conn->read_size += tmp;
1604                                 size -= tmp;
1605                         }
1606                         sBUG_ON(size != 0);
1607                 } else {
1608                         /*
1609                          * There are no problems with the safety from concurrent
1610                          * accesses to dummy_page, since for ISCSI_RESERVED_TAG
1611                          * the data only read and then discarded.
1612                          */
1613                         for (i = 0; i < (signed)ISCSI_CONN_IOV_MAX; i++) {
1614                                 conn->read_iov[i].iov_base =
1615                                         (void __force __user *)(page_address(dummy_page));
1616                                 tmp = min_t(u32, size, PAGE_SIZE);
1617                                 conn->read_iov[i].iov_len = tmp;
1618                                 conn->read_size += tmp;
1619                                 size -= tmp;
1620                         }
1621
1622                         /* We already checked size in check_segment_length() */
1623                         sBUG_ON(size != 0);
1624                 }
1625
1626                 conn->read_msg.msg_iovlen = i;
1627                 TRACE_DBG("msg_iov=%p, msg_iovlen=%zd", conn->read_msg.msg_iov,
1628                         conn->read_msg.msg_iovlen);
1629         }
1630
1631 out:
1632         return err;
1633 }
1634
1635 int cmnd_rx_continue(struct iscsi_cmnd *req)
1636 {
1637         struct iscsi_conn *conn = req->conn;
1638         struct iscsi_session *session = conn->session;
1639         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1640         struct scst_cmd *scst_cmd = req->scst_cmd;
1641         scst_data_direction dir;
1642         bool unsolicited_data_expected = false;
1643         int res = 0;
1644
1645         TRACE_ENTRY();
1646
1647         TRACE_DBG("scsi command: %x", req_hdr->scb[0]);
1648
1649         EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_AFTER_PREPROC);
1650
1651         dir = scst_cmd_get_data_direction(scst_cmd);
1652
1653         /*
1654          * Check for preliminary completion here to save R2Ts. For TASK QUEUE
1655          * FULL statuses that might be a big performance win.
1656          */
1657         if (unlikely(scst_cmd_prelim_completed(scst_cmd) ||
1658             unlikely(req->prelim_compl_flags != 0))) {
1659                 /*
1660                  * If necessary, ISCSI_CMD_ABORTED will be set by
1661                  * iscsi_xmit_response().
1662                  */
1663                 res = iscsi_preliminary_complete(req, req, true);
1664                 goto trace;
1665         }
1666
1667         /* For prelim completed commands sg&K can be already set! */
1668
1669         if (dir != SCST_DATA_BIDI) {
1670                 req->sg = scst_cmd_get_sg(scst_cmd);
1671                 req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
1672                 req->bufflen = scst_cmd_get_bufflen(scst_cmd);
1673         } else {
1674                 req->sg = scst_cmd_get_in_sg(scst_cmd);
1675                 req->sg_cnt = scst_cmd_get_in_sg_cnt(scst_cmd);
1676                 req->bufflen = scst_cmd_get_in_bufflen(scst_cmd);
1677         }
1678
1679         if (dir & SCST_DATA_WRITE) {
1680                 unsolicited_data_expected = !(req_hdr->flags & ISCSI_CMD_FINAL);
1681
1682                 if (unlikely(session->sess_param.initial_r2t &&
1683                     unsolicited_data_expected)) {
1684                         PRINT_ERROR("Initiator %s violated negotiated "
1685                                 "parameters: initial R2T is required (ITT %x, "
1686                                 "op  %x)", session->initiator_name,
1687                                 cmnd_itt(req), req_hdr->scb[0]);
1688                         goto out_close;
1689                 }
1690
1691                 if (unlikely(!session->sess_param.immediate_data &&
1692                     req->pdu.datasize)) {
1693                         PRINT_ERROR("Initiator %s violated negotiated "
1694                                 "parameters: forbidden immediate data sent "
1695                                 "(ITT %x, op  %x)", session->initiator_name,
1696                                 cmnd_itt(req), req_hdr->scb[0]);
1697                         goto out_close;
1698                 }
1699
1700                 if (unlikely(session->sess_param.first_burst_length < req->pdu.datasize)) {
1701                         PRINT_ERROR("Initiator %s violated negotiated "
1702                                 "parameters: immediate data len (%d) > "
1703                                 "first_burst_length (%d) (ITT %x, op  %x)",
1704                                 session->initiator_name,
1705                                 req->pdu.datasize,
1706                                 session->sess_param.first_burst_length,
1707                                 cmnd_itt(req), req_hdr->scb[0]);
1708                         goto out_close;
1709                 }
1710
1711                 req->r2t_len_to_receive = be32_to_cpu(req_hdr->data_length) -
1712                                           req->pdu.datasize;
1713
1714                 if (unlikely(req->r2t_len_to_receive > req->bufflen)) {
1715                         PRINT_ERROR("req->r2t_len_to_receive %d > req->bufflen "
1716                                 "%d", req->r2t_len_to_receive, req->bufflen);
1717                         goto out_close;
1718                 }
1719
1720                 res = cmnd_insert_data_wait_hash(req);
1721                 if (unlikely(res != 0)) {
1722                         /*
1723                          * We have to close connection, because otherwise a data
1724                          * corruption is possible if we allow to receive data
1725                          * for this request in another request with dublicated
1726                          * ITT.
1727                          */
1728                         goto out_close;
1729                 }
1730
1731                 if (unsolicited_data_expected) {
1732                         req->outstanding_r2t = 1;
1733                         req->r2t_len_to_send = req->r2t_len_to_receive -
1734                                 min_t(unsigned int,
1735                                       session->sess_param.first_burst_length -
1736                                                 req->pdu.datasize,
1737                                       req->r2t_len_to_receive);
1738                 } else
1739                         req->r2t_len_to_send = req->r2t_len_to_receive;
1740
1741                 req_add_to_write_timeout_list(req);
1742
1743                 if (req->pdu.datasize) {
1744                         res = cmnd_prepare_recv_pdu(conn, req, 0, req->pdu.datasize);
1745                         /* For performance better to send R2Ts ASAP */
1746                         if (likely(res == 0) && (req->r2t_len_to_send != 0))
1747                                 send_r2t(req);
1748                 }
1749         } else {
1750                 if (unlikely(!(req_hdr->flags & ISCSI_CMD_FINAL) ||
1751                              req->pdu.datasize)) {
1752                         PRINT_ERROR("Unexpected unsolicited data (ITT %x "
1753                                 "CDB %x", cmnd_itt(req), req_hdr->scb[0]);
1754                         set_scst_preliminary_status_rsp(req, true,
1755                                 SCST_LOAD_SENSE(iscsi_sense_unexpected_unsolicited_data));
1756                 }
1757         }
1758
1759 trace:
1760         TRACE_DBG("req=%p, dir=%d, unsolicited_data_expected=%d, "
1761                 "r2t_len_to_receive=%d, r2t_len_to_send=%d, bufflen=%d, "
1762                 "own_sg %d", req, dir, unsolicited_data_expected,
1763                 req->r2t_len_to_receive, req->r2t_len_to_send, req->bufflen,
1764                 req->own_sg);
1765
1766 out:
1767         TRACE_EXIT_RES(res);
1768         return res;
1769
1770 out_close:
1771         mark_conn_closed(conn);
1772         res = -EINVAL;
1773         goto out;
1774 }
1775
1776 static int scsi_cmnd_start(struct iscsi_cmnd *req)
1777 {
1778         struct iscsi_conn *conn = req->conn;
1779         struct iscsi_session *session = conn->session;
1780         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1781         struct scst_cmd *scst_cmd;
1782         scst_data_direction dir;
1783         struct iscsi_ahs_hdr *ahdr;
1784         int res = 0;
1785
1786         TRACE_ENTRY();
1787
1788         TRACE_DBG("scsi command: %x", req_hdr->scb[0]);
1789
1790         TRACE_DBG("Incrementing active_cmds (cmd %p, sess %p, "
1791                 "new value %d)", req, session,
1792                 atomic_read(&session->active_cmds)+1);
1793         atomic_inc(&session->active_cmds);
1794         req->dec_active_cmnds = 1;
1795
1796         scst_cmd = scst_rx_cmd(session->scst_sess,
1797                 (uint8_t *)&req_hdr->lun, sizeof(req_hdr->lun),
1798                 req_hdr->scb, sizeof(req_hdr->scb), SCST_NON_ATOMIC);
1799         if (scst_cmd == NULL) {
1800                 res = create_preliminary_status_rsp(req, SAM_STAT_BUSY,
1801                         NULL, 0);
1802                 goto out;
1803         }
1804
1805         req->scst_cmd = scst_cmd;
1806         scst_cmd_set_tag(scst_cmd, req_hdr->itt);
1807         scst_cmd_set_tgt_priv(scst_cmd, req);
1808
1809         if ((req_hdr->flags & ISCSI_CMD_READ) &&
1810             (req_hdr->flags & ISCSI_CMD_WRITE)) {
1811                 int sz = cmnd_read_size(req);
1812                 if (unlikely(sz < 0)) {
1813                         PRINT_ERROR("%s", "BIDI data transfer, but initiator "
1814                                 "not supplied Bidirectional Read Expected Data "
1815                                 "Transfer Length AHS");
1816                         set_scst_preliminary_status_rsp(req, true,
1817                            SCST_LOAD_SENSE(scst_sense_parameter_value_invalid));
1818                 } else {
1819                         req->read_size = sz;
1820                         dir = SCST_DATA_BIDI;
1821                         scst_cmd_set_expected(scst_cmd, dir, sz);
1822                         scst_cmd_set_expected_in_transfer_len(scst_cmd,
1823                                 be32_to_cpu(req_hdr->data_length));
1824 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
1825                         scst_cmd_set_tgt_need_alloc_data_buf(scst_cmd);
1826 #endif
1827                 }
1828         } else if (req_hdr->flags & ISCSI_CMD_READ) {
1829                 req->read_size = be32_to_cpu(req_hdr->data_length);
1830                 dir = SCST_DATA_READ;
1831                 scst_cmd_set_expected(scst_cmd, dir, req->read_size);
1832 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
1833                 scst_cmd_set_tgt_need_alloc_data_buf(scst_cmd);
1834 #endif
1835         } else if (req_hdr->flags & ISCSI_CMD_WRITE) {
1836                 dir = SCST_DATA_WRITE;
1837                 scst_cmd_set_expected(scst_cmd, dir,
1838                         be32_to_cpu(req_hdr->data_length));
1839         } else {
1840                 dir = SCST_DATA_NONE;
1841                 scst_cmd_set_expected(scst_cmd, dir, 0);
1842         }
1843
1844         switch (req_hdr->flags & ISCSI_CMD_ATTR_MASK) {
1845         case ISCSI_CMD_SIMPLE:
1846                 scst_cmd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1847                 break;
1848         case ISCSI_CMD_HEAD_OF_QUEUE:
1849                 scst_cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1850                 break;
1851         case ISCSI_CMD_ORDERED:
1852                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1853                 break;
1854         case ISCSI_CMD_ACA:
1855                 scst_cmd->queue_type = SCST_CMD_QUEUE_ACA;
1856                 break;
1857         case ISCSI_CMD_UNTAGGED:
1858                 scst_cmd->queue_type = SCST_CMD_QUEUE_UNTAGGED;
1859                 break;
1860         default:
1861                 PRINT_ERROR("Unknown task code %x, use ORDERED instead",
1862                         req_hdr->flags & ISCSI_CMD_ATTR_MASK);
1863                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1864                 break;
1865         }
1866
1867         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1868         scst_cmd_set_tgt_sn(scst_cmd, req_hdr->cmd_sn);
1869
1870         ahdr = (struct iscsi_ahs_hdr *)req->pdu.ahs;
1871         if (ahdr != NULL) {
1872                 uint8_t *p = (uint8_t *)ahdr;
1873                 unsigned int size = 0;
1874                 do {
1875                         int s;
1876
1877                         ahdr = (struct iscsi_ahs_hdr *)p;
1878
1879                         if (ahdr->ahstype == ISCSI_AHSTYPE_CDB) {
1880                                 struct iscsi_cdb_ahdr *eca =
1881                                         (struct iscsi_cdb_ahdr *)ahdr;
1882                                 scst_cmd_set_ext_cdb(scst_cmd, eca->cdb,
1883                                         be16_to_cpu(ahdr->ahslength) - 1);
1884                                 break;
1885                         }
1886                         s = 3 + be16_to_cpu(ahdr->ahslength);
1887                         s = (s + 3) & -4;
1888                         size += s;
1889                         p += s;
1890                 } while (size < req->pdu.ahssize);
1891         }
1892
1893         TRACE_DBG("START Command (itt %x, queue_type %d)",
1894                 req_hdr->itt, scst_cmd->queue_type);
1895         req->scst_state = ISCSI_CMD_STATE_RX_CMD;
1896         conn->rx_task = current;
1897         scst_cmd_init_stage1_done(scst_cmd, SCST_CONTEXT_DIRECT, 0);
1898
1899         if (req->scst_state != ISCSI_CMD_STATE_RX_CMD)
1900                 res = cmnd_rx_continue(req);
1901         else {
1902                 TRACE_DBG("Delaying req %p post processing (scst_state %d)",
1903                         req, req->scst_state);
1904                 res = 1;
1905         }
1906
1907 out:
1908         TRACE_EXIT_RES(res);
1909         return res;
1910 }
1911
1912 static int data_out_start(struct iscsi_cmnd *cmnd)
1913 {
1914         struct iscsi_conn *conn = cmnd->conn;
1915         struct iscsi_data_out_hdr *req_hdr =
1916                 (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1917         struct iscsi_cmnd *orig_req;
1918 #if 0
1919         struct iscsi_hdr *orig_req_hdr;
1920 #endif
1921         u32 offset = be32_to_cpu(req_hdr->buffer_offset);
1922         int res = 0;
1923
1924         TRACE_ENTRY();
1925
1926         /*
1927          * There is no race with send_r2t() and conn_abort(), since
1928          * all functions called from single read thread
1929          */
1930         iscsi_extracheck_is_rd_thread(cmnd->conn);
1931
1932         update_stat_sn(cmnd);
1933
1934         orig_req = cmnd_find_data_wait_hash(conn, req_hdr->itt);
1935         cmnd->cmd_req = orig_req;
1936         if (unlikely(orig_req == NULL)) {
1937                 /*
1938                  * It shouldn't happen, since we don't abort any request until
1939                  * we received all related PDUs from the initiator or timeout
1940                  * them. Let's quietly drop such PDUs.
1941                  */
1942                 TRACE_MGMT_DBG("Unable to find scsi task ITT %x",
1943                         cmnd_itt(cmnd));
1944                 res = iscsi_preliminary_complete(cmnd, cmnd, true);
1945                 goto out;
1946         }
1947
1948         if (unlikely(orig_req->r2t_len_to_receive < cmnd->pdu.datasize)) {
1949                 if (orig_req->prelim_compl_flags != 0) {
1950                         /* We can have fake r2t_len_to_receive */
1951                         goto go;
1952                 }
1953                 PRINT_ERROR("Data size (%d) > R2T length to receive (%d)",
1954                         cmnd->pdu.datasize, orig_req->r2t_len_to_receive);
1955                 set_scst_preliminary_status_rsp(orig_req, false,
1956                         SCST_LOAD_SENSE(iscsi_sense_incorrect_amount_of_data));
1957                 goto go;
1958         }
1959
1960         /* Crazy iSCSI spec requires us to make this unneeded check */
1961 #if 0 /* ...but some initiators (Windows) don't care to correctly set it */
1962         orig_req_hdr = &orig_req->pdu.bhs;
1963         if (unlikely(orig_req_hdr->lun != req_hdr->lun)) {
1964                 PRINT_ERROR("Wrong LUN (%lld) in Data-Out PDU (expected %lld), "
1965                         "orig_req %p, cmnd %p", (unsigned long long)req_hdr->lun,
1966                         (unsigned long long)orig_req_hdr->lun, orig_req, cmnd);
1967                 create_reject_rsp(orig_req, ISCSI_REASON_PROTOCOL_ERROR, false);
1968                 goto go;
1969         }
1970 #endif
1971
1972 go:
1973         if (req_hdr->flags & ISCSI_FLG_FINAL)
1974                 orig_req->outstanding_r2t--;
1975
1976         if (unlikely(orig_req->prelim_compl_flags != 0)) {
1977                 res = iscsi_preliminary_complete(cmnd, orig_req, true);
1978                 goto out;
1979         }
1980
1981         TRACE_WRITE("cmnd %p, orig_req %p, offset %u, datasize %u", cmnd,
1982                 orig_req, offset, cmnd->pdu.datasize);
1983
1984         res = cmnd_prepare_recv_pdu(conn, orig_req, offset, cmnd->pdu.datasize);
1985
1986 out:
1987         TRACE_EXIT_RES(res);
1988         return res;
1989 }
1990
1991 static void data_out_end(struct iscsi_cmnd *cmnd)
1992 {
1993         struct iscsi_data_out_hdr *req_hdr =
1994                 (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1995         struct iscsi_cmnd *req;
1996
1997         TRACE_ENTRY();
1998
1999         EXTRACHECKS_BUG_ON(cmnd == NULL);
2000         req = cmnd->cmd_req;
2001         if (unlikely(req == NULL))
2002                 goto out;
2003
2004         TRACE_DBG("cmnd %p, req %p", cmnd, req);
2005
2006         iscsi_extracheck_is_rd_thread(cmnd->conn);
2007
2008         if (!(cmnd->conn->ddigest_type & DIGEST_NONE) &&
2009             !cmnd->ddigest_checked) {
2010                 cmd_add_on_rx_ddigest_list(req, cmnd);
2011                 cmnd_get(cmnd);
2012         }
2013
2014         /*
2015          * Now we received the data and can adjust r2t_len_to_receive of the
2016          * orig req. We couldn't do it earlier, because it will break data
2017          * receiving errors recovery (calls of iscsi_fail_data_waiting_cmnd()).
2018          */
2019         req->r2t_len_to_receive -= cmnd->pdu.datasize;
2020
2021         if (unlikely(req->prelim_compl_flags != 0)) {
2022                 /*
2023                  * We might need to wait for one or more PDUs. Let's simplify
2024                  * other code.
2025                  */
2026                 req->r2t_len_to_receive = req->outstanding_r2t;
2027                 req->r2t_len_to_send = 0;
2028         }
2029
2030         TRACE_DBG("req %p, FINAL %x, outstanding_r2t %d, r2t_len_to_receive %d,"
2031                 " r2t_len_to_send %d", req, req_hdr->flags & ISCSI_FLG_FINAL,
2032                 req->outstanding_r2t, req->r2t_len_to_receive,
2033                 req->r2t_len_to_send);
2034
2035         if (!(req_hdr->flags & ISCSI_FLG_FINAL))
2036                 goto out;
2037
2038         if (req->r2t_len_to_receive == 0) {
2039                 if (!req->pending)
2040                         iscsi_restart_cmnd(req);
2041         } else if (req->r2t_len_to_send != 0)
2042                 send_r2t(req);
2043
2044 out:
2045         TRACE_EXIT();
2046         return;
2047 }
2048
2049 /* Might be called under target_mutex and cmd_list_lock */
2050 static void __cmnd_abort(struct iscsi_cmnd *cmnd)
2051 {
2052         unsigned long timeout_time = jiffies + ISCSI_TM_DATA_WAIT_SCHED_TIMEOUT;
2053         struct iscsi_conn *conn = cmnd->conn;
2054
2055         TRACE_MGMT_DBG("Aborting cmd %p, scst_cmd %p (scst state %x, "
2056                 "ref_cnt %d, on_write_timeout_list %d, write_start %ld, ITT %x, "
2057                 "sn %u, op %x, r2t_len_to_receive %d, r2t_len_to_send %d, "
2058                 "CDB op %x, size to write %u, outstanding_r2t %d, "
2059                 "sess->exp_cmd_sn %u, conn %p, rd_task %p)",
2060                 cmnd, cmnd->scst_cmd, cmnd->scst_state,
2061                 atomic_read(&cmnd->ref_cnt), cmnd->on_write_timeout_list,
2062                 cmnd->write_start, cmnd_itt(cmnd), cmnd->pdu.bhs.sn,
2063                 cmnd_opcode(cmnd), cmnd->r2t_len_to_receive,
2064                 cmnd->r2t_len_to_send, cmnd_scsicode(cmnd),
2065                 cmnd_write_size(cmnd), cmnd->outstanding_r2t,
2066                 cmnd->conn->session->exp_cmd_sn, cmnd->conn,
2067                 cmnd->conn->rd_task);
2068
2069 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
2070         TRACE_MGMT_DBG("net_ref_cnt %d", atomic_read(&cmnd->net_ref_cnt));
2071 #endif
2072
2073         /*
2074          * Lock to sync with iscsi_check_tm_data_wait_timeouts(), including
2075          * CMD_ABORTED bit set.
2076          */
2077         spin_lock_bh(&iscsi_rd_lock);
2078
2079         /*
2080          * We suppose that preliminary commands completion is tested by
2081          * comparing prelim_compl_flags with 0. Otherwise a race is possible,
2082          * like sending command in SCST core as PRELIM_COMPLETED, while it
2083          * wasn't aborted in it yet and have as the result a wrong success
2084          * status sent to the initiator.
2085          */
2086         set_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags);
2087
2088         TRACE_MGMT_DBG("Setting conn_tm_active for conn %p", conn);
2089         conn->conn_tm_active = 1;
2090
2091         spin_unlock_bh(&iscsi_rd_lock);
2092
2093         /*
2094          * We need the lock to sync with req_add_to_write_timeout_list() and
2095          * close races for rsp_timer.expires.
2096          */
2097         spin_lock_bh(&conn->write_list_lock);
2098         if (!timer_pending(&conn->rsp_timer) ||
2099             time_after(conn->rsp_timer.expires, timeout_time)) {
2100                 TRACE_MGMT_DBG("Mod timer on %ld (conn %p)", timeout_time,
2101                         conn);
2102                 mod_timer(&conn->rsp_timer, timeout_time);
2103         } else
2104                 TRACE_MGMT_DBG("Timer for conn %p is going to fire on %ld "
2105                         "(timeout time %ld)", conn, conn->rsp_timer.expires,
2106                         timeout_time);
2107         spin_unlock_bh(&conn->write_list_lock);
2108
2109         return;
2110 }
2111
2112 /* Must be called from the read or conn close thread */
2113 static int cmnd_abort(struct iscsi_cmnd *req, int *status)
2114 {
2115         struct iscsi_task_mgt_hdr *req_hdr =
2116                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2117         struct iscsi_cmnd *cmnd;
2118         int res = -1;
2119
2120         req_hdr->ref_cmd_sn = be32_to_cpu(req_hdr->ref_cmd_sn);
2121
2122         if (!before(req_hdr->ref_cmd_sn, req_hdr->cmd_sn)) {
2123                 TRACE(TRACE_MGMT, "ABORT TASK: RefCmdSN(%u) > CmdSN(%u)",
2124                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn);
2125                 *status = ISCSI_RESPONSE_UNKNOWN_TASK;
2126                 goto out;
2127         }
2128
2129         cmnd = cmnd_find_itt_get(req->conn, req_hdr->rtt);
2130         if (cmnd) {
2131                 struct iscsi_conn *conn = cmnd->conn;
2132                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
2133
2134                 if (req_hdr->lun != hdr->lun) {
2135                         PRINT_ERROR("ABORT TASK: LUN mismatch: req LUN "
2136                                     "%llx, cmd LUN %llx, rtt %u",
2137                                     (long long unsigned int)req_hdr->lun,
2138                                     (long long unsigned int)hdr->lun,
2139                                     req_hdr->rtt);
2140                         *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2141                         goto out_put;
2142                 }
2143
2144                 if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
2145                         if (req_hdr->ref_cmd_sn != req_hdr->cmd_sn) {
2146                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != TM "
2147                                         "cmd CmdSN(%u) for immediate command "
2148                                         "%p", req_hdr->ref_cmd_sn,
2149                                         req_hdr->cmd_sn, cmnd);
2150                                 *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2151                                 goto out_put;
2152                         }
2153                 } else {
2154                         if (req_hdr->ref_cmd_sn != hdr->cmd_sn) {
2155                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != "
2156                                         "CmdSN(%u) for command %p",
2157                                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
2158                                         cmnd);
2159                                 *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2160                                 goto out_put;
2161                         }
2162                 }
2163
2164                 if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
2165                     (req_hdr->cmd_sn == hdr->cmd_sn)) {
2166                         PRINT_ERROR("ABORT TASK: SN mismatch: req SN %x, "
2167                                 "cmd SN %x, rtt %u", req_hdr->cmd_sn,
2168                                 hdr->cmd_sn, req_hdr->rtt);
2169                         *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2170                         goto out_put;
2171                 }
2172
2173                 spin_lock_bh(&conn->cmd_list_lock);
2174                 __cmnd_abort(cmnd);
2175                 spin_unlock_bh(&conn->cmd_list_lock);
2176
2177                 cmnd_put(cmnd);
2178                 res = 0;
2179         } else {
2180                 TRACE_MGMT_DBG("cmd RTT %x not found", req_hdr->rtt);
2181                 /*
2182                  * iSCSI RFC:
2183                  *
2184                  * b)  If the Referenced Task Tag does not identify an existing task,
2185                  * but if the CmdSN indicated by the RefCmdSN field in the Task
2186                  * Management function request is within the valid CmdSN window
2187                  * and less than the CmdSN of the Task Management function
2188                  * request itself, then targets must consider the CmdSN received
2189                  * and return the "Function complete" response.
2190                  *
2191                  * c)  If the Referenced Task Tag does not identify an existing task
2192                  * and if the CmdSN indicated by the RefCmdSN field in the Task
2193                  * Management function request is outside the valid CmdSN window,
2194                  * then targets must return the "Task does not exist" response.
2195                  *
2196                  * 128 seems to be a good "window".
2197                  */
2198                 if (between(req_hdr->ref_cmd_sn, req_hdr->cmd_sn - 128,
2199                             req_hdr->cmd_sn)) {
2200                         *status = ISCSI_RESPONSE_FUNCTION_COMPLETE;
2201                         res = 0;
2202                 } else
2203                         *status = ISCSI_RESPONSE_UNKNOWN_TASK;
2204         }
2205
2206 out:
2207         return res;
2208
2209 out_put:
2210         cmnd_put(cmnd);
2211         goto out;
2212 }
2213
2214 /* Must be called from the read or conn close thread */
2215 static int target_abort(struct iscsi_cmnd *req, int all)
2216 {
2217         struct iscsi_target *target = req->conn->session->target;
2218         struct iscsi_task_mgt_hdr *req_hdr =
2219                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2220         struct iscsi_session *session;
2221         struct iscsi_conn *conn;
2222         struct iscsi_cmnd *cmnd;
2223
2224         mutex_lock(&target->target_mutex);
2225
2226         list_for_each_entry(session, &target->session_list,
2227                             session_list_entry) {
2228                 list_for_each_entry(conn, &session->conn_list,
2229                                     conn_list_entry) {
2230                         spin_lock_bh(&conn->cmd_list_lock);
2231                         list_for_each_entry(cmnd, &conn->cmd_list,
2232                                             cmd_list_entry) {
2233                                 if (cmnd == req)
2234                                         continue;
2235                                 if (all)
2236                                         __cmnd_abort(cmnd);
2237                                 else if (req_hdr->lun == cmnd_hdr(cmnd)->lun)
2238                                         __cmnd_abort(cmnd);
2239                         }
2240                         spin_unlock_bh(&conn->cmd_list_lock);
2241                 }
2242         }
2243
2244         mutex_unlock(&target->target_mutex);
2245         return 0;
2246 }
2247
2248 /* Must be called from the read or conn close thread */
2249 static void task_set_abort(struct iscsi_cmnd *req)
2250 {
2251         struct iscsi_session *session = req->conn->session;
2252         struct iscsi_task_mgt_hdr *req_hdr =
2253                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2254         struct iscsi_target *target = session->target;
2255         struct iscsi_conn *conn;
2256         struct iscsi_cmnd *cmnd;
2257
2258         mutex_lock(&target->target_mutex);
2259
2260         list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
2261                 spin_lock_bh(&conn->cmd_list_lock);
2262                 list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
2263                         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
2264                         if (cmnd == req)
2265                                 continue;
2266                         if (req_hdr->lun != hdr->lun)
2267                                 continue;
2268                         if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
2269                             req_hdr->cmd_sn == hdr->cmd_sn)
2270                                 continue;
2271                         __cmnd_abort(cmnd);
2272                 }
2273                 spin_unlock_bh(&conn->cmd_list_lock);
2274         }
2275
2276         mutex_unlock(&target->target_mutex);
2277         return;
2278 }
2279
2280 /* Must be called from the read or conn close thread */
2281 void conn_abort(struct iscsi_conn *conn)
2282 {
2283         struct iscsi_cmnd *cmnd;
2284
2285         TRACE_MGMT_DBG("Aborting conn %p", conn);
2286
2287         iscsi_extracheck_is_rd_thread(conn);
2288
2289         spin_lock_bh(&conn->cmd_list_lock);
2290 again:
2291         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
2292                 __cmnd_abort(cmnd);
2293                 if (cmnd->r2t_len_to_receive != 0) {
2294                         if (!cmnd_get_check(cmnd)) {
2295                                 spin_unlock_bh(&conn->cmd_list_lock);
2296
2297                                 /* ToDo: this is racy for MC/S */
2298                                 iscsi_fail_data_waiting_cmnd(cmnd);
2299
2300                                 cmnd_put(cmnd);
2301
2302                                 /*
2303                                  * We are in the read thread, so we may not
2304                                  * worry that after cmnd release conn gets
2305                                  * released as well.
2306                                  */
2307                                 spin_lock_bh(&conn->cmd_list_lock);
2308                                 goto again;
2309                         }
2310                 }
2311         }
2312         spin_unlock_bh(&conn->cmd_list_lock);
2313
2314         return;
2315 }
2316
2317 static void execute_task_management(struct iscsi_cmnd *req)
2318 {
2319         struct iscsi_conn *conn = req->conn;
2320         struct iscsi_session *sess = conn->session;
2321         struct iscsi_task_mgt_hdr *req_hdr =
2322                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2323         int rc, status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2324         int function = req_hdr->function & ISCSI_FUNCTION_MASK;
2325         struct scst_rx_mgmt_params params;
2326
2327         TRACE(TRACE_MGMT, "TM fn %d", function);
2328
2329         TRACE_MGMT_DBG("TM req %p, ITT %x, RTT %x, sn %u, con %p", req,
2330                 cmnd_itt(req), req_hdr->rtt, req_hdr->cmd_sn, conn);
2331
2332         iscsi_extracheck_is_rd_thread(conn);
2333
2334         spin_lock(&sess->sn_lock);
2335         sess->tm_active++;
2336         sess->tm_sn = req_hdr->cmd_sn;
2337         if (sess->tm_rsp != NULL) {
2338                 struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
2339
2340                 TRACE_MGMT_DBG("Dropping delayed TM rsp %p", tm_rsp);
2341
2342                 sess->tm_rsp = NULL;
2343                 sess->tm_active--;
2344
2345                 spin_unlock(&sess->sn_lock);
2346
2347                 sBUG_ON(sess->tm_active < 0);
2348
2349                 rsp_cmnd_release(tm_rsp);
2350         } else
2351                 spin_unlock(&sess->sn_lock);
2352
2353         memset(&params, 0, sizeof(params));
2354         params.atomic = SCST_NON_ATOMIC;
2355         params.tgt_priv = req;
2356
2357         if ((function != ISCSI_FUNCTION_ABORT_TASK) &&
2358             (req_hdr->rtt != ISCSI_RESERVED_TAG)) {
2359                 PRINT_ERROR("Invalid RTT %x (TM fn %d)", req_hdr->rtt,
2360                         function);
2361                 rc = -1;
2362                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2363                 goto reject;
2364         }
2365
2366         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
2367
2368         switch (function) {
2369         case ISCSI_FUNCTION_ABORT_TASK:
2370                 rc = cmnd_abort(req, &status);
2371                 if (rc == 0) {
2372                         params.fn = SCST_ABORT_TASK;
2373                         params.tag = req_hdr->rtt;
2374                         params.tag_set = 1;
2375                         params.lun = (uint8_t *)&req_hdr->lun;
2376                         params.lun_len = sizeof(req_hdr->lun);
2377                         params.lun_set = 1;
2378                         params.cmd_sn = req_hdr->cmd_sn;
2379                         params.cmd_sn_set = 1;
2380                         rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2381                                 &params);
2382                         status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2383                 }
2384                 break;
2385         case ISCSI_FUNCTION_ABORT_TASK_SET:
2386                 task_set_abort(req);
2387                 params.fn = SCST_ABORT_TASK_SET;
2388                 params.lun = (uint8_t *)&req_hdr->lun;
2389                 params.lun_len = sizeof(req_hdr->lun);
2390                 params.lun_set = 1;
2391                 params.cmd_sn = req_hdr->cmd_sn;
2392                 params.cmd_sn_set = 1;
2393                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2394                         &params);
2395                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2396                 break;
2397         case ISCSI_FUNCTION_CLEAR_TASK_SET:
2398                 task_set_abort(req);
2399                 params.fn = SCST_CLEAR_TASK_SET;
2400                 params.lun = (uint8_t *)&req_hdr->lun;
2401                 params.lun_len = sizeof(req_hdr->lun);
2402                 params.lun_set = 1;
2403                 params.cmd_sn = req_hdr->cmd_sn;
2404                 params.cmd_sn_set = 1;
2405                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2406                         &params);
2407                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2408                 break;
2409         case ISCSI_FUNCTION_CLEAR_ACA:
2410                 params.fn = SCST_CLEAR_ACA;
2411                 params.lun = (uint8_t *)&req_hdr->lun;
2412                 params.lun_len = sizeof(req_hdr->lun);
2413                 params.lun_set = 1;
2414                 params.cmd_sn = req_hdr->cmd_sn;
2415                 params.cmd_sn_set = 1;
2416                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2417                         &params);
2418                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2419                 break;
2420         case ISCSI_FUNCTION_TARGET_COLD_RESET:
2421         case ISCSI_FUNCTION_TARGET_WARM_RESET:
2422                 target_abort(req, 1);
2423                 params.fn = SCST_TARGET_RESET;
2424                 params.cmd_sn = req_hdr->cmd_sn;
2425                 params.cmd_sn_set = 1;
2426                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2427                         &params);
2428                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2429                 break;
2430         case ISCSI_FUNCTION_LOGICAL_UNIT_RESET:
2431                 target_abort(req, 0);
2432                 params.fn = SCST_LUN_RESET;
2433                 params.lun = (uint8_t *)&req_hdr->lun;
2434                 params.lun_len = sizeof(req_hdr->lun);
2435                 params.lun_set = 1;
2436                 params.cmd_sn = req_hdr->cmd_sn;
2437                 params.cmd_sn_set = 1;
2438                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2439                         &params);
2440                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2441                 break;
2442         case ISCSI_FUNCTION_TASK_REASSIGN:
2443                 rc = -1;
2444                 status = ISCSI_RESPONSE_ALLEGIANCE_REASSIGNMENT_UNSUPPORTED;
2445                 break;
2446         default:
2447                 PRINT_ERROR("Unknown TM function %d", function);
2448                 rc = -1;
2449                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2450                 break;
2451         }
2452
2453 reject:
2454         if (rc != 0)
2455                 iscsi_send_task_mgmt_resp(req, status);
2456
2457         return;
2458 }
2459
2460 static void noop_out_exec(struct iscsi_cmnd *req)
2461 {
2462         struct iscsi_cmnd *rsp;
2463         struct iscsi_nop_in_hdr *rsp_hdr;
2464
2465         TRACE_ENTRY();
2466
2467         TRACE_DBG("%p", req);
2468
2469         if (cmnd_itt(req) != cpu_to_be32(ISCSI_RESERVED_TAG)) {
2470                 rsp = iscsi_alloc_main_rsp(req);
2471
2472                 rsp_hdr = (struct iscsi_nop_in_hdr *)&rsp->pdu.bhs;
2473                 rsp_hdr->opcode = ISCSI_OP_NOOP_IN;
2474                 rsp_hdr->flags = ISCSI_FLG_FINAL;
2475                 rsp_hdr->itt = req->pdu.bhs.itt;
2476                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
2477
2478                 if (req->pdu.datasize)
2479                         sBUG_ON(req->sg == NULL);
2480                 else
2481                         sBUG_ON(req->sg != NULL);
2482
2483                 if (req->sg) {
2484                         rsp->sg = req->sg;
2485                         rsp->sg_cnt = req->sg_cnt;
2486                         rsp->bufflen = req->bufflen;
2487                 }
2488
2489                 /* We already checked it in check_segment_length() */
2490                 sBUG_ON(get_pgcnt(req->pdu.datasize, 0) > ISCSI_CONN_IOV_MAX);
2491
2492                 rsp->pdu.datasize = req->pdu.datasize;
2493         }
2494
2495         req_cmnd_release(req);
2496
2497         TRACE_EXIT();
2498         return;
2499 }
2500
2501 static void logout_exec(struct iscsi_cmnd *req)
2502 {
2503         struct iscsi_logout_req_hdr *req_hdr;
2504         struct iscsi_cmnd *rsp;
2505         struct iscsi_logout_rsp_hdr *rsp_hdr;
2506
2507         PRINT_INFO("Logout received from initiator %s",
2508                 req->conn->session->initiator_name);
2509         TRACE_DBG("%p", req);
2510
2511         req_hdr = (struct iscsi_logout_req_hdr *)&req->pdu.bhs;
2512         rsp = iscsi_alloc_main_rsp(req);
2513         rsp_hdr = (struct iscsi_logout_rsp_hdr *)&rsp->pdu.bhs;
2514         rsp_hdr->opcode = ISCSI_OP_LOGOUT_RSP;
2515         rsp_hdr->flags = ISCSI_FLG_FINAL;
2516         rsp_hdr->itt = req_hdr->itt;
2517         rsp->should_close_conn = 1;
2518
2519         req_cmnd_release(req);
2520
2521         return;
2522 }
2523
2524 static void iscsi_cmnd_exec(struct iscsi_cmnd *cmnd)
2525 {
2526         TRACE_ENTRY();
2527
2528         TRACE_DBG("cmnd %p, op %x, SN %u", cmnd, cmnd_opcode(cmnd),
2529                 cmnd->pdu.bhs.sn);
2530
2531         iscsi_extracheck_is_rd_thread(cmnd->conn);
2532
2533         if (cmnd_opcode(cmnd) == ISCSI_OP_SCSI_CMD) {
2534                 if (cmnd->r2t_len_to_receive == 0)
2535                         iscsi_restart_cmnd(cmnd);
2536                 else if (cmnd->r2t_len_to_send != 0)
2537                         send_r2t(cmnd);
2538                 goto out;
2539         }
2540
2541         if (cmnd->prelim_compl_flags != 0) {
2542                 TRACE_MGMT_DBG("Terminating prelim completed non-SCSI cmnd %p "
2543                         "(op %x)", cmnd, cmnd_opcode(cmnd));
2544                 req_cmnd_release(cmnd);
2545                 goto out;
2546         }
2547
2548         switch (cmnd_opcode(cmnd)) {
2549         case ISCSI_OP_NOOP_OUT:
2550                 noop_out_exec(cmnd);
2551                 break;
2552         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2553                 execute_task_management(cmnd);
2554                 break;
2555         case ISCSI_OP_LOGOUT_CMD:
2556                 logout_exec(cmnd);
2557                 break;
2558         default:
2559                 PRINT_CRIT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
2560                 sBUG();
2561                 break;
2562         }
2563
2564 out:
2565         TRACE_EXIT();
2566         return;
2567 }
2568
2569 static void set_cork(struct socket *sock, int on)
2570 {
2571         int opt = on;
2572         mm_segment_t oldfs;
2573
2574         oldfs = get_fs();
2575         set_fs(get_ds());
2576         sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK,
2577                               (void __force __user *)&opt, sizeof(opt));
2578         set_fs(oldfs);
2579         return;
2580 }
2581
2582 void cmnd_tx_start(struct iscsi_cmnd *cmnd)
2583 {
2584         struct iscsi_conn *conn = cmnd->conn;
2585
2586         TRACE_DBG("conn %p, cmnd %p, opcode %x", conn, cmnd, cmnd_opcode(cmnd));
2587         iscsi_cmnd_set_length(&cmnd->pdu);
2588
2589         iscsi_extracheck_is_wr_thread(conn);
2590
2591         set_cork(conn->sock, 1);
2592
2593         conn->write_iop = conn->write_iov;
2594         conn->write_iop->iov_base = (void __force __user *)(&cmnd->pdu.bhs);
2595         conn->write_iop->iov_len = sizeof(cmnd->pdu.bhs);
2596         conn->write_iop_used = 1;
2597         conn->write_size = sizeof(cmnd->pdu.bhs) + cmnd->pdu.datasize;
2598         conn->write_offset = 0;
2599
2600         switch (cmnd_opcode(cmnd)) {
2601         case ISCSI_OP_NOOP_IN:
2602                 cmnd_set_sn(cmnd, 1);
2603                 break;
2604         case ISCSI_OP_SCSI_RSP:
2605                 cmnd_set_sn(cmnd, 1);
2606                 break;
2607         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2608                 cmnd_set_sn(cmnd, 1);
2609                 break;
2610         case ISCSI_OP_TEXT_RSP:
2611                 cmnd_set_sn(cmnd, 1);
2612                 break;
2613         case ISCSI_OP_SCSI_DATA_IN:
2614         {
2615                 struct iscsi_data_in_hdr *rsp =
2616                         (struct iscsi_data_in_hdr *)&cmnd->pdu.bhs;
2617                 u32 offset = cpu_to_be32(rsp->buffer_offset);
2618
2619                 TRACE_DBG("cmnd %p, offset %u, datasize %u, bufflen %u", cmnd,
2620                         offset, cmnd->pdu.datasize, cmnd->bufflen);
2621
2622                 sBUG_ON(offset > cmnd->bufflen);
2623                 sBUG_ON(offset + cmnd->pdu.datasize > cmnd->bufflen);
2624
2625                 conn->write_offset = offset;
2626
2627                 cmnd_set_sn(cmnd, (rsp->flags & ISCSI_FLG_FINAL) ? 1 : 0);
2628                 break;
2629         }
2630         case ISCSI_OP_LOGOUT_RSP:
2631                 cmnd_set_sn(cmnd, 1);
2632                 break;
2633         case ISCSI_OP_R2T:
2634                 cmnd->pdu.bhs.sn = cmnd_set_sn(cmnd, 0);
2635                 break;
2636         case ISCSI_OP_ASYNC_MSG:
2637                 cmnd_set_sn(cmnd, 1);
2638                 break;
2639         case ISCSI_OP_REJECT:
2640                 cmnd_set_sn(cmnd, 1);
2641                 break;
2642         default:
2643                 PRINT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
2644                 break;
2645         }
2646
2647         iscsi_dump_pdu(&cmnd->pdu);
2648         return;
2649 }
2650
2651 void cmnd_tx_end(struct iscsi_cmnd *cmnd)
2652 {
2653         struct iscsi_conn *conn = cmnd->conn;
2654
2655         TRACE_DBG("%p:%x (should_close_conn %d, should_close_all_conn %d)",
2656                 cmnd, cmnd_opcode(cmnd), cmnd->should_close_conn,
2657                 cmnd->should_close_all_conn);
2658
2659 #ifdef CONFIG_SCST_EXTRACHECKS
2660         switch (cmnd_opcode(cmnd)) {
2661         case ISCSI_OP_NOOP_IN:
2662         case ISCSI_OP_SCSI_RSP:
2663         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2664         case ISCSI_OP_TEXT_RSP:
2665         case ISCSI_OP_R2T:
2666         case ISCSI_OP_ASYNC_MSG:
2667         case ISCSI_OP_REJECT:
2668         case ISCSI_OP_SCSI_DATA_IN:
2669         case ISCSI_OP_LOGOUT_RSP:
2670                 break;
2671         default:
2672                 PRINT_CRIT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2673                 sBUG();
2674                 break;
2675         }
2676 #endif
2677
2678         if (unlikely(cmnd->should_close_conn)) {
2679                 if (cmnd->should_close_all_conn) {
2680                         PRINT_INFO("Closing all connections for target %x at "
2681                                 "initiator's %s request",
2682                                 cmnd->conn->session->target->tid,
2683                                 conn->session->initiator_name);
2684                         target_del_all_sess(cmnd->conn->session->target, 0);
2685                 } else {
2686                         PRINT_INFO("Closing connection at initiator's %s "
2687                                 "request", conn->session->initiator_name);
2688                         mark_conn_closed(conn);
2689                 }
2690         }
2691
2692         set_cork(cmnd->conn->sock, 0);
2693         return;
2694 }
2695
2696 /*
2697  * Push the command for execution. This functions reorders the commands.
2698  * Called from the read thread.
2699  *
2700  * Basically, since we don't support MC/S and TCP guarantees data delivery
2701  * order, all that SN's stuff isn't needed at all (commands delivery order is
2702  * a natural commands execution order), but insane iSCSI spec requires
2703  * us to check it and we have to, because some crazy initiators can rely
2704  * on the SN's based order and reorder requests during sending. For all other
2705  * normal initiators all that code is a NOP.
2706  */
2707 static void iscsi_push_cmnd(struct iscsi_cmnd *cmnd)
2708 {
2709         struct iscsi_session *session = cmnd->conn->session;
2710         struct list_head *entry;
2711         u32 cmd_sn;
2712
2713         TRACE_DBG("cmnd %p, iSCSI opcode %x, sn %u, exp sn %u", cmnd,
2714                 cmnd_opcode(cmnd), cmnd->pdu.bhs.sn, session->exp_cmd_sn);
2715
2716         iscsi_extracheck_is_rd_thread(cmnd->conn);
2717
2718         sBUG_ON(cmnd->parent_req != NULL);
2719
2720         if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
2721                 TRACE_DBG("Immediate cmd %p (cmd_sn %u)", cmnd,
2722                         cmnd->pdu.bhs.sn);
2723                 iscsi_cmnd_exec(cmnd);
2724                 goto out;
2725         }
2726
2727         spin_lock(&session->sn_lock);
2728
2729         cmd_sn = cmnd->pdu.bhs.sn;
2730         if (cmd_sn == session->exp_cmd_sn) {
2731                 while (1) {
2732                         session->exp_cmd_sn = ++cmd_sn;
2733
2734                         if (unlikely(session->tm_active > 0)) {
2735                                 if (before(cmd_sn, session->tm_sn)) {
2736                                         struct iscsi_conn *conn = cmnd->conn;
2737
2738                                         spin_unlock(&session->sn_lock);
2739
2740                                         spin_lock_bh(&conn->cmd_list_lock);
2741                                         __cmnd_abort(cmnd);
2742                                         spin_unlock_bh(&conn->cmd_list_lock);
2743
2744                                         spin_lock(&session->sn_lock);
2745                                 }
2746                                 iscsi_check_send_delayed_tm_resp(session);
2747                         }
2748
2749                         spin_unlock(&session->sn_lock);
2750
2751                         iscsi_cmnd_exec(cmnd);
2752
2753                         spin_lock(&session->sn_lock);
2754
2755                         if (list_empty(&session->pending_list))
2756                                 break;
2757                         cmnd = list_entry(session->pending_list.next,
2758                                           struct iscsi_cmnd,
2759                                           pending_list_entry);
2760                         if (cmnd->pdu.bhs.sn != cmd_sn)
2761                                 break;
2762
2763                         list_del(&cmnd->pending_list_entry);
2764                         cmnd->pending = 0;
2765
2766                         TRACE_MGMT_DBG("Processing pending cmd %p (cmd_sn %u)",
2767                                 cmnd, cmd_sn);
2768                 }
2769         } else {
2770                 int drop = 0;
2771
2772                 TRACE_DBG("Pending cmd %p (cmd_sn %u, exp_cmd_sn %u)",
2773                         cmnd, cmd_sn, session->exp_cmd_sn);
2774
2775                 /*
2776                  * iSCSI RFC 3720: "The target MUST silently ignore any
2777                  * non-immediate command outside of [from ExpCmdSN to MaxCmdSN
2778                  * inclusive] range". But we won't honor the MaxCmdSN
2779                  * requirement, because, since we adjust MaxCmdSN from the
2780                  * separate write thread, rarely it is possible that initiator
2781                  * can legally send command with CmdSN>MaxSN. But it won't
2782                  * hurt anything, in the worst case it will lead to
2783                  * additional QUEUE FULL status.
2784                  */
2785
2786                 if (unlikely(before(cmd_sn, session->exp_cmd_sn))) {
2787                         PRINT_ERROR("Unexpected cmd_sn (%u,%u)", cmd_sn,
2788                                 session->exp_cmd_sn);
2789                         drop = 1;
2790                 }
2791
2792 #if 0
2793                 if (unlikely(after(cmd_sn, session->exp_cmd_sn +
2794                                         iscsi_get_allowed_cmds(session)))) {
2795                         TRACE_MGMT_DBG("Too large cmd_sn %u (exp_cmd_sn %u, "
2796                                 "max_sn %u)", cmd_sn, session->exp_cmd_sn,
2797                                 iscsi_get_allowed_cmds(session));
2798                 }
2799 #endif
2800
2801                 spin_unlock(&session->sn_lock);
2802
2803                 if (unlikely(drop)) {
2804                         req_cmnd_release_force(cmnd);
2805                         goto out;
2806                 }
2807
2808                 if (unlikely(test_bit(ISCSI_CMD_ABORTED,
2809                                         &cmnd->prelim_compl_flags))) {
2810                         struct iscsi_cmnd *tm_clone;
2811
2812                         TRACE_MGMT_DBG("Pending aborted cmnd %p, creating TM "
2813                                 "clone (scst cmd %p, state %d)", cmnd,
2814                                 cmnd->scst_cmd, cmnd->scst_state);
2815
2816                         tm_clone = cmnd_alloc(cmnd->conn, NULL);
2817                         if (tm_clone != NULL) {
2818                                 set_bit(ISCSI_CMD_ABORTED,
2819                                         &tm_clone->prelim_compl_flags);
2820                                 tm_clone->pdu = cmnd->pdu;
2821
2822                                 TRACE_MGMT_DBG("TM clone %p created",
2823                                                tm_clone);
2824
2825                                 iscsi_cmnd_exec(cmnd);
2826                                 cmnd = tm_clone;
2827                         } else
2828                                 PRINT_ERROR("%s", "Unable to create TM clone");
2829                 }
2830
2831                 TRACE_MGMT_DBG("Pending cmnd %p (op %x, sn %u, exp sn %u)",
2832                         cmnd, cmnd_opcode(cmnd), cmd_sn, session->exp_cmd_sn);
2833
2834                 spin_lock(&session->sn_lock);
2835                 list_for_each(entry, &session->pending_list) {
2836                         struct iscsi_cmnd *tmp =
2837                                 list_entry(entry, struct iscsi_cmnd,
2838                                            pending_list_entry);
2839                         if (before(cmd_sn, tmp->pdu.bhs.sn))
2840                                 break;
2841                 }
2842                 list_add_tail(&cmnd->pending_list_entry, entry);
2843                 cmnd->pending = 1;
2844         }
2845
2846         spin_unlock(&session->sn_lock);
2847 out:
2848         return;
2849 }
2850
2851 static int check_segment_length(struct iscsi_cmnd *cmnd)
2852 {
2853         struct iscsi_conn *conn = cmnd->conn;
2854         struct iscsi_session *session = conn->session;
2855
2856         if (unlikely(cmnd->pdu.datasize > session->sess_param.max_recv_data_length)) {
2857                 PRINT_ERROR("Initiator %s violated negotiated parameters: "
2858                         "data too long (ITT %x, datasize %u, "
2859                         "max_recv_data_length %u", session->initiator_name,
2860                         cmnd_itt(cmnd), cmnd->pdu.datasize,
2861                         session->sess_param.max_recv_data_length);
2862                 mark_conn_closed(conn);
2863                 return -EINVAL;
2864         }
2865         return 0;
2866 }
2867
2868 int cmnd_rx_start(struct iscsi_cmnd *cmnd)
2869 {
2870         int res, rc;
2871
2872         iscsi_dump_pdu(&cmnd->pdu);
2873
2874         res = check_segment_length(cmnd);
2875         if (res != 0)
2876                 goto out;
2877
2878         switch (cmnd_opcode(cmnd)) {
2879         case ISCSI_OP_SCSI_CMD:
2880                 res = scsi_cmnd_start(cmnd);
2881                 if (unlikely(res < 0))
2882                         goto out;
2883                 spin_lock(&cmnd->conn->session->sn_lock);
2884                 __update_stat_sn(cmnd);
2885                 rc = check_cmd_sn(cmnd);
2886                 spin_unlock(&cmnd->conn->session->sn_lock);
2887                 break;
2888         case ISCSI_OP_SCSI_DATA_OUT:
2889                 res = data_out_start(cmnd);
2890                 goto out;
2891         case ISCSI_OP_NOOP_OUT:
2892                 rc = noop_out_start(cmnd);
2893                 break;
2894         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2895         case ISCSI_OP_LOGOUT_CMD:
2896                 spin_lock(&cmnd->conn->session->sn_lock);
2897                 __update_stat_sn(cmnd);
2898                 rc = check_cmd_sn(cmnd);
2899                 spin_unlock(&cmnd->conn->session->sn_lock);
2900                 break;
2901         case ISCSI_OP_TEXT_CMD:
2902         case ISCSI_OP_SNACK_CMD:
2903         default:
2904                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2905                 break;
2906         }
2907
2908         if (unlikely(rc < 0)) {
2909                 PRINT_ERROR("Error %d (iSCSI opcode %x, ITT %x)", rc,
2910                         cmnd_opcode(cmnd), cmnd_itt(cmnd));
2911                 res = create_reject_rsp(cmnd, -rc, true);
2912         }
2913
2914 out:
2915         TRACE_EXIT_RES(res);
2916         return res;
2917 }
2918
2919 void cmnd_rx_end(struct iscsi_cmnd *cmnd)
2920 {
2921         TRACE_ENTRY();
2922
2923         TRACE_DBG("cmnd %p, opcode %x", cmnd, cmnd_opcode(cmnd));
2924
2925         switch (cmnd_opcode(cmnd)) {
2926         case ISCSI_OP_SCSI_CMD:
2927         case ISCSI_OP_NOOP_OUT:
2928         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2929         case ISCSI_OP_LOGOUT_CMD:
2930                 iscsi_push_cmnd(cmnd);
2931                 goto out;
2932         case ISCSI_OP_SCSI_DATA_OUT:
2933                 data_out_end(cmnd);
2934                 break;
2935         default:
2936                 PRINT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
2937                 break;
2938         }
2939
2940         req_cmnd_release(cmnd);
2941
2942 out:
2943         TRACE_EXIT();
2944         return;
2945 }
2946
2947 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
2948 static int iscsi_alloc_data_buf(struct scst_cmd *cmd)
2949 {
2950         /*
2951          * sock->ops->sendpage() is async zero copy operation,
2952          * so we must be sure not to free and reuse
2953          * the command's buffer before the sending was completed
2954          * by the network layers. It is possible only if we
2955          * don't use SGV cache.
2956          */
2957         EXTRACHECKS_BUG_ON(!(scst_cmd_get_data_direction(cmd) & SCST_DATA_READ));
2958         scst_cmd_set_no_sgv(cmd);
2959         return 1;
2960 }
2961 #endif
2962
2963 static void iscsi_preprocessing_done(struct scst_cmd *scst_cmd)
2964 {
2965         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
2966                                 scst_cmd_get_tgt_priv(scst_cmd);
2967
2968         TRACE_DBG("req %p", req);
2969
2970         if (req->conn->rx_task == current)
2971                 req->scst_state = ISCSI_CMD_STATE_AFTER_PREPROC;
2972         else {
2973                 /*
2974                  * We wait for the state change without any protection, so
2975                  * without cmnd_get() it is possible that req will die
2976                  * "immediately" after the state assignment and
2977                  * iscsi_make_conn_rd_active() will operate on dead data.
2978                  * We use the ordered version of cmnd_get(), because "get"
2979                  * must be done before the state assignment.
2980                  *
2981                  * We protected from the race on calling cmnd_rx_continue(),
2982                  * because there can be only one read thread processing
2983                  * connection.
2984                  */
2985                 cmnd_get_ordered(req);
2986                 req->scst_state = ISCSI_CMD_STATE_AFTER_PREPROC;
2987                 iscsi_make_conn_rd_active(req->conn);
2988                 if (unlikely(req->conn->closing)) {
2989                         TRACE_DBG("Waking up closing conn %p", req->conn);
2990                         wake_up(&req->conn->read_state_waitQ);
2991                 }
2992                 cmnd_put(req);
2993         }
2994
2995         return;
2996 }
2997
2998 /*
2999  * No locks.
3000  *
3001  * IMPORTANT! Connection conn must be protected by additional conn_get()
3002  * upon entrance in this function, because otherwise it could be destroyed
3003  * inside as a result of iscsi_send(), which releases sent commands.
3004  */
3005 static void iscsi_try_local_processing(struct iscsi_conn *conn)
3006 {
3007         int local;
3008
3009         TRACE_ENTRY();
3010
3011         spin_lock_bh(&iscsi_wr_lock);
3012         switch (conn->wr_state) {
3013         case ISCSI_CONN_WR_STATE_IN_LIST:
3014                 list_del(&conn->wr_list_entry);
3015                 /* go through */
3016         case ISCSI_CONN_WR_STATE_IDLE:
3017 #ifdef CONFIG_SCST_EXTRACHECKS
3018                 conn->wr_task = current;
3019 #endif
3020                 conn->wr_state = ISCSI_CONN_WR_STATE_PROCESSING;
3021                 conn->wr_space_ready = 0;
3022                 local = 1;
3023                 break;
3024         default:
3025                 local = 0;
3026                 break;
3027         }
3028         spin_unlock_bh(&iscsi_wr_lock);
3029
3030         if (local) {
3031                 int rc = 1;
3032
3033                 if (test_write_ready(conn))
3034                         rc = iscsi_send(conn);
3035
3036                 spin_lock_bh(&iscsi_wr_lock);
3037 #ifdef CONFIG_SCST_EXTRACHECKS
3038                 conn->wr_task = NULL;
3039 #endif
3040                 if ((rc <= 0) || test_write_ready(conn)) {
3041                         list_add_tail(&conn->wr_list_entry, &iscsi_wr_list);
3042                         conn->wr_state = ISCSI_CONN_WR_STATE_IN_LIST;
3043                         wake_up(&iscsi_wr_waitQ);
3044                 } else
3045                         conn->wr_state = ISCSI_CONN_WR_STATE_IDLE;
3046                 spin_unlock_bh(&iscsi_wr_lock);
3047         }
3048
3049         TRACE_EXIT();
3050         return;
3051 }
3052
3053 static int iscsi_xmit_response(struct scst_cmd *scst_cmd)
3054 {
3055         int is_send_status = scst_cmd_get_is_send_status(scst_cmd);
3056         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
3057                                         scst_cmd_get_tgt_priv(scst_cmd);
3058         struct iscsi_conn *conn = req->conn;
3059         int status = scst_cmd_get_status(scst_cmd);
3060         u8 *sense = scst_cmd_get_sense_buffer(scst_cmd);
3061         int sense_len = scst_cmd_get_sense_buffer_len(scst_cmd);
3062
3063         if (unlikely(scst_cmd_atomic(scst_cmd)))
3064                 return SCST_TGT_RES_NEED_THREAD_CTX;
3065
3066         scst_cmd_set_tgt_priv(scst_cmd, NULL);
3067
3068         EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_RESTARTED);
3069
3070         if (unlikely(scst_cmd_aborted(scst_cmd)))
3071                 set_bit(ISCSI_CMD_ABORTED, &req->prelim_compl_flags);
3072
3073         if (unlikely(req->prelim_compl_flags != 0)) {
3074                 if (test_bit(ISCSI_CMD_ABORTED, &req->prelim_compl_flags)) {
3075                         TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
3076                                 req->scst_cmd);
3077                         scst_set_delivery_status(req->scst_cmd,
3078                                 SCST_CMD_DELIVERY_ABORTED);
3079                         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
3080                         req_cmnd_release_force(req);
3081                         goto out;
3082                 }
3083
3084                 TRACE_DBG("Prelim completed req %p", req);
3085
3086                 /*
3087                  * We could preliminary have finished req before we
3088                  * knew its device, so check if we return correct sense
3089                  * format.
3090                  */
3091                 scst_check_convert_sense(scst_cmd);
3092
3093                 if (!req->own_sg) {
3094                         req->sg = scst_cmd_get_sg(scst_cmd);
3095                         req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
3096                 }
3097         } else {
3098                 EXTRACHECKS_BUG_ON(req->own_sg);
3099                 req->sg = scst_cmd_get_sg(scst_cmd);
3100                 req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
3101         }
3102
3103         req->bufflen = scst_cmd_get_resp_data_len(scst_cmd);
3104
3105         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
3106
3107         TRACE_DBG("req %p, is_send_status=%x, req->bufflen=%d, req->sg=%p, "
3108                 "req->sg_cnt %d", req, is_send_status, req->bufflen, req->sg,
3109                 req->sg_cnt);
3110
3111         EXTRACHECKS_BUG_ON(req->hashed);
3112         if (req->main_rsp != NULL)
3113                 EXTRACHECKS_BUG_ON(cmnd_opcode(req->main_rsp) != ISCSI_OP_REJECT);
3114
3115         if (unlikely((req->bufflen != 0) && !is_send_status)) {
3116                 PRINT_CRIT_ERROR("%s", "Sending DATA without STATUS is "
3117                         "unsupported");
3118                 scst_set_cmd_error(scst_cmd,
3119                         SCST_LOAD_SENSE(scst_sense_hardw_error));
3120                 sBUG(); /* ToDo */
3121         }
3122
3123         if (req->bufflen != 0) {
3124                 /*
3125                  * Check above makes sure that is_send_status is set,
3126                  * so status is valid here, but in future that could change.
3127                  * ToDo
3128                  */
3129                 if ((status != SAM_STAT_CHECK_CONDITION) &&
3130                     ((cmnd_hdr(req)->flags & (ISCSI_CMD_WRITE|ISCSI_CMD_READ)) !=
3131                                 (ISCSI_CMD_WRITE|ISCSI_CMD_READ))) {
3132                         send_data_rsp(req, status, is_send_status);
3133                 } else {
3134                         struct iscsi_cmnd *rsp;
3135                         send_data_rsp(req, 0, 0);
3136                         if (is_send_status) {
3137                                 rsp = create_status_rsp(req, status, sense,
3138                                         sense_len, true);
3139                                 iscsi_cmnd_init_write(rsp, 0);
3140                         }
3141                 }
3142         } else if (is_send_status) {
3143                 struct iscsi_cmnd *rsp;
3144                 rsp = create_status_rsp(req, status, sense, sense_len, false);
3145                 iscsi_cmnd_init_write(rsp, 0);
3146         }
3147 #ifdef CONFIG_SCST_EXTRACHECKS
3148         else
3149                 sBUG();
3150 #endif
3151
3152         /*
3153          * "_ordered" here to protect from reorder, which can lead to
3154          * preliminary connection destroy in req_cmnd_release(). Just in
3155          * case, actually, because reordering shouldn't go so far, but who
3156          * knows..
3157          */
3158         conn_get_ordered(conn);
3159         req_cmnd_release(req);
3160         iscsi_try_local_processing(conn);
3161         conn_put(conn);
3162
3163 out:
3164         return SCST_TGT_RES_SUCCESS;
3165 }
3166
3167 /* Called under sn_lock */
3168 static bool iscsi_is_delay_tm_resp(struct iscsi_cmnd *rsp)
3169 {
3170         bool res = 0;
3171         struct iscsi_task_mgt_hdr *req_hdr =
3172                 (struct iscsi_task_mgt_hdr *)&rsp->parent_req->pdu.bhs;
3173         int function = req_hdr->function & ISCSI_FUNCTION_MASK;
3174         struct iscsi_session *sess = rsp->conn->session;
3175
3176         TRACE_ENTRY();
3177
3178         /* This should be checked for immediate TM commands as well */
3179
3180         switch (function) {
3181         default:
3182                 if (before(sess->exp_cmd_sn, req_hdr->cmd_sn))
3183                         res = 1;
3184                 break;
3185         }
3186
3187         TRACE_EXIT_RES(res);
3188         return res;
3189 }
3190
3191 /* Called under sn_lock, but might drop it inside, then reaquire */
3192 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess)
3193         __acquires(&sn_lock)
3194         __releases(&sn_lock)
3195 {
3196         struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
3197
3198         TRACE_ENTRY();
3199
3200         if (tm_rsp == NULL)
3201                 goto out;
3202
3203         if (iscsi_is_delay_tm_resp(tm_rsp))
3204                 goto out;
3205
3206         TRACE_MGMT_DBG("Sending delayed rsp %p", tm_rsp);
3207
3208         sess->tm_rsp = NULL;
3209         sess->tm_active--;
3210
3211         spin_unlock(&sess->sn_lock);
3212
3213         sBUG_ON(sess->tm_active < 0);
3214
3215         iscsi_cmnd_init_write(tm_rsp, ISCSI_INIT_WRITE_WAKE);
3216
3217         spin_lock(&sess->sn_lock);
3218
3219 out:
3220         TRACE_EXIT();
3221         return;
3222 }
3223
3224 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status)
3225 {
3226         struct iscsi_cmnd *rsp;
3227         struct iscsi_task_mgt_hdr *req_hdr =
3228                                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
3229         struct iscsi_task_rsp_hdr *rsp_hdr;
3230         struct iscsi_session *sess = req->conn->session;
3231         int fn = req_hdr->function & ISCSI_FUNCTION_MASK;
3232
3233         TRACE_ENTRY();
3234
3235         TRACE_MGMT_DBG("TM req %p finished", req);
3236         TRACE(TRACE_MGMT, "TM fn %d finished, status %d", fn, status);
3237
3238         rsp = iscsi_alloc_rsp(req);
3239         rsp_hdr = (struct iscsi_task_rsp_hdr *)&rsp->pdu.bhs;
3240
3241         rsp_hdr->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
3242         rsp_hdr->flags = ISCSI_FLG_FINAL;
3243         rsp_hdr->itt = req_hdr->itt;
3244         rsp_hdr->response = status;
3245
3246         if (fn == ISCSI_FUNCTION_TARGET_COLD_RESET) {
3247                 rsp->should_close_conn = 1;
3248                 rsp->should_close_all_conn = 1;
3249         }
3250
3251         sBUG_ON(sess->tm_rsp != NULL);
3252
3253         spin_lock(&sess->sn_lock);
3254         if (iscsi_is_delay_tm_resp(rsp)) {
3255                 TRACE_MGMT_DBG("Delaying TM fn %d response %p "
3256                         "(req %p), because not all affected commands "
3257                         "received (TM cmd sn %u, exp sn %u)",
3258                         req_hdr->function & ISCSI_FUNCTION_MASK, rsp, req,
3259                         req_hdr->cmd_sn, sess->exp_cmd_sn);
3260                 sess->tm_rsp = rsp;
3261                 spin_unlock(&sess->sn_lock);
3262                 goto out_release;
3263         }
3264         sess->tm_active--;
3265         spin_unlock(&sess->sn_lock);
3266
3267         sBUG_ON(sess->tm_active < 0);
3268
3269         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_WAKE);
3270
3271 out_release:
3272         req_cmnd_release(req);
3273
3274         TRACE_EXIT();
3275         return;
3276 }
3277
3278 static inline int iscsi_get_mgmt_response(int status)
3279 {
3280         switch (status) {
3281         case SCST_MGMT_STATUS_SUCCESS:
3282                 return ISCSI_RESPONSE_FUNCTION_COMPLETE;
3283
3284         case SCST_MGMT_STATUS_TASK_NOT_EXIST:
3285                 return ISCSI_RESPONSE_UNKNOWN_TASK;
3286
3287         case SCST_MGMT_STATUS_LUN_NOT_EXIST:
3288                 return ISCSI_RESPONSE_UNKNOWN_LUN;
3289
3290         case SCST_MGMT_STATUS_FN_NOT_SUPPORTED:
3291                 return ISCSI_RESPONSE_FUNCTION_UNSUPPORTED;
3292
3293         case SCST_MGMT_STATUS_REJECTED:
3294         case SCST_MGMT_STATUS_FAILED:
3295         default:
3296                 return ISCSI_RESPONSE_FUNCTION_REJECTED;
3297         }
3298 }
3299
3300 static void iscsi_task_mgmt_fn_done(struct scst_mgmt_cmd *scst_mcmd)
3301 {
3302         int fn = scst_mgmt_cmd_get_fn(scst_mcmd);
3303         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
3304                                 scst_mgmt_cmd_get_tgt_priv(scst_mcmd);
3305         int status =
3306                 iscsi_get_mgmt_response(scst_mgmt_cmd_get_status(scst_mcmd));
3307
3308         if ((status == ISCSI_RESPONSE_UNKNOWN_TASK) &&
3309             (fn == SCST_ABORT_TASK)) {
3310                 /* If we are here, we found the task, so must succeed */
3311                 status = ISCSI_RESPONSE_FUNCTION_COMPLETE;
3312         }
3313
3314         TRACE_MGMT_DBG("req %p, scst_mcmd %p, fn %d, scst status %d, status %d",
3315                 req, scst_mcmd, fn, scst_mgmt_cmd_get_status(scst_mcmd),
3316                 status);
3317
3318         switch (fn) {
3319         case SCST_NEXUS_LOSS_SESS:
3320         case SCST_ABORT_ALL_TASKS_SESS:
3321                 /* They are internal */
3322                 break;
3323         default:
3324                 iscsi_send_task_mgmt_resp(req, status);
3325                 scst_mgmt_cmd_set_tgt_priv(scst_mcmd, NULL);
3326                 break;
3327         }
3328         return;
3329 }
3330
3331 static int iscsi_scsi_aen(struct scst_aen *aen)
3332 {
3333         int res = SCST_AEN_RES_SUCCESS;
3334         uint64_t lun = scst_aen_get_lun(aen);
3335         const uint8_t *sense = scst_aen_get_sense(aen);
3336         int sense_len = scst_aen_get_sense_len(aen);
3337         struct iscsi_session *sess = scst_sess_get_tgt_priv(
3338                                         scst_aen_get_sess(aen));
3339         struct iscsi_conn *conn;
3340         bool found;
3341         struct iscsi_cmnd *fake_req, *rsp;
3342         struct iscsi_async_msg_hdr *rsp_hdr;
3343         struct scatterlist *sg;
3344
3345         TRACE_ENTRY();
3346
3347         TRACE_MGMT_DBG("SCSI AEN to sess %p (initiator %s)", sess,
3348                 sess->initiator_name);
3349
3350         mutex_lock(&sess->target->target_mutex);
3351
3352         found = false;
3353         list_for_each_entry_reverse(conn, &sess->conn_list, conn_list_entry) {
3354                 if (!test_bit(ISCSI_CONN_SHUTTINGDOWN, &conn->conn_aflags) &&
3355                     (conn->conn_reinst_successor == NULL)) {
3356                         found = true;
3357                         break;
3358                 }
3359         }
3360         if (!found) {
3361                 TRACE_MGMT_DBG("Unable to find alive conn for sess %p", sess);
3362                 goto out_err;
3363         }
3364
3365         /* Create a fake request */
3366         fake_req = cmnd_alloc(conn, NULL);
3367         if (fake_req == NULL) {
3368                 PRINT_ERROR("%s", "Unable to alloc fake AEN request");
3369                 goto out_err;
3370         }
3371
3372         mutex_unlock(&sess->target->target_mutex);
3373
3374         rsp = iscsi_alloc_main_rsp(fake_req);
3375         if (rsp == NULL) {
3376                 PRINT_ERROR("%s", "Unable to alloc AEN rsp");
3377                 goto out_err_free_req;
3378         }
3379
3380         fake_req->scst_state = ISCSI_CMD_STATE_AEN;
3381         fake_req->scst_aen = aen;
3382
3383         rsp_hdr = (struct iscsi_async_msg_hdr *)&rsp->pdu.bhs;
3384
3385         rsp_hdr->opcode = ISCSI_OP_ASYNC_MSG;
3386         rsp_hdr->flags = ISCSI_FLG_FINAL;
3387         rsp_hdr->lun = lun; /* it's already in SCSI form */
3388         rsp_hdr->ffffffff = 0xffffffff;
3389         rsp_hdr->async_event = ISCSI_ASYNC_SCSI;
3390
3391         sg = rsp->sg = rsp->rsp_sg;
3392         rsp->sg_cnt = 2;
3393         rsp->own_sg = 1;
3394
3395         sg_init_table(sg, 2);
3396         sg_set_buf(&sg[0], &rsp->sense_hdr, sizeof(rsp->sense_hdr));
3397         sg_set_buf(&sg[1], sense, sense_len);
3398
3399         rsp->sense_hdr.length = cpu_to_be16(sense_len);
3400         rsp->pdu.datasize = sizeof(rsp->sense_hdr) + sense_len;
3401         rsp->bufflen = rsp->pdu.datasize;
3402
3403         req_cmnd_release(fake_req);
3404
3405 out:
3406         TRACE_EXIT_RES(res);
3407         return res;
3408
3409 out_err_free_req:
3410         req_cmnd_release(fake_req);
3411
3412 out_err:
3413         mutex_unlock(&sess->target->target_mutex);
3414         res = SCST_AEN_RES_FAILED;
3415         goto out;
3416 }
3417
3418 static int iscsi_report_aen(struct scst_aen *aen)
3419 {
3420         int res;
3421         int event_fn = scst_aen_get_event_fn(aen);
3422
3423         TRACE_ENTRY();
3424
3425         switch (event_fn) {
3426         case SCST_AEN_SCSI:
3427                 res = iscsi_scsi_aen(aen);
3428                 break;
3429         default:
3430                 TRACE_MGMT_DBG("Unsupported AEN %d", event_fn);
3431                 res = SCST_AEN_RES_NOT_SUPPORTED;
3432                 break;
3433         }
3434
3435         TRACE_EXIT_RES(res);
3436         return res;
3437 }
3438
3439 static int iscsi_target_detect(struct scst_tgt_template *templ)
3440 {
3441         /* Nothing to do */
3442         return 0;
3443 }
3444
3445 static int iscsi_target_release(struct scst_tgt *scst_tgt)
3446 {
3447         /* Nothing to do */
3448         return 0;
3449 }
3450
3451 #if !defined(CONFIG_SCST_PROC) && \
3452         (defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING))
3453 static struct scst_trace_log iscsi_local_trace_tbl[] = {
3454     { TRACE_D_WRITE,            "d_write" },
3455     { TRACE_CONN_OC,            "conn" },
3456     { TRACE_CONN_OC_DBG,        "conn_dbg" },
3457     { TRACE_D_IOV,              "iov" },
3458     { TRACE_D_DUMP_PDU,         "pdu" },
3459     { TRACE_NET_PG,             "net_page" },
3460     { 0,                        NULL }
3461 };
3462
3463 #define ISCSI_TRACE_TLB_HELP    ", d_read, d_write, conn, conn_dbg, iov, pdu, net_page"
3464 #endif
3465
3466 struct scst_tgt_template iscsi_template = {
3467         .name = "iscsi",
3468         .sg_tablesize = 0xFFFF /* no limit */,
3469         .threads_num = 0,
3470         .no_clustering = 1,
3471         .xmit_response_atomic = 0,
3472 #ifndef CONFIG_SCST_PROC
3473         .tgtt_attrs = iscsi_attrs,
3474         .tgt_attrs = iscsi_tgt_attrs,
3475         .sess_attrs = iscsi_sess_attrs,
3476         .enable_tgt = iscsi_enable_target,
3477         .is_tgt_enabled = iscsi_is_target_enabled,
3478 #endif
3479 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
3480         .default_trace_flags = ISCSI_DEFAULT_LOG_FLAGS,
3481         .trace_flags = &trace_flag,
3482 #if !defined(CONFIG_SCST_PROC) && \
3483         (defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING))
3484         .trace_tbl = iscsi_local_trace_tbl,
3485         .trace_tbl_help = ISCSI_TRACE_TLB_HELP,
3486 #endif
3487 #endif
3488         .detect = iscsi_target_detect,
3489         .release = iscsi_target_release,
3490         .xmit_response = iscsi_xmit_response,
3491 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3492         .alloc_data_buf = iscsi_alloc_data_buf,
3493 #endif
3494         .preprocessing_done = iscsi_preprocessing_done,
3495         .pre_exec = iscsi_pre_exec,
3496         .task_mgmt_affected_cmds_done = iscsi_task_mgmt_affected_cmds_done,
3497         .task_mgmt_fn_done = iscsi_task_mgmt_fn_done,
3498         .report_aen = iscsi_report_aen,
3499 };
3500
3501 static __init int iscsi_run_threads(int count, char *name, int (*fn)(void *))
3502 {
3503         int res = 0;
3504         int i;
3505         struct iscsi_thread_t *thr;
3506
3507         for (i = 0; i < count; i++) {
3508                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
3509                 if (!thr) {
3510                         res = -ENOMEM;
3511                         PRINT_ERROR("Failed to allocate thr %d", res);
3512                         goto out;
3513                 }
3514                 thr->thr = kthread_run(fn, NULL, "%s%d", name, i);
3515                 if (IS_ERR(thr->thr)) {
3516                         res = PTR_ERR(thr->thr);
3517                         PRINT_ERROR("kthread_create() failed: %d", res);
3518                         kfree(thr);
3519                         goto out;
3520                 }
3521                 list_add_tail(&thr->threads_list_entry, &iscsi_threads_list);
3522         }
3523
3524 out:
3525         return res;
3526 }
3527
3528 static void iscsi_stop_threads(void)
3529 {
3530         struct iscsi_thread_t *t, *tmp;
3531
3532         list_for_each_entry_safe(t, tmp, &iscsi_threads_list,
3533                                 threads_list_entry) {
3534                 int rc = kthread_stop(t->thr);
3535                 if (rc < 0)
3536                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
3537                 list_del(&t->threads_list_entry);
3538                 kfree(t);
3539         }
3540         return;
3541 }
3542
3543 static int __init iscsi_init(void)
3544 {
3545         int err = 0;
3546         int num;
3547
3548         PRINT_INFO("iSCSI SCST Target - version %s", ISCSI_VERSION_STRING);
3549
3550         dummy_page = alloc_pages(GFP_KERNEL, 0);
3551         if (dummy_page == NULL) {
3552                 PRINT_ERROR("%s", "Dummy page allocation failed");
3553                 goto out;
3554         }
3555
3556         sg_init_table(&dummy_sg, 1);
3557         sg_set_page(&dummy_sg, dummy_page, PAGE_SIZE, 0);
3558
3559 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3560         err = net_set_get_put_page_callbacks(iscsi_get_page_callback,
3561                         iscsi_put_page_callback);
3562         if (err != 0) {
3563                 PRINT_INFO("Unable to set page callbackes: %d", err);
3564                 goto out_free_dummy;
3565         }
3566 #else
3567 #ifndef GENERATING_UPSTREAM_PATCH
3568         PRINT_WARNING("%s",
3569                 "CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION "
3570                 "not enabled in your kernel. ISCSI-SCST will be working with "
3571                 "not the best performance. Refer README file for details.");
3572 #endif
3573 #endif
3574
3575         ctr_major = register_chrdev(0, ctr_name, &ctr_fops);
3576         if (ctr_major < 0) {
3577                 PRINT_ERROR("failed to register the control device %d",
3578                             ctr_major);
3579                 err = ctr_major;
3580                 goto out_callb;
3581         }
3582
3583         err = event_init();
3584         if (err < 0)
3585                 goto out_reg;
3586
3587         iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS);
3588         if (!iscsi_cmnd_cache) {
3589                 err = -ENOMEM;
3590                 goto out_event;
3591         }
3592
3593         err = scst_register_target_template(&iscsi_template);
3594         if (err < 0)
3595                 goto out_kmem;
3596
3597         iscsi_template_registered = 1;
3598
3599 #ifdef CONFIG_SCST_PROC
3600         err = iscsi_procfs_init();
3601         if (err < 0)
3602                 goto out_reg_tmpl;
3603 #endif
3604
3605         num = max((int)num_online_cpus(), 2);
3606
3607         err = iscsi_run_threads(num, "iscsird", istrd);
3608         if (err != 0)
3609                 goto out_thr;
3610
3611         err = iscsi_run_threads(num, "iscsiwr", istwr);
3612         if (err != 0)
3613                 goto out_thr;
3614
3615 out:
3616         return err;
3617