Fixed the following category of checkpatch complaints:
[mirror/scst/.git] / iscsi-scst / kernel / iscsi.c
1 /*
2  *  Copyright (C) 2002 - 2003 Ardis Technolgies <roman@ardistech.com>
3  *  Copyright (C) 2007 - 2009 Vladislav Bolkhovitin
4  *  Copyright (C) 2007 - 2009 ID7 Ltd.
5  *
6  *  This program is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU General Public License
8  *  as published by the Free Software Foundation, version 2
9  *  of the License.
10  *
11  *  This program is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  *  GNU General Public License for more details.
15  */
16
17 #include <linux/module.h>
18 #include <linux/hash.h>
19 #include <linux/kthread.h>
20 #include <linux/scatterlist.h>
21 #include <net/tcp.h>
22 #include <scsi/scsi.h>
23
24 #include "iscsi.h"
25 #include "digest.h"
26
27 #ifndef GENERATING_UPSTREAM_PATCH
28 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
29 #warning "Patch put_page_callback-<kernel-version>.patch not applied on your\
30  kernel or CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION\
31  config option not set. ISCSI-SCST will be working with not the best\
32  performance. Refer README file for details."
33 #endif
34 #endif
35
36 #define ISCSI_INIT_WRITE_WAKE           0x1
37
38 static int ctr_major;
39 static char ctr_name[] = "iscsi-scst-ctl";
40 static int iscsi_template_registered;
41
42 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
43 unsigned long iscsi_trace_flag = ISCSI_DEFAULT_LOG_FLAGS;
44 #endif
45
46 static struct kmem_cache *iscsi_cmnd_cache;
47
48 DEFINE_SPINLOCK(iscsi_rd_lock);
49 LIST_HEAD(iscsi_rd_list);
50 DECLARE_WAIT_QUEUE_HEAD(iscsi_rd_waitQ);
51
52 DEFINE_SPINLOCK(iscsi_wr_lock);
53 LIST_HEAD(iscsi_wr_list);
54 DECLARE_WAIT_QUEUE_HEAD(iscsi_wr_waitQ);
55
56 static struct page *dummy_page;
57 static struct scatterlist dummy_sg;
58
59 struct iscsi_thread_t {
60         struct task_struct *thr;
61         struct list_head threads_list_entry;
62 };
63
64 static LIST_HEAD(iscsi_threads_list);
65
66 static void cmnd_remove_data_wait_hash(struct iscsi_cmnd *cmnd);
67 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status);
68 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess);
69 static void req_cmnd_release(struct iscsi_cmnd *req);
70 static int iscsi_preliminary_complete(struct iscsi_cmnd *req,
71         struct iscsi_cmnd *orig_req, bool get_data);
72 static int cmnd_insert_data_wait_hash(struct iscsi_cmnd *cmnd);
73 static void __cmnd_abort(struct iscsi_cmnd *cmnd);
74 static void iscsi_set_resid(struct iscsi_cmnd *rsp, bool bufflen_set);
75 static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags);
76
77 static void req_del_from_write_timeout_list(struct iscsi_cmnd *req)
78 {
79         struct iscsi_conn *conn;
80
81         TRACE_ENTRY();
82
83         if (!req->on_write_timeout_list)
84                 goto out;
85
86         conn = req->conn;
87
88         TRACE_DBG("Deleting cmd %p from conn %p write_timeout_list",
89                 req, conn);
90
91         spin_lock_bh(&conn->write_list_lock);
92         list_del(&req->write_timeout_list_entry);
93         req->on_write_timeout_list = 0;
94         spin_unlock_bh(&conn->write_list_lock);
95
96 out:
97         TRACE_EXIT();
98         return;
99 }
100
101 static inline u32 cmnd_write_size(struct iscsi_cmnd *cmnd)
102 {
103         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
104
105         if (hdr->flags & ISCSI_CMD_WRITE)
106                 return be32_to_cpu(hdr->data_length);
107         return 0;
108 }
109
110 static inline int cmnd_read_size(struct iscsi_cmnd *cmnd)
111 {
112         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
113
114         if (hdr->flags & ISCSI_CMD_READ) {
115                 struct iscsi_ahs_hdr *ahdr;
116
117                 if (!(hdr->flags & ISCSI_CMD_WRITE))
118                         return be32_to_cpu(hdr->data_length);
119
120                 ahdr = (struct iscsi_ahs_hdr *)cmnd->pdu.ahs;
121                 if (ahdr != NULL) {
122                         uint8_t *p = (uint8_t *)ahdr;
123                         unsigned int size = 0;
124                         do {
125                                 int s;
126
127                                 ahdr = (struct iscsi_ahs_hdr *)p;
128
129                                 if (ahdr->ahstype == ISCSI_AHSTYPE_RLENGTH) {
130                                         struct iscsi_rlength_ahdr *rh =
131                                               (struct iscsi_rlength_ahdr *)ahdr;
132                                         return be32_to_cpu(rh->read_length);
133                                 }
134
135                                 s = 3 + be16_to_cpu(ahdr->ahslength);
136                                 s = (s + 3) & -4;
137                                 size += s;
138                                 p += s;
139                         } while (size < cmnd->pdu.ahssize);
140                 }
141                 return -1;
142         }
143         return 0;
144 }
145
146 void iscsi_restart_cmnd(struct iscsi_cmnd *cmnd)
147 {
148         int status;
149
150         TRACE_ENTRY();
151
152         EXTRACHECKS_BUG_ON(cmnd->r2t_len_to_receive != 0);
153         EXTRACHECKS_BUG_ON(cmnd->r2t_len_to_send != 0);
154
155         iscsi_extracheck_is_rd_thread(cmnd->conn);
156
157         req_del_from_write_timeout_list(cmnd);
158
159         /*
160          * Let's remove cmnd from the hash earlier to keep it smaller.
161          * See also corresponding comment in req_cmnd_release().
162          */
163         if (cmnd->hashed)
164                 cmnd_remove_data_wait_hash(cmnd);
165
166         if (unlikely(test_bit(ISCSI_CONN_REINSTATING,
167                         &cmnd->conn->conn_aflags))) {
168                 struct iscsi_target *target = cmnd->conn->session->target;
169                 bool get_out;
170
171                 mutex_lock(&target->target_mutex);
172
173                 get_out = test_bit(ISCSI_CONN_REINSTATING,
174                                 &cmnd->conn->conn_aflags);
175                 /* Let's don't look dead */
176                 if (scst_cmd_get_cdb(cmnd->scst_cmd)[0] == TEST_UNIT_READY)
177                         get_out = false;
178
179                 if (!get_out)
180                         goto unlock_cont;
181
182                 TRACE_MGMT_DBG("Pending cmnd %p, because conn %p is "
183                         "reinstated", cmnd, cmnd->conn);
184
185                 cmnd->scst_state = ISCSI_CMD_STATE_REINST_PENDING;
186                 list_add_tail(&cmnd->reinst_pending_cmd_list_entry,
187                         &cmnd->conn->reinst_pending_cmd_list);
188
189 unlock_cont:
190                 mutex_unlock(&target->target_mutex);
191
192                 if (get_out)
193                         goto out;
194         }
195
196         if (unlikely(cmnd->prelim_compl_flags != 0)) {
197                 if (test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags)) {
198                         TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted", cmnd,
199                                 cmnd->scst_cmd);
200                         req_cmnd_release_force(cmnd);
201                         goto out;
202                 }
203
204                 if (cmnd->scst_cmd == NULL) {
205                         TRACE_MGMT_DBG("Finishing preliminary completed cmd %p "
206                                 "with NULL scst_cmd", cmnd);
207                         req_cmnd_release(cmnd);
208                         goto out;
209                 }
210                 EXTRACHECKS_BUG_ON(scst_cmd_get_status(cmnd->scst_cmd) == 0);
211
212                 status = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
213         } else
214                 status = SCST_PREPROCESS_STATUS_SUCCESS;
215
216         cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
217
218         scst_restart_cmd(cmnd->scst_cmd, status, SCST_CONTEXT_THREAD);
219
220 out:
221         TRACE_EXIT();
222         return;
223 }
224
225 void iscsi_fail_data_waiting_cmnd(struct iscsi_cmnd *cmnd)
226 {
227         TRACE_ENTRY();
228
229         TRACE_MGMT_DBG("Failing data waiting cmnd %p", cmnd);
230
231         /*
232          * There is no race with conn_abort(), since all functions
233          * called from single read thread
234          */
235         iscsi_extracheck_is_rd_thread(cmnd->conn);
236         cmnd->r2t_len_to_receive = 0;
237         cmnd->r2t_len_to_send = 0;
238
239         req_cmnd_release_force(cmnd);
240
241         TRACE_EXIT();
242         return;
243 }
244
245 struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn,
246                               struct iscsi_cmnd *parent)
247 {
248         struct iscsi_cmnd *cmnd;
249
250         /* ToDo: __GFP_NOFAIL?? */
251 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
252         cmnd = kmem_cache_alloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
253         memset(cmnd, 0, sizeof(*cmnd));
254 #else
255         cmnd = kmem_cache_zalloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
256 #endif
257
258         atomic_set(&cmnd->ref_cnt, 1);
259         cmnd->scst_state = ISCSI_CMD_STATE_NEW;
260         cmnd->conn = conn;
261         cmnd->parent_req = parent;
262
263         if (parent == NULL) {
264                 conn_get(conn);
265
266 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
267                 atomic_set(&cmnd->net_ref_cnt, 0);
268 #endif
269                 INIT_LIST_HEAD(&cmnd->rsp_cmd_list);
270                 INIT_LIST_HEAD(&cmnd->rx_ddigest_cmd_list);
271                 cmnd->target_task_tag = cpu_to_be32(ISCSI_RESERVED_TAG);
272
273                 spin_lock_bh(&conn->cmd_list_lock);
274                 list_add_tail(&cmnd->cmd_list_entry, &conn->cmd_list);
275                 spin_unlock_bh(&conn->cmd_list_lock);
276         }
277
278         TRACE_DBG("conn %p, parent %p, cmnd %p", conn, parent, cmnd);
279         return cmnd;
280 }
281
282 /* Frees a command. Also frees the additional header. */
283 static void cmnd_free(struct iscsi_cmnd *cmnd)
284 {
285         TRACE_ENTRY();
286
287         TRACE_DBG("cmnd %p", cmnd);
288
289         if (unlikely(test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags))) {
290                 TRACE_MGMT_DBG("Free aborted cmd %p (scst cmd %p, state %d, "
291                         "parent_req %p)", cmnd, cmnd->scst_cmd,
292                         cmnd->scst_state, cmnd->parent_req);
293         }
294
295         /* Catch users from cmd_list or rsp_cmd_list */
296         EXTRACHECKS_BUG_ON(atomic_read(&cmnd->ref_cnt) != 0);
297
298         kfree(cmnd->pdu.ahs);
299
300 #ifdef CONFIG_SCST_EXTRACHECKS
301         if (unlikely(cmnd->on_write_list || cmnd->on_write_timeout_list)) {
302                 struct iscsi_scsi_cmd_hdr *req = cmnd_hdr(cmnd);
303
304                 PRINT_CRIT_ERROR("cmnd %p still on some list?, %x, %x, %x, "
305                         "%x, %x, %x, %x", cmnd, req->opcode, req->scb[0],
306                         req->flags, req->itt, be32_to_cpu(req->data_length),
307                         req->cmd_sn, be32_to_cpu(cmnd->pdu.datasize));
308
309                 if (unlikely(cmnd->parent_req)) {
310                         struct iscsi_scsi_cmd_hdr *preq =
311                                         cmnd_hdr(cmnd->parent_req);
312                         PRINT_CRIT_ERROR("%p %x %u", preq, preq->opcode,
313                                 preq->scb[0]);
314                 }
315                 sBUG();
316         }
317 #endif
318
319         kmem_cache_free(iscsi_cmnd_cache, cmnd);
320
321         TRACE_EXIT();
322         return;
323 }
324
325 /* Might be called under some lock and on SIRQ */
326 void cmnd_done(struct iscsi_cmnd *cmnd)
327 {
328         TRACE_ENTRY();
329
330         TRACE_DBG("cmnd %p", cmnd);
331
332         if (unlikely(test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags))) {
333                 TRACE_MGMT_DBG("Done aborted cmd %p (scst cmd %p, state %d, "
334                         "parent_req %p)", cmnd, cmnd->scst_cmd,
335                         cmnd->scst_state, cmnd->parent_req);
336         }
337
338         EXTRACHECKS_BUG_ON(cmnd->on_rx_digest_list);
339         EXTRACHECKS_BUG_ON(cmnd->hashed);
340
341         req_del_from_write_timeout_list(cmnd);
342
343         if (cmnd->parent_req == NULL) {
344                 struct iscsi_conn *conn = cmnd->conn;
345                 struct iscsi_cmnd *rsp, *t;
346
347                 TRACE_DBG("Deleting req %p from conn %p", cmnd, conn);
348
349                 spin_lock_bh(&conn->cmd_list_lock);
350                 list_del(&cmnd->cmd_list_entry);
351                 spin_unlock_bh(&conn->cmd_list_lock);
352
353                 conn_put(conn);
354
355                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rx_ddigest_cmd_list));
356
357                 /* Order between above and below code is important! */
358
359                 if ((cmnd->scst_cmd != NULL) || (cmnd->scst_aen != NULL)) {
360                         switch (cmnd->scst_state) {
361                         case ISCSI_CMD_STATE_PROCESSED:
362                                 TRACE_DBG("cmd %p PROCESSED", cmnd);
363                                 scst_tgt_cmd_done(cmnd->scst_cmd,
364                                         SCST_CONTEXT_DIRECT_ATOMIC);
365                                 break;
366
367                         case ISCSI_CMD_STATE_AFTER_PREPROC:
368                         {
369                                 /* It can be for some aborted commands */
370                                 struct scst_cmd *scst_cmd = cmnd->scst_cmd;
371                                 TRACE_DBG("cmd %p AFTER_PREPROC", cmnd);
372                                 cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
373                                 cmnd->scst_cmd = NULL;
374                                 scst_restart_cmd(scst_cmd,
375                                         SCST_PREPROCESS_STATUS_ERROR_FATAL,
376                                         SCST_CONTEXT_THREAD);
377                                 break;
378                         }
379
380                         case ISCSI_CMD_STATE_AEN:
381                                 TRACE_DBG("cmd %p AEN PROCESSED", cmnd);
382                                 scst_aen_done(cmnd->scst_aen);
383                                 break;
384
385                         case ISCSI_CMD_STATE_OUT_OF_SCST_PRELIM_COMPL:
386                                 break;
387
388                         default:
389                                 PRINT_CRIT_ERROR("Unexpected cmnd scst state "
390                                         "%d", cmnd->scst_state);
391                                 sBUG();
392                                 break;
393                         }
394                 }
395
396                 if (cmnd->own_sg) {
397                         TRACE_DBG("own_sg for req %p", cmnd);
398                         if (cmnd->sg != &dummy_sg)
399                                 scst_free(cmnd->sg, cmnd->sg_cnt);
400 #ifdef CONFIG_SCST_DEBUG
401                         cmnd->own_sg = 0;
402                         cmnd->sg = NULL;
403                         cmnd->sg_cnt = -1;
404 #endif
405                 }
406
407                 if (cmnd->dec_active_cmnds) {
408                         struct iscsi_session *sess = cmnd->conn->session;
409                         TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
410                                 "new value %d)", cmnd, sess,
411                                 atomic_read(&sess->active_cmds)-1);
412                         atomic_dec(&sess->active_cmds);
413 #ifdef CONFIG_SCST_EXTRACHECKS
414                         if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
415                                 PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
416                                         atomic_read(&sess->active_cmds));
417                                 sBUG();
418                         }
419 #endif
420                 }
421
422                 list_for_each_entry_safe(rsp, t, &cmnd->rsp_cmd_list,
423                                         rsp_cmd_list_entry) {
424                         cmnd_free(rsp);
425                 }
426
427                 cmnd_free(cmnd);
428         } else {
429                 if (cmnd->own_sg) {
430                         TRACE_DBG("own_sg for rsp %p", cmnd);
431                         if ((cmnd->sg != &dummy_sg) && (cmnd->sg != cmnd->rsp_sg))
432                                 scst_free(cmnd->sg, cmnd->sg_cnt);
433 #ifdef CONFIG_SCST_DEBUG
434                         cmnd->own_sg = 0;
435                         cmnd->sg = NULL;
436                         cmnd->sg_cnt = -1;
437 #endif
438                 }
439
440                 EXTRACHECKS_BUG_ON(cmnd->dec_active_cmnds);
441
442                 if (cmnd == cmnd->parent_req->main_rsp) {
443                         TRACE_DBG("Finishing main rsp %p (req %p)", cmnd,
444                                 cmnd->parent_req);
445                         cmnd->parent_req->main_rsp = NULL;
446                 }
447
448                 cmnd_put(cmnd->parent_req);
449                 /*
450                  * rsp will be freed on the last parent's put and can already
451                  * be freed!!
452                  */
453         }
454
455         TRACE_EXIT();
456         return;
457 }
458
459 /*
460  * Corresponding conn may also get destroyed after this function, except only
461  * if it's called from the read thread!
462  *
463  * It can't be called in parallel with iscsi_cmnds_init_write()!
464  */
465 void req_cmnd_release_force(struct iscsi_cmnd *req)
466 {
467         struct iscsi_cmnd *rsp, *t;
468         struct iscsi_conn *conn = req->conn;
469         LIST_HEAD(cmds_list);
470
471         TRACE_ENTRY();
472
473         TRACE_MGMT_DBG("req %p", req);
474
475         sBUG_ON(req == conn->read_cmnd);
476
477         spin_lock_bh(&conn->write_list_lock);
478         list_for_each_entry_safe(rsp, t, &conn->write_list, write_list_entry) {
479                 if (rsp->parent_req != req)
480                         continue;
481
482                 cmd_del_from_write_list(rsp);
483
484                 list_add_tail(&rsp->write_list_entry, &cmds_list);
485         }
486         spin_unlock_bh(&conn->write_list_lock);
487
488         list_for_each_entry_safe(rsp, t, &cmds_list, write_list_entry) {
489                 TRACE_MGMT_DBG("Putting write rsp %p", rsp);
490                 list_del(&rsp->write_list_entry);
491                 cmnd_put(rsp);
492         }
493
494         /* Supposed nobody can add responses in the list anymore */
495         list_for_each_entry_reverse(rsp, &req->rsp_cmd_list,
496                         rsp_cmd_list_entry) {
497                 bool r;
498
499                 if (rsp->force_cleanup_done)
500                         continue;
501
502                 rsp->force_cleanup_done = 1;
503
504                 if (cmnd_get_check(rsp))
505                         continue;
506
507                 spin_lock_bh(&conn->write_list_lock);
508                 r = rsp->on_write_list || rsp->write_processing_started;
509                 spin_unlock_bh(&conn->write_list_lock);
510
511                 cmnd_put(rsp);
512
513                 if (r)
514                         continue;
515
516                 /*
517                  * If both on_write_list and write_processing_started not set,
518                  * we can safely put() rsp.
519                  */
520                 TRACE_MGMT_DBG("Putting rsp %p", rsp);
521                 cmnd_put(rsp);
522         }
523
524         if (req->main_rsp != NULL) {
525                 TRACE_MGMT_DBG("Putting main rsp %p", req->main_rsp);
526                 cmnd_put(req->main_rsp);
527                 req->main_rsp = NULL;
528         }
529
530         req_cmnd_release(req);
531
532         TRACE_EXIT();
533         return;
534 }
535
536 /*
537  * Corresponding conn may also get destroyed after this function, except only
538  * if it's called from the read thread!
539  */
540 static void req_cmnd_release(struct iscsi_cmnd *req)
541 {
542         struct iscsi_cmnd *c, *t;
543
544         TRACE_ENTRY();
545
546         TRACE_DBG("req %p", req);
547
548 #ifdef CONFIG_SCST_EXTRACHECKS
549         sBUG_ON(req->release_called);
550         req->release_called = 1;
551 #endif
552
553         if (unlikely(test_bit(ISCSI_CMD_ABORTED, &req->prelim_compl_flags))) {
554                 TRACE_MGMT_DBG("Release aborted req cmd %p (scst cmd %p, "
555                         "state %d)", req, req->scst_cmd, req->scst_state);
556         }
557
558         sBUG_ON(req->parent_req != NULL);
559
560         /*
561          * We have to remove hashed req from the hash list before sending
562          * response. Otherwise we can have a race, when for some reason cmd's
563          * release (and, hence, removal from the hash) is delayed after the
564          * transmission and initiator sends cmd with the same ITT, hence
565          * the new command will be erroneously rejected as a duplicate.
566          */
567         if (unlikely(req->hashed)) {
568                 /* It sometimes can happen during errors recovery */
569                 cmnd_remove_data_wait_hash(req);
570         }
571
572         if (unlikely(req->main_rsp != NULL)) {
573                 TRACE_DBG("Sending main rsp %p", req->main_rsp);
574                 iscsi_cmnd_init_write(req->main_rsp, ISCSI_INIT_WRITE_WAKE);
575                 req->main_rsp = NULL;
576         }
577
578         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
579                                 rx_ddigest_cmd_list_entry) {
580                 cmd_del_from_rx_ddigest_list(c);
581                 cmnd_put(c);
582         }
583
584         EXTRACHECKS_BUG_ON(req->pending);
585
586         if (req->dec_active_cmnds) {
587                 struct iscsi_session *sess = req->conn->session;
588                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
589                         "new value %d)", req, sess,
590                         atomic_read(&sess->active_cmds)-1);
591                 atomic_dec(&sess->active_cmds);
592                 req->dec_active_cmnds = 0;
593 #ifdef CONFIG_SCST_EXTRACHECKS
594                 if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
595                         PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
596                                 atomic_read(&sess->active_cmds));
597                         sBUG();
598                 }
599 #endif
600         }
601
602         cmnd_put(req);
603
604         TRACE_EXIT();
605         return;
606 }
607
608 /*
609  * Corresponding conn may also get destroyed after this function, except only
610  * if it's called from the read thread!
611  */
612 void rsp_cmnd_release(struct iscsi_cmnd *cmnd)
613 {
614         TRACE_DBG("%p", cmnd);
615
616 #ifdef CONFIG_SCST_EXTRACHECKS
617         sBUG_ON(cmnd->release_called);
618         cmnd->release_called = 1;
619 #endif
620
621         EXTRACHECKS_BUG_ON(cmnd->parent_req == NULL);
622
623         cmnd_put(cmnd);
624         return;
625 }
626
627 static struct iscsi_cmnd *iscsi_alloc_rsp(struct iscsi_cmnd *parent)
628 {
629         struct iscsi_cmnd *rsp;
630
631         TRACE_ENTRY();
632
633         rsp = cmnd_alloc(parent->conn, parent);
634
635         TRACE_DBG("Adding rsp %p to parent %p", rsp, parent);
636         list_add_tail(&rsp->rsp_cmd_list_entry, &parent->rsp_cmd_list);
637
638         cmnd_get(parent);
639
640         TRACE_EXIT_HRES((unsigned long)rsp);
641         return rsp;
642 }
643
644 static inline struct iscsi_cmnd *iscsi_alloc_main_rsp(struct iscsi_cmnd *parent)
645 {
646         struct iscsi_cmnd *rsp;
647
648         TRACE_ENTRY();
649
650         EXTRACHECKS_BUG_ON(parent->main_rsp != NULL);
651
652         rsp = iscsi_alloc_rsp(parent);
653         parent->main_rsp = rsp;
654
655         TRACE_EXIT_HRES((unsigned long)rsp);
656         return rsp;
657 }
658
659 static void iscsi_cmnds_init_write(struct list_head *send, int flags)
660 {
661         struct iscsi_cmnd *rsp = list_entry(send->next, struct iscsi_cmnd,
662                                                 write_list_entry);
663         struct iscsi_conn *conn = rsp->conn;
664         struct list_head *pos, *next;
665
666         sBUG_ON(list_empty(send));
667
668         if (!(conn->ddigest_type & DIGEST_NONE)) {
669                 list_for_each(pos, send) {
670                         rsp = list_entry(pos, struct iscsi_cmnd,
671                                                 write_list_entry);
672
673                         if (rsp->pdu.datasize != 0) {
674                                 TRACE_DBG("Doing data digest (%p:%x)", rsp,
675                                         cmnd_opcode(rsp));
676                                 digest_tx_data(rsp);
677                         }
678                 }
679         }
680
681         spin_lock_bh(&conn->write_list_lock);
682         list_for_each_safe(pos, next, send) {
683                 rsp = list_entry(pos, struct iscsi_cmnd, write_list_entry);
684
685                 TRACE_DBG("%p:%x", rsp, cmnd_opcode(rsp));
686
687                 sBUG_ON(conn != rsp->conn);
688
689                 list_del(&rsp->write_list_entry);
690                 cmd_add_on_write_list(conn, rsp);
691         }
692         spin_unlock_bh(&conn->write_list_lock);
693
694         if (flags & ISCSI_INIT_WRITE_WAKE)
695                 iscsi_make_conn_wr_active(conn);
696
697         return;
698 }
699
700 static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags)
701 {
702         LIST_HEAD(head);
703
704 #ifdef CONFIG_SCST_EXTRACHECKS
705         if (unlikely(rsp->on_write_list)) {
706                 PRINT_CRIT_ERROR("cmd already on write list (%x %x %x "
707                         "%u %u %d %d", cmnd_itt(rsp),
708                         cmnd_opcode(rsp), cmnd_scsicode(rsp),
709                         rsp->hdigest, rsp->ddigest,
710                         list_empty(&rsp->rsp_cmd_list), rsp->hashed);
711                 sBUG();
712         }
713 #endif
714         list_add_tail(&rsp->write_list_entry, &head);
715         iscsi_cmnds_init_write(&head, flags);
716         return;
717 }
718
719 static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
720 {
721         struct iscsi_cmnd *rsp;
722         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
723         struct iscsi_data_in_hdr *rsp_hdr;
724         u32 pdusize, expsize, size, offset, sn;
725         LIST_HEAD(send);
726
727         TRACE_DBG("req %p", req);
728
729         pdusize = req->conn->session->sess_param.max_xmit_data_length;
730         expsize = req->read_size;
731         size = min(expsize, (u32)req->bufflen);
732         offset = 0;
733         sn = 0;
734
735         while (1) {
736                 rsp = iscsi_alloc_rsp(req);
737                 TRACE_DBG("rsp %p", rsp);
738                 rsp->sg = req->sg;
739                 rsp->sg_cnt = req->sg_cnt;
740                 rsp->bufflen = req->bufflen;
741                 rsp_hdr = (struct iscsi_data_in_hdr *)&rsp->pdu.bhs;
742
743                 rsp_hdr->opcode = ISCSI_OP_SCSI_DATA_IN;
744                 rsp_hdr->itt = req_hdr->itt;
745                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
746                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
747                 rsp_hdr->data_sn = cpu_to_be32(sn);
748
749                 if (size <= pdusize) {
750                         TRACE_DBG("offset %d, size %d", offset, size);
751                         rsp->pdu.datasize = size;
752                         if (send_status) {
753                                 unsigned int scsisize;
754
755                                 TRACE_DBG("status %x", status);
756
757                                 EXTRACHECKS_BUG_ON((cmnd_hdr(req)->flags & ISCSI_CMD_WRITE) != 0);
758
759                                 rsp_hdr->flags = ISCSI_FLG_FINAL | ISCSI_FLG_STATUS;
760                                 rsp_hdr->cmd_status = status;
761
762                                 scsisize = req->bufflen;
763                                 if (scsisize < expsize) {
764                                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
765                                         size = expsize - scsisize;
766                                 } else if (scsisize > expsize) {
767                                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
768                                         size = scsisize - expsize;
769                                 } else
770                                         size = 0;
771                                 rsp_hdr->residual_count = cpu_to_be32(size);
772                         }
773                         list_add_tail(&rsp->write_list_entry, &send);
774                         break;
775                 }
776
777                 TRACE_DBG("pdusize %d, offset %d, size %d", pdusize, offset,
778                         size);
779
780                 rsp->pdu.datasize = pdusize;
781
782                 size -= pdusize;
783                 offset += pdusize;
784                 sn++;
785
786                 list_add_tail(&rsp->write_list_entry, &send);
787         }
788         iscsi_cmnds_init_write(&send, 0);
789         return;
790 }
791
792 static void iscsi_init_status_rsp(struct iscsi_cmnd *rsp,
793         int status, const u8 *sense_buf, int sense_len, bool bufflen_set)
794 {
795         struct iscsi_cmnd *req = rsp->parent_req;
796         struct iscsi_scsi_rsp_hdr *rsp_hdr;
797         struct scatterlist *sg;
798
799         TRACE_ENTRY();
800
801         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
802         rsp_hdr->opcode = ISCSI_OP_SCSI_RSP;
803         rsp_hdr->flags = ISCSI_FLG_FINAL;
804         rsp_hdr->response = ISCSI_RESPONSE_COMMAND_COMPLETED;
805         rsp_hdr->cmd_status = status;
806         rsp_hdr->itt = cmnd_hdr(req)->itt;
807
808         if (SCST_SENSE_VALID(sense_buf)) {
809                 TRACE_DBG("%s", "SENSE VALID");
810
811                 sg = rsp->sg = rsp->rsp_sg;
812                 rsp->sg_cnt = 2;
813                 rsp->own_sg = 1;
814
815                 sg_init_table(sg, 2);
816                 sg_set_buf(&sg[0], &rsp->sense_hdr, sizeof(rsp->sense_hdr));
817                 sg_set_buf(&sg[1], sense_buf, sense_len);
818
819                 rsp->sense_hdr.length = cpu_to_be16(sense_len);
820
821                 rsp->pdu.datasize = sizeof(rsp->sense_hdr) + sense_len;
822                 rsp->bufflen = rsp->pdu.datasize;
823         } else {
824                 rsp->pdu.datasize = 0;
825                 rsp->bufflen = 0;
826         }
827
828         iscsi_set_resid(rsp, bufflen_set);
829
830         TRACE_EXIT();
831         return;
832 }
833
834 static inline struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req,
835         int status, const u8 *sense_buf, int sense_len, bool bufflen_set)
836 {
837         struct iscsi_cmnd *rsp;
838
839         TRACE_ENTRY();
840
841         rsp = iscsi_alloc_rsp(req);
842         TRACE_DBG("rsp %p", rsp);
843
844         iscsi_init_status_rsp(rsp, status, sense_buf, sense_len, bufflen_set);
845
846         TRACE_EXIT_HRES((unsigned long)rsp);
847         return rsp;
848 }
849
850 static struct iscsi_cmnd *create_prelim_status_rsp(struct iscsi_cmnd *req,
851         int status, const u8 *sense_buf, int sense_len)
852 {
853         struct iscsi_cmnd *rsp;
854
855         TRACE_ENTRY();
856
857         rsp = iscsi_alloc_main_rsp(req);
858         TRACE_DBG("main rsp %p", rsp);
859
860         iscsi_init_status_rsp(rsp, status, sense_buf, sense_len, false);
861
862         TRACE_EXIT_HRES((unsigned long)rsp);
863         return rsp;
864 }
865
866 static int iscsi_set_prelim_r2t_len_to_receive(struct iscsi_cmnd *req)
867 {
868         struct iscsi_hdr *req_hdr = &req->pdu.bhs;
869         int res = 0;
870
871         TRACE_ENTRY();
872
873         if (req_hdr->flags & ISCSI_CMD_FINAL)
874                 goto out;
875
876         res = cmnd_insert_data_wait_hash(req);
877         if (res != 0) {
878                 /*
879                  * We have to close connection, because otherwise a data
880                  * corruption is possible if we allow to receive data
881                  * for this request in another request with dublicated ITT.
882                  */
883                 mark_conn_closed(req->conn);
884                 goto out;
885         }
886
887         /*
888          * We need to wait for one or more PDUs. Let's simplify
889          * other code and pretend we need to receive 1 byte.
890          * In data_out_start() we will correct it.
891          */
892         if (req->outstanding_r2t == 0) {
893                 req->outstanding_r2t = 1;
894                 req_add_to_write_timeout_list(req);
895         }
896         req->r2t_len_to_receive = 1;
897         req->r2t_len_to_send = 0;
898
899         TRACE_DBG("req %p, op %x, outstanding_r2t %d, r2t_len_to_receive %d, "
900                 "r2t_len_to_send %d", req, cmnd_opcode(req),
901                 req->outstanding_r2t, req->r2t_len_to_receive,
902                 req->r2t_len_to_send);
903
904 out:
905         TRACE_EXIT_RES(res);
906         return res;
907 }
908
909 static int create_preliminary_status_rsp(struct iscsi_cmnd *req,
910         int status, const u8 *sense_buf, int sense_len)
911 {
912         int res = 0;
913         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
914
915         TRACE_ENTRY();
916
917         if (req->prelim_compl_flags != 0) {
918                 TRACE_MGMT_DBG("req %p already prelim completed", req);
919                 goto out;
920         }
921
922         req->scst_state = ISCSI_CMD_STATE_OUT_OF_SCST_PRELIM_COMPL;
923
924         if ((req_hdr->flags & ISCSI_CMD_READ) &&
925             (req_hdr->flags & ISCSI_CMD_WRITE)) {
926                 int sz = cmnd_read_size(req);
927                 if (sz > 0)
928                         req->read_size = sz;
929         } else if (req_hdr->flags & ISCSI_CMD_READ)
930                 req->read_size = be32_to_cpu(req_hdr->data_length);
931
932         create_prelim_status_rsp(req, status, sense_buf, sense_len);
933         res = iscsi_preliminary_complete(req, req, true);
934
935 out:
936         TRACE_EXIT_RES(res);
937         return res;
938 }
939
940 static int set_scst_preliminary_status_rsp(struct iscsi_cmnd *req,
941         bool get_data, int key, int asc, int ascq)
942 {
943         int res = 0;
944
945         TRACE_ENTRY();
946
947         if (req->scst_cmd == NULL) {
948                 /* There must be already error set */
949                 goto complete;
950         }
951
952         scst_set_cmd_error(req->scst_cmd, key, asc, ascq);
953
954 complete:
955         res = iscsi_preliminary_complete(req, req, get_data);
956
957         TRACE_EXIT_RES(res);
958         return res;
959 }
960
961 static int create_reject_rsp(struct iscsi_cmnd *req, int reason, bool get_data)
962 {
963         int res = 0;
964         struct iscsi_cmnd *rsp;
965         struct iscsi_reject_hdr *rsp_hdr;
966         struct scatterlist *sg;
967
968         TRACE_ENTRY();
969
970         TRACE_MGMT_DBG("Reject: req %p, reason %x", req, reason);
971
972         if (cmnd_opcode(req) == ISCSI_OP_SCSI_CMD) {
973                 if (req->scst_cmd == NULL) {
974                         /* BUSY status must be already set */
975                         struct iscsi_scsi_rsp_hdr *rsp_hdr;
976                         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&req->main_rsp->pdu.bhs;
977                         sBUG_ON(rsp_hdr->cmd_status == 0);
978                         /*
979                          * Let's not send REJECT here. The initiator will retry
980                          * and, hopefully, next time we will not fail allocating
981                          * scst_cmd, so we will then send the REJECT.
982                          */
983                         goto out;
984                 } else
985                         set_scst_preliminary_status_rsp(req, get_data,
986                                 SCST_LOAD_SENSE(scst_sense_invalid_message));
987         }
988
989         rsp = iscsi_alloc_main_rsp(req);
990         rsp_hdr = (struct iscsi_reject_hdr *)&rsp->pdu.bhs;
991
992         rsp_hdr->opcode = ISCSI_OP_REJECT;
993         rsp_hdr->ffffffff = ISCSI_RESERVED_TAG;
994         rsp_hdr->reason = reason;
995
996         sg = rsp->sg = rsp->rsp_sg;
997         rsp->sg_cnt = 1;
998         rsp->own_sg = 1;
999         sg_init_one(sg, &req->pdu.bhs, sizeof(struct iscsi_hdr));
1000         rsp->bufflen = rsp->pdu.datasize = sizeof(struct iscsi_hdr);
1001
1002         res = iscsi_preliminary_complete(req, req, true);
1003
1004 out:
1005         TRACE_EXIT_RES(res);
1006         return res;
1007 }
1008
1009 static inline int iscsi_get_allowed_cmds(struct iscsi_session *sess)
1010 {
1011         int res = max(-1, (int)sess->max_queued_cmnds -
1012                                 atomic_read(&sess->active_cmds)-1);
1013         TRACE_DBG("allowed cmds %d (sess %p, active_cmds %d)", res,
1014                 sess, atomic_read(&sess->active_cmds));
1015         return res;
1016 }
1017
1018 static u32 cmnd_set_sn(struct iscsi_cmnd *cmnd, int set_stat_sn)
1019 {
1020         struct iscsi_conn *conn = cmnd->conn;
1021         struct iscsi_session *sess = conn->session;
1022         u32 res;
1023
1024         spin_lock(&sess->sn_lock);
1025
1026         if (set_stat_sn)
1027                 cmnd->pdu.bhs.sn = cpu_to_be32(conn->stat_sn++);
1028         cmnd->pdu.bhs.exp_sn = cpu_to_be32(sess->exp_cmd_sn);
1029         cmnd->pdu.bhs.max_sn = cpu_to_be32(sess->exp_cmd_sn +
1030                                  iscsi_get_allowed_cmds(sess));
1031
1032         res = cpu_to_be32(conn->stat_sn);
1033
1034         spin_unlock(&sess->sn_lock);
1035         return res;
1036 }
1037
1038 /* Called under sn_lock */
1039 static void __update_stat_sn(struct iscsi_cmnd *cmnd)
1040 {
1041         struct iscsi_conn *conn = cmnd->conn;
1042         u32 exp_stat_sn;
1043
1044         cmnd->pdu.bhs.exp_sn = exp_stat_sn = be32_to_cpu(cmnd->pdu.bhs.exp_sn);
1045         TRACE_DBG("%x,%x", cmnd_opcode(cmnd), exp_stat_sn);
1046         if ((int)(exp_stat_sn - conn->exp_stat_sn) > 0 &&
1047             (int)(exp_stat_sn - conn->stat_sn) <= 0) {
1048                 /* free pdu resources */
1049                 cmnd->conn->exp_stat_sn = exp_stat_sn;
1050         }
1051         return;
1052 }
1053
1054 static inline void update_stat_sn(struct iscsi_cmnd *cmnd)
1055 {
1056         spin_lock(&cmnd->conn->session->sn_lock);
1057         __update_stat_sn(cmnd);
1058         spin_unlock(&cmnd->conn->session->sn_lock);
1059         return;
1060 }
1061
1062 /* Called under sn_lock */
1063 static int check_cmd_sn(struct iscsi_cmnd *cmnd)
1064 {
1065         struct iscsi_session *session = cmnd->conn->session;
1066         u32 cmd_sn;
1067
1068         cmnd->pdu.bhs.sn = cmd_sn = be32_to_cpu(cmnd->pdu.bhs.sn);
1069         TRACE_DBG("%d(%d)", cmd_sn, session->exp_cmd_sn);
1070         if (likely((s32)(cmd_sn - session->exp_cmd_sn) >= 0))
1071                 return 0;
1072         PRINT_ERROR("sequence error (%x,%x)", cmd_sn, session->exp_cmd_sn);
1073         return -ISCSI_REASON_PROTOCOL_ERROR;
1074 }
1075
1076 static struct iscsi_cmnd *cmnd_find_itt_get(struct iscsi_conn *conn, u32 itt)
1077 {
1078         struct iscsi_cmnd *cmnd, *found_cmnd = NULL;
1079
1080         spin_lock_bh(&conn->cmd_list_lock);
1081         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1082                 if ((cmnd->pdu.bhs.itt == itt) && !cmnd_get_check(cmnd)) {
1083                         found_cmnd = cmnd;
1084                         break;
1085                 }
1086         }
1087         spin_unlock_bh(&conn->cmd_list_lock);
1088
1089         return found_cmnd;
1090 }
1091
1092 /**
1093  ** We use the ITT hash only to find original request PDU for subsequent
1094  ** Data-Out PDUs.
1095  **/
1096
1097 /* Must be called under cmnd_data_wait_hash_lock */
1098 static struct iscsi_cmnd *__cmnd_find_data_wait_hash(struct iscsi_conn *conn,
1099         u32 itt)
1100 {
1101         struct list_head *head;
1102         struct iscsi_cmnd *cmnd;
1103
1104         head = &conn->session->cmnd_data_wait_hash[cmnd_hashfn(itt)];
1105
1106         list_for_each_entry(cmnd, head, hash_list_entry) {
1107                 if (cmnd->pdu.bhs.itt == itt)
1108                         return cmnd;
1109         }
1110         return NULL;
1111 }
1112
1113 static struct iscsi_cmnd *cmnd_find_data_wait_hash(struct iscsi_conn *conn,
1114         u32 itt)
1115 {
1116         struct iscsi_cmnd *res;
1117         struct iscsi_session *session = conn->session;
1118
1119         spin_lock(&session->cmnd_data_wait_hash_lock);
1120         res = __cmnd_find_data_wait_hash(conn, itt);
1121         spin_unlock(&session->cmnd_data_wait_hash_lock);
1122
1123         return res;
1124 }
1125
1126 static inline u32 get_next_ttt(struct iscsi_conn *conn)
1127 {
1128         u32 ttt;
1129         struct iscsi_session *session = conn->session;
1130
1131         /* Not compatible with MC/S! */
1132
1133         iscsi_extracheck_is_rd_thread(conn);
1134
1135         if (unlikely(session->next_ttt == ISCSI_RESERVED_TAG))
1136                 session->next_ttt++;
1137         ttt = session->next_ttt++;
1138
1139         return ttt;
1140 }
1141
1142 static int cmnd_insert_data_wait_hash(struct iscsi_cmnd *cmnd)
1143 {
1144         struct iscsi_session *session = cmnd->conn->session;
1145         struct iscsi_cmnd *tmp;
1146         struct list_head *head;
1147         int err = 0;
1148         u32 itt = cmnd->pdu.bhs.itt;
1149
1150         if (unlikely(cmnd->hashed)) {
1151                 /* It can be for preliminary completed commands */
1152                 goto out;
1153         }
1154
1155         /*
1156          * We don't need TTT, because ITT/buffer_offset pair is sufficient
1157          * to find out the original request and buffer for Data-Out PDUs, but
1158          * crazy iSCSI spec requires us to send this superfluous field in
1159          * R2T PDUs and some initiators may rely on it.
1160          */
1161         cmnd->target_task_tag = get_next_ttt(cmnd->conn);
1162
1163         TRACE_DBG("%p:%x", cmnd, itt);
1164         if (unlikely(itt == ISCSI_RESERVED_TAG)) {
1165                 PRINT_ERROR("%s", "ITT is RESERVED_TAG");
1166                 PRINT_BUFFER("Incorrect BHS", &cmnd->pdu.bhs,
1167                         sizeof(cmnd->pdu.bhs));
1168                 err = -ISCSI_REASON_PROTOCOL_ERROR;
1169                 goto out;
1170         }
1171
1172         spin_lock(&session->cmnd_data_wait_hash_lock);
1173
1174         head = &session->cmnd_data_wait_hash[cmnd_hashfn(itt)];
1175
1176         tmp = __cmnd_find_data_wait_hash(cmnd->conn, itt);
1177         if (likely(!tmp)) {
1178                 TRACE_DBG("Adding cmnd %p to the hash (ITT %x)", cmnd,
1179                         cmnd_itt(cmnd));
1180                 list_add_tail(&cmnd->hash_list_entry, head);
1181                 cmnd->hashed = 1;
1182         } else {
1183                 PRINT_ERROR("Task %x in progress, cmnd %p", itt, cmnd);
1184                 err = -ISCSI_REASON_TASK_IN_PROGRESS;
1185         }
1186
1187         spin_unlock(&session->cmnd_data_wait_hash_lock);
1188
1189 out:
1190         return err;
1191 }
1192
1193 static void cmnd_remove_data_wait_hash(struct iscsi_cmnd *cmnd)
1194 {
1195         struct iscsi_session *session = cmnd->conn->session;
1196         struct iscsi_cmnd *tmp;
1197
1198         spin_lock(&session->cmnd_data_wait_hash_lock);
1199
1200         tmp = __cmnd_find_data_wait_hash(cmnd->conn, cmnd->pdu.bhs.itt);
1201
1202         if (likely(tmp && tmp == cmnd)) {
1203                 TRACE_DBG("Deleting cmnd %p from the hash (ITT %x)", cmnd,
1204                         cmnd_itt(cmnd));
1205                 list_del(&cmnd->hash_list_entry);
1206                 cmnd->hashed = 0;
1207         } else
1208                 PRINT_ERROR("%p:%x not found", cmnd, cmnd_itt(cmnd));
1209
1210         spin_unlock(&session->cmnd_data_wait_hash_lock);
1211
1212         return;
1213 }
1214
1215 static void cmnd_prepare_get_rejected_immed_data(struct iscsi_cmnd *cmnd)
1216 {
1217         struct iscsi_conn *conn = cmnd->conn;
1218         struct scatterlist *sg = cmnd->sg;
1219         char __user *addr;
1220         u32 size;
1221         unsigned int i;
1222
1223         TRACE_ENTRY();
1224
1225         TRACE_DBG_FLAG(iscsi_get_flow_ctrl_or_mgmt_dbg_log_flag(cmnd),
1226                 "Skipping (cmnd %p, ITT %x, op %x, cmd op %x, "
1227                 "datasize %u, scst_cmd %p, scst state %d)", cmnd,
1228                 cmnd_itt(cmnd), cmnd_opcode(cmnd), cmnd_hdr(cmnd)->scb[0],
1229                 cmnd->pdu.datasize, cmnd->scst_cmd, cmnd->scst_state);
1230
1231         iscsi_extracheck_is_rd_thread(conn);
1232
1233         size = cmnd->pdu.datasize;
1234         if (!size)
1235                 goto out;
1236
1237         /* We already checked pdu.datasize in check_segment_length() */
1238
1239         if (sg == NULL) {
1240                 /*
1241                  * There are no problems with the safety from concurrent
1242                  * accesses to dummy_page in dummy_sg, since data only
1243                  * will be read and then discarded.
1244                  */
1245                 sg = cmnd->sg = &dummy_sg;
1246                 cmnd->bufflen = PAGE_SIZE;
1247                 cmnd->own_sg = 1;
1248         }
1249
1250         addr = (char __force __user *)(page_address(sg_page(&sg[0])));
1251         sBUG_ON(addr == NULL);
1252         conn->read_size = size;
1253         for (i = 0; size > PAGE_SIZE; i++, size -= cmnd->bufflen) {
1254                 /* We already checked pdu.datasize in check_segment_length() */
1255                 sBUG_ON(i >= ISCSI_CONN_IOV_MAX);
1256                 conn->read_iov[i].iov_base = addr;
1257                 conn->read_iov[i].iov_len = cmnd->bufflen;
1258         }
1259         conn->read_iov[i].iov_base = addr;
1260         conn->read_iov[i].iov_len = size;
1261         conn->read_msg.msg_iov = conn->read_iov;
1262         conn->read_msg.msg_iovlen = ++i;
1263
1264 out:
1265         TRACE_EXIT();
1266         return;
1267 }
1268
1269 static void iscsi_set_resid(struct iscsi_cmnd *rsp, bool bufflen_set)
1270 {
1271         struct iscsi_cmnd *req = rsp->parent_req;
1272         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1273         struct iscsi_scsi_rsp_hdr *rsp_hdr;
1274         int resid, resp_len, in_resp_len;
1275
1276         if ((req_hdr->flags & ISCSI_CMD_READ) &&
1277             (req_hdr->flags & ISCSI_CMD_WRITE)) {
1278                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
1279
1280                 if (bufflen_set) {
1281                         resp_len = req->bufflen;
1282                         if (req->scst_cmd != NULL)
1283                                 in_resp_len = scst_cmd_get_in_bufflen(req->scst_cmd);
1284                         else
1285                                 in_resp_len = 0;
1286                 } else {
1287                         resp_len = 0;
1288                         in_resp_len = 0;
1289                 }
1290
1291                 resid = be32_to_cpu(req_hdr->data_length) - in_resp_len;
1292                 if (resid > 0) {
1293                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
1294                         rsp_hdr->residual_count = cpu_to_be32(resid);
1295                 } else if (resid < 0) {
1296                         resid = -resid;
1297                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
1298                         rsp_hdr->residual_count = cpu_to_be32(resid);
1299                 }
1300
1301                 resid = req->read_size - resp_len;
1302                 if (resid > 0) {
1303                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_UNDERFLOW;
1304                         rsp_hdr->bi_residual_count = cpu_to_be32(resid);
1305                 } else if (resid < 0) {
1306                         resid = -resid;
1307                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_OVERFLOW;
1308                         rsp_hdr->bi_residual_count = cpu_to_be32(resid);
1309                 }
1310         } else {
1311                 if (bufflen_set)
1312                         resp_len = req->bufflen;
1313                 else
1314                         resp_len = 0;
1315
1316                 resid = req->read_size - resp_len;
1317                 if (resid > 0) {
1318                         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
1319                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
1320                         rsp_hdr->residual_count = cpu_to_be32(resid);
1321                 } else if (resid < 0) {
1322                         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
1323                         resid = -resid;
1324                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
1325                         rsp_hdr->residual_count = cpu_to_be32(resid);
1326                 }
1327         }
1328         return;
1329 }
1330
1331 static int iscsi_preliminary_complete(struct iscsi_cmnd *req,
1332         struct iscsi_cmnd *orig_req, bool get_data)
1333 {
1334         struct iscsi_hdr *req_hdr = &req->pdu.bhs;
1335         int res = 0;
1336         bool set_r2t_len;
1337
1338         TRACE_ENTRY();
1339
1340         TRACE_DBG_FLAG(iscsi_get_flow_ctrl_or_mgmt_dbg_log_flag(orig_req),
1341                 "Prelim completed req %p, orig_req %p (FINAL %x, "
1342                 "outstanding_r2t %d)", req, orig_req,
1343                 (req_hdr->flags & ISCSI_CMD_FINAL), orig_req->outstanding_r2t);
1344
1345         iscsi_extracheck_is_rd_thread(req->conn);
1346         sBUG_ON(req->parent_req != NULL);
1347
1348         if (test_bit(ISCSI_CMD_PRELIM_COMPLETED, &req->prelim_compl_flags)) {
1349                 TRACE_MGMT_DBG("req %p already prelim completed", req);
1350                 /* To not try to get data twice */
1351                 get_data = false;
1352         }
1353
1354         set_r2t_len = !req->hashed &&
1355                       (cmnd_opcode(req) == ISCSI_OP_SCSI_CMD) &&
1356                       !test_bit(ISCSI_CMD_PRELIM_COMPLETED,
1357                                 &orig_req->prelim_compl_flags);
1358         set_bit(ISCSI_CMD_PRELIM_COMPLETED, &orig_req->prelim_compl_flags);
1359
1360         TRACE_DBG("get_data %d, set_r2t_len %d", get_data, set_r2t_len);
1361
1362         if (get_data)
1363                 cmnd_prepare_get_rejected_immed_data(req);
1364
1365         if (set_r2t_len)
1366                 res = iscsi_set_prelim_r2t_len_to_receive(orig_req);
1367
1368         TRACE_EXIT_RES(res);
1369         return res;
1370 }
1371
1372 static int cmnd_prepare_recv_pdu(struct iscsi_conn *conn,
1373         struct iscsi_cmnd *cmd, u32 offset, u32 size)
1374 {
1375         struct scatterlist *sg = cmd->sg;
1376         unsigned int bufflen = cmd->bufflen;
1377         unsigned int idx, i;
1378         char __user *addr;
1379         int res = 0;
1380
1381         TRACE_DBG("%p %u,%u", cmd->sg, offset, size);
1382
1383         iscsi_extracheck_is_rd_thread(conn);
1384
1385         if (unlikely((offset >= bufflen) ||
1386                      (offset + size > bufflen))) {
1387                 PRINT_ERROR("Wrong ltn (%u %u %u)", offset, size, bufflen);
1388                 mark_conn_closed(conn);
1389                 res = -EIO;
1390                 goto out;
1391         }
1392
1393         offset += sg[0].offset;
1394         idx = offset >> PAGE_SHIFT;
1395         offset &= ~PAGE_MASK;
1396
1397         conn->read_msg.msg_iov = conn->read_iov;
1398         conn->read_size = size;
1399
1400         i = 0;
1401         while (1) {
1402                 addr = (char __force __user *)(page_address(sg_page(&sg[idx])));
1403                 sBUG_ON(addr == NULL);
1404                 conn->read_iov[i].iov_base = addr + offset;
1405                 if (offset + size <= PAGE_SIZE) {
1406                         TRACE_DBG("idx=%d, offset=%u, size=%d, addr=%p",
1407                                 idx, offset, size, addr);
1408                         conn->read_iov[i].iov_len = size;
1409                         conn->read_msg.msg_iovlen = ++i;
1410                         break;
1411                 }
1412                 conn->read_iov[i].iov_len = PAGE_SIZE - offset;
1413                 TRACE_DBG("idx=%d, offset=%u, size=%d, iov_len=%zd, addr=%p",
1414                         idx, offset, size, conn->read_iov[i].iov_len, addr);
1415                 size -= conn->read_iov[i].iov_len;
1416                 if (unlikely(++i >= ISCSI_CONN_IOV_MAX)) {
1417                         PRINT_ERROR("Initiator %s violated negotiated "
1418                                 "parameters by sending too much data (size "
1419                                 "left %d)", conn->session->initiator_name,
1420                                 size);
1421                         mark_conn_closed(conn);
1422                         res = -EINVAL;
1423                         break;
1424                 }
1425                 idx++;
1426                 offset = sg[idx].offset;
1427         }
1428         TRACE_DBG("msg_iov=%p, msg_iovlen=%zd",
1429                 conn->read_msg.msg_iov, conn->read_msg.msg_iovlen);
1430
1431 out:
1432         return res;
1433 }
1434
1435 static void send_r2t(struct iscsi_cmnd *req)
1436 {
1437         struct iscsi_session *sess = req->conn->session;
1438         struct iscsi_cmnd *rsp;
1439         struct iscsi_r2t_hdr *rsp_hdr;
1440         u32 offset, burst;
1441         LIST_HEAD(send);
1442
1443         TRACE_ENTRY();
1444
1445         EXTRACHECKS_BUG_ON(req->r2t_len_to_send == 0);
1446
1447         /*
1448          * There is no race with data_out_start() and conn_abort(), since
1449          * all functions called from single read thread
1450          */
1451         iscsi_extracheck_is_rd_thread(req->conn);
1452
1453         /*
1454          * We don't need to check for PRELIM_COMPLETED here, because for such
1455          * commands we set r2t_len_to_send = 0, hence made sure we won't
1456          * called here.
1457          */
1458
1459         EXTRACHECKS_BUG_ON(req->outstanding_r2t >
1460                            sess->sess_param.max_outstanding_r2t);
1461
1462         if (req->outstanding_r2t == sess->sess_param.max_outstanding_r2t)
1463                 goto out;
1464
1465         burst = sess->sess_param.max_burst_length;
1466         offset = be32_to_cpu(cmnd_hdr(req)->data_length) -
1467                         req->r2t_len_to_send;
1468
1469         do {
1470                 rsp = iscsi_alloc_rsp(req);
1471                 rsp->pdu.bhs.ttt = req->target_task_tag;
1472                 rsp_hdr = (struct iscsi_r2t_hdr *)&rsp->pdu.bhs;
1473                 rsp_hdr->opcode = ISCSI_OP_R2T;
1474                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1475                 rsp_hdr->lun = cmnd_hdr(req)->lun;
1476                 rsp_hdr->itt = cmnd_hdr(req)->itt;
1477                 rsp_hdr->r2t_sn = cpu_to_be32(req->r2t_sn++);
1478                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
1479                 if (req->r2t_len_to_send > burst) {
1480                         rsp_hdr->data_length = cpu_to_be32(burst);
1481                         req->r2t_len_to_send -= burst;
1482                         offset += burst;
1483                 } else {
1484                         rsp_hdr->data_length = cpu_to_be32(req->r2t_len_to_send);
1485                         req->r2t_len_to_send = 0;
1486                 }
1487
1488                 TRACE_WRITE("req %p, data_length %u, buffer_offset %u, "
1489                         "r2t_sn %u, outstanding_r2t %u", req,
1490                         be32_to_cpu(rsp_hdr->data_length),
1491                         be32_to_cpu(rsp_hdr->buffer_offset),
1492                         be32_to_cpu(rsp_hdr->r2t_sn), req->outstanding_r2t);
1493
1494                 list_add_tail(&rsp->write_list_entry, &send);
1495                 req->outstanding_r2t++;
1496
1497         } while ((req->outstanding_r2t < sess->sess_param.max_outstanding_r2t) &&
1498                  (req->r2t_len_to_send != 0));
1499
1500         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_WAKE);
1501
1502 out:
1503         TRACE_EXIT();
1504         return;
1505 }
1506
1507 static int iscsi_pre_exec(struct scst_cmd *scst_cmd)
1508 {
1509         int res = SCST_PREPROCESS_STATUS_SUCCESS;
1510         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
1511                 scst_cmd_get_tgt_priv(scst_cmd);
1512         struct iscsi_cmnd *c, *t;
1513
1514         TRACE_ENTRY();
1515
1516         EXTRACHECKS_BUG_ON(scst_cmd_atomic(scst_cmd));
1517
1518         /* If data digest isn't used this list will be empty */
1519         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
1520                                 rx_ddigest_cmd_list_entry) {
1521                 TRACE_DBG("Checking digest of RX ddigest cmd %p", c);
1522                 if (digest_rx_data(c) != 0) {
1523                         scst_set_cmd_error(scst_cmd,
1524                                 SCST_LOAD_SENSE(iscsi_sense_crc_error));
1525                         res = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
1526                         /*
1527                          * The rest of rx_ddigest_cmd_list will be freed
1528                          * in req_cmnd_release()
1529                          */
1530                         goto out;
1531                 }
1532                 cmd_del_from_rx_ddigest_list(c);
1533                 cmnd_put(c);
1534         }
1535
1536 out:
1537         TRACE_EXIT_RES(res);
1538         return res;
1539 }
1540
1541 static int noop_out_start(struct iscsi_cmnd *cmnd)
1542 {
1543         struct iscsi_conn *conn = cmnd->conn;
1544         struct iscsi_hdr *req_hdr = &cmnd->pdu.bhs;
1545         u32 size, tmp;
1546         int i, err = 0;
1547
1548         TRACE_DBG("%p", cmnd);
1549
1550         iscsi_extracheck_is_rd_thread(conn);
1551
1552         if (!(req_hdr->flags & ISCSI_FLG_FINAL)) {
1553                 PRINT_ERROR("%s", "Initiator sent NOP-Out with not a single "
1554                         "PDU");
1555                 err = -ISCSI_REASON_PROTOCOL_ERROR;
1556                 goto out;
1557         }
1558
1559         if (cmnd_itt(cmnd) == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1560                 if (unlikely(!(cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE)))
1561                         PRINT_ERROR("%s", "Initiator sent RESERVED tag for "
1562                                 "non-immediate NOP-Out command");
1563         }
1564
1565         spin_lock(&conn->session->sn_lock);
1566         __update_stat_sn(cmnd);
1567         err = check_cmd_sn(cmnd);
1568         spin_unlock(&conn->session->sn_lock);
1569         if (unlikely(err))
1570                 goto out;
1571
1572         size = cmnd->pdu.datasize;
1573
1574         if (size) {
1575                 conn->read_msg.msg_iov = conn->read_iov;
1576                 if (cmnd->pdu.bhs.itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1577                         struct scatterlist *sg;
1578
1579                         cmnd->sg = sg = scst_alloc(size, GFP_KERNEL,
1580                                                 &cmnd->sg_cnt);
1581                         if (sg == NULL) {
1582                                 TRACE(TRACE_OUT_OF_MEM, "Allocating buffer for"
1583                                       " %d NOP-Out payload failed", size);
1584                                 err = -ISCSI_REASON_OUT_OF_RESOURCES;
1585                                 goto out;
1586                         }
1587
1588                         /* We already checked it in check_segment_length() */
1589                         sBUG_ON(cmnd->sg_cnt > (signed)ISCSI_CONN_IOV_MAX);
1590
1591                         cmnd->own_sg = 1;
1592                         cmnd->bufflen = size;
1593
1594                         for (i = 0; i < cmnd->sg_cnt; i++) {
1595                                 conn->read_iov[i].iov_base =
1596                                         (void __force __user *)(page_address(sg_page(&sg[i])));
1597                                 tmp = min_t(u32, size, PAGE_SIZE);
1598                                 conn->read_iov[i].iov_len = tmp;
1599                                 conn->read_size += tmp;
1600                                 size -= tmp;
1601                         }
1602                         sBUG_ON(size != 0);
1603                 } else {
1604                         /*
1605                          * There are no problems with the safety from concurrent
1606                          * accesses to dummy_page, since for ISCSI_RESERVED_TAG
1607                          * the data only read and then discarded.
1608                          */
1609                         for (i = 0; i < (signed)ISCSI_CONN_IOV_MAX; i++) {
1610                                 conn->read_iov[i].iov_base =
1611                                         (void __force __user *)(page_address(dummy_page));
1612                                 tmp = min_t(u32, size, PAGE_SIZE);
1613                                 conn->read_iov[i].iov_len = tmp;
1614                                 conn->read_size += tmp;
1615                                 size -= tmp;
1616                         }
1617
1618                         /* We already checked size in check_segment_length() */
1619                         sBUG_ON(size != 0);
1620                 }
1621
1622                 conn->read_msg.msg_iovlen = i;
1623                 TRACE_DBG("msg_iov=%p, msg_iovlen=%zd", conn->read_msg.msg_iov,
1624                         conn->read_msg.msg_iovlen);
1625         }
1626
1627 out:
1628         return err;
1629 }
1630
1631 int cmnd_rx_continue(struct iscsi_cmnd *req)
1632 {
1633         struct iscsi_conn *conn = req->conn;
1634         struct iscsi_session *session = conn->session;
1635         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1636         struct scst_cmd *scst_cmd = req->scst_cmd;
1637         scst_data_direction dir;
1638         bool unsolicited_data_expected = false;
1639         int res = 0;
1640
1641         TRACE_ENTRY();
1642
1643         TRACE_DBG("scsi command: %x", req_hdr->scb[0]);
1644
1645         EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_AFTER_PREPROC);
1646
1647         dir = scst_cmd_get_data_direction(scst_cmd);
1648
1649         /* Check prelim_compl_flags here to save R2Ts */
1650         if (unlikely(scst_cmd_completed(scst_cmd) ||
1651             unlikely(req->prelim_compl_flags != 0) ||
1652             unlikely(scst_cmd_aborted(scst_cmd)))) {
1653                 if (scst_cmd_aborted(scst_cmd))
1654                         set_bit(ISCSI_CMD_ABORTED, &req->prelim_compl_flags);
1655                 res = iscsi_preliminary_complete(req, req, true);
1656                 goto trace;
1657         }
1658
1659         /* For prelim completed commands sg&K can be already set! */
1660
1661         if (dir != SCST_DATA_BIDI) {
1662                 req->sg = scst_cmd_get_sg(scst_cmd);
1663                 req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
1664                 req->bufflen = scst_cmd_get_bufflen(scst_cmd);
1665         } else {
1666                 req->sg = scst_cmd_get_in_sg(scst_cmd);
1667                 req->sg_cnt = scst_cmd_get_in_sg_cnt(scst_cmd);
1668                 req->bufflen = scst_cmd_get_in_bufflen(scst_cmd);
1669         }
1670
1671         if (dir & SCST_DATA_WRITE) {
1672                 unsolicited_data_expected = !(req_hdr->flags & ISCSI_CMD_FINAL);
1673
1674                 if (unlikely(session->sess_param.initial_r2t &&
1675                     unsolicited_data_expected)) {
1676                         PRINT_ERROR("Initiator %s violated negotiated "
1677                                 "parameters: initial R2T is required (ITT %x, "
1678                                 "op  %x)", session->initiator_name,
1679                                 cmnd_itt(req), req_hdr->scb[0]);
1680                         goto out_close;
1681                 }
1682
1683                 if (unlikely(!session->sess_param.immediate_data &&
1684                     req->pdu.datasize)) {
1685                         PRINT_ERROR("Initiator %s violated negotiated "
1686                                 "parameters: forbidden immediate data sent "
1687                                 "(ITT %x, op  %x)", session->initiator_name,
1688                                 cmnd_itt(req), req_hdr->scb[0]);
1689                         goto out_close;
1690                 }
1691
1692                 if (unlikely(session->sess_param.first_burst_length < req->pdu.datasize)) {
1693                         PRINT_ERROR("Initiator %s violated negotiated "
1694                                 "parameters: immediate data len (%d) > "
1695                                 "first_burst_length (%d) (ITT %x, op  %x)",
1696                                 session->initiator_name,
1697                                 req->pdu.datasize,
1698                                 session->sess_param.first_burst_length,
1699                                 cmnd_itt(req), req_hdr->scb[0]);
1700                         goto out_close;
1701                 }
1702
1703                 req->r2t_len_to_receive = be32_to_cpu(req_hdr->data_length) -
1704                                           req->pdu.datasize;
1705
1706                 if (unlikely(req->r2t_len_to_receive > req->bufflen)) {
1707                         PRINT_ERROR("req->r2t_len_to_receive %d > req->bufflen "
1708                                 "%d", req->r2t_len_to_receive, req->bufflen);
1709                         goto out_close;
1710                 }
1711
1712                 res = cmnd_insert_data_wait_hash(req);
1713                 if (unlikely(res != 0)) {
1714                         /*
1715                          * We have to close connection, because otherwise a data
1716                          * corruption is possible if we allow to receive data
1717                          * for this request in another request with dublicated
1718                          * ITT.
1719                          */
1720                         goto out_close;
1721                 }
1722
1723                 if (unsolicited_data_expected) {
1724                         req->outstanding_r2t = 1;
1725                         req->r2t_len_to_send = req->r2t_len_to_receive -
1726                                 min_t(unsigned int,
1727                                       session->sess_param.first_burst_length -
1728                                                 req->pdu.datasize,
1729                                       req->r2t_len_to_receive);
1730                 } else
1731                         req->r2t_len_to_send = req->r2t_len_to_receive;
1732
1733                 req_add_to_write_timeout_list(req);
1734
1735                 if (req->pdu.datasize) {
1736                         res = cmnd_prepare_recv_pdu(conn, req, 0, req->pdu.datasize);
1737                         /* For performance better to send R2Ts ASAP */
1738                         if (likely(res == 0) && (req->r2t_len_to_send != 0))
1739                                 send_r2t(req);
1740                 }
1741         } else {
1742                 if (unlikely(!(req_hdr->flags & ISCSI_CMD_FINAL) ||
1743                              req->pdu.datasize)) {
1744                         PRINT_ERROR("Unexpected unsolicited data (ITT %x "
1745                                 "CDB %x", cmnd_itt(req), req_hdr->scb[0]);
1746                         set_scst_preliminary_status_rsp(req, true,
1747                                 SCST_LOAD_SENSE(iscsi_sense_unexpected_unsolicited_data));
1748                 }
1749         }
1750
1751 trace:
1752         TRACE_DBG("req=%p, dir=%d, unsolicited_data_expected=%d, "
1753                 "r2t_len_to_receive=%d, r2t_len_to_send=%d, bufflen=%d, "
1754                 "own_sg %d", req, dir, unsolicited_data_expected,
1755                 req->r2t_len_to_receive, req->r2t_len_to_send, req->bufflen,
1756                 req->own_sg);
1757
1758 out:
1759         TRACE_EXIT_RES(res);
1760         return res;
1761
1762 out_close:
1763         mark_conn_closed(conn);
1764         res = -EINVAL;
1765         goto out;
1766 }
1767
1768 static int scsi_cmnd_start(struct iscsi_cmnd *req)
1769 {
1770         struct iscsi_conn *conn = req->conn;
1771         struct iscsi_session *session = conn->session;
1772         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1773         struct scst_cmd *scst_cmd;
1774         scst_data_direction dir;
1775         struct iscsi_ahs_hdr *ahdr;
1776         int res = 0;
1777
1778         TRACE_ENTRY();
1779
1780         TRACE_DBG("scsi command: %x", req_hdr->scb[0]);
1781
1782         TRACE_DBG("Incrementing active_cmds (cmd %p, sess %p, "
1783                 "new value %d)", req, session,
1784                 atomic_read(&session->active_cmds)+1);
1785         atomic_inc(&session->active_cmds);
1786         req->dec_active_cmnds = 1;
1787
1788         scst_cmd = scst_rx_cmd(session->scst_sess,
1789                 (uint8_t *)&req_hdr->lun, sizeof(req_hdr->lun),
1790                 req_hdr->scb, sizeof(req_hdr->scb), SCST_NON_ATOMIC);
1791         if (scst_cmd == NULL) {
1792                 res = create_preliminary_status_rsp(req, SAM_STAT_BUSY,
1793                         NULL, 0);
1794                 goto out;
1795         }
1796
1797         req->scst_cmd = scst_cmd;
1798         scst_cmd_set_tag(scst_cmd, req_hdr->itt);
1799         scst_cmd_set_tgt_priv(scst_cmd, req);
1800
1801         if ((req_hdr->flags & ISCSI_CMD_READ) &&
1802             (req_hdr->flags & ISCSI_CMD_WRITE)) {
1803                 int sz = cmnd_read_size(req);
1804                 if (unlikely(sz < 0)) {
1805                         PRINT_ERROR("%s", "BIDI data transfer, but initiator "
1806                                 "not supplied Bidirectional Read Expected Data "
1807                                 "Transfer Length AHS");
1808                         set_scst_preliminary_status_rsp(req, true,
1809                            SCST_LOAD_SENSE(scst_sense_parameter_value_invalid));
1810                 } else {
1811                         req->read_size = sz;
1812                         dir = SCST_DATA_BIDI;
1813                         scst_cmd_set_expected(scst_cmd, dir, sz);
1814                         scst_cmd_set_expected_in_transfer_len(scst_cmd,
1815                                 be32_to_cpu(req_hdr->data_length));
1816 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
1817                         scst_cmd_set_tgt_need_alloc_data_buf(scst_cmd);
1818 #endif
1819                 }
1820         } else if (req_hdr->flags & ISCSI_CMD_READ) {
1821                 req->read_size = be32_to_cpu(req_hdr->data_length);
1822                 dir = SCST_DATA_READ;
1823                 scst_cmd_set_expected(scst_cmd, dir, req->read_size);
1824 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
1825                 scst_cmd_set_tgt_need_alloc_data_buf(scst_cmd);
1826 #endif
1827         } else if (req_hdr->flags & ISCSI_CMD_WRITE) {
1828                 dir = SCST_DATA_WRITE;
1829                 scst_cmd_set_expected(scst_cmd, dir,
1830                         be32_to_cpu(req_hdr->data_length));
1831         } else {
1832                 dir = SCST_DATA_NONE;
1833                 scst_cmd_set_expected(scst_cmd, dir, 0);
1834         }
1835
1836         switch (req_hdr->flags & ISCSI_CMD_ATTR_MASK) {
1837         case ISCSI_CMD_SIMPLE:
1838                 scst_cmd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1839                 break;
1840         case ISCSI_CMD_HEAD_OF_QUEUE:
1841                 scst_cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1842                 break;
1843         case ISCSI_CMD_ORDERED:
1844                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1845                 break;
1846         case ISCSI_CMD_ACA:
1847                 scst_cmd->queue_type = SCST_CMD_QUEUE_ACA;
1848                 break;
1849         case ISCSI_CMD_UNTAGGED:
1850                 scst_cmd->queue_type = SCST_CMD_QUEUE_UNTAGGED;
1851                 break;
1852         default:
1853                 PRINT_ERROR("Unknown task code %x, use ORDERED instead",
1854                         req_hdr->flags & ISCSI_CMD_ATTR_MASK);
1855                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1856                 break;
1857         }
1858
1859         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1860         scst_cmd_set_tgt_sn(scst_cmd, req_hdr->cmd_sn);
1861
1862         ahdr = (struct iscsi_ahs_hdr *)req->pdu.ahs;
1863         if (ahdr != NULL) {
1864                 uint8_t *p = (uint8_t *)ahdr;
1865                 unsigned int size = 0;
1866                 do {
1867                         int s;
1868
1869                         ahdr = (struct iscsi_ahs_hdr *)p;
1870
1871                         if (ahdr->ahstype == ISCSI_AHSTYPE_CDB) {
1872                                 struct iscsi_cdb_ahdr *eca =
1873                                         (struct iscsi_cdb_ahdr *)ahdr;
1874                                 scst_cmd_set_ext_cdb(scst_cmd, eca->cdb,
1875                                         be16_to_cpu(ahdr->ahslength) - 1);
1876                                 break;
1877                         }
1878                         s = 3 + be16_to_cpu(ahdr->ahslength);
1879                         s = (s + 3) & -4;
1880                         size += s;
1881                         p += s;
1882                 } while (size < req->pdu.ahssize);
1883         }
1884
1885         TRACE_DBG("START Command (itt %x, queue_type %d)",
1886                 req_hdr->itt, scst_cmd->queue_type);
1887         req->scst_state = ISCSI_CMD_STATE_RX_CMD;
1888         conn->rx_task = current;
1889         scst_cmd_init_stage1_done(scst_cmd, SCST_CONTEXT_DIRECT, 0);
1890
1891         if (req->scst_state != ISCSI_CMD_STATE_RX_CMD)
1892                 res = cmnd_rx_continue(req);
1893         else {
1894                 TRACE_DBG("Delaying req %p post processing (scst_state %d)",
1895                         req, req->scst_state);
1896                 res = 1;
1897         }
1898
1899 out:
1900         TRACE_EXIT_RES(res);
1901         return res;
1902 }
1903
1904 static int data_out_start(struct iscsi_cmnd *cmnd)
1905 {
1906         struct iscsi_conn *conn = cmnd->conn;
1907         struct iscsi_data_out_hdr *req_hdr =
1908                 (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1909         struct iscsi_cmnd *orig_req;
1910         struct iscsi_hdr *orig_req_hdr;
1911         u32 offset = be32_to_cpu(req_hdr->buffer_offset);
1912         int res = 0;
1913
1914         TRACE_ENTRY();
1915
1916         /*
1917          * There is no race with send_r2t() and conn_abort(), since
1918          * all functions called from single read thread
1919          */
1920         iscsi_extracheck_is_rd_thread(cmnd->conn);
1921
1922         update_stat_sn(cmnd);
1923
1924         orig_req = cmnd_find_data_wait_hash(conn, req_hdr->itt);
1925         cmnd->cmd_req = orig_req;
1926         if (unlikely(orig_req == NULL)) {
1927                 /*
1928                  * It shouldn't happen, since we don't abort any request until
1929                  * we received all related PDUs from the initiator or timeout
1930                  * them. Let's quietly drop such PDUs.
1931                  */
1932                 TRACE(TRACE_MGMT_MINOR, "Unable to find scsi task ITT %x",
1933                         cmnd_itt(cmnd));
1934                 res = iscsi_preliminary_complete(cmnd, cmnd, true);
1935                 goto out;
1936         }
1937
1938         if (unlikely(orig_req->r2t_len_to_receive < cmnd->pdu.datasize)) {
1939                 if (orig_req->prelim_compl_flags != 0) {
1940                         /* We can have fake r2t_len_to_receive */
1941                         goto go;
1942                 }
1943                 PRINT_ERROR("Data size (%d) > R2T length to receive (%d)",
1944                         cmnd->pdu.datasize, orig_req->r2t_len_to_receive);
1945                 set_scst_preliminary_status_rsp(orig_req, false,
1946                         SCST_LOAD_SENSE(iscsi_sense_incorrect_amount_of_data));
1947                 goto go;
1948         }
1949
1950         /* Crazy iSCSI spec requires us to make this unneeded check */
1951         orig_req_hdr = &orig_req->pdu.bhs;
1952         if (unlikely(orig_req_hdr->lun != req_hdr->lun)) {
1953                 PRINT_ERROR("Wrong LUN (%lld) in Data-Out PDU (expected %lld), "
1954                         "orig_req %p, cmnd %p", (unsigned long long)req_hdr->lun,
1955                         (unsigned long long)orig_req_hdr->lun, orig_req, cmnd);
1956                 create_reject_rsp(orig_req, ISCSI_REASON_PROTOCOL_ERROR, false);
1957                 goto go;
1958         }
1959
1960 go:
1961         if (req_hdr->flags & ISCSI_FLG_FINAL)
1962                 orig_req->outstanding_r2t--;
1963
1964         if (unlikely(orig_req->prelim_compl_flags != 0)) {
1965                 res = iscsi_preliminary_complete(cmnd, orig_req, true);
1966                 goto out;
1967         }
1968
1969         TRACE_WRITE("cmnd %p, orig_req %p, offset %u, datasize %u", cmnd,
1970                 orig_req, offset, cmnd->pdu.datasize);
1971
1972         res = cmnd_prepare_recv_pdu(conn, orig_req, offset, cmnd->pdu.datasize);
1973
1974 out:
1975         TRACE_EXIT_RES(res);
1976         return res;
1977 }
1978
1979 static void data_out_end(struct iscsi_cmnd *cmnd)
1980 {
1981         struct iscsi_data_out_hdr *req_hdr =
1982                 (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1983         struct iscsi_cmnd *req;
1984
1985         TRACE_ENTRY();
1986
1987         EXTRACHECKS_BUG_ON(cmnd == NULL);
1988         req = cmnd->cmd_req;
1989         if (unlikely(req == NULL))
1990                 goto out;
1991
1992         TRACE_DBG("cmnd %p, req %p", cmnd, req);
1993
1994         iscsi_extracheck_is_rd_thread(cmnd->conn);
1995
1996         if (!(cmnd->conn->ddigest_type & DIGEST_NONE) &&
1997             !cmnd->ddigest_checked) {
1998                 cmd_add_on_rx_ddigest_list(req, cmnd);
1999                 cmnd_get(cmnd);
2000         }
2001
2002         /*
2003          * Now we received the data and can adjust r2t_len_to_receive of the
2004          * orig req. We couldn't do it earlier, because it will break data
2005          * receiving errors recovery (calls of iscsi_fail_data_waiting_cmnd()).
2006          */
2007         req->r2t_len_to_receive -= cmnd->pdu.datasize;
2008
2009         if (unlikely(req->prelim_compl_flags != 0)) {
2010                 /*
2011                  * We might need to wait for one or more PDUs. Let's simplify
2012                  * other code.
2013                  */
2014                 req->r2t_len_to_receive = req->outstanding_r2t;
2015                 req->r2t_len_to_send = 0;
2016         }
2017
2018         TRACE_DBG("req %p, FINAL %x, outstanding_r2t %d, r2t_len_to_receive %d,"
2019                 " r2t_len_to_send %d", req, req_hdr->flags & ISCSI_FLG_FINAL,
2020                 req->outstanding_r2t, req->r2t_len_to_receive,
2021                 req->r2t_len_to_send);
2022
2023         if (!(req_hdr->flags & ISCSI_FLG_FINAL))
2024                 goto out;
2025
2026         if (req->r2t_len_to_receive == 0) {
2027                 if (!req->pending)
2028                         iscsi_restart_cmnd(req);
2029         } else if (req->r2t_len_to_send != 0)
2030                 send_r2t(req);
2031
2032 out:
2033         TRACE_EXIT();
2034         return;
2035 }
2036
2037 /* Might be called under target_mutex and cmd_list_lock */
2038 static void __cmnd_abort(struct iscsi_cmnd *cmnd)
2039 {
2040         unsigned long timeout_time = jiffies + ISCSI_TM_DATA_WAIT_SCHED_TIMEOUT;
2041         struct iscsi_conn *conn = cmnd->conn;
2042
2043         TRACE_MGMT_DBG("Aborting cmd %p, scst_cmd %p (scst state %x, "
2044                 "ref_cnt %d, on_write_timeout_list %d, write_start %ld, ITT %x, "
2045                 "sn %u, op %x, r2t_len_to_receive %d, r2t_len_to_send %d, "
2046                 "CDB op %x, size to write %u, outstanding_r2t %d, "
2047                 "sess->exp_cmd_sn %u, conn %p, rd_task %p)",
2048                 cmnd, cmnd->scst_cmd, cmnd->scst_state,
2049                 atomic_read(&cmnd->ref_cnt), cmnd->on_write_timeout_list,
2050                 cmnd->write_start, cmnd_itt(cmnd), cmnd->pdu.bhs.sn,
2051                 cmnd_opcode(cmnd), cmnd->r2t_len_to_receive,
2052                 cmnd->r2t_len_to_send, cmnd_scsicode(cmnd),
2053                 cmnd_write_size(cmnd), cmnd->outstanding_r2t,
2054                 cmnd->conn->session->exp_cmd_sn, cmnd->conn,
2055                 cmnd->conn->rd_task);
2056
2057 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
2058         TRACE_MGMT_DBG("net_ref_cnt %d", atomic_read(&cmnd->net_ref_cnt));
2059 #endif
2060
2061         /*
2062          * Lock to sync with iscsi_check_tm_data_wait_timeouts(), including
2063          * CMD_ABORTED bit set.
2064          */
2065         spin_lock_bh(&iscsi_rd_lock);
2066
2067         /*
2068          * We suppose that preliminary commands completion is tested by
2069          * comparing prelim_compl_flags with 0. Otherwise a race is possible,
2070          * like sending command in SCST core as PRELIM_COMPLETED, while it
2071          * wasn't aborted in it yet and have as the result a wrong success
2072          * status sent to the initiator.
2073          */
2074         set_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags);
2075
2076         TRACE_MGMT_DBG("Setting conn_tm_active for conn %p", conn);
2077         conn->conn_tm_active = 1;
2078
2079         spin_unlock_bh(&iscsi_rd_lock);
2080
2081         /*
2082          * We need the lock to sync with req_add_to_write_timeout_list() and
2083          * close races for rsp_timer.expires.
2084          */
2085         spin_lock_bh(&conn->write_list_lock);
2086         if (!timer_pending(&conn->rsp_timer) ||
2087             time_after(conn->rsp_timer.expires, timeout_time)) {
2088                 TRACE_MGMT_DBG("Mod timer on %ld (conn %p)", timeout_time,
2089                         conn);
2090                 mod_timer(&conn->rsp_timer, timeout_time);
2091         } else
2092                 TRACE_MGMT_DBG("Timer for conn %p is going to fire on %ld "
2093                         "(timeout time %ld)", conn, conn->rsp_timer.expires,
2094                         timeout_time);
2095         spin_unlock_bh(&conn->write_list_lock);
2096
2097         return;
2098 }
2099
2100 /* Must be called from the read or conn close thread */
2101 static int cmnd_abort(struct iscsi_cmnd *req, int *status)
2102 {
2103         struct iscsi_task_mgt_hdr *req_hdr =
2104                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2105         struct iscsi_cmnd *cmnd;
2106         int res = -1;
2107
2108         req_hdr->ref_cmd_sn = be32_to_cpu(req_hdr->ref_cmd_sn);
2109
2110         if (!before(req_hdr->ref_cmd_sn, req_hdr->cmd_sn)) {
2111                 TRACE(TRACE_MGMT, "ABORT TASK: RefCmdSN(%u) > CmdSN(%u)",
2112                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn);
2113                 *status = ISCSI_RESPONSE_UNKNOWN_TASK;
2114                 goto out;
2115         }
2116
2117         cmnd = cmnd_find_itt_get(req->conn, req_hdr->rtt);
2118         if (cmnd) {
2119                 struct iscsi_conn *conn = cmnd->conn;
2120                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
2121
2122                 if (req_hdr->lun != hdr->lun) {
2123                         PRINT_ERROR("ABORT TASK: LUN mismatch: req LUN "
2124                                     "%llx, cmd LUN %llx, rtt %u",
2125                                     (long long unsigned int)req_hdr->lun,
2126                                     (long long unsigned int)hdr->lun,
2127                                     req_hdr->rtt);
2128                         *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2129                         goto out_put;
2130                 }
2131
2132                 if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
2133                         if (req_hdr->ref_cmd_sn != req_hdr->cmd_sn) {
2134                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != TM "
2135                                         "cmd CmdSN(%u) for immediate command "
2136                                         "%p", req_hdr->ref_cmd_sn,
2137                                         req_hdr->cmd_sn, cmnd);
2138                                 *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2139                                 goto out_put;
2140                         }
2141                 } else {
2142                         if (req_hdr->ref_cmd_sn != hdr->cmd_sn) {
2143                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != "
2144                                         "CmdSN(%u) for command %p",
2145                                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
2146                                         cmnd);
2147                                 *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2148                                 goto out_put;
2149                         }
2150                 }
2151
2152                 if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
2153                     (req_hdr->cmd_sn == hdr->cmd_sn)) {
2154                         PRINT_ERROR("ABORT TASK: SN mismatch: req SN %x, "
2155                                 "cmd SN %x, rtt %u", req_hdr->cmd_sn,
2156                                 hdr->cmd_sn, req_hdr->rtt);
2157                         *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2158                         goto out_put;
2159                 }
2160
2161                 spin_lock_bh(&conn->cmd_list_lock);
2162                 __cmnd_abort(cmnd);
2163                 spin_unlock_bh(&conn->cmd_list_lock);
2164
2165                 cmnd_put(cmnd);
2166                 res = 0;
2167         } else {
2168                 TRACE_MGMT_DBG("cmd RTT %x not found", req_hdr->rtt);
2169                 /*
2170                  * iSCSI RFC:
2171                  *
2172                  * b)  If the Referenced Task Tag does not identify an existing task,
2173                  * but if the CmdSN indicated by the RefCmdSN field in the Task
2174                  * Management function request is within the valid CmdSN window
2175                  * and less than the CmdSN of the Task Management function
2176                  * request itself, then targets must consider the CmdSN received
2177                  * and return the "Function complete" response.
2178                  *
2179                  * c)  If the Referenced Task Tag does not identify an existing task
2180                  * and if the CmdSN indicated by the RefCmdSN field in the Task
2181                  * Management function request is outside the valid CmdSN window,
2182                  * then targets must return the "Task does not exist" response.
2183                  *
2184                  * 128 seems to be a good "window".
2185                  */
2186                 if (between(req_hdr->ref_cmd_sn, req_hdr->cmd_sn - 128,
2187                             req_hdr->cmd_sn)) {
2188                         *status = ISCSI_RESPONSE_FUNCTION_COMPLETE;
2189                         res = 0;
2190                 } else
2191                         *status = ISCSI_RESPONSE_UNKNOWN_TASK;
2192         }
2193
2194 out:
2195         return res;
2196
2197 out_put:
2198         cmnd_put(cmnd);
2199         goto out;
2200 }
2201
2202 /* Must be called from the read or conn close thread */
2203 static int target_abort(struct iscsi_cmnd *req, int all)
2204 {
2205         struct iscsi_target *target = req->conn->session->target;
2206         struct iscsi_task_mgt_hdr *req_hdr =
2207                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2208         struct iscsi_session *session;
2209         struct iscsi_conn *conn;
2210         struct iscsi_cmnd *cmnd;
2211
2212         mutex_lock(&target->target_mutex);
2213
2214         list_for_each_entry(session, &target->session_list,
2215                             session_list_entry) {
2216                 list_for_each_entry(conn, &session->conn_list,
2217                                     conn_list_entry) {
2218                         spin_lock_bh(&conn->cmd_list_lock);
2219                         list_for_each_entry(cmnd, &conn->cmd_list,
2220                                             cmd_list_entry) {
2221                                 if (cmnd == req)
2222                                         continue;
2223                                 if (all)
2224                                         __cmnd_abort(cmnd);
2225                                 else if (req_hdr->lun == cmnd_hdr(cmnd)->lun)
2226                                         __cmnd_abort(cmnd);
2227                         }
2228                         spin_unlock_bh(&conn->cmd_list_lock);
2229                 }
2230         }
2231
2232         mutex_unlock(&target->target_mutex);
2233         return 0;
2234 }
2235
2236 /* Must be called from the read or conn close thread */
2237 static void task_set_abort(struct iscsi_cmnd *req)
2238 {
2239         struct iscsi_session *session = req->conn->session;
2240         struct iscsi_task_mgt_hdr *req_hdr =
2241                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2242         struct iscsi_target *target = session->target;
2243         struct iscsi_conn *conn;
2244         struct iscsi_cmnd *cmnd;
2245
2246         mutex_lock(&target->target_mutex);
2247
2248         list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
2249                 spin_lock_bh(&conn->cmd_list_lock);
2250                 list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
2251                         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
2252                         if (cmnd == req)
2253                                 continue;
2254                         if (req_hdr->lun != hdr->lun)
2255                                 continue;
2256                         if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
2257                             req_hdr->cmd_sn == hdr->cmd_sn)
2258                                 continue;
2259                         __cmnd_abort(cmnd);
2260                 }
2261                 spin_unlock_bh(&conn->cmd_list_lock);
2262         }
2263
2264         mutex_unlock(&target->target_mutex);
2265         return;
2266 }
2267
2268 /* Must be called from the read or conn close thread */
2269 void conn_abort(struct iscsi_conn *conn)
2270 {
2271         struct iscsi_cmnd *cmnd;
2272
2273         TRACE_MGMT_DBG("Aborting conn %p", conn);
2274
2275         iscsi_extracheck_is_rd_thread(conn);
2276
2277         spin_lock_bh(&conn->cmd_list_lock);
2278 again:
2279         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
2280                 __cmnd_abort(cmnd);
2281                 if (cmnd->r2t_len_to_receive != 0) {
2282                         if (!cmnd_get_check(cmnd)) {
2283                                 spin_unlock_bh(&conn->cmd_list_lock);
2284
2285                                 /* ToDo: this is racy for MC/S */
2286                                 iscsi_fail_data_waiting_cmnd(cmnd);
2287
2288                                 cmnd_put(cmnd);
2289
2290                                 /*
2291                                  * We are in the read thread, so we may not
2292                                  * worry that after cmnd release conn gets
2293                                  * released as well.
2294                                  */
2295                                 spin_lock_bh(&conn->cmd_list_lock);
2296                                 goto again;
2297                         }
2298                 }
2299         }
2300         spin_unlock_bh(&conn->cmd_list_lock);
2301
2302         return;
2303 }
2304
2305 static void execute_task_management(struct iscsi_cmnd *req)
2306 {
2307         struct iscsi_conn *conn = req->conn;
2308         struct iscsi_session *sess = conn->session;
2309         struct iscsi_task_mgt_hdr *req_hdr =
2310                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2311         int rc, status, function = req_hdr->function & ISCSI_FUNCTION_MASK;
2312         struct scst_rx_mgmt_params params;
2313
2314         TRACE((function == ISCSI_FUNCTION_ABORT_TASK) ?
2315                         TRACE_MGMT_MINOR : TRACE_MGMT,
2316                 "TM fn %d", function);
2317
2318         TRACE_MGMT_DBG("TM req %p, ITT %x, RTT %x, sn %u, con %p", req,
2319                 cmnd_itt(req), req_hdr->rtt, req_hdr->cmd_sn, conn);
2320
2321         iscsi_extracheck_is_rd_thread(conn);
2322
2323         spin_lock(&sess->sn_lock);
2324         sess->tm_active++;
2325         sess->tm_sn = req_hdr->cmd_sn;
2326         if (sess->tm_rsp != NULL) {
2327                 struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
2328
2329                 TRACE(TRACE_MGMT_MINOR, "Dropping delayed TM rsp %p", tm_rsp);
2330
2331                 sess->tm_rsp = NULL;
2332                 sess->tm_active--;
2333
2334                 spin_unlock(&sess->sn_lock);
2335
2336                 sBUG_ON(sess->tm_active < 0);
2337
2338                 rsp_cmnd_release(tm_rsp);
2339         } else
2340                 spin_unlock(&sess->sn_lock);
2341
2342         memset(&params, 0, sizeof(params));
2343         params.atomic = SCST_NON_ATOMIC;
2344         params.tgt_priv = req;
2345
2346         if ((function != ISCSI_FUNCTION_ABORT_TASK) &&
2347             (req_hdr->rtt != ISCSI_RESERVED_TAG)) {
2348                 PRINT_ERROR("Invalid RTT %x (TM fn %d)", req_hdr->rtt,
2349                         function);
2350                 rc = -1;
2351                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2352                 goto reject;
2353         }
2354
2355         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
2356
2357         switch (function) {
2358         case ISCSI_FUNCTION_ABORT_TASK:
2359                 rc = cmnd_abort(req, &status);
2360                 if (rc == 0) {
2361                         params.fn = SCST_ABORT_TASK;
2362                         params.tag = req_hdr->rtt;
2363                         params.tag_set = 1;
2364                         params.lun = (uint8_t *)&req_hdr->lun;
2365                         params.lun_len = sizeof(req_hdr->lun);
2366                         params.lun_set = 1;
2367                         params.cmd_sn = req_hdr->cmd_sn;
2368                         params.cmd_sn_set = 1;
2369                         rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2370                                 &params);
2371                         status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2372                 }
2373                 break;
2374         case ISCSI_FUNCTION_ABORT_TASK_SET:
2375                 task_set_abort(req);
2376                 params.fn = SCST_ABORT_TASK_SET;
2377                 params.lun = (uint8_t *)&req_hdr->lun;
2378                 params.lun_len = sizeof(req_hdr->lun);
2379                 params.lun_set = 1;
2380                 params.cmd_sn = req_hdr->cmd_sn;
2381                 params.cmd_sn_set = 1;
2382                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2383                         &params);
2384                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2385                 break;
2386         case ISCSI_FUNCTION_CLEAR_TASK_SET:
2387                 task_set_abort(req);
2388                 params.fn = SCST_CLEAR_TASK_SET;
2389                 params.lun = (uint8_t *)&req_hdr->lun;
2390                 params.lun_len = sizeof(req_hdr->lun);
2391                 params.lun_set = 1;
2392                 params.cmd_sn = req_hdr->cmd_sn;
2393                 params.cmd_sn_set = 1;
2394                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2395                         &params);
2396                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2397                 break;
2398         case ISCSI_FUNCTION_CLEAR_ACA:
2399                 params.fn = SCST_CLEAR_ACA;
2400                 params.lun = (uint8_t *)&req_hdr->lun;
2401                 params.lun_len = sizeof(req_hdr->lun);
2402                 params.lun_set = 1;
2403                 params.cmd_sn = req_hdr->cmd_sn;
2404                 params.cmd_sn_set = 1;
2405                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2406                         &params);
2407                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2408                 break;
2409         case ISCSI_FUNCTION_TARGET_COLD_RESET:
2410         case ISCSI_FUNCTION_TARGET_WARM_RESET:
2411                 target_abort(req, 1);
2412                 params.fn = SCST_TARGET_RESET;
2413                 params.cmd_sn = req_hdr->cmd_sn;
2414                 params.cmd_sn_set = 1;
2415                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2416                         &params);
2417                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2418                 break;
2419         case ISCSI_FUNCTION_LOGICAL_UNIT_RESET:
2420                 target_abort(req, 0);
2421                 params.fn = SCST_LUN_RESET;
2422                 params.lun = (uint8_t *)&req_hdr->lun;
2423                 params.lun_len = sizeof(req_hdr->lun);
2424                 params.lun_set = 1;
2425                 params.cmd_sn = req_hdr->cmd_sn;
2426                 params.cmd_sn_set = 1;
2427                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2428                         &params);
2429                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2430                 break;
2431         case ISCSI_FUNCTION_TASK_REASSIGN:
2432                 rc = -1;
2433                 status = ISCSI_RESPONSE_ALLEGIANCE_REASSIGNMENT_UNSUPPORTED;
2434                 break;
2435         default:
2436                 PRINT_ERROR("Unknown TM function %d", function);
2437                 rc = -1;
2438                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2439                 break;
2440         }
2441
2442 reject:
2443         if (rc != 0)
2444                 iscsi_send_task_mgmt_resp(req, status);
2445
2446         return;
2447 }
2448
2449 static void noop_out_exec(struct iscsi_cmnd *req)
2450 {
2451         struct iscsi_cmnd *rsp;
2452         struct iscsi_nop_in_hdr *rsp_hdr;
2453
2454         TRACE_ENTRY();
2455
2456         TRACE_DBG("%p", req);
2457
2458         if (cmnd_itt(req) != cpu_to_be32(ISCSI_RESERVED_TAG)) {
2459                 rsp = iscsi_alloc_main_rsp(req);
2460
2461                 rsp_hdr = (struct iscsi_nop_in_hdr *)&rsp->pdu.bhs;
2462                 rsp_hdr->opcode = ISCSI_OP_NOOP_IN;
2463                 rsp_hdr->flags = ISCSI_FLG_FINAL;
2464                 rsp_hdr->itt = req->pdu.bhs.itt;
2465                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
2466
2467                 if (req->pdu.datasize)
2468                         sBUG_ON(req->sg == NULL);
2469                 else
2470                         sBUG_ON(req->sg != NULL);
2471
2472                 if (req->sg) {
2473                         rsp->sg = req->sg;
2474                         rsp->sg_cnt = req->sg_cnt;
2475                         rsp->bufflen = req->bufflen;
2476                 }
2477
2478                 /* We already checked it in check_segment_length() */
2479                 sBUG_ON(get_pgcnt(req->pdu.datasize, 0) > ISCSI_CONN_IOV_MAX);
2480
2481                 rsp->pdu.datasize = req->pdu.datasize;
2482         }
2483
2484         req_cmnd_release(req);
2485
2486         TRACE_EXIT();
2487         return;
2488 }
2489
2490 static void logout_exec(struct iscsi_cmnd *req)
2491 {
2492         struct iscsi_logout_req_hdr *req_hdr;
2493         struct iscsi_cmnd *rsp;
2494         struct iscsi_logout_rsp_hdr *rsp_hdr;
2495
2496         PRINT_INFO("Logout received from initiator %s",
2497                 req->conn->session->initiator_name);
2498         TRACE_DBG("%p", req);
2499
2500         req_hdr = (struct iscsi_logout_req_hdr *)&req->pdu.bhs;
2501         rsp = iscsi_alloc_main_rsp(req);
2502         rsp_hdr = (struct iscsi_logout_rsp_hdr *)&rsp->pdu.bhs;
2503         rsp_hdr->opcode = ISCSI_OP_LOGOUT_RSP;
2504         rsp_hdr->flags = ISCSI_FLG_FINAL;
2505         rsp_hdr->itt = req_hdr->itt;
2506         rsp->should_close_conn = 1;
2507
2508         req_cmnd_release(req);
2509
2510         return;
2511 }
2512
2513 static void iscsi_cmnd_exec(struct iscsi_cmnd *cmnd)
2514 {
2515         TRACE_ENTRY();
2516
2517         TRACE_DBG("cmnd %p, op %x, SN %u", cmnd, cmnd_opcode(cmnd),
2518                 cmnd->pdu.bhs.sn);
2519
2520         iscsi_extracheck_is_rd_thread(cmnd->conn);
2521
2522         if (cmnd_opcode(cmnd) == ISCSI_OP_SCSI_CMD) {
2523                 if (cmnd->r2t_len_to_receive == 0)
2524                         iscsi_restart_cmnd(cmnd);
2525                 else if (cmnd->r2t_len_to_send != 0)
2526                         send_r2t(cmnd);
2527                 goto out;
2528         }
2529
2530         if (cmnd->prelim_compl_flags != 0) {
2531                 TRACE_MGMT_DBG("Terminating prelim completed non-SCSI cmnd %p "
2532                         "(op %x)", cmnd, cmnd_opcode(cmnd));
2533                 req_cmnd_release(cmnd);
2534                 goto out;
2535         }
2536
2537         switch (cmnd_opcode(cmnd)) {
2538         case ISCSI_OP_NOOP_OUT:
2539                 noop_out_exec(cmnd);
2540                 break;
2541         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2542                 execute_task_management(cmnd);
2543                 break;
2544         case ISCSI_OP_LOGOUT_CMD:
2545                 logout_exec(cmnd);
2546                 break;
2547         default:
2548                 PRINT_CRIT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
2549                 sBUG();
2550                 break;
2551         }
2552
2553 out:
2554         TRACE_EXIT();
2555         return;
2556 }
2557
2558 static void set_cork(struct socket *sock, int on)
2559 {
2560         int opt = on;
2561         mm_segment_t oldfs;
2562
2563         oldfs = get_fs();
2564         set_fs(get_ds());
2565         sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK,
2566                               (void __force __user *)&opt, sizeof(opt));
2567         set_fs(oldfs);
2568         return;
2569 }
2570
2571 void cmnd_tx_start(struct iscsi_cmnd *cmnd)
2572 {
2573         struct iscsi_conn *conn = cmnd->conn;
2574
2575         TRACE_DBG("conn %p, cmnd %p, opcode %x", conn, cmnd, cmnd_opcode(cmnd));
2576         iscsi_cmnd_set_length(&cmnd->pdu);
2577
2578         iscsi_extracheck_is_wr_thread(conn);
2579
2580         set_cork(conn->sock, 1);
2581
2582         conn->write_iop = conn->write_iov;
2583         conn->write_iop->iov_base = (void __force __user *)(&cmnd->pdu.bhs);
2584         conn->write_iop->iov_len = sizeof(cmnd->pdu.bhs);
2585         conn->write_iop_used = 1;
2586         conn->write_size = sizeof(cmnd->pdu.bhs) + cmnd->pdu.datasize;
2587         conn->write_offset = 0;
2588
2589         switch (cmnd_opcode(cmnd)) {
2590         case ISCSI_OP_NOOP_IN:
2591                 cmnd_set_sn(cmnd, 1);
2592                 break;
2593         case ISCSI_OP_SCSI_RSP:
2594                 cmnd_set_sn(cmnd, 1);
2595                 break;
2596         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2597                 cmnd_set_sn(cmnd, 1);
2598                 break;
2599         case ISCSI_OP_TEXT_RSP:
2600                 cmnd_set_sn(cmnd, 1);
2601                 break;
2602         case ISCSI_OP_SCSI_DATA_IN:
2603         {
2604                 struct iscsi_data_in_hdr *rsp =
2605                         (struct iscsi_data_in_hdr *)&cmnd->pdu.bhs;
2606                 u32 offset = cpu_to_be32(rsp->buffer_offset);
2607
2608                 TRACE_DBG("cmnd %p, offset %u, datasize %u, bufflen %u", cmnd,
2609                         offset, cmnd->pdu.datasize, cmnd->bufflen);
2610
2611                 sBUG_ON(offset > cmnd->bufflen);
2612                 sBUG_ON(offset + cmnd->pdu.datasize > cmnd->bufflen);
2613
2614                 conn->write_offset = offset;
2615
2616                 cmnd_set_sn(cmnd, (rsp->flags & ISCSI_FLG_FINAL) ? 1 : 0);
2617                 break;
2618         }
2619         case ISCSI_OP_LOGOUT_RSP:
2620                 cmnd_set_sn(cmnd, 1);
2621                 break;
2622         case ISCSI_OP_R2T:
2623                 cmnd->pdu.bhs.sn = cmnd_set_sn(cmnd, 0);
2624                 break;
2625         case ISCSI_OP_ASYNC_MSG:
2626                 cmnd_set_sn(cmnd, 1);
2627                 break;
2628         case ISCSI_OP_REJECT:
2629                 cmnd_set_sn(cmnd, 1);
2630                 break;
2631         default:
2632                 PRINT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
2633                 break;
2634         }
2635
2636         iscsi_dump_pdu(&cmnd->pdu);
2637         return;
2638 }
2639
2640 void cmnd_tx_end(struct iscsi_cmnd *cmnd)
2641 {
2642         struct iscsi_conn *conn = cmnd->conn;
2643
2644         TRACE_DBG("%p:%x (should_close_conn %d, should_close_all_conn %d)",
2645                 cmnd, cmnd_opcode(cmnd), cmnd->should_close_conn,
2646                 cmnd->should_close_all_conn);
2647
2648 #ifdef CONFIG_SCST_EXTRACHECKS
2649         switch (cmnd_opcode(cmnd)) {
2650         case ISCSI_OP_NOOP_IN:
2651         case ISCSI_OP_SCSI_RSP:
2652         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2653         case ISCSI_OP_TEXT_RSP:
2654         case ISCSI_OP_R2T:
2655         case ISCSI_OP_ASYNC_MSG:
2656         case ISCSI_OP_REJECT:
2657         case ISCSI_OP_SCSI_DATA_IN:
2658         case ISCSI_OP_LOGOUT_RSP:
2659                 break;
2660         default:
2661                 PRINT_CRIT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2662                 sBUG();
2663                 break;
2664         }
2665 #endif
2666
2667         if (unlikely(cmnd->should_close_conn)) {
2668                 if (cmnd->should_close_all_conn) {
2669                         PRINT_INFO("Closing all connections for target %x at "
2670                                 "initiator's %s request",
2671                                 cmnd->conn->session->target->tid,
2672                                 conn->session->initiator_name);
2673                         target_del_all_sess(cmnd->conn->session->target, 0);
2674                 } else {
2675                         PRINT_INFO("Closing connection at initiator's %s "
2676                                 "request", conn->session->initiator_name);
2677                         mark_conn_closed(conn);
2678                 }
2679         }
2680
2681         set_cork(cmnd->conn->sock, 0);
2682         return;
2683 }
2684
2685 /*
2686  * Push the command for execution. This functions reorders the commands.
2687  * Called from the read thread.
2688  *
2689  * Basically, since we don't support MC/S and TCP guarantees data delivery
2690  * order, all that SN's stuff isn't needed at all (commands delivery order is
2691  * a natural commands execution order), but insane iSCSI spec requires
2692  * us to check it and we have to, because some crazy initiators can rely
2693  * on the SN's based order and reorder requests during sending. For all other
2694  * normal initiators all that code is a NOP.
2695  */
2696 static void iscsi_push_cmnd(struct iscsi_cmnd *cmnd)
2697 {
2698         struct iscsi_session *session = cmnd->conn->session;
2699         struct list_head *entry;
2700         u32 cmd_sn;
2701
2702         TRACE_DBG("cmnd %p, iSCSI opcode %x, sn %u, exp sn %u", cmnd,
2703                 cmnd_opcode(cmnd), cmnd->pdu.bhs.sn, session->exp_cmd_sn);
2704
2705         iscsi_extracheck_is_rd_thread(cmnd->conn);
2706
2707         sBUG_ON(cmnd->parent_req != NULL);
2708
2709         if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
2710                 TRACE_DBG("Immediate cmd %p (cmd_sn %u)", cmnd,
2711                         cmnd->pdu.bhs.sn);
2712                 iscsi_cmnd_exec(cmnd);
2713                 goto out;
2714         }
2715
2716         spin_lock(&session->sn_lock);
2717
2718         cmd_sn = cmnd->pdu.bhs.sn;
2719         if (cmd_sn == session->exp_cmd_sn) {
2720                 while (1) {
2721                         session->exp_cmd_sn = ++cmd_sn;
2722
2723                         if (unlikely(session->tm_active > 0)) {
2724                                 if (before(cmd_sn, session->tm_sn)) {
2725                                         struct iscsi_conn *conn = cmnd->conn;
2726
2727                                         spin_unlock(&session->sn_lock);
2728
2729                                         spin_lock_bh(&conn->cmd_list_lock);
2730                                         __cmnd_abort(cmnd);
2731                                         spin_unlock_bh(&conn->cmd_list_lock);
2732
2733                                         spin_lock(&session->sn_lock);
2734                                 }
2735                                 iscsi_check_send_delayed_tm_resp(session);
2736                         }
2737
2738                         spin_unlock(&session->sn_lock);
2739
2740                         iscsi_cmnd_exec(cmnd);
2741
2742                         spin_lock(&session->sn_lock);
2743
2744                         if (list_empty(&session->pending_list))
2745                                 break;
2746                         cmnd = list_entry(session->pending_list.next,
2747                                           struct iscsi_cmnd,
2748                                           pending_list_entry);
2749                         if (cmnd->pdu.bhs.sn != cmd_sn)
2750                                 break;
2751
2752                         list_del(&cmnd->pending_list_entry);
2753                         cmnd->pending = 0;
2754
2755                         TRACE_MGMT_DBG("Processing pending cmd %p (cmd_sn %u)",
2756                                 cmnd, cmd_sn);
2757                 }
2758         } else {
2759                 int drop = 0;
2760
2761                 TRACE_DBG("Pending cmd %p (cmd_sn %u, exp_cmd_sn %u)",
2762                         cmnd, cmd_sn, session->exp_cmd_sn);
2763
2764                 /*
2765                  * iSCSI RFC 3720: "The target MUST silently ignore any
2766                  * non-immediate command outside of [from ExpCmdSN to MaxCmdSN
2767                  * inclusive] range". But we won't honor the MaxCmdSN
2768                  * requirement, because, since we adjust MaxCmdSN from the
2769                  * separate write thread, rarely it is possible that initiator
2770                  * can legally send command with CmdSN>MaxSN. But it won't
2771                  * hurt anything, in the worst case it will lead to
2772                  * additional QUEUE FULL status.
2773                  */
2774
2775                 if (unlikely(before(cmd_sn, session->exp_cmd_sn))) {
2776                         PRINT_ERROR("Unexpected cmd_sn (%u,%u)", cmd_sn,
2777                                 session->exp_cmd_sn);
2778                         drop = 1;
2779                 }
2780
2781 #if 0
2782                 if (unlikely(after(cmd_sn, session->exp_cmd_sn +
2783                                         iscsi_get_allowed_cmds(session)))) {
2784                         TRACE_MGMT_DBG("Too large cmd_sn %u (exp_cmd_sn %u, "
2785                                 "max_sn %u)", cmd_sn, session->exp_cmd_sn,
2786                                 iscsi_get_allowed_cmds(session));
2787                 }
2788 #endif
2789
2790                 spin_unlock(&session->sn_lock);
2791
2792                 if (unlikely(drop)) {
2793                         req_cmnd_release_force(cmnd);
2794                         goto out;
2795                 }
2796
2797                 if (unlikely(test_bit(ISCSI_CMD_ABORTED,
2798                                         &cmnd->prelim_compl_flags))) {
2799                         struct iscsi_cmnd *tm_clone;
2800
2801                         TRACE_MGMT_DBG("Pending aborted cmnd %p, creating TM "
2802                                 "clone (scst cmd %p, state %d)", cmnd,
2803                                 cmnd->scst_cmd, cmnd->scst_state);
2804
2805                         tm_clone = cmnd_alloc(cmnd->conn, NULL);
2806                         if (tm_clone != NULL) {
2807                                 set_bit(ISCSI_CMD_ABORTED,
2808                                         &tm_clone->prelim_compl_flags);
2809                                 tm_clone->pdu = cmnd->pdu;
2810
2811                                 TRACE_MGMT_DBG("TM clone %p created",
2812                                                tm_clone);
2813
2814                                 iscsi_cmnd_exec(cmnd);
2815                                 cmnd = tm_clone;
2816                         } else
2817                                 PRINT_ERROR("%s", "Unable to create TM clone");
2818                 }
2819
2820                 TRACE_MGMT_DBG("Pending cmnd %p (op %x, sn %u, exp sn %u)",
2821                         cmnd, cmnd_opcode(cmnd), cmd_sn, session->exp_cmd_sn);
2822
2823                 spin_lock(&session->sn_lock);
2824                 list_for_each(entry, &session->pending_list) {
2825                         struct iscsi_cmnd *tmp =
2826                                 list_entry(entry, struct iscsi_cmnd,
2827                                            pending_list_entry);
2828                         if (before(cmd_sn, tmp->pdu.bhs.sn))
2829                                 break;
2830                 }
2831                 list_add_tail(&cmnd->pending_list_entry, entry);
2832                 cmnd->pending = 1;
2833         }
2834
2835         spin_unlock(&session->sn_lock);
2836 out:
2837         return;
2838 }
2839
2840 static int check_segment_length(struct iscsi_cmnd *cmnd)
2841 {
2842         struct iscsi_conn *conn = cmnd->conn;
2843         struct iscsi_session *session = conn->session;
2844
2845         if (unlikely(cmnd->pdu.datasize > session->sess_param.max_recv_data_length)) {
2846                 PRINT_ERROR("Initiator %s violated negotiated parameters: "
2847                         "data too long (ITT %x, datasize %u, "
2848                         "max_recv_data_length %u", session->initiator_name,
2849                         cmnd_itt(cmnd), cmnd->pdu.datasize,
2850                         session->sess_param.max_recv_data_length);
2851                 mark_conn_closed(conn);
2852                 return -EINVAL;
2853         }
2854         return 0;
2855 }
2856
2857 int cmnd_rx_start(struct iscsi_cmnd *cmnd)
2858 {
2859         int res, rc;
2860
2861         iscsi_dump_pdu(&cmnd->pdu);
2862
2863         res = check_segment_length(cmnd);
2864         if (res != 0)
2865                 goto out;
2866
2867         switch (cmnd_opcode(cmnd)) {
2868         case ISCSI_OP_SCSI_CMD:
2869                 res = scsi_cmnd_start(cmnd);
2870                 if (unlikely(res < 0))
2871                         goto out;
2872                 spin_lock(&cmnd->conn->session->sn_lock);
2873                 __update_stat_sn(cmnd);
2874                 rc = check_cmd_sn(cmnd);
2875                 spin_unlock(&cmnd->conn->session->sn_lock);
2876                 break;
2877         case ISCSI_OP_SCSI_DATA_OUT:
2878                 res = data_out_start(cmnd);
2879                 goto out;
2880         case ISCSI_OP_NOOP_OUT:
2881                 rc = noop_out_start(cmnd);
2882                 break;
2883         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2884         case ISCSI_OP_LOGOUT_CMD:
2885                 spin_lock(&cmnd->conn->session->sn_lock);
2886                 __update_stat_sn(cmnd);
2887                 rc = check_cmd_sn(cmnd);
2888                 spin_unlock(&cmnd->conn->session->sn_lock);
2889                 break;
2890         case ISCSI_OP_TEXT_CMD:
2891         case ISCSI_OP_SNACK_CMD:
2892         default:
2893                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2894                 break;
2895         }
2896
2897         if (unlikely(rc < 0)) {
2898                 PRINT_ERROR("Error %d (iSCSI opcode %x, ITT %x)", rc,
2899                         cmnd_opcode(cmnd), cmnd_itt(cmnd));
2900                 res = create_reject_rsp(cmnd, -rc, true);
2901         }
2902
2903 out:
2904         TRACE_EXIT_RES(res);
2905         return res;
2906 }
2907
2908 void cmnd_rx_end(struct iscsi_cmnd *cmnd)
2909 {
2910         TRACE_ENTRY();
2911
2912         TRACE_DBG("cmnd %p, opcode %x", cmnd, cmnd_opcode(cmnd));
2913
2914         switch (cmnd_opcode(cmnd)) {
2915         case ISCSI_OP_SCSI_CMD:
2916         case ISCSI_OP_NOOP_OUT:
2917         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2918         case ISCSI_OP_LOGOUT_CMD:
2919                 iscsi_push_cmnd(cmnd);
2920                 goto out;
2921         case ISCSI_OP_SCSI_DATA_OUT:
2922                 data_out_end(cmnd);
2923                 break;
2924         default:
2925                 PRINT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
2926                 break;
2927         }
2928
2929         req_cmnd_release(cmnd);
2930
2931 out:
2932         TRACE_EXIT();
2933         return;
2934 }
2935
2936 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
2937 static int iscsi_alloc_data_buf(struct scst_cmd *cmd)
2938 {
2939         /*
2940          * sock->ops->sendpage() is async zero copy operation,
2941          * so we must be sure not to free and reuse
2942          * the command's buffer before the sending was completed
2943          * by the network layers. It is possible only if we
2944          * don't use SGV cache.
2945          */
2946         EXTRACHECKS_BUG_ON(!(scst_cmd_get_data_direction(cmd) & SCST_DATA_READ));
2947         scst_cmd_set_no_sgv(cmd);
2948         return 1;
2949 }
2950 #endif
2951
2952 static void iscsi_preprocessing_done(struct scst_cmd *scst_cmd)
2953 {
2954         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
2955                                 scst_cmd_get_tgt_priv(scst_cmd);
2956
2957         TRACE_DBG("req %p", req);
2958
2959         if (req->conn->rx_task == current)
2960                 req->scst_state = ISCSI_CMD_STATE_AFTER_PREPROC;
2961         else {
2962                 /*
2963                  * We wait for the state change without any protection, so
2964                  * without cmnd_get() it is possible that req will die
2965                  * "immediately" after the state assignment and
2966                  * iscsi_make_conn_rd_active() will operate on dead data.
2967                  * We use the ordered version of cmnd_get(), because "get"
2968                  * must be done before the state assignment.
2969                  *
2970                  * We protected from the race on calling cmnd_rx_continue(),
2971                  * because there can be only one read thread processing
2972                  * connection.
2973                  */
2974                 cmnd_get_ordered(req);
2975                 req->scst_state = ISCSI_CMD_STATE_AFTER_PREPROC;
2976                 iscsi_make_conn_rd_active(req->conn);
2977                 if (unlikely(req->conn->closing)) {
2978                         TRACE_DBG("Waking up closing conn %p", req->conn);
2979                         wake_up(&req->conn->read_state_waitQ);
2980                 }
2981                 cmnd_put(req);
2982         }
2983
2984         return;
2985 }
2986
2987 /*
2988  * No locks.
2989  *
2990  * IMPORTANT! Connection conn must be protected by additional conn_get()
2991  * upon entrance in this function, because otherwise it could be destroyed
2992  * inside as a result of iscsi_send(), which releases sent commands.
2993  */
2994 static void iscsi_try_local_processing(struct iscsi_conn *conn)
2995 {
2996         int local;
2997
2998         TRACE_ENTRY();
2999
3000         spin_lock_bh(&iscsi_wr_lock);
3001         switch (conn->wr_state) {
3002         case ISCSI_CONN_WR_STATE_IN_LIST:
3003                 list_del(&conn->wr_list_entry);
3004                 /* go through */
3005         case ISCSI_CONN_WR_STATE_IDLE:
3006 #ifdef CONFIG_SCST_EXTRACHECKS
3007                 conn->wr_task = current;
3008 #endif
3009                 conn->wr_state = ISCSI_CONN_WR_STATE_PROCESSING;
3010                 conn->wr_space_ready = 0;
3011                 local = 1;
3012                 break;
3013         default:
3014                 local = 0;
3015                 break;
3016         }
3017         spin_unlock_bh(&iscsi_wr_lock);
3018
3019         if (local) {
3020                 int rc = 1;
3021
3022                 if (test_write_ready(conn))
3023                         rc = iscsi_send(conn);
3024
3025                 spin_lock_bh(&iscsi_wr_lock);
3026 #ifdef CONFIG_SCST_EXTRACHECKS
3027                 conn->wr_task = NULL;
3028 #endif
3029                 if ((rc <= 0) || test_write_ready(conn)) {
3030                         list_add_tail(&conn->wr_list_entry, &iscsi_wr_list);
3031                         conn->wr_state = ISCSI_CONN_WR_STATE_IN_LIST;
3032                         wake_up(&iscsi_wr_waitQ);
3033                 } else
3034                         conn->wr_state = ISCSI_CONN_WR_STATE_IDLE;
3035                 spin_unlock_bh(&iscsi_wr_lock);
3036         }
3037
3038         TRACE_EXIT();
3039         return;
3040 }
3041
3042 static int iscsi_xmit_response(struct scst_cmd *scst_cmd)
3043 {
3044         int is_send_status = scst_cmd_get_is_send_status(scst_cmd);
3045         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
3046                                         scst_cmd_get_tgt_priv(scst_cmd);
3047         struct iscsi_conn *conn = req->conn;
3048         int status = scst_cmd_get_status(scst_cmd);
3049         u8 *sense = scst_cmd_get_sense_buffer(scst_cmd);
3050         int sense_len = scst_cmd_get_sense_buffer_len(scst_cmd);
3051
3052         if (unlikely(scst_cmd_atomic(scst_cmd)))
3053                 return SCST_TGT_RES_NEED_THREAD_CTX;
3054
3055         scst_cmd_set_tgt_priv(scst_cmd, NULL);
3056
3057         EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_RESTARTED);
3058
3059         if (unlikely(scst_cmd_aborted(scst_cmd)))
3060                 set_bit(ISCSI_CMD_ABORTED, &req->prelim_compl_flags);
3061
3062         if (unlikely(req->prelim_compl_flags != 0)) {
3063                 if (test_bit(ISCSI_CMD_ABORTED, &req->prelim_compl_flags)) {
3064                         TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
3065                                 req->scst_cmd);
3066                         scst_set_delivery_status(req->scst_cmd,
3067                                 SCST_CMD_DELIVERY_ABORTED);
3068                         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
3069                         req_cmnd_release_force(req);
3070                         goto out;
3071                 }
3072
3073                 TRACE_DBG("Prelim completed req %p", req);
3074
3075                 /* To make sure we didn't miss anything above */
3076                 sBUG_ON(status == 0);
3077
3078                 /*
3079                  * We could preliminary have finished req before we
3080                  * knew its device, so check if we return correct sense
3081                  * format.
3082                  */
3083                 scst_check_convert_sense(scst_cmd);
3084
3085                 if (!req->own_sg) {
3086                         req->sg = scst_cmd_get_sg(scst_cmd);
3087                         req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
3088                 }
3089         } else {
3090                 EXTRACHECKS_BUG_ON(req->own_sg);
3091                 req->sg = scst_cmd_get_sg(scst_cmd);
3092                 req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
3093         }
3094
3095         req->bufflen = scst_cmd_get_resp_data_len(scst_cmd);
3096
3097         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
3098
3099         TRACE_DBG("req %p, is_send_status=%x, req->bufflen=%d, req->sg=%p, "
3100                 "req->sg_cnt %d", req, is_send_status, req->bufflen, req->sg,
3101                 req->sg_cnt);
3102
3103         EXTRACHECKS_BUG_ON(req->hashed);
3104         if (req->main_rsp != NULL)
3105                 EXTRACHECKS_BUG_ON(cmnd_opcode(req->main_rsp) != ISCSI_OP_REJECT);
3106
3107         if (unlikely((req->bufflen != 0) && !is_send_status)) {
3108                 PRINT_CRIT_ERROR("%s", "Sending DATA without STATUS is "
3109                         "unsupported");
3110                 scst_set_cmd_error(scst_cmd,
3111                         SCST_LOAD_SENSE(scst_sense_hardw_error));
3112                 sBUG(); /* ToDo */
3113         }
3114
3115         if (req->bufflen != 0) {
3116                 /*
3117                  * Check above makes sure that is_send_status is set,
3118                  * so status is valid here, but in future that could change.
3119                  * ToDo
3120                  */
3121                 if ((status != SAM_STAT_CHECK_CONDITION) &&
3122                     ((cmnd_hdr(req)->flags & (ISCSI_CMD_WRITE|ISCSI_CMD_READ)) !=
3123                                 (ISCSI_CMD_WRITE|ISCSI_CMD_READ))) {
3124                         send_data_rsp(req, status, is_send_status);
3125                 } else {
3126                         struct iscsi_cmnd *rsp;
3127                         send_data_rsp(req, 0, 0);
3128                         if (is_send_status) {
3129                                 rsp = create_status_rsp(req, status, sense,
3130                                         sense_len, true);
3131                                 iscsi_cmnd_init_write(rsp, 0);
3132                         }
3133                 }
3134         } else if (is_send_status) {
3135                 struct iscsi_cmnd *rsp;
3136                 rsp = create_status_rsp(req, status, sense, sense_len, false);
3137                 iscsi_cmnd_init_write(rsp, 0);
3138         }
3139 #ifdef CONFIG_SCST_EXTRACHECKS
3140         else
3141                 sBUG();
3142 #endif
3143
3144         /*
3145          * "_ordered" here to protect from reorder, which can lead to
3146          * preliminary connection destroy in req_cmnd_release(). Just in
3147          * case, actually, because reordering shouldn't go so far, but who
3148          * knows..
3149          */
3150         conn_get_ordered(conn);
3151         req_cmnd_release(req);
3152         iscsi_try_local_processing(conn);
3153         conn_put(conn);
3154
3155 out:
3156         return SCST_TGT_RES_SUCCESS;
3157 }
3158
3159 /* Called under sn_lock */
3160 static bool iscsi_is_delay_tm_resp(struct iscsi_cmnd *rsp)
3161 {
3162         bool res = 0;
3163         struct iscsi_task_mgt_hdr *req_hdr =
3164                 (struct iscsi_task_mgt_hdr *)&rsp->parent_req->pdu.bhs;
3165         int function = req_hdr->function & ISCSI_FUNCTION_MASK;
3166         struct iscsi_session *sess = rsp->conn->session;
3167
3168         TRACE_ENTRY();
3169
3170         /* This should be checked for immediate TM commands as well */
3171
3172         switch (function) {
3173         default:
3174                 if (before(sess->exp_cmd_sn, req_hdr->cmd_sn))
3175                         res = 1;
3176                 break;
3177         }
3178
3179         TRACE_EXIT_RES(res);
3180         return res;
3181 }
3182
3183 /* Called under sn_lock, but might drop it inside, then reaquire */
3184 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess)
3185         __acquires(&sn_lock)
3186         __releases(&sn_lock)
3187 {
3188         struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
3189
3190         TRACE_ENTRY();
3191
3192         if (tm_rsp == NULL)
3193                 goto out;
3194
3195         if (iscsi_is_delay_tm_resp(tm_rsp))
3196                 goto out;
3197
3198         TRACE(TRACE_MGMT_MINOR, "Sending delayed rsp %p", tm_rsp);
3199
3200         sess->tm_rsp = NULL;
3201         sess->tm_active--;
3202
3203         spin_unlock(&sess->sn_lock);
3204
3205         sBUG_ON(sess->tm_active < 0);
3206
3207         iscsi_cmnd_init_write(tm_rsp, ISCSI_INIT_WRITE_WAKE);
3208
3209         spin_lock(&sess->sn_lock);
3210
3211 out:
3212         TRACE_EXIT();
3213         return;
3214 }
3215
3216 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status)
3217 {
3218         struct iscsi_cmnd *rsp;
3219         struct iscsi_task_mgt_hdr *req_hdr =
3220                                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
3221         struct iscsi_task_rsp_hdr *rsp_hdr;
3222         struct iscsi_session *sess = req->conn->session;
3223         int fn = req_hdr->function & ISCSI_FUNCTION_MASK;
3224
3225         TRACE_ENTRY();
3226
3227         TRACE_MGMT_DBG("TM req %p finished", req);
3228         TRACE((req_hdr->function == ISCSI_FUNCTION_ABORT_TASK) ?
3229                          TRACE_MGMT_MINOR : TRACE_MGMT,
3230                 "TM fn %d finished, status %d", fn, status);
3231
3232         rsp = iscsi_alloc_rsp(req);
3233         rsp_hdr = (struct iscsi_task_rsp_hdr *)&rsp->pdu.bhs;
3234
3235         rsp_hdr->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
3236         rsp_hdr->flags = ISCSI_FLG_FINAL;
3237         rsp_hdr->itt = req_hdr->itt;
3238         rsp_hdr->response = status;
3239
3240         if (fn == ISCSI_FUNCTION_TARGET_COLD_RESET) {
3241                 rsp->should_close_conn = 1;
3242                 rsp->should_close_all_conn = 1;
3243         }
3244
3245         sBUG_ON(sess->tm_rsp != NULL);
3246
3247         spin_lock(&sess->sn_lock);
3248         if (iscsi_is_delay_tm_resp(rsp)) {
3249                 TRACE(TRACE_MGMT_MINOR, "Delaying TM fn %d response %p "
3250                         "(req %p), because not all affected commands received "
3251                         "(TM cmd sn %u, exp sn %u)",
3252                         req_hdr->function & ISCSI_FUNCTION_MASK, rsp, req,
3253                         req_hdr->cmd_sn, sess->exp_cmd_sn);
3254                 sess->tm_rsp = rsp;
3255                 spin_unlock(&sess->sn_lock);
3256                 goto out_release;
3257         }
3258         sess->tm_active--;
3259         spin_unlock(&sess->sn_lock);
3260
3261         sBUG_ON(sess->tm_active < 0);
3262
3263         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_WAKE);
3264
3265 out_release:
3266         req_cmnd_release(req);
3267
3268         TRACE_EXIT();
3269         return;
3270 }
3271
3272 static inline int iscsi_get_mgmt_response(int status)
3273 {
3274         switch (status) {
3275         case SCST_MGMT_STATUS_SUCCESS:
3276                 return ISCSI_RESPONSE_FUNCTION_COMPLETE;
3277
3278         case SCST_MGMT_STATUS_TASK_NOT_EXIST:
3279                 return ISCSI_RESPONSE_UNKNOWN_TASK;
3280
3281         case SCST_MGMT_STATUS_LUN_NOT_EXIST:
3282                 return ISCSI_RESPONSE_UNKNOWN_LUN;
3283
3284         case SCST_MGMT_STATUS_FN_NOT_SUPPORTED:
3285                 return ISCSI_RESPONSE_FUNCTION_UNSUPPORTED;
3286
3287         case SCST_MGMT_STATUS_REJECTED:
3288         case SCST_MGMT_STATUS_FAILED:
3289         default:
3290                 return ISCSI_RESPONSE_FUNCTION_REJECTED;
3291         }
3292 }
3293
3294 static void iscsi_task_mgmt_fn_done(struct scst_mgmt_cmd *scst_mcmd)
3295 {
3296         int fn = scst_mgmt_cmd_get_fn(scst_mcmd);
3297         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
3298                                 scst_mgmt_cmd_get_tgt_priv(scst_mcmd);
3299         int status =
3300                 iscsi_get_mgmt_response(scst_mgmt_cmd_get_status(scst_mcmd));
3301
3302         if ((status == ISCSI_RESPONSE_UNKNOWN_TASK) &&
3303             (fn == SCST_ABORT_TASK)) {
3304                 /* If we are here, we found the task, so must succeed */
3305                 status = ISCSI_RESPONSE_FUNCTION_COMPLETE;
3306         }
3307
3308         TRACE_MGMT_DBG("req %p, scst_mcmd %p, fn %d, scst status %d, status %d",
3309                 req, scst_mcmd, fn, scst_mgmt_cmd_get_status(scst_mcmd),
3310                 status);
3311
3312         switch (fn) {
3313         case SCST_NEXUS_LOSS_SESS:
3314         case SCST_ABORT_ALL_TASKS_SESS:
3315                 /* They are internal */
3316                 break;
3317         default:
3318                 iscsi_send_task_mgmt_resp(req, status);
3319                 scst_mgmt_cmd_set_tgt_priv(scst_mcmd, NULL);
3320                 break;
3321         }
3322         return;
3323 }
3324
3325 static int iscsi_scsi_aen(struct scst_aen *aen)
3326 {
3327         int res = SCST_AEN_RES_SUCCESS;
3328         uint64_t lun = scst_aen_get_lun(aen);
3329         const uint8_t *sense = scst_aen_get_sense(aen);
3330         int sense_len = scst_aen_get_sense_len(aen);
3331         struct iscsi_session *sess = scst_sess_get_tgt_priv(
3332                                         scst_aen_get_sess(aen));
3333         struct iscsi_conn *conn;
3334         bool found;
3335         struct iscsi_cmnd *fake_req, *rsp;
3336         struct iscsi_async_msg_hdr *rsp_hdr;
3337         struct scatterlist *sg;
3338
3339         TRACE_ENTRY();
3340
3341         TRACE_MGMT_DBG("SCSI AEN to sess %p (initiator %s)", sess,
3342                 sess->initiator_name);
3343
3344         mutex_lock(&sess->target->target_mutex);
3345
3346         found = false;
3347         list_for_each_entry_reverse(conn, &sess->conn_list, conn_list_entry) {
3348                 if (!test_bit(ISCSI_CONN_SHUTTINGDOWN, &conn->conn_aflags) &&
3349                     (conn->conn_reinst_successor == NULL)) {
3350                         found = true;
3351                         break;
3352                 }
3353         }
3354         if (!found) {
3355                 TRACE_MGMT_DBG("Unable to find alive conn for sess %p", sess);
3356                 goto out_err;
3357         }
3358
3359         /* Create a fake request */
3360         fake_req = cmnd_alloc(conn, NULL);
3361         if (fake_req == NULL) {
3362                 PRINT_ERROR("%s", "Unable to alloc fake AEN request");
3363                 goto out_err;
3364         }
3365
3366         mutex_unlock(&sess->target->target_mutex);
3367
3368         rsp = iscsi_alloc_main_rsp(fake_req);
3369         if (rsp == NULL) {
3370                 PRINT_ERROR("%s", "Unable to alloc AEN rsp");
3371                 goto out_err_free_req;
3372         }
3373
3374         fake_req->scst_state = ISCSI_CMD_STATE_AEN;
3375         fake_req->scst_aen = aen;
3376
3377         rsp_hdr = (struct iscsi_async_msg_hdr *)&rsp->pdu.bhs;
3378
3379         rsp_hdr->opcode = ISCSI_OP_ASYNC_MSG;
3380         rsp_hdr->flags = ISCSI_FLG_FINAL;
3381         rsp_hdr->lun = lun; /* it's already in SCSI form */
3382         rsp_hdr->ffffffff = 0xffffffff;
3383         rsp_hdr->async_event = ISCSI_ASYNC_SCSI;
3384
3385         sg = rsp->sg = rsp->rsp_sg;
3386         rsp->sg_cnt = 2;
3387         rsp->own_sg = 1;
3388
3389         sg_init_table(sg, 2);
3390         sg_set_buf(&sg[0], &rsp->sense_hdr, sizeof(rsp->sense_hdr));
3391         sg_set_buf(&sg[1], sense, sense_len);
3392
3393         rsp->sense_hdr.length = cpu_to_be16(sense_len);
3394         rsp->pdu.datasize = sizeof(rsp->sense_hdr) + sense_len;
3395         rsp->bufflen = rsp->pdu.datasize;
3396
3397         req_cmnd_release(fake_req);
3398
3399 out:
3400         TRACE_EXIT_RES(res);
3401         return res;
3402
3403 out_err_free_req:
3404         req_cmnd_release(fake_req);
3405
3406 out_err:
3407         mutex_unlock(&sess->target->target_mutex);
3408         res = SCST_AEN_RES_FAILED;
3409         goto out;
3410 }
3411
3412 static int iscsi_report_aen(struct scst_aen *aen)
3413 {
3414         int res;
3415         int event_fn = scst_aen_get_event_fn(aen);
3416
3417         TRACE_ENTRY();
3418
3419         switch (event_fn) {
3420         case SCST_AEN_SCSI:
3421                 res = iscsi_scsi_aen(aen);
3422                 break;
3423         default:
3424                 TRACE_MGMT_DBG("Unsupported AEN %d", event_fn);
3425                 res = SCST_AEN_RES_NOT_SUPPORTED;
3426                 break;
3427         }
3428
3429         TRACE_EXIT_RES(res);
3430         return res;
3431 }
3432
3433 static int iscsi_target_detect(struct scst_tgt_template *templ)
3434 {
3435         /* Nothing to do */
3436         return 0;
3437 }
3438
3439 static int iscsi_target_release(struct scst_tgt *scst_tgt)
3440 {
3441         /* Nothing to do */
3442         return 0;
3443 }
3444
3445 #ifndef CONFIG_SCST_PROC
3446 static struct scst_trace_log iscsi_local_trace_tbl[] = {
3447     { TRACE_D_WRITE,            "d_write" },
3448     { TRACE_CONN_OC,            "conn" },
3449     { TRACE_CONN_OC_DBG,        "conn_dbg" },
3450     { TRACE_D_IOV,              "iov" },
3451     { TRACE_D_DUMP_PDU,         "pdu" },
3452     { TRACE_NET_PG,             "net_page" },
3453     { 0,                        NULL }
3454 };
3455
3456 #define ISCSI_TRACE_TLB_HELP    ", d_read, d_write, conn, conn_dbg, iov, pdu, net_page"
3457 #endif
3458
3459 struct scst_tgt_template iscsi_template = {
3460         .name = "iscsi",
3461         .sg_tablesize = 0xFFFF /* no limit */,
3462         .threads_num = 0,
3463         .no_clustering = 1,
3464         .xmit_response_atomic = 0,
3465 #ifndef CONFIG_SCST_PROC
3466         .tgtt_attrs = iscsi_attrs,
3467         .tgt_attrs = iscsi_tgt_attrs,
3468         .sess_attrs = iscsi_sess_attrs,
3469         .enable_tgt = iscsi_enable_target,
3470         .is_tgt_enabled = iscsi_is_target_enabled,
3471 #endif
3472 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
3473         .default_trace_flags = ISCSI_DEFAULT_LOG_FLAGS,
3474         .trace_flags = &trace_flag,
3475 #ifndef CONFIG_SCST_PROC
3476         .trace_tbl = iscsi_local_trace_tbl,
3477         .trace_tbl_help = ISCSI_TRACE_TLB_HELP,
3478 #endif
3479 #endif
3480         .detect = iscsi_target_detect,
3481         .release = iscsi_target_release,
3482         .xmit_response = iscsi_xmit_response,
3483 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3484         .alloc_data_buf = iscsi_alloc_data_buf,
3485 #endif
3486         .preprocessing_done = iscsi_preprocessing_done,
3487         .pre_exec = iscsi_pre_exec,
3488         .task_mgmt_affected_cmds_done = iscsi_task_mgmt_affected_cmds_done,
3489         .task_mgmt_fn_done = iscsi_task_mgmt_fn_done,
3490         .report_aen = iscsi_report_aen,
3491 };
3492
3493 static __init int iscsi_run_threads(int count, char *name, int (*fn)(void *))
3494 {
3495         int res = 0;
3496         int i;
3497         struct iscsi_thread_t *thr;
3498
3499         for (i = 0; i < count; i++) {
3500                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
3501                 if (!thr) {
3502                         res = -ENOMEM;
3503                         PRINT_ERROR("Failed to allocate thr %d", res);
3504                         goto out;
3505                 }
3506                 thr->thr = kthread_run(fn, NULL, "%s%d", name, i);
3507                 if (IS_ERR(thr->thr)) {
3508                         res = PTR_ERR(thr->thr);
3509                         PRINT_ERROR("kthread_create() failed: %d", res);
3510                         kfree(thr);
3511                         goto out;
3512                 }
3513                 list_add_tail(&thr->threads_list_entry, &iscsi_threads_list);
3514         }
3515
3516 out:
3517         return res;
3518 }
3519
3520 static void iscsi_stop_threads(void)
3521 {
3522         struct iscsi_thread_t *t, *tmp;
3523
3524         list_for_each_entry_safe(t, tmp, &iscsi_threads_list,
3525                                 threads_list_entry) {
3526                 int rc = kthread_stop(t->thr);
3527                 if (rc < 0)
3528                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
3529                 list_del(&t->threads_list_entry);
3530                 kfree(t);
3531         }
3532         return;
3533 }
3534
3535 static int __init iscsi_init(void)
3536 {
3537         int err = 0;
3538         int num;
3539
3540         PRINT_INFO("iSCSI SCST Target - version %s", ISCSI_VERSION_STRING);
3541
3542         dummy_page = alloc_pages(GFP_KERNEL, 0);
3543         if (dummy_page == NULL) {
3544                 PRINT_ERROR("%s", "Dummy page allocation failed");
3545                 goto out;
3546         }
3547
3548         sg_init_table(&dummy_sg, 1);
3549         sg_set_page(&dummy_sg, dummy_page, PAGE_SIZE, 0);
3550
3551 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3552         err = net_set_get_put_page_callbacks(iscsi_get_page_callback,
3553                         iscsi_put_page_callback);
3554         if (err != 0) {
3555                 PRINT_INFO("Unable to set page callbackes: %d", err);
3556                 goto out_free_dummy;
3557         }
3558 #else
3559 #ifndef GENERATING_UPSTREAM_PATCH
3560         PRINT_WARNING("%s",
3561                 "CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION "
3562                 "not enabled in your kernel. ISCSI-SCST will be working with "
3563                 "not the best performance. Refer README file for details.");
3564 #endif
3565 #endif
3566
3567         ctr_major = register_chrdev(0, ctr_name, &ctr_fops);
3568         if (ctr_major < 0) {
3569                 PRINT_ERROR("failed to register the control device %d",
3570                             ctr_major);
3571                 err = ctr_major;
3572                 goto out_callb;
3573         }
3574
3575         err = event_init();
3576         if (err < 0)
3577                 goto out_reg;
3578
3579         iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS);
3580         if (!iscsi_cmnd_cache) {
3581                 err = -ENOMEM;
3582                 goto out_event;
3583         }
3584
3585         err = scst_register_target_template(&iscsi_template);
3586         if (err < 0)
3587                 goto out_kmem;
3588
3589         iscsi_template_registered = 1;
3590
3591 #ifdef CONFIG_SCST_PROC
3592         err = iscsi_procfs_init();
3593         if (err < 0)
3594                 goto out_reg_tmpl;
3595 #endif
3596
3597         num = max((int)num_online_cpus(), 2);
3598
3599         err = iscsi_run_threads(num, "iscsird", istrd);
3600         if (err != 0)
3601                 goto out_thr;
3602
3603         err = iscsi_run_threads(num, "iscsiwr", istwr);
3604         if (err != 0)
3605                 goto out_thr;
3606
3607 out:
3608         return err;
3609
3610 out_thr:
3611 #ifdef CONFIG_SCST_PROC
3612         iscsi_procfs_exit();
3613 #endif
3614         iscsi_stop_threads();
3615
3616 #ifdef CONFIG_SCST_PROC
3617 out_reg_tmpl:
3618 #endif
3619         scst_unregister_target_template(&iscsi_template);
3620
3621 out_kmem:
3622         kmem_cache_destroy(iscsi_cmnd_cache);
3623
3624 out_event:
3625         event_exit();
3626
3627 out_reg:
3628         unregister_chrdev(ctr_major, ctr_name);