Sysfs interface for targets made uniform. ISCSI-SCST made confirming the uniformity...
[mirror/scst/.git] / iscsi-scst / kernel / iscsi.c
1 /*
2  *  Copyright (C) 2002 - 2003 Ardis Technolgies <roman@ardistech.com>
3  *  Copyright (C) 2007 - 2009 Vladislav Bolkhovitin
4  *  Copyright (C) 2007 - 2009 ID7 Ltd.
5  *
6  *  This program is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU General Public License
8  *  as published by the Free Software Foundation, version 2
9  *  of the License.
10  *
11  *  This program is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  *  GNU General Public License for more details.
15  */
16
17 #include <linux/module.h>
18 #include <linux/hash.h>
19 #include <linux/kthread.h>
20 #include <linux/scatterlist.h>
21 #include <net/tcp.h>
22 #include <scsi/scsi.h>
23
24 #include "iscsi.h"
25 #include "digest.h"
26
27 #ifndef GENERATING_UPSTREAM_PATCH
28 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
29 #warning "Patch put_page_callback-<kernel-version>.patch not applied on your\
30  kernel or CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION\
31  config option not set. ISCSI-SCST will be working with not the best\
32  performance. Refer README file for details."
33 #endif
34 #endif
35
36 #define ISCSI_INIT_WRITE_WAKE           0x1
37
38 static int ctr_major;
39 static char ctr_name[] = "iscsi-scst-ctl";
40
41 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
42 unsigned long iscsi_trace_flag = ISCSI_DEFAULT_LOG_FLAGS;
43 #endif
44
45 static struct kmem_cache *iscsi_cmnd_cache;
46
47 DEFINE_SPINLOCK(iscsi_rd_lock);
48 LIST_HEAD(iscsi_rd_list);
49 DECLARE_WAIT_QUEUE_HEAD(iscsi_rd_waitQ);
50
51 DEFINE_SPINLOCK(iscsi_wr_lock);
52 LIST_HEAD(iscsi_wr_list);
53 DECLARE_WAIT_QUEUE_HEAD(iscsi_wr_waitQ);
54
55 static struct page *dummy_page;
56 static struct scatterlist dummy_sg;
57
58 struct iscsi_thread_t {
59         struct task_struct *thr;
60         struct list_head threads_list_entry;
61 };
62
63 static LIST_HEAD(iscsi_threads_list);
64
65 static void cmnd_remove_data_wait_hash(struct iscsi_cmnd *cmnd);
66 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status);
67 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess);
68 static void req_cmnd_release(struct iscsi_cmnd *req);
69 static int iscsi_preliminary_complete(struct iscsi_cmnd *req,
70         struct iscsi_cmnd *orig_req, bool get_data);
71 static int cmnd_insert_data_wait_hash(struct iscsi_cmnd *cmnd);
72 static void __cmnd_abort(struct iscsi_cmnd *cmnd);
73 static void iscsi_set_resid(struct iscsi_cmnd *rsp, bool bufflen_set);
74 static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags);
75
76 static void req_del_from_write_timeout_list(struct iscsi_cmnd *req)
77 {
78         struct iscsi_conn *conn;
79
80         TRACE_ENTRY();
81
82         if (!req->on_write_timeout_list)
83                 goto out;
84
85         conn = req->conn;
86
87         TRACE_DBG("Deleting cmd %p from conn %p write_timeout_list",
88                 req, conn);
89
90         spin_lock_bh(&conn->write_list_lock);
91         list_del(&req->write_timeout_list_entry);
92         req->on_write_timeout_list = 0;
93         spin_unlock_bh(&conn->write_list_lock);
94
95 out:
96         TRACE_EXIT();
97         return;
98 }
99
100 static inline u32 cmnd_write_size(struct iscsi_cmnd *cmnd)
101 {
102         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
103
104         if (hdr->flags & ISCSI_CMD_WRITE)
105                 return be32_to_cpu(hdr->data_length);
106         return 0;
107 }
108
109 static inline int cmnd_read_size(struct iscsi_cmnd *cmnd)
110 {
111         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
112
113         if (hdr->flags & ISCSI_CMD_READ) {
114                 struct iscsi_ahs_hdr *ahdr;
115
116                 if (!(hdr->flags & ISCSI_CMD_WRITE))
117                         return be32_to_cpu(hdr->data_length);
118
119                 ahdr = (struct iscsi_ahs_hdr *)cmnd->pdu.ahs;
120                 if (ahdr != NULL) {
121                         uint8_t *p = (uint8_t *)ahdr;
122                         unsigned int size = 0;
123                         do {
124                                 int s;
125
126                                 ahdr = (struct iscsi_ahs_hdr *)p;
127
128                                 if (ahdr->ahstype == ISCSI_AHSTYPE_RLENGTH) {
129                                         struct iscsi_rlength_ahdr *rh =
130                                               (struct iscsi_rlength_ahdr *)ahdr;
131                                         return be32_to_cpu(rh->read_length);
132                                 }
133
134                                 s = 3 + be16_to_cpu(ahdr->ahslength);
135                                 s = (s + 3) & -4;
136                                 size += s;
137                                 p += s;
138                         } while (size < cmnd->pdu.ahssize);
139                 }
140                 return -1;
141         }
142         return 0;
143 }
144
145 void iscsi_restart_cmnd(struct iscsi_cmnd *cmnd)
146 {
147         int status;
148
149         TRACE_ENTRY();
150
151         EXTRACHECKS_BUG_ON(cmnd->r2t_len_to_receive != 0);
152         EXTRACHECKS_BUG_ON(cmnd->r2t_len_to_send != 0);
153
154         iscsi_extracheck_is_rd_thread(cmnd->conn);
155
156         req_del_from_write_timeout_list(cmnd);
157
158         /*
159          * Let's remove cmnd from the hash earlier to keep it smaller.
160          * See also corresponding comment in req_cmnd_release().
161          */
162         if (cmnd->hashed)
163                 cmnd_remove_data_wait_hash(cmnd);
164
165         if (unlikely(test_bit(ISCSI_CONN_REINSTATING,
166                         &cmnd->conn->conn_aflags))) {
167                 struct iscsi_target *target = cmnd->conn->session->target;
168                 bool get_out;
169
170                 mutex_lock(&target->target_mutex);
171
172                 get_out = test_bit(ISCSI_CONN_REINSTATING,
173                                 &cmnd->conn->conn_aflags);
174                 /* Let's don't look dead */
175                 if (scst_cmd_get_cdb(cmnd->scst_cmd)[0] == TEST_UNIT_READY)
176                         get_out = false;
177
178                 if (!get_out)
179                         goto unlock_cont;
180
181                 TRACE_MGMT_DBG("Pending cmnd %p, because conn %p is "
182                         "reinstated", cmnd, cmnd->conn);
183
184                 cmnd->scst_state = ISCSI_CMD_STATE_REINST_PENDING;
185                 list_add_tail(&cmnd->reinst_pending_cmd_list_entry,
186                         &cmnd->conn->reinst_pending_cmd_list);
187
188 unlock_cont:
189                 mutex_unlock(&target->target_mutex);
190
191                 if (get_out)
192                         goto out;
193         }
194
195         if (unlikely(cmnd->prelim_compl_flags != 0)) {
196                 if (test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags)) {
197                         TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted", cmnd,
198                                 cmnd->scst_cmd);
199                         req_cmnd_release_force(cmnd);
200                         goto out;
201                 }
202
203                 if (cmnd->scst_cmd == NULL) {
204                         TRACE_MGMT_DBG("Finishing preliminary completed cmd %p "
205                                 "with NULL scst_cmd", cmnd);
206                         req_cmnd_release(cmnd);
207                         goto out;
208                 }
209
210                 status = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
211         } else
212                 status = SCST_PREPROCESS_STATUS_SUCCESS;
213
214         cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
215
216         scst_restart_cmd(cmnd->scst_cmd, status, SCST_CONTEXT_THREAD);
217
218 out:
219         TRACE_EXIT();
220         return;
221 }
222
223 void iscsi_fail_data_waiting_cmnd(struct iscsi_cmnd *cmnd)
224 {
225         TRACE_ENTRY();
226
227         TRACE_MGMT_DBG("Failing data waiting cmnd %p", cmnd);
228
229         /*
230          * There is no race with conn_abort(), since all functions
231          * called from single read thread
232          */
233         iscsi_extracheck_is_rd_thread(cmnd->conn);
234         cmnd->r2t_len_to_receive = 0;
235         cmnd->r2t_len_to_send = 0;
236
237         req_cmnd_release_force(cmnd);
238
239         TRACE_EXIT();
240         return;
241 }
242
243 struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn,
244                               struct iscsi_cmnd *parent)
245 {
246         struct iscsi_cmnd *cmnd;
247
248         /* ToDo: __GFP_NOFAIL?? */
249 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
250         cmnd = kmem_cache_alloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
251         memset(cmnd, 0, sizeof(*cmnd));
252 #else
253         cmnd = kmem_cache_zalloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
254 #endif
255
256         atomic_set(&cmnd->ref_cnt, 1);
257         cmnd->scst_state = ISCSI_CMD_STATE_NEW;
258         cmnd->conn = conn;
259         cmnd->parent_req = parent;
260
261         if (parent == NULL) {
262                 conn_get(conn);
263
264 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
265                 atomic_set(&cmnd->net_ref_cnt, 0);
266 #endif
267                 INIT_LIST_HEAD(&cmnd->rsp_cmd_list);
268                 INIT_LIST_HEAD(&cmnd->rx_ddigest_cmd_list);
269                 cmnd->target_task_tag = cpu_to_be32(ISCSI_RESERVED_TAG);
270
271                 spin_lock_bh(&conn->cmd_list_lock);
272                 list_add_tail(&cmnd->cmd_list_entry, &conn->cmd_list);
273                 spin_unlock_bh(&conn->cmd_list_lock);
274         }
275
276         TRACE_DBG("conn %p, parent %p, cmnd %p", conn, parent, cmnd);
277         return cmnd;
278 }
279
280 /* Frees a command. Also frees the additional header. */
281 static void cmnd_free(struct iscsi_cmnd *cmnd)
282 {
283         TRACE_ENTRY();
284
285         TRACE_DBG("cmnd %p", cmnd);
286
287         if (unlikely(test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags))) {
288                 TRACE_MGMT_DBG("Free aborted cmd %p (scst cmd %p, state %d, "
289                         "parent_req %p)", cmnd, cmnd->scst_cmd,
290                         cmnd->scst_state, cmnd->parent_req);
291         }
292
293         /* Catch users from cmd_list or rsp_cmd_list */
294         EXTRACHECKS_BUG_ON(atomic_read(&cmnd->ref_cnt) != 0);
295
296         kfree(cmnd->pdu.ahs);
297
298 #ifdef CONFIG_SCST_EXTRACHECKS
299         if (unlikely(cmnd->on_write_list || cmnd->on_write_timeout_list)) {
300                 struct iscsi_scsi_cmd_hdr *req = cmnd_hdr(cmnd);
301
302                 PRINT_CRIT_ERROR("cmnd %p still on some list?, %x, %x, %x, "
303                         "%x, %x, %x, %x", cmnd, req->opcode, req->scb[0],
304                         req->flags, req->itt, be32_to_cpu(req->data_length),
305                         req->cmd_sn, be32_to_cpu(cmnd->pdu.datasize));
306
307                 if (unlikely(cmnd->parent_req)) {
308                         struct iscsi_scsi_cmd_hdr *preq =
309                                         cmnd_hdr(cmnd->parent_req);
310                         PRINT_CRIT_ERROR("%p %x %u", preq, preq->opcode,
311                                 preq->scb[0]);
312                 }
313                 sBUG();
314         }
315 #endif
316
317         kmem_cache_free(iscsi_cmnd_cache, cmnd);
318
319         TRACE_EXIT();
320         return;
321 }
322
323 /* Might be called under some lock and on SIRQ */
324 void cmnd_done(struct iscsi_cmnd *cmnd)
325 {
326         TRACE_ENTRY();
327
328         TRACE_DBG("cmnd %p", cmnd);
329
330         if (unlikely(test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags))) {
331                 TRACE_MGMT_DBG("Done aborted cmd %p (scst cmd %p, state %d, "
332                         "parent_req %p)", cmnd, cmnd->scst_cmd,
333                         cmnd->scst_state, cmnd->parent_req);
334         }
335
336         EXTRACHECKS_BUG_ON(cmnd->on_rx_digest_list);
337         EXTRACHECKS_BUG_ON(cmnd->hashed);
338
339         req_del_from_write_timeout_list(cmnd);
340
341         if (cmnd->parent_req == NULL) {
342                 struct iscsi_conn *conn = cmnd->conn;
343                 struct iscsi_cmnd *rsp, *t;
344
345                 TRACE_DBG("Deleting req %p from conn %p", cmnd, conn);
346
347                 spin_lock_bh(&conn->cmd_list_lock);
348                 list_del(&cmnd->cmd_list_entry);
349                 spin_unlock_bh(&conn->cmd_list_lock);
350
351                 conn_put(conn);
352
353                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rx_ddigest_cmd_list));
354
355                 /* Order between above and below code is important! */
356
357                 if ((cmnd->scst_cmd != NULL) || (cmnd->scst_aen != NULL)) {
358                         switch (cmnd->scst_state) {
359                         case ISCSI_CMD_STATE_PROCESSED:
360                                 TRACE_DBG("cmd %p PROCESSED", cmnd);
361                                 scst_tgt_cmd_done(cmnd->scst_cmd,
362                                         SCST_CONTEXT_DIRECT_ATOMIC);
363                                 break;
364
365                         case ISCSI_CMD_STATE_AFTER_PREPROC:
366                         {
367                                 /* It can be for some aborted commands */
368                                 struct scst_cmd *scst_cmd = cmnd->scst_cmd;
369                                 TRACE_DBG("cmd %p AFTER_PREPROC", cmnd);
370                                 cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
371                                 cmnd->scst_cmd = NULL;
372                                 scst_restart_cmd(scst_cmd,
373                                         SCST_PREPROCESS_STATUS_ERROR_FATAL,
374                                         SCST_CONTEXT_THREAD);
375                                 break;
376                         }
377
378                         case ISCSI_CMD_STATE_AEN:
379                                 TRACE_DBG("cmd %p AEN PROCESSED", cmnd);
380                                 scst_aen_done(cmnd->scst_aen);
381                                 break;
382
383                         case ISCSI_CMD_STATE_OUT_OF_SCST_PRELIM_COMPL:
384                                 break;
385
386                         default:
387                                 PRINT_CRIT_ERROR("Unexpected cmnd scst state "
388                                         "%d", cmnd->scst_state);
389                                 sBUG();
390                                 break;
391                         }
392                 }
393
394                 if (cmnd->own_sg) {
395                         TRACE_DBG("own_sg for req %p", cmnd);
396                         if (cmnd->sg != &dummy_sg)
397                                 scst_free(cmnd->sg, cmnd->sg_cnt);
398 #ifdef CONFIG_SCST_DEBUG
399                         cmnd->own_sg = 0;
400                         cmnd->sg = NULL;
401                         cmnd->sg_cnt = -1;
402 #endif
403                 }
404
405                 if (cmnd->dec_active_cmnds) {
406                         struct iscsi_session *sess = cmnd->conn->session;
407                         TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
408                                 "new value %d)", cmnd, sess,
409                                 atomic_read(&sess->active_cmds)-1);
410                         atomic_dec(&sess->active_cmds);
411 #ifdef CONFIG_SCST_EXTRACHECKS
412                         if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
413                                 PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
414                                         atomic_read(&sess->active_cmds));
415                                 sBUG();
416                         }
417 #endif
418                 }
419
420                 list_for_each_entry_safe(rsp, t, &cmnd->rsp_cmd_list,
421                                         rsp_cmd_list_entry) {
422                         cmnd_free(rsp);
423                 }
424
425                 cmnd_free(cmnd);
426         } else {
427                 if (cmnd->own_sg) {
428                         TRACE_DBG("own_sg for rsp %p", cmnd);
429                         if ((cmnd->sg != &dummy_sg) && (cmnd->sg != cmnd->rsp_sg))
430                                 scst_free(cmnd->sg, cmnd->sg_cnt);
431 #ifdef CONFIG_SCST_DEBUG
432                         cmnd->own_sg = 0;
433                         cmnd->sg = NULL;
434                         cmnd->sg_cnt = -1;
435 #endif
436                 }
437
438                 EXTRACHECKS_BUG_ON(cmnd->dec_active_cmnds);
439
440                 if (cmnd == cmnd->parent_req->main_rsp) {
441                         TRACE_DBG("Finishing main rsp %p (req %p)", cmnd,
442                                 cmnd->parent_req);
443                         cmnd->parent_req->main_rsp = NULL;
444                 }
445
446                 cmnd_put(cmnd->parent_req);
447                 /*
448                  * rsp will be freed on the last parent's put and can already
449                  * be freed!!
450                  */
451         }
452
453         TRACE_EXIT();
454         return;
455 }
456
457 /*
458  * Corresponding conn may also get destroyed after this function, except only
459  * if it's called from the read thread!
460  *
461  * It can't be called in parallel with iscsi_cmnds_init_write()!
462  */
463 void req_cmnd_release_force(struct iscsi_cmnd *req)
464 {
465         struct iscsi_cmnd *rsp, *t;
466         struct iscsi_conn *conn = req->conn;
467         LIST_HEAD(cmds_list);
468
469         TRACE_ENTRY();
470
471         TRACE_MGMT_DBG("req %p", req);
472
473         sBUG_ON(req == conn->read_cmnd);
474
475         spin_lock_bh(&conn->write_list_lock);
476         list_for_each_entry_safe(rsp, t, &conn->write_list, write_list_entry) {
477                 if (rsp->parent_req != req)
478                         continue;
479
480                 cmd_del_from_write_list(rsp);
481
482                 list_add_tail(&rsp->write_list_entry, &cmds_list);
483         }
484         spin_unlock_bh(&conn->write_list_lock);
485
486         list_for_each_entry_safe(rsp, t, &cmds_list, write_list_entry) {
487                 TRACE_MGMT_DBG("Putting write rsp %p", rsp);
488                 list_del(&rsp->write_list_entry);
489                 cmnd_put(rsp);
490         }
491
492         /* Supposed nobody can add responses in the list anymore */
493         list_for_each_entry_reverse(rsp, &req->rsp_cmd_list,
494                         rsp_cmd_list_entry) {
495                 bool r;
496
497                 if (rsp->force_cleanup_done)
498                         continue;
499
500                 rsp->force_cleanup_done = 1;
501
502                 if (cmnd_get_check(rsp))
503                         continue;
504
505                 spin_lock_bh(&conn->write_list_lock);
506                 r = rsp->on_write_list || rsp->write_processing_started;
507                 spin_unlock_bh(&conn->write_list_lock);
508
509                 cmnd_put(rsp);
510
511                 if (r)
512                         continue;
513
514                 /*
515                  * If both on_write_list and write_processing_started not set,
516                  * we can safely put() rsp.
517                  */
518                 TRACE_MGMT_DBG("Putting rsp %p", rsp);
519                 cmnd_put(rsp);
520         }
521
522         if (req->main_rsp != NULL) {
523                 TRACE_MGMT_DBG("Putting main rsp %p", req->main_rsp);
524                 cmnd_put(req->main_rsp);
525                 req->main_rsp = NULL;
526         }
527
528         req_cmnd_release(req);
529
530         TRACE_EXIT();
531         return;
532 }
533
534 /*
535  * Corresponding conn may also get destroyed after this function, except only
536  * if it's called from the read thread!
537  */
538 static void req_cmnd_release(struct iscsi_cmnd *req)
539 {
540         struct iscsi_cmnd *c, *t;
541
542         TRACE_ENTRY();
543
544         TRACE_DBG("req %p", req);
545
546 #ifdef CONFIG_SCST_EXTRACHECKS
547         sBUG_ON(req->release_called);
548         req->release_called = 1;
549 #endif
550
551         if (unlikely(test_bit(ISCSI_CMD_ABORTED, &req->prelim_compl_flags))) {
552                 TRACE_MGMT_DBG("Release aborted req cmd %p (scst cmd %p, "
553                         "state %d)", req, req->scst_cmd, req->scst_state);
554         }
555
556         sBUG_ON(req->parent_req != NULL);
557
558         /*
559          * We have to remove hashed req from the hash list before sending
560          * response. Otherwise we can have a race, when for some reason cmd's
561          * release (and, hence, removal from the hash) is delayed after the
562          * transmission and initiator sends cmd with the same ITT, hence
563          * the new command will be erroneously rejected as a duplicate.
564          */
565         if (unlikely(req->hashed)) {
566                 /* It sometimes can happen during errors recovery */
567                 cmnd_remove_data_wait_hash(req);
568         }
569
570         if (unlikely(req->main_rsp != NULL)) {
571                 TRACE_DBG("Sending main rsp %p", req->main_rsp);
572                 iscsi_cmnd_init_write(req->main_rsp, ISCSI_INIT_WRITE_WAKE);
573                 req->main_rsp = NULL;
574         }
575
576         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
577                                 rx_ddigest_cmd_list_entry) {
578                 cmd_del_from_rx_ddigest_list(c);
579                 cmnd_put(c);
580         }
581
582         EXTRACHECKS_BUG_ON(req->pending);
583
584         if (req->dec_active_cmnds) {
585                 struct iscsi_session *sess = req->conn->session;
586                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
587                         "new value %d)", req, sess,
588                         atomic_read(&sess->active_cmds)-1);
589                 atomic_dec(&sess->active_cmds);
590                 req->dec_active_cmnds = 0;
591 #ifdef CONFIG_SCST_EXTRACHECKS
592                 if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
593                         PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
594                                 atomic_read(&sess->active_cmds));
595                         sBUG();
596                 }
597 #endif
598         }
599
600         cmnd_put(req);
601
602         TRACE_EXIT();
603         return;
604 }
605
606 /*
607  * Corresponding conn may also get destroyed after this function, except only
608  * if it's called from the read thread!
609  */
610 void rsp_cmnd_release(struct iscsi_cmnd *cmnd)
611 {
612         TRACE_DBG("%p", cmnd);
613
614 #ifdef CONFIG_SCST_EXTRACHECKS
615         sBUG_ON(cmnd->release_called);
616         cmnd->release_called = 1;
617 #endif
618
619         EXTRACHECKS_BUG_ON(cmnd->parent_req == NULL);
620
621         cmnd_put(cmnd);
622         return;
623 }
624
625 static struct iscsi_cmnd *iscsi_alloc_rsp(struct iscsi_cmnd *parent)
626 {
627         struct iscsi_cmnd *rsp;
628
629         TRACE_ENTRY();
630
631         rsp = cmnd_alloc(parent->conn, parent);
632
633         TRACE_DBG("Adding rsp %p to parent %p", rsp, parent);
634         list_add_tail(&rsp->rsp_cmd_list_entry, &parent->rsp_cmd_list);
635
636         cmnd_get(parent);
637
638         TRACE_EXIT_HRES((unsigned long)rsp);
639         return rsp;
640 }
641
642 static inline struct iscsi_cmnd *iscsi_alloc_main_rsp(struct iscsi_cmnd *parent)
643 {
644         struct iscsi_cmnd *rsp;
645
646         TRACE_ENTRY();
647
648         EXTRACHECKS_BUG_ON(parent->main_rsp != NULL);
649
650         rsp = iscsi_alloc_rsp(parent);
651         parent->main_rsp = rsp;
652
653         TRACE_EXIT_HRES((unsigned long)rsp);
654         return rsp;
655 }
656
657 static void iscsi_cmnds_init_write(struct list_head *send, int flags)
658 {
659         struct iscsi_cmnd *rsp = list_entry(send->next, struct iscsi_cmnd,
660                                                 write_list_entry);
661         struct iscsi_conn *conn = rsp->conn;
662         struct list_head *pos, *next;
663
664         sBUG_ON(list_empty(send));
665
666         if (!(conn->ddigest_type & DIGEST_NONE)) {
667                 list_for_each(pos, send) {
668                         rsp = list_entry(pos, struct iscsi_cmnd,
669                                                 write_list_entry);
670
671                         if (rsp->pdu.datasize != 0) {
672                                 TRACE_DBG("Doing data digest (%p:%x)", rsp,
673                                         cmnd_opcode(rsp));
674                                 digest_tx_data(rsp);
675                         }
676                 }
677         }
678
679         spin_lock_bh(&conn->write_list_lock);
680         list_for_each_safe(pos, next, send) {
681                 rsp = list_entry(pos, struct iscsi_cmnd, write_list_entry);
682
683                 TRACE_DBG("%p:%x", rsp, cmnd_opcode(rsp));
684
685                 sBUG_ON(conn != rsp->conn);
686
687                 list_del(&rsp->write_list_entry);
688                 cmd_add_on_write_list(conn, rsp);
689         }
690         spin_unlock_bh(&conn->write_list_lock);
691
692         if (flags & ISCSI_INIT_WRITE_WAKE)
693                 iscsi_make_conn_wr_active(conn);
694
695         return;
696 }
697
698 static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags)
699 {
700         LIST_HEAD(head);
701
702 #ifdef CONFIG_SCST_EXTRACHECKS
703         if (unlikely(rsp->on_write_list)) {
704                 PRINT_CRIT_ERROR("cmd already on write list (%x %x %x "
705                         "%u %u %d %d", cmnd_itt(rsp),
706                         cmnd_opcode(rsp), cmnd_scsicode(rsp),
707                         rsp->hdigest, rsp->ddigest,
708                         list_empty(&rsp->rsp_cmd_list), rsp->hashed);
709                 sBUG();
710         }
711 #endif
712         list_add_tail(&rsp->write_list_entry, &head);
713         iscsi_cmnds_init_write(&head, flags);
714         return;
715 }
716
717 static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
718 {
719         struct iscsi_cmnd *rsp;
720         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
721         struct iscsi_data_in_hdr *rsp_hdr;
722         u32 pdusize, expsize, size, offset, sn;
723         LIST_HEAD(send);
724
725         TRACE_DBG("req %p", req);
726
727         pdusize = req->conn->session->sess_params.max_xmit_data_length;
728         expsize = req->read_size;
729         size = min(expsize, (u32)req->bufflen);
730         offset = 0;
731         sn = 0;
732
733         while (1) {
734                 rsp = iscsi_alloc_rsp(req);
735                 TRACE_DBG("rsp %p", rsp);
736                 rsp->sg = req->sg;
737                 rsp->sg_cnt = req->sg_cnt;
738                 rsp->bufflen = req->bufflen;
739                 rsp_hdr = (struct iscsi_data_in_hdr *)&rsp->pdu.bhs;
740
741                 rsp_hdr->opcode = ISCSI_OP_SCSI_DATA_IN;
742                 rsp_hdr->itt = req_hdr->itt;
743                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
744                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
745                 rsp_hdr->data_sn = cpu_to_be32(sn);
746
747                 if (size <= pdusize) {
748                         TRACE_DBG("offset %d, size %d", offset, size);
749                         rsp->pdu.datasize = size;
750                         if (send_status) {
751                                 unsigned int scsisize;
752
753                                 TRACE_DBG("status %x", status);
754
755                                 EXTRACHECKS_BUG_ON((cmnd_hdr(req)->flags & ISCSI_CMD_WRITE) != 0);
756
757                                 rsp_hdr->flags = ISCSI_FLG_FINAL | ISCSI_FLG_STATUS;
758                                 rsp_hdr->cmd_status = status;
759
760                                 scsisize = req->bufflen;
761                                 if (scsisize < expsize) {
762                                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
763                                         size = expsize - scsisize;
764                                 } else if (scsisize > expsize) {
765                                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
766                                         size = scsisize - expsize;
767                                 } else
768                                         size = 0;
769                                 rsp_hdr->residual_count = cpu_to_be32(size);
770                         }
771                         list_add_tail(&rsp->write_list_entry, &send);
772                         break;
773                 }
774
775                 TRACE_DBG("pdusize %d, offset %d, size %d", pdusize, offset,
776                         size);
777
778                 rsp->pdu.datasize = pdusize;
779
780                 size -= pdusize;
781                 offset += pdusize;
782                 sn++;
783
784                 list_add_tail(&rsp->write_list_entry, &send);
785         }
786         iscsi_cmnds_init_write(&send, 0);
787         return;
788 }
789
790 static void iscsi_init_status_rsp(struct iscsi_cmnd *rsp,
791         int status, const u8 *sense_buf, int sense_len, bool bufflen_set)
792 {
793         struct iscsi_cmnd *req = rsp->parent_req;
794         struct iscsi_scsi_rsp_hdr *rsp_hdr;
795         struct scatterlist *sg;
796
797         TRACE_ENTRY();
798
799         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
800         rsp_hdr->opcode = ISCSI_OP_SCSI_RSP;
801         rsp_hdr->flags = ISCSI_FLG_FINAL;
802         rsp_hdr->response = ISCSI_RESPONSE_COMMAND_COMPLETED;
803         rsp_hdr->cmd_status = status;
804         rsp_hdr->itt = cmnd_hdr(req)->itt;
805
806         if (SCST_SENSE_VALID(sense_buf)) {
807                 TRACE_DBG("%s", "SENSE VALID");
808
809                 sg = rsp->sg = rsp->rsp_sg;
810                 rsp->sg_cnt = 2;
811                 rsp->own_sg = 1;
812
813                 sg_init_table(sg, 2);
814                 sg_set_buf(&sg[0], &rsp->sense_hdr, sizeof(rsp->sense_hdr));
815                 sg_set_buf(&sg[1], sense_buf, sense_len);
816
817                 rsp->sense_hdr.length = cpu_to_be16(sense_len);
818
819                 rsp->pdu.datasize = sizeof(rsp->sense_hdr) + sense_len;
820                 rsp->bufflen = rsp->pdu.datasize;
821         } else {
822                 rsp->pdu.datasize = 0;
823                 rsp->bufflen = 0;
824         }
825
826         iscsi_set_resid(rsp, bufflen_set);
827
828         TRACE_EXIT();
829         return;
830 }
831
832 static inline struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req,
833         int status, const u8 *sense_buf, int sense_len, bool bufflen_set)
834 {
835         struct iscsi_cmnd *rsp;
836
837         TRACE_ENTRY();
838
839         rsp = iscsi_alloc_rsp(req);
840         TRACE_DBG("rsp %p", rsp);
841
842         iscsi_init_status_rsp(rsp, status, sense_buf, sense_len, bufflen_set);
843
844         TRACE_EXIT_HRES((unsigned long)rsp);
845         return rsp;
846 }
847
848 static struct iscsi_cmnd *create_prelim_status_rsp(struct iscsi_cmnd *req,
849         int status, const u8 *sense_buf, int sense_len)
850 {
851         struct iscsi_cmnd *rsp;
852
853         TRACE_ENTRY();
854
855         rsp = iscsi_alloc_main_rsp(req);
856         TRACE_DBG("main rsp %p", rsp);
857
858         iscsi_init_status_rsp(rsp, status, sense_buf, sense_len, false);
859
860         TRACE_EXIT_HRES((unsigned long)rsp);
861         return rsp;
862 }
863
864 static int iscsi_set_prelim_r2t_len_to_receive(struct iscsi_cmnd *req)
865 {
866         struct iscsi_hdr *req_hdr = &req->pdu.bhs;
867         int res = 0;
868
869         TRACE_ENTRY();
870
871         if (req_hdr->flags & ISCSI_CMD_FINAL)
872                 goto out;
873
874         res = cmnd_insert_data_wait_hash(req);
875         if (res != 0) {
876                 /*
877                  * We have to close connection, because otherwise a data
878                  * corruption is possible if we allow to receive data
879                  * for this request in another request with dublicated ITT.
880                  */
881                 mark_conn_closed(req->conn);
882                 goto out;
883         }
884
885         /*
886          * We need to wait for one or more PDUs. Let's simplify
887          * other code and pretend we need to receive 1 byte.
888          * In data_out_start() we will correct it.
889          */
890         if (req->outstanding_r2t == 0) {
891                 req->outstanding_r2t = 1;
892                 req_add_to_write_timeout_list(req);
893         }
894         req->r2t_len_to_receive = 1;
895         req->r2t_len_to_send = 0;
896
897         TRACE_DBG("req %p, op %x, outstanding_r2t %d, r2t_len_to_receive %d, "
898                 "r2t_len_to_send %d", req, cmnd_opcode(req),
899                 req->outstanding_r2t, req->r2t_len_to_receive,
900                 req->r2t_len_to_send);
901
902 out:
903         TRACE_EXIT_RES(res);
904         return res;
905 }
906
907 static int create_preliminary_status_rsp(struct iscsi_cmnd *req,
908         int status, const u8 *sense_buf, int sense_len)
909 {
910         int res = 0;
911         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
912
913         TRACE_ENTRY();
914
915         if (req->prelim_compl_flags != 0) {
916                 TRACE_MGMT_DBG("req %p already prelim completed", req);
917                 goto out;
918         }
919
920         req->scst_state = ISCSI_CMD_STATE_OUT_OF_SCST_PRELIM_COMPL;
921
922         if ((req_hdr->flags & ISCSI_CMD_READ) &&
923             (req_hdr->flags & ISCSI_CMD_WRITE)) {
924                 int sz = cmnd_read_size(req);
925                 if (sz > 0)
926                         req->read_size = sz;
927         } else if (req_hdr->flags & ISCSI_CMD_READ)
928                 req->read_size = be32_to_cpu(req_hdr->data_length);
929
930         create_prelim_status_rsp(req, status, sense_buf, sense_len);
931         res = iscsi_preliminary_complete(req, req, true);
932
933 out:
934         TRACE_EXIT_RES(res);
935         return res;
936 }
937
938 static int set_scst_preliminary_status_rsp(struct iscsi_cmnd *req,
939         bool get_data, int key, int asc, int ascq)
940 {
941         int res = 0;
942
943         TRACE_ENTRY();
944
945         if (req->scst_cmd == NULL) {
946                 /* There must be already error set */
947                 goto complete;
948         }
949
950         scst_set_cmd_error(req->scst_cmd, key, asc, ascq);
951
952 complete:
953         res = iscsi_preliminary_complete(req, req, get_data);
954
955         TRACE_EXIT_RES(res);
956         return res;
957 }
958
959 static int create_reject_rsp(struct iscsi_cmnd *req, int reason, bool get_data)
960 {
961         int res = 0;
962         struct iscsi_cmnd *rsp;
963         struct iscsi_reject_hdr *rsp_hdr;
964         struct scatterlist *sg;
965
966         TRACE_ENTRY();
967
968         TRACE_MGMT_DBG("Reject: req %p, reason %x", req, reason);
969
970         if (cmnd_opcode(req) == ISCSI_OP_SCSI_CMD) {
971                 if (req->scst_cmd == NULL) {
972                         /* BUSY status must be already set */
973                         struct iscsi_scsi_rsp_hdr *rsp_hdr;
974                         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&req->main_rsp->pdu.bhs;
975                         sBUG_ON(rsp_hdr->cmd_status == 0);
976                         /*
977                          * Let's not send REJECT here. The initiator will retry
978                          * and, hopefully, next time we will not fail allocating
979                          * scst_cmd, so we will then send the REJECT.
980                          */
981                         goto out;
982                 } else
983                         set_scst_preliminary_status_rsp(req, get_data,
984                                 SCST_LOAD_SENSE(scst_sense_invalid_message));
985         }
986
987         rsp = iscsi_alloc_main_rsp(req);
988         rsp_hdr = (struct iscsi_reject_hdr *)&rsp->pdu.bhs;
989
990         rsp_hdr->opcode = ISCSI_OP_REJECT;
991         rsp_hdr->ffffffff = ISCSI_RESERVED_TAG;
992         rsp_hdr->reason = reason;
993
994         sg = rsp->sg = rsp->rsp_sg;
995         rsp->sg_cnt = 1;
996         rsp->own_sg = 1;
997         sg_init_one(sg, &req->pdu.bhs, sizeof(struct iscsi_hdr));
998         rsp->bufflen = rsp->pdu.datasize = sizeof(struct iscsi_hdr);
999
1000         res = iscsi_preliminary_complete(req, req, true);
1001
1002 out:
1003         TRACE_EXIT_RES(res);
1004         return res;
1005 }
1006
1007 static inline int iscsi_get_allowed_cmds(struct iscsi_session *sess)
1008 {
1009         int res = max(-1, (int)sess->tgt_params.queued_cmnds -
1010                                 atomic_read(&sess->active_cmds)-1);
1011         TRACE_DBG("allowed cmds %d (sess %p, active_cmds %d)", res,
1012                 sess, atomic_read(&sess->active_cmds));
1013         return res;
1014 }
1015
1016 static u32 cmnd_set_sn(struct iscsi_cmnd *cmnd, int set_stat_sn)
1017 {
1018         struct iscsi_conn *conn = cmnd->conn;
1019         struct iscsi_session *sess = conn->session;
1020         u32 res;
1021
1022         spin_lock(&sess->sn_lock);
1023
1024         if (set_stat_sn)
1025                 cmnd->pdu.bhs.sn = cpu_to_be32(conn->stat_sn++);
1026         cmnd->pdu.bhs.exp_sn = cpu_to_be32(sess->exp_cmd_sn);
1027         cmnd->pdu.bhs.max_sn = cpu_to_be32(sess->exp_cmd_sn +
1028                                  iscsi_get_allowed_cmds(sess));
1029
1030         res = cpu_to_be32(conn->stat_sn);
1031
1032         spin_unlock(&sess->sn_lock);
1033         return res;
1034 }
1035
1036 /* Called under sn_lock */
1037 static void __update_stat_sn(struct iscsi_cmnd *cmnd)
1038 {
1039         struct iscsi_conn *conn = cmnd->conn;
1040         u32 exp_stat_sn;
1041
1042         cmnd->pdu.bhs.exp_sn = exp_stat_sn = be32_to_cpu(cmnd->pdu.bhs.exp_sn);
1043         TRACE_DBG("%x,%x", cmnd_opcode(cmnd), exp_stat_sn);
1044         if ((int)(exp_stat_sn - conn->exp_stat_sn) > 0 &&
1045             (int)(exp_stat_sn - conn->stat_sn) <= 0) {
1046                 /* free pdu resources */
1047                 cmnd->conn->exp_stat_sn = exp_stat_sn;
1048         }
1049         return;
1050 }
1051
1052 static inline void update_stat_sn(struct iscsi_cmnd *cmnd)
1053 {
1054         spin_lock(&cmnd->conn->session->sn_lock);
1055         __update_stat_sn(cmnd);
1056         spin_unlock(&cmnd->conn->session->sn_lock);
1057         return;
1058 }
1059
1060 /* Called under sn_lock */
1061 static int check_cmd_sn(struct iscsi_cmnd *cmnd)
1062 {
1063         struct iscsi_session *session = cmnd->conn->session;
1064         u32 cmd_sn;
1065
1066         cmnd->pdu.bhs.sn = cmd_sn = be32_to_cpu(cmnd->pdu.bhs.sn);
1067         TRACE_DBG("%d(%d)", cmd_sn, session->exp_cmd_sn);
1068         if (likely((s32)(cmd_sn - session->exp_cmd_sn) >= 0))
1069                 return 0;
1070         PRINT_ERROR("sequence error (%x,%x)", cmd_sn, session->exp_cmd_sn);
1071         return -ISCSI_REASON_PROTOCOL_ERROR;
1072 }
1073
1074 static struct iscsi_cmnd *cmnd_find_itt_get(struct iscsi_conn *conn, u32 itt)
1075 {
1076         struct iscsi_cmnd *cmnd, *found_cmnd = NULL;
1077
1078         spin_lock_bh(&conn->cmd_list_lock);
1079         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1080                 if ((cmnd->pdu.bhs.itt == itt) && !cmnd_get_check(cmnd)) {
1081                         found_cmnd = cmnd;
1082                         break;
1083                 }
1084         }
1085         spin_unlock_bh(&conn->cmd_list_lock);
1086
1087         return found_cmnd;
1088 }
1089
1090 /**
1091  ** We use the ITT hash only to find original request PDU for subsequent
1092  ** Data-Out PDUs.
1093  **/
1094
1095 /* Must be called under cmnd_data_wait_hash_lock */
1096 static struct iscsi_cmnd *__cmnd_find_data_wait_hash(struct iscsi_conn *conn,
1097         u32 itt)
1098 {
1099         struct list_head *head;
1100         struct iscsi_cmnd *cmnd;
1101
1102         head = &conn->session->cmnd_data_wait_hash[cmnd_hashfn(itt)];
1103
1104         list_for_each_entry(cmnd, head, hash_list_entry) {
1105                 if (cmnd->pdu.bhs.itt == itt)
1106                         return cmnd;
1107         }
1108         return NULL;
1109 }
1110
1111 static struct iscsi_cmnd *cmnd_find_data_wait_hash(struct iscsi_conn *conn,
1112         u32 itt)
1113 {
1114         struct iscsi_cmnd *res;
1115         struct iscsi_session *session = conn->session;
1116
1117         spin_lock(&session->cmnd_data_wait_hash_lock);
1118         res = __cmnd_find_data_wait_hash(conn, itt);
1119         spin_unlock(&session->cmnd_data_wait_hash_lock);
1120
1121         return res;
1122 }
1123
1124 static inline u32 get_next_ttt(struct iscsi_conn *conn)
1125 {
1126         u32 ttt;
1127         struct iscsi_session *session = conn->session;
1128
1129         /* Not compatible with MC/S! */
1130
1131         iscsi_extracheck_is_rd_thread(conn);
1132
1133         if (unlikely(session->next_ttt == ISCSI_RESERVED_TAG))
1134                 session->next_ttt++;
1135         ttt = session->next_ttt++;
1136
1137         return ttt;
1138 }
1139
1140 static int cmnd_insert_data_wait_hash(struct iscsi_cmnd *cmnd)
1141 {
1142         struct iscsi_session *session = cmnd->conn->session;
1143         struct iscsi_cmnd *tmp;
1144         struct list_head *head;
1145         int err = 0;
1146         u32 itt = cmnd->pdu.bhs.itt;
1147
1148         if (unlikely(cmnd->hashed)) {
1149                 /* It can be for preliminary completed commands */
1150                 goto out;
1151         }
1152
1153         /*
1154          * We don't need TTT, because ITT/buffer_offset pair is sufficient
1155          * to find out the original request and buffer for Data-Out PDUs, but
1156          * crazy iSCSI spec requires us to send this superfluous field in
1157          * R2T PDUs and some initiators may rely on it.
1158          */
1159         cmnd->target_task_tag = get_next_ttt(cmnd->conn);
1160
1161         TRACE_DBG("%p:%x", cmnd, itt);
1162         if (unlikely(itt == ISCSI_RESERVED_TAG)) {
1163                 PRINT_ERROR("%s", "ITT is RESERVED_TAG");
1164                 PRINT_BUFFER("Incorrect BHS", &cmnd->pdu.bhs,
1165                         sizeof(cmnd->pdu.bhs));
1166                 err = -ISCSI_REASON_PROTOCOL_ERROR;
1167                 goto out;
1168         }
1169
1170         spin_lock(&session->cmnd_data_wait_hash_lock);
1171
1172         head = &session->cmnd_data_wait_hash[cmnd_hashfn(itt)];
1173
1174         tmp = __cmnd_find_data_wait_hash(cmnd->conn, itt);
1175         if (likely(!tmp)) {
1176                 TRACE_DBG("Adding cmnd %p to the hash (ITT %x)", cmnd,
1177                         cmnd_itt(cmnd));
1178                 list_add_tail(&cmnd->hash_list_entry, head);
1179                 cmnd->hashed = 1;
1180         } else {
1181                 PRINT_ERROR("Task %x in progress, cmnd %p", itt, cmnd);
1182                 err = -ISCSI_REASON_TASK_IN_PROGRESS;
1183         }
1184
1185         spin_unlock(&session->cmnd_data_wait_hash_lock);
1186
1187 out:
1188         return err;
1189 }
1190
1191 static void cmnd_remove_data_wait_hash(struct iscsi_cmnd *cmnd)
1192 {
1193         struct iscsi_session *session = cmnd->conn->session;
1194         struct iscsi_cmnd *tmp;
1195
1196         spin_lock(&session->cmnd_data_wait_hash_lock);
1197
1198         tmp = __cmnd_find_data_wait_hash(cmnd->conn, cmnd->pdu.bhs.itt);
1199
1200         if (likely(tmp && tmp == cmnd)) {
1201                 TRACE_DBG("Deleting cmnd %p from the hash (ITT %x)", cmnd,
1202                         cmnd_itt(cmnd));
1203                 list_del(&cmnd->hash_list_entry);
1204                 cmnd->hashed = 0;
1205         } else
1206                 PRINT_ERROR("%p:%x not found", cmnd, cmnd_itt(cmnd));
1207
1208         spin_unlock(&session->cmnd_data_wait_hash_lock);
1209
1210         return;
1211 }
1212
1213 static void cmnd_prepare_get_rejected_immed_data(struct iscsi_cmnd *cmnd)
1214 {
1215         struct iscsi_conn *conn = cmnd->conn;
1216         struct scatterlist *sg = cmnd->sg;
1217         char __user *addr;
1218         u32 size;
1219         unsigned int i;
1220
1221         TRACE_ENTRY();
1222
1223         TRACE_DBG_FLAG(iscsi_get_flow_ctrl_or_mgmt_dbg_log_flag(cmnd),
1224                 "Skipping (cmnd %p, ITT %x, op %x, cmd op %x, "
1225                 "datasize %u, scst_cmd %p, scst state %d)", cmnd,
1226                 cmnd_itt(cmnd), cmnd_opcode(cmnd), cmnd_hdr(cmnd)->scb[0],
1227                 cmnd->pdu.datasize, cmnd->scst_cmd, cmnd->scst_state);
1228
1229         iscsi_extracheck_is_rd_thread(conn);
1230
1231         size = cmnd->pdu.datasize;
1232         if (!size)
1233                 goto out;
1234
1235         /* We already checked pdu.datasize in check_segment_length() */
1236
1237         if (sg == NULL) {
1238                 /*
1239                  * There are no problems with the safety from concurrent
1240                  * accesses to dummy_page in dummy_sg, since data only
1241                  * will be read and then discarded.
1242                  */
1243                 sg = cmnd->sg = &dummy_sg;
1244                 cmnd->bufflen = PAGE_SIZE;
1245                 cmnd->own_sg = 1;
1246         }
1247
1248         addr = (char __force __user *)(page_address(sg_page(&sg[0])));
1249         sBUG_ON(addr == NULL);
1250         conn->read_size = size;
1251         for (i = 0; size > PAGE_SIZE; i++, size -= cmnd->bufflen) {
1252                 /* We already checked pdu.datasize in check_segment_length() */
1253                 sBUG_ON(i >= ISCSI_CONN_IOV_MAX);
1254                 conn->read_iov[i].iov_base = addr;
1255                 conn->read_iov[i].iov_len = cmnd->bufflen;
1256         }
1257         conn->read_iov[i].iov_base = addr;
1258         conn->read_iov[i].iov_len = size;
1259         conn->read_msg.msg_iov = conn->read_iov;
1260         conn->read_msg.msg_iovlen = ++i;
1261
1262 out:
1263         TRACE_EXIT();
1264         return;
1265 }
1266
1267 static void iscsi_set_resid(struct iscsi_cmnd *rsp, bool bufflen_set)
1268 {
1269         struct iscsi_cmnd *req = rsp->parent_req;
1270         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1271         struct iscsi_scsi_rsp_hdr *rsp_hdr;
1272         int resid, resp_len, in_resp_len;
1273
1274         if ((req_hdr->flags & ISCSI_CMD_READ) &&
1275             (req_hdr->flags & ISCSI_CMD_WRITE)) {
1276                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
1277
1278                 if (bufflen_set) {
1279                         resp_len = req->bufflen;
1280                         if (req->scst_cmd != NULL)
1281                                 in_resp_len = scst_cmd_get_in_bufflen(req->scst_cmd);
1282                         else
1283                                 in_resp_len = 0;
1284                 } else {
1285                         resp_len = 0;
1286                         in_resp_len = 0;
1287                 }
1288
1289                 resid = be32_to_cpu(req_hdr->data_length) - in_resp_len;
1290                 if (resid > 0) {
1291                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
1292                         rsp_hdr->residual_count = cpu_to_be32(resid);
1293                 } else if (resid < 0) {
1294                         resid = -resid;
1295                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
1296                         rsp_hdr->residual_count = cpu_to_be32(resid);
1297                 }
1298
1299                 resid = req->read_size - resp_len;
1300                 if (resid > 0) {
1301                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_UNDERFLOW;
1302                         rsp_hdr->bi_residual_count = cpu_to_be32(resid);
1303                 } else if (resid < 0) {
1304                         resid = -resid;
1305                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_OVERFLOW;
1306                         rsp_hdr->bi_residual_count = cpu_to_be32(resid);
1307                 }
1308         } else {
1309                 if (bufflen_set)
1310                         resp_len = req->bufflen;
1311                 else
1312                         resp_len = 0;
1313
1314                 resid = req->read_size - resp_len;
1315                 if (resid > 0) {
1316                         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
1317                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
1318                         rsp_hdr->residual_count = cpu_to_be32(resid);
1319                 } else if (resid < 0) {
1320                         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
1321                         resid = -resid;
1322                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
1323                         rsp_hdr->residual_count = cpu_to_be32(resid);
1324                 }
1325         }
1326         return;
1327 }
1328
1329 static int iscsi_preliminary_complete(struct iscsi_cmnd *req,
1330         struct iscsi_cmnd *orig_req, bool get_data)
1331 {
1332         int res = 0;
1333         bool set_r2t_len;
1334
1335         TRACE_ENTRY();
1336
1337 #ifdef CONFIG_SCST_DEBUG
1338         {
1339                 struct iscsi_hdr *req_hdr = &req->pdu.bhs;
1340                 TRACE_DBG_FLAG(iscsi_get_flow_ctrl_or_mgmt_dbg_log_flag(orig_req),
1341                         "Prelim completed req %p, orig_req %p (FINAL %x, "
1342                         "outstanding_r2t %d)", req, orig_req,
1343                         (req_hdr->flags & ISCSI_CMD_FINAL),
1344                         orig_req->outstanding_r2t);
1345         }
1346 #endif
1347
1348         iscsi_extracheck_is_rd_thread(req->conn);
1349         sBUG_ON(req->parent_req != NULL);
1350
1351         if (test_bit(ISCSI_CMD_PRELIM_COMPLETED, &req->prelim_compl_flags)) {
1352                 TRACE_MGMT_DBG("req %p already prelim completed", req);
1353                 /* To not try to get data twice */
1354                 get_data = false;
1355         }
1356
1357         set_r2t_len = !req->hashed &&
1358                       (cmnd_opcode(req) == ISCSI_OP_SCSI_CMD) &&
1359                       !test_bit(ISCSI_CMD_PRELIM_COMPLETED,
1360                                 &orig_req->prelim_compl_flags);
1361         set_bit(ISCSI_CMD_PRELIM_COMPLETED, &orig_req->prelim_compl_flags);
1362
1363         TRACE_DBG("get_data %d, set_r2t_len %d", get_data, set_r2t_len);
1364
1365         if (get_data)
1366                 cmnd_prepare_get_rejected_immed_data(req);
1367
1368         if (set_r2t_len)
1369                 res = iscsi_set_prelim_r2t_len_to_receive(orig_req);
1370
1371         TRACE_EXIT_RES(res);
1372         return res;
1373 }
1374
1375 static int cmnd_prepare_recv_pdu(struct iscsi_conn *conn,
1376         struct iscsi_cmnd *cmd, u32 offset, u32 size)
1377 {
1378         struct scatterlist *sg = cmd->sg;
1379         unsigned int bufflen = cmd->bufflen;
1380         unsigned int idx, i;
1381         char __user *addr;
1382         int res = 0;
1383
1384         TRACE_DBG("%p %u,%u", cmd->sg, offset, size);
1385
1386         iscsi_extracheck_is_rd_thread(conn);
1387
1388         if (unlikely((offset >= bufflen) ||
1389                      (offset + size > bufflen))) {
1390                 PRINT_ERROR("Wrong ltn (%u %u %u)", offset, size, bufflen);
1391                 mark_conn_closed(conn);
1392                 res = -EIO;
1393                 goto out;
1394         }
1395
1396         offset += sg[0].offset;
1397         idx = offset >> PAGE_SHIFT;
1398         offset &= ~PAGE_MASK;
1399
1400         conn->read_msg.msg_iov = conn->read_iov;
1401         conn->read_size = size;
1402
1403         i = 0;
1404         while (1) {
1405                 addr = (char __force __user *)(page_address(sg_page(&sg[idx])));
1406                 sBUG_ON(addr == NULL);
1407                 conn->read_iov[i].iov_base = addr + offset;
1408                 if (offset + size <= PAGE_SIZE) {
1409                         TRACE_DBG("idx=%d, offset=%u, size=%d, addr=%p",
1410                                 idx, offset, size, addr);
1411                         conn->read_iov[i].iov_len = size;
1412                         conn->read_msg.msg_iovlen = ++i;
1413                         break;
1414                 }
1415                 conn->read_iov[i].iov_len = PAGE_SIZE - offset;
1416                 TRACE_DBG("idx=%d, offset=%u, size=%d, iov_len=%zd, addr=%p",
1417                         idx, offset, size, conn->read_iov[i].iov_len, addr);
1418                 size -= conn->read_iov[i].iov_len;
1419                 if (unlikely(++i >= ISCSI_CONN_IOV_MAX)) {
1420                         PRINT_ERROR("Initiator %s violated negotiated "
1421                                 "parameters by sending too much data (size "
1422                                 "left %d)", conn->session->initiator_name,
1423                                 size);
1424                         mark_conn_closed(conn);
1425                         res = -EINVAL;
1426                         break;
1427                 }
1428                 idx++;
1429                 offset = sg[idx].offset;
1430         }
1431         TRACE_DBG("msg_iov=%p, msg_iovlen=%zd",
1432                 conn->read_msg.msg_iov, conn->read_msg.msg_iovlen);
1433
1434 out:
1435         return res;
1436 }
1437
1438 static void send_r2t(struct iscsi_cmnd *req)
1439 {
1440         struct iscsi_session *sess = req->conn->session;
1441         struct iscsi_cmnd *rsp;
1442         struct iscsi_r2t_hdr *rsp_hdr;
1443         u32 offset, burst;
1444         LIST_HEAD(send);
1445
1446         TRACE_ENTRY();
1447
1448         EXTRACHECKS_BUG_ON(req->r2t_len_to_send == 0);
1449
1450         /*
1451          * There is no race with data_out_start() and conn_abort(), since
1452          * all functions called from single read thread
1453          */
1454         iscsi_extracheck_is_rd_thread(req->conn);
1455
1456         /*
1457          * We don't need to check for PRELIM_COMPLETED here, because for such
1458          * commands we set r2t_len_to_send = 0, hence made sure we won't
1459          * called here.
1460          */
1461
1462         EXTRACHECKS_BUG_ON(req->outstanding_r2t >
1463                            sess->sess_params.max_outstanding_r2t);
1464
1465         if (req->outstanding_r2t == sess->sess_params.max_outstanding_r2t)
1466                 goto out;
1467
1468         burst = sess->sess_params.max_burst_length;
1469         offset = be32_to_cpu(cmnd_hdr(req)->data_length) -
1470                         req->r2t_len_to_send;
1471
1472         do {
1473                 rsp = iscsi_alloc_rsp(req);
1474                 rsp->pdu.bhs.ttt = req->target_task_tag;
1475                 rsp_hdr = (struct iscsi_r2t_hdr *)&rsp->pdu.bhs;
1476                 rsp_hdr->opcode = ISCSI_OP_R2T;
1477                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1478                 rsp_hdr->lun = cmnd_hdr(req)->lun;
1479                 rsp_hdr->itt = cmnd_hdr(req)->itt;
1480                 rsp_hdr->r2t_sn = cpu_to_be32(req->r2t_sn++);
1481                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
1482                 if (req->r2t_len_to_send > burst) {
1483                         rsp_hdr->data_length = cpu_to_be32(burst);
1484                         req->r2t_len_to_send -= burst;
1485                         offset += burst;
1486                 } else {
1487                         rsp_hdr->data_length = cpu_to_be32(req->r2t_len_to_send);
1488                         req->r2t_len_to_send = 0;
1489                 }
1490
1491                 TRACE_WRITE("req %p, data_length %u, buffer_offset %u, "
1492                         "r2t_sn %u, outstanding_r2t %u", req,
1493                         be32_to_cpu(rsp_hdr->data_length),
1494                         be32_to_cpu(rsp_hdr->buffer_offset),
1495                         be32_to_cpu(rsp_hdr->r2t_sn), req->outstanding_r2t);
1496
1497                 list_add_tail(&rsp->write_list_entry, &send);
1498                 req->outstanding_r2t++;
1499
1500         } while ((req->outstanding_r2t < sess->sess_params.max_outstanding_r2t) &&
1501                  (req->r2t_len_to_send != 0));
1502
1503         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_WAKE);
1504
1505 out:
1506         TRACE_EXIT();
1507         return;
1508 }
1509
1510 static int iscsi_pre_exec(struct scst_cmd *scst_cmd)
1511 {
1512         int res = SCST_PREPROCESS_STATUS_SUCCESS;
1513         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
1514                 scst_cmd_get_tgt_priv(scst_cmd);
1515         struct iscsi_cmnd *c, *t;
1516
1517         TRACE_ENTRY();
1518
1519         EXTRACHECKS_BUG_ON(scst_cmd_atomic(scst_cmd));
1520
1521         /* If data digest isn't used this list will be empty */
1522         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
1523                                 rx_ddigest_cmd_list_entry) {
1524                 TRACE_DBG("Checking digest of RX ddigest cmd %p", c);
1525                 if (digest_rx_data(c) != 0) {
1526                         scst_set_cmd_error(scst_cmd,
1527                                 SCST_LOAD_SENSE(iscsi_sense_crc_error));
1528                         res = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
1529                         /*
1530                          * The rest of rx_ddigest_cmd_list will be freed
1531                          * in req_cmnd_release()
1532                          */
1533                         goto out;
1534                 }
1535                 cmd_del_from_rx_ddigest_list(c);
1536                 cmnd_put(c);
1537         }
1538
1539 out:
1540         TRACE_EXIT_RES(res);
1541         return res;
1542 }
1543
1544 static int noop_out_start(struct iscsi_cmnd *cmnd)
1545 {
1546         struct iscsi_conn *conn = cmnd->conn;
1547         struct iscsi_hdr *req_hdr = &cmnd->pdu.bhs;
1548         u32 size, tmp;
1549         int i, err = 0;
1550
1551         TRACE_DBG("%p", cmnd);
1552
1553         iscsi_extracheck_is_rd_thread(conn);
1554
1555         if (!(req_hdr->flags & ISCSI_FLG_FINAL)) {
1556                 PRINT_ERROR("%s", "Initiator sent NOP-Out with not a single "
1557                         "PDU");
1558                 err = -ISCSI_REASON_PROTOCOL_ERROR;
1559                 goto out;
1560         }
1561
1562         if (cmnd_itt(cmnd) == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1563                 if (unlikely(!(cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE)))
1564                         PRINT_ERROR("%s", "Initiator sent RESERVED tag for "
1565                                 "non-immediate NOP-Out command");
1566         }
1567
1568         spin_lock(&conn->session->sn_lock);
1569         __update_stat_sn(cmnd);
1570         err = check_cmd_sn(cmnd);
1571         spin_unlock(&conn->session->sn_lock);
1572         if (unlikely(err))
1573                 goto out;
1574
1575         size = cmnd->pdu.datasize;
1576
1577         if (size) {
1578                 conn->read_msg.msg_iov = conn->read_iov;
1579                 if (cmnd->pdu.bhs.itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1580                         struct scatterlist *sg;
1581
1582                         cmnd->sg = sg = scst_alloc(size, GFP_KERNEL,
1583                                                 &cmnd->sg_cnt);
1584                         if (sg == NULL) {
1585                                 TRACE(TRACE_OUT_OF_MEM, "Allocating buffer for"
1586                                       " %d NOP-Out payload failed", size);
1587                                 err = -ISCSI_REASON_OUT_OF_RESOURCES;
1588                                 goto out;
1589                         }
1590
1591                         /* We already checked it in check_segment_length() */
1592                         sBUG_ON(cmnd->sg_cnt > (signed)ISCSI_CONN_IOV_MAX);
1593
1594                         cmnd->own_sg = 1;
1595                         cmnd->bufflen = size;
1596
1597                         for (i = 0; i < cmnd->sg_cnt; i++) {
1598                                 conn->read_iov[i].iov_base =
1599                                         (void __force __user *)(page_address(sg_page(&sg[i])));
1600                                 tmp = min_t(u32, size, PAGE_SIZE);
1601                                 conn->read_iov[i].iov_len = tmp;
1602                                 conn->read_size += tmp;
1603                                 size -= tmp;
1604                         }
1605                         sBUG_ON(size != 0);
1606                 } else {
1607                         /*
1608                          * There are no problems with the safety from concurrent
1609                          * accesses to dummy_page, since for ISCSI_RESERVED_TAG
1610                          * the data only read and then discarded.
1611                          */
1612                         for (i = 0; i < (signed)ISCSI_CONN_IOV_MAX; i++) {
1613                                 conn->read_iov[i].iov_base =
1614                                         (void __force __user *)(page_address(dummy_page));
1615                                 tmp = min_t(u32, size, PAGE_SIZE);
1616                                 conn->read_iov[i].iov_len = tmp;
1617                                 conn->read_size += tmp;
1618                                 size -= tmp;
1619                         }
1620
1621                         /* We already checked size in check_segment_length() */
1622                         sBUG_ON(size != 0);
1623                 }
1624
1625                 conn->read_msg.msg_iovlen = i;
1626                 TRACE_DBG("msg_iov=%p, msg_iovlen=%zd", conn->read_msg.msg_iov,
1627                         conn->read_msg.msg_iovlen);
1628         }
1629
1630 out:
1631         return err;
1632 }
1633
1634 int cmnd_rx_continue(struct iscsi_cmnd *req)
1635 {
1636         struct iscsi_conn *conn = req->conn;
1637         struct iscsi_session *session = conn->session;
1638         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1639         struct scst_cmd *scst_cmd = req->scst_cmd;
1640         scst_data_direction dir;
1641         bool unsolicited_data_expected = false;
1642         int res = 0;
1643
1644         TRACE_ENTRY();
1645
1646         TRACE_DBG("scsi command: %x", req_hdr->scb[0]);
1647
1648         EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_AFTER_PREPROC);
1649
1650         dir = scst_cmd_get_data_direction(scst_cmd);
1651
1652         /*
1653          * Check for preliminary completion here to save R2Ts. For TASK QUEUE
1654          * FULL statuses that might be a big performance win.
1655          */
1656         if (unlikely(scst_cmd_prelim_completed(scst_cmd) ||
1657             unlikely(req->prelim_compl_flags != 0))) {
1658                 /*
1659                  * If necessary, ISCSI_CMD_ABORTED will be set by
1660                  * iscsi_xmit_response().
1661                  */
1662                 res = iscsi_preliminary_complete(req, req, true);
1663                 goto trace;
1664         }
1665
1666         /* For prelim completed commands sg&K can be already set! */
1667
1668         if (dir != SCST_DATA_BIDI) {
1669                 req->sg = scst_cmd_get_sg(scst_cmd);
1670                 req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
1671                 req->bufflen = scst_cmd_get_bufflen(scst_cmd);
1672         } else {
1673                 req->sg = scst_cmd_get_in_sg(scst_cmd);
1674                 req->sg_cnt = scst_cmd_get_in_sg_cnt(scst_cmd);
1675                 req->bufflen = scst_cmd_get_in_bufflen(scst_cmd);
1676         }
1677
1678         if (dir & SCST_DATA_WRITE) {
1679                 unsolicited_data_expected = !(req_hdr->flags & ISCSI_CMD_FINAL);
1680
1681                 if (unlikely(session->sess_params.initial_r2t &&
1682                     unsolicited_data_expected)) {
1683                         PRINT_ERROR("Initiator %s violated negotiated "
1684                                 "parameters: initial R2T is required (ITT %x, "
1685                                 "op  %x)", session->initiator_name,
1686                                 cmnd_itt(req), req_hdr->scb[0]);
1687                         goto out_close;
1688                 }
1689
1690                 if (unlikely(!session->sess_params.immediate_data &&
1691                     req->pdu.datasize)) {
1692                         PRINT_ERROR("Initiator %s violated negotiated "
1693                                 "parameters: forbidden immediate data sent "
1694                                 "(ITT %x, op  %x)", session->initiator_name,
1695                                 cmnd_itt(req), req_hdr->scb[0]);
1696                         goto out_close;
1697                 }
1698
1699                 if (unlikely(session->sess_params.first_burst_length < req->pdu.datasize)) {
1700                         PRINT_ERROR("Initiator %s violated negotiated "
1701                                 "parameters: immediate data len (%d) > "
1702                                 "first_burst_length (%d) (ITT %x, op  %x)",
1703                                 session->initiator_name,
1704                                 req->pdu.datasize,
1705                                 session->sess_params.first_burst_length,
1706                                 cmnd_itt(req), req_hdr->scb[0]);
1707                         goto out_close;
1708                 }
1709
1710                 req->r2t_len_to_receive = be32_to_cpu(req_hdr->data_length) -
1711                                           req->pdu.datasize;
1712
1713                 if (unlikely(req->r2t_len_to_receive > req->bufflen)) {
1714                         PRINT_ERROR("req->r2t_len_to_receive %d > req->bufflen "
1715                                 "%d", req->r2t_len_to_receive, req->bufflen);
1716                         goto out_close;
1717                 }
1718
1719                 res = cmnd_insert_data_wait_hash(req);
1720                 if (unlikely(res != 0)) {
1721                         /*
1722                          * We have to close connection, because otherwise a data
1723                          * corruption is possible if we allow to receive data
1724                          * for this request in another request with dublicated
1725                          * ITT.
1726                          */
1727                         goto out_close;
1728                 }
1729
1730                 if (unsolicited_data_expected) {
1731                         req->outstanding_r2t = 1;
1732                         req->r2t_len_to_send = req->r2t_len_to_receive -
1733                                 min_t(unsigned int,
1734                                       session->sess_params.first_burst_length -
1735                                                 req->pdu.datasize,
1736                                       req->r2t_len_to_receive);
1737                 } else
1738                         req->r2t_len_to_send = req->r2t_len_to_receive;
1739
1740                 req_add_to_write_timeout_list(req);
1741
1742                 if (req->pdu.datasize) {
1743                         res = cmnd_prepare_recv_pdu(conn, req, 0, req->pdu.datasize);
1744                         /* For performance better to send R2Ts ASAP */
1745                         if (likely(res == 0) && (req->r2t_len_to_send != 0))
1746                                 send_r2t(req);
1747                 }
1748         } else {
1749                 if (unlikely(!(req_hdr->flags & ISCSI_CMD_FINAL) ||
1750                              req->pdu.datasize)) {
1751                         PRINT_ERROR("Unexpected unsolicited data (ITT %x "
1752                                 "CDB %x", cmnd_itt(req), req_hdr->scb[0]);
1753                         set_scst_preliminary_status_rsp(req, true,
1754                                 SCST_LOAD_SENSE(iscsi_sense_unexpected_unsolicited_data));
1755                 }
1756         }
1757
1758 trace:
1759         TRACE_DBG("req=%p, dir=%d, unsolicited_data_expected=%d, "
1760                 "r2t_len_to_receive=%d, r2t_len_to_send=%d, bufflen=%d, "
1761                 "own_sg %d", req, dir, unsolicited_data_expected,
1762                 req->r2t_len_to_receive, req->r2t_len_to_send, req->bufflen,
1763                 req->own_sg);
1764
1765 out:
1766         TRACE_EXIT_RES(res);
1767         return res;
1768
1769 out_close:
1770         mark_conn_closed(conn);
1771         res = -EINVAL;
1772         goto out;
1773 }
1774
1775 static int scsi_cmnd_start(struct iscsi_cmnd *req)
1776 {
1777         struct iscsi_conn *conn = req->conn;
1778         struct iscsi_session *session = conn->session;
1779         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1780         struct scst_cmd *scst_cmd;
1781         scst_data_direction dir;
1782         struct iscsi_ahs_hdr *ahdr;
1783         int res = 0;
1784
1785         TRACE_ENTRY();
1786
1787         TRACE_DBG("scsi command: %x", req_hdr->scb[0]);
1788
1789         TRACE_DBG("Incrementing active_cmds (cmd %p, sess %p, "
1790                 "new value %d)", req, session,
1791                 atomic_read(&session->active_cmds)+1);
1792         atomic_inc(&session->active_cmds);
1793         req->dec_active_cmnds = 1;
1794
1795         scst_cmd = scst_rx_cmd(session->scst_sess,
1796                 (uint8_t *)&req_hdr->lun, sizeof(req_hdr->lun),
1797                 req_hdr->scb, sizeof(req_hdr->scb), SCST_NON_ATOMIC);
1798         if (scst_cmd == NULL) {
1799                 res = create_preliminary_status_rsp(req, SAM_STAT_BUSY,
1800                         NULL, 0);
1801                 goto out;
1802         }
1803
1804         req->scst_cmd = scst_cmd;
1805         scst_cmd_set_tag(scst_cmd, req_hdr->itt);
1806         scst_cmd_set_tgt_priv(scst_cmd, req);
1807
1808         if ((req_hdr->flags & ISCSI_CMD_READ) &&
1809             (req_hdr->flags & ISCSI_CMD_WRITE)) {
1810                 int sz = cmnd_read_size(req);
1811                 if (unlikely(sz < 0)) {
1812                         PRINT_ERROR("%s", "BIDI data transfer, but initiator "
1813                                 "not supplied Bidirectional Read Expected Data "
1814                                 "Transfer Length AHS");
1815                         set_scst_preliminary_status_rsp(req, true,
1816                            SCST_LOAD_SENSE(scst_sense_parameter_value_invalid));
1817                 } else {
1818                         req->read_size = sz;
1819                         dir = SCST_DATA_BIDI;
1820                         scst_cmd_set_expected(scst_cmd, dir, sz);
1821                         scst_cmd_set_expected_in_transfer_len(scst_cmd,
1822                                 be32_to_cpu(req_hdr->data_length));
1823 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
1824                         scst_cmd_set_tgt_need_alloc_data_buf(scst_cmd);
1825 #endif
1826                 }
1827         } else if (req_hdr->flags & ISCSI_CMD_READ) {
1828                 req->read_size = be32_to_cpu(req_hdr->data_length);
1829                 dir = SCST_DATA_READ;
1830                 scst_cmd_set_expected(scst_cmd, dir, req->read_size);
1831 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
1832                 scst_cmd_set_tgt_need_alloc_data_buf(scst_cmd);
1833 #endif
1834         } else if (req_hdr->flags & ISCSI_CMD_WRITE) {
1835                 dir = SCST_DATA_WRITE;
1836                 scst_cmd_set_expected(scst_cmd, dir,
1837                         be32_to_cpu(req_hdr->data_length));
1838         } else {
1839                 dir = SCST_DATA_NONE;
1840                 scst_cmd_set_expected(scst_cmd, dir, 0);
1841         }
1842
1843         switch (req_hdr->flags & ISCSI_CMD_ATTR_MASK) {
1844         case ISCSI_CMD_SIMPLE:
1845                 scst_cmd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1846                 break;
1847         case ISCSI_CMD_HEAD_OF_QUEUE:
1848                 scst_cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1849                 break;
1850         case ISCSI_CMD_ORDERED:
1851                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1852                 break;
1853         case ISCSI_CMD_ACA:
1854                 scst_cmd->queue_type = SCST_CMD_QUEUE_ACA;
1855                 break;
1856         case ISCSI_CMD_UNTAGGED:
1857                 scst_cmd->queue_type = SCST_CMD_QUEUE_UNTAGGED;
1858                 break;
1859         default:
1860                 PRINT_ERROR("Unknown task code %x, use ORDERED instead",
1861                         req_hdr->flags & ISCSI_CMD_ATTR_MASK);
1862                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1863                 break;
1864         }
1865
1866         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1867         scst_cmd_set_tgt_sn(scst_cmd, req_hdr->cmd_sn);
1868
1869         ahdr = (struct iscsi_ahs_hdr *)req->pdu.ahs;
1870         if (ahdr != NULL) {
1871                 uint8_t *p = (uint8_t *)ahdr;
1872                 unsigned int size = 0;
1873                 do {
1874                         int s;
1875
1876                         ahdr = (struct iscsi_ahs_hdr *)p;
1877
1878                         if (ahdr->ahstype == ISCSI_AHSTYPE_CDB) {
1879                                 struct iscsi_cdb_ahdr *eca =
1880                                         (struct iscsi_cdb_ahdr *)ahdr;
1881                                 scst_cmd_set_ext_cdb(scst_cmd, eca->cdb,
1882                                         be16_to_cpu(ahdr->ahslength) - 1);
1883                                 break;
1884                         }
1885                         s = 3 + be16_to_cpu(ahdr->ahslength);
1886                         s = (s + 3) & -4;
1887                         size += s;
1888                         p += s;
1889                 } while (size < req->pdu.ahssize);
1890         }
1891
1892         TRACE_DBG("START Command (itt %x, queue_type %d)",
1893                 req_hdr->itt, scst_cmd->queue_type);
1894         req->scst_state = ISCSI_CMD_STATE_RX_CMD;
1895         conn->rx_task = current;
1896         scst_cmd_init_stage1_done(scst_cmd, SCST_CONTEXT_DIRECT, 0);
1897
1898         if (req->scst_state != ISCSI_CMD_STATE_RX_CMD)
1899                 res = cmnd_rx_continue(req);
1900         else {
1901                 TRACE_DBG("Delaying req %p post processing (scst_state %d)",
1902                         req, req->scst_state);
1903                 res = 1;
1904         }
1905
1906 out:
1907         TRACE_EXIT_RES(res);
1908         return res;
1909 }
1910
1911 static int data_out_start(struct iscsi_cmnd *cmnd)
1912 {
1913         struct iscsi_conn *conn = cmnd->conn;
1914         struct iscsi_data_out_hdr *req_hdr =
1915                 (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1916         struct iscsi_cmnd *orig_req;
1917 #if 0
1918         struct iscsi_hdr *orig_req_hdr;
1919 #endif
1920         u32 offset = be32_to_cpu(req_hdr->buffer_offset);
1921         int res = 0;
1922
1923         TRACE_ENTRY();
1924
1925         /*
1926          * There is no race with send_r2t() and conn_abort(), since
1927          * all functions called from single read thread
1928          */
1929         iscsi_extracheck_is_rd_thread(cmnd->conn);
1930
1931         update_stat_sn(cmnd);
1932
1933         orig_req = cmnd_find_data_wait_hash(conn, req_hdr->itt);
1934         cmnd->cmd_req = orig_req;
1935         if (unlikely(orig_req == NULL)) {
1936                 /*
1937                  * It shouldn't happen, since we don't abort any request until
1938                  * we received all related PDUs from the initiator or timeout
1939                  * them. Let's quietly drop such PDUs.
1940                  */
1941                 TRACE_MGMT_DBG("Unable to find scsi task ITT %x",
1942                         cmnd_itt(cmnd));
1943                 res = iscsi_preliminary_complete(cmnd, cmnd, true);
1944                 goto out;
1945         }
1946
1947         if (unlikely(orig_req->r2t_len_to_receive < cmnd->pdu.datasize)) {
1948                 if (orig_req->prelim_compl_flags != 0) {
1949                         /* We can have fake r2t_len_to_receive */
1950                         goto go;
1951                 }
1952                 PRINT_ERROR("Data size (%d) > R2T length to receive (%d)",
1953                         cmnd->pdu.datasize, orig_req->r2t_len_to_receive);
1954                 set_scst_preliminary_status_rsp(orig_req, false,
1955                         SCST_LOAD_SENSE(iscsi_sense_incorrect_amount_of_data));
1956                 goto go;
1957         }
1958
1959         /* Crazy iSCSI spec requires us to make this unneeded check */
1960 #if 0 /* ...but some initiators (Windows) don't care to correctly set it */
1961         orig_req_hdr = &orig_req->pdu.bhs;
1962         if (unlikely(orig_req_hdr->lun != req_hdr->lun)) {
1963                 PRINT_ERROR("Wrong LUN (%lld) in Data-Out PDU (expected %lld), "
1964                         "orig_req %p, cmnd %p", (unsigned long long)req_hdr->lun,
1965                         (unsigned long long)orig_req_hdr->lun, orig_req, cmnd);
1966                 create_reject_rsp(orig_req, ISCSI_REASON_PROTOCOL_ERROR, false);
1967                 goto go;
1968         }
1969 #endif
1970
1971 go:
1972         if (req_hdr->flags & ISCSI_FLG_FINAL)
1973                 orig_req->outstanding_r2t--;
1974
1975         if (unlikely(orig_req->prelim_compl_flags != 0)) {
1976                 res = iscsi_preliminary_complete(cmnd, orig_req, true);
1977                 goto out;
1978         }
1979
1980         TRACE_WRITE("cmnd %p, orig_req %p, offset %u, datasize %u", cmnd,
1981                 orig_req, offset, cmnd->pdu.datasize);
1982
1983         res = cmnd_prepare_recv_pdu(conn, orig_req, offset, cmnd->pdu.datasize);
1984
1985 out:
1986         TRACE_EXIT_RES(res);
1987         return res;
1988 }
1989
1990 static void data_out_end(struct iscsi_cmnd *cmnd)
1991 {
1992         struct iscsi_data_out_hdr *req_hdr =
1993                 (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1994         struct iscsi_cmnd *req;
1995
1996         TRACE_ENTRY();
1997
1998         EXTRACHECKS_BUG_ON(cmnd == NULL);
1999         req = cmnd->cmd_req;
2000         if (unlikely(req == NULL))
2001                 goto out;
2002
2003         TRACE_DBG("cmnd %p, req %p", cmnd, req);
2004
2005         iscsi_extracheck_is_rd_thread(cmnd->conn);
2006
2007         if (!(cmnd->conn->ddigest_type & DIGEST_NONE) &&
2008             !cmnd->ddigest_checked) {
2009                 cmd_add_on_rx_ddigest_list(req, cmnd);
2010                 cmnd_get(cmnd);
2011         }
2012
2013         /*
2014          * Now we received the data and can adjust r2t_len_to_receive of the
2015          * orig req. We couldn't do it earlier, because it will break data
2016          * receiving errors recovery (calls of iscsi_fail_data_waiting_cmnd()).
2017          */
2018         req->r2t_len_to_receive -= cmnd->pdu.datasize;
2019
2020         if (unlikely(req->prelim_compl_flags != 0)) {
2021                 /*
2022                  * We might need to wait for one or more PDUs. Let's simplify
2023                  * other code.
2024                  */
2025                 req->r2t_len_to_receive = req->outstanding_r2t;
2026                 req->r2t_len_to_send = 0;
2027         }
2028
2029         TRACE_DBG("req %p, FINAL %x, outstanding_r2t %d, r2t_len_to_receive %d,"
2030                 " r2t_len_to_send %d", req, req_hdr->flags & ISCSI_FLG_FINAL,
2031                 req->outstanding_r2t, req->r2t_len_to_receive,
2032                 req->r2t_len_to_send);
2033
2034         if (!(req_hdr->flags & ISCSI_FLG_FINAL))
2035                 goto out;
2036
2037         if (req->r2t_len_to_receive == 0) {
2038                 if (!req->pending)
2039                         iscsi_restart_cmnd(req);
2040         } else if (req->r2t_len_to_send != 0)
2041                 send_r2t(req);
2042
2043 out:
2044         TRACE_EXIT();
2045         return;
2046 }
2047
2048 /* Might be called under target_mutex and cmd_list_lock */
2049 static void __cmnd_abort(struct iscsi_cmnd *cmnd)
2050 {
2051         unsigned long timeout_time = jiffies + ISCSI_TM_DATA_WAIT_SCHED_TIMEOUT;
2052         struct iscsi_conn *conn = cmnd->conn;
2053
2054         TRACE_MGMT_DBG("Aborting cmd %p, scst_cmd %p (scst state %x, "
2055                 "ref_cnt %d, on_write_timeout_list %d, write_start %ld, ITT %x, "
2056                 "sn %u, op %x, r2t_len_to_receive %d, r2t_len_to_send %d, "
2057                 "CDB op %x, size to write %u, outstanding_r2t %d, "
2058                 "sess->exp_cmd_sn %u, conn %p, rd_task %p)",
2059                 cmnd, cmnd->scst_cmd, cmnd->scst_state,
2060                 atomic_read(&cmnd->ref_cnt), cmnd->on_write_timeout_list,
2061                 cmnd->write_start, cmnd_itt(cmnd), cmnd->pdu.bhs.sn,
2062                 cmnd_opcode(cmnd), cmnd->r2t_len_to_receive,
2063                 cmnd->r2t_len_to_send, cmnd_scsicode(cmnd),
2064                 cmnd_write_size(cmnd), cmnd->outstanding_r2t,
2065                 cmnd->conn->session->exp_cmd_sn, cmnd->conn,
2066                 cmnd->conn->rd_task);
2067
2068 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
2069         TRACE_MGMT_DBG("net_ref_cnt %d", atomic_read(&cmnd->net_ref_cnt));
2070 #endif
2071
2072         /*
2073          * Lock to sync with iscsi_check_tm_data_wait_timeouts(), including
2074          * CMD_ABORTED bit set.
2075          */
2076         spin_lock_bh(&iscsi_rd_lock);
2077
2078         /*
2079          * We suppose that preliminary commands completion is tested by
2080          * comparing prelim_compl_flags with 0. Otherwise a race is possible,
2081          * like sending command in SCST core as PRELIM_COMPLETED, while it
2082          * wasn't aborted in it yet and have as the result a wrong success
2083          * status sent to the initiator.
2084          */
2085         set_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags);
2086
2087         TRACE_MGMT_DBG("Setting conn_tm_active for conn %p", conn);
2088         conn->conn_tm_active = 1;
2089
2090         spin_unlock_bh(&iscsi_rd_lock);
2091
2092         /*
2093          * We need the lock to sync with req_add_to_write_timeout_list() and
2094          * close races for rsp_timer.expires.
2095          */
2096         spin_lock_bh(&conn->write_list_lock);
2097         if (!timer_pending(&conn->rsp_timer) ||
2098             time_after(conn->rsp_timer.expires, timeout_time)) {
2099                 TRACE_MGMT_DBG("Mod timer on %ld (conn %p)", timeout_time,
2100                         conn);
2101                 mod_timer(&conn->rsp_timer, timeout_time);
2102         } else
2103                 TRACE_MGMT_DBG("Timer for conn %p is going to fire on %ld "
2104                         "(timeout time %ld)", conn, conn->rsp_timer.expires,
2105                         timeout_time);
2106         spin_unlock_bh(&conn->write_list_lock);
2107
2108         return;
2109 }
2110
2111 /* Must be called from the read or conn close thread */
2112 static int cmnd_abort(struct iscsi_cmnd *req, int *status)
2113 {
2114         struct iscsi_task_mgt_hdr *req_hdr =
2115                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2116         struct iscsi_cmnd *cmnd;
2117         int res = -1;
2118
2119         req_hdr->ref_cmd_sn = be32_to_cpu(req_hdr->ref_cmd_sn);
2120
2121         if (!before(req_hdr->ref_cmd_sn, req_hdr->cmd_sn)) {
2122                 TRACE(TRACE_MGMT, "ABORT TASK: RefCmdSN(%u) > CmdSN(%u)",
2123                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn);
2124                 *status = ISCSI_RESPONSE_UNKNOWN_TASK;
2125                 goto out;
2126         }
2127
2128         cmnd = cmnd_find_itt_get(req->conn, req_hdr->rtt);
2129         if (cmnd) {
2130                 struct iscsi_conn *conn = cmnd->conn;
2131                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
2132
2133                 if (req_hdr->lun != hdr->lun) {
2134                         PRINT_ERROR("ABORT TASK: LUN mismatch: req LUN "
2135                                     "%llx, cmd LUN %llx, rtt %u",
2136                                     (long long unsigned int)req_hdr->lun,
2137                                     (long long unsigned int)hdr->lun,
2138                                     req_hdr->rtt);
2139                         *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2140                         goto out_put;
2141                 }
2142
2143                 if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
2144                         if (req_hdr->ref_cmd_sn != req_hdr->cmd_sn) {
2145                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != TM "
2146                                         "cmd CmdSN(%u) for immediate command "
2147                                         "%p", req_hdr->ref_cmd_sn,
2148                                         req_hdr->cmd_sn, cmnd);
2149                                 *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2150                                 goto out_put;
2151                         }
2152                 } else {
2153                         if (req_hdr->ref_cmd_sn != hdr->cmd_sn) {
2154                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != "
2155                                         "CmdSN(%u) for command %p",
2156                                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
2157                                         cmnd);
2158                                 *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2159                                 goto out_put;
2160                         }
2161                 }
2162
2163                 if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
2164                     (req_hdr->cmd_sn == hdr->cmd_sn)) {
2165                         PRINT_ERROR("ABORT TASK: SN mismatch: req SN %x, "
2166                                 "cmd SN %x, rtt %u", req_hdr->cmd_sn,
2167                                 hdr->cmd_sn, req_hdr->rtt);
2168                         *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2169                         goto out_put;
2170                 }
2171
2172                 spin_lock_bh(&conn->cmd_list_lock);
2173                 __cmnd_abort(cmnd);
2174                 spin_unlock_bh(&conn->cmd_list_lock);
2175
2176                 cmnd_put(cmnd);
2177                 res = 0;
2178         } else {
2179                 TRACE_MGMT_DBG("cmd RTT %x not found", req_hdr->rtt);
2180                 /*
2181                  * iSCSI RFC:
2182                  *
2183                  * b)  If the Referenced Task Tag does not identify an existing task,
2184                  * but if the CmdSN indicated by the RefCmdSN field in the Task
2185                  * Management function request is within the valid CmdSN window
2186                  * and less than the CmdSN of the Task Management function
2187                  * request itself, then targets must consider the CmdSN received
2188                  * and return the "Function complete" response.
2189                  *
2190                  * c)  If the Referenced Task Tag does not identify an existing task
2191                  * and if the CmdSN indicated by the RefCmdSN field in the Task
2192                  * Management function request is outside the valid CmdSN window,
2193                  * then targets must return the "Task does not exist" response.
2194                  *
2195                  * 128 seems to be a good "window".
2196                  */
2197                 if (between(req_hdr->ref_cmd_sn, req_hdr->cmd_sn - 128,
2198                             req_hdr->cmd_sn)) {
2199                         *status = ISCSI_RESPONSE_FUNCTION_COMPLETE;
2200                         res = 0;
2201                 } else
2202                         *status = ISCSI_RESPONSE_UNKNOWN_TASK;
2203         }
2204
2205 out:
2206         return res;
2207
2208 out_put:
2209         cmnd_put(cmnd);
2210         goto out;
2211 }
2212
2213 /* Must be called from the read or conn close thread */
2214 static int target_abort(struct iscsi_cmnd *req, int all)
2215 {
2216         struct iscsi_target *target = req->conn->session->target;
2217         struct iscsi_task_mgt_hdr *req_hdr =
2218                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2219         struct iscsi_session *session;
2220         struct iscsi_conn *conn;
2221         struct iscsi_cmnd *cmnd;
2222
2223         mutex_lock(&target->target_mutex);
2224
2225         list_for_each_entry(session, &target->session_list,
2226                             session_list_entry) {
2227                 list_for_each_entry(conn, &session->conn_list,
2228                                     conn_list_entry) {
2229                         spin_lock_bh(&conn->cmd_list_lock);
2230                         list_for_each_entry(cmnd, &conn->cmd_list,
2231                                             cmd_list_entry) {
2232                                 if (cmnd == req)
2233                                         continue;
2234                                 if (all)
2235                                         __cmnd_abort(cmnd);
2236                                 else if (req_hdr->lun == cmnd_hdr(cmnd)->lun)
2237                                         __cmnd_abort(cmnd);
2238                         }
2239                         spin_unlock_bh(&conn->cmd_list_lock);
2240                 }
2241         }
2242
2243         mutex_unlock(&target->target_mutex);
2244         return 0;
2245 }
2246
2247 /* Must be called from the read or conn close thread */
2248 static void task_set_abort(struct iscsi_cmnd *req)
2249 {
2250         struct iscsi_session *session = req->conn->session;
2251         struct iscsi_task_mgt_hdr *req_hdr =
2252                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2253         struct iscsi_target *target = session->target;
2254         struct iscsi_conn *conn;
2255         struct iscsi_cmnd *cmnd;
2256
2257         mutex_lock(&target->target_mutex);
2258
2259         list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
2260                 spin_lock_bh(&conn->cmd_list_lock);
2261                 list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
2262                         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
2263                         if (cmnd == req)
2264                                 continue;
2265                         if (req_hdr->lun != hdr->lun)
2266                                 continue;
2267                         if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
2268                             req_hdr->cmd_sn == hdr->cmd_sn)
2269                                 continue;
2270                         __cmnd_abort(cmnd);
2271                 }
2272                 spin_unlock_bh(&conn->cmd_list_lock);
2273         }
2274
2275         mutex_unlock(&target->target_mutex);
2276         return;
2277 }
2278
2279 /* Must be called from the read or conn close thread */
2280 void conn_abort(struct iscsi_conn *conn)
2281 {
2282         struct iscsi_cmnd *cmnd;
2283
2284         TRACE_MGMT_DBG("Aborting conn %p", conn);
2285
2286         iscsi_extracheck_is_rd_thread(conn);
2287
2288         spin_lock_bh(&conn->cmd_list_lock);
2289 again:
2290         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
2291                 __cmnd_abort(cmnd);
2292                 if (cmnd->r2t_len_to_receive != 0) {
2293                         if (!cmnd_get_check(cmnd)) {
2294                                 spin_unlock_bh(&conn->cmd_list_lock);
2295
2296                                 /* ToDo: this is racy for MC/S */
2297                                 iscsi_fail_data_waiting_cmnd(cmnd);
2298
2299                                 cmnd_put(cmnd);
2300
2301                                 /*
2302                                  * We are in the read thread, so we may not
2303                                  * worry that after cmnd release conn gets
2304                                  * released as well.
2305                                  */
2306                                 spin_lock_bh(&conn->cmd_list_lock);
2307                                 goto again;
2308                         }
2309                 }
2310         }
2311         spin_unlock_bh(&conn->cmd_list_lock);
2312
2313         return;
2314 }
2315
2316 static void execute_task_management(struct iscsi_cmnd *req)
2317 {
2318         struct iscsi_conn *conn = req->conn;
2319         struct iscsi_session *sess = conn->session;
2320         struct iscsi_task_mgt_hdr *req_hdr =
2321                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2322         int rc, status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2323         int function = req_hdr->function & ISCSI_FUNCTION_MASK;
2324         struct scst_rx_mgmt_params params;
2325
2326         TRACE(TRACE_MGMT, "TM fn %d", function);
2327
2328         TRACE_MGMT_DBG("TM req %p, ITT %x, RTT %x, sn %u, con %p", req,
2329                 cmnd_itt(req), req_hdr->rtt, req_hdr->cmd_sn, conn);
2330
2331         iscsi_extracheck_is_rd_thread(conn);
2332
2333         spin_lock(&sess->sn_lock);
2334         sess->tm_active++;
2335         sess->tm_sn = req_hdr->cmd_sn;
2336         if (sess->tm_rsp != NULL) {
2337                 struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
2338
2339                 TRACE_MGMT_DBG("Dropping delayed TM rsp %p", tm_rsp);
2340
2341                 sess->tm_rsp = NULL;
2342                 sess->tm_active--;
2343
2344                 spin_unlock(&sess->sn_lock);
2345
2346                 sBUG_ON(sess->tm_active < 0);
2347
2348                 rsp_cmnd_release(tm_rsp);
2349         } else
2350                 spin_unlock(&sess->sn_lock);
2351
2352         memset(&params, 0, sizeof(params));
2353         params.atomic = SCST_NON_ATOMIC;
2354         params.tgt_priv = req;
2355
2356         if ((function != ISCSI_FUNCTION_ABORT_TASK) &&
2357             (req_hdr->rtt != ISCSI_RESERVED_TAG)) {
2358                 PRINT_ERROR("Invalid RTT %x (TM fn %d)", req_hdr->rtt,
2359                         function);
2360                 rc = -1;
2361                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2362                 goto reject;
2363         }
2364
2365         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
2366
2367         switch (function) {
2368         case ISCSI_FUNCTION_ABORT_TASK:
2369                 rc = cmnd_abort(req, &status);
2370                 if (rc == 0) {
2371                         params.fn = SCST_ABORT_TASK;
2372                         params.tag = req_hdr->rtt;
2373                         params.tag_set = 1;
2374                         params.lun = (uint8_t *)&req_hdr->lun;
2375                         params.lun_len = sizeof(req_hdr->lun);
2376                         params.lun_set = 1;
2377                         params.cmd_sn = req_hdr->cmd_sn;
2378                         params.cmd_sn_set = 1;
2379                         rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2380                                 &params);
2381                         status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2382                 }
2383                 break;
2384         case ISCSI_FUNCTION_ABORT_TASK_SET:
2385                 task_set_abort(req);
2386                 params.fn = SCST_ABORT_TASK_SET;
2387                 params.lun = (uint8_t *)&req_hdr->lun;
2388                 params.lun_len = sizeof(req_hdr->lun);
2389                 params.lun_set = 1;
2390                 params.cmd_sn = req_hdr->cmd_sn;
2391                 params.cmd_sn_set = 1;
2392                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2393                         &params);
2394                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2395                 break;
2396         case ISCSI_FUNCTION_CLEAR_TASK_SET:
2397                 task_set_abort(req);
2398                 params.fn = SCST_CLEAR_TASK_SET;
2399                 params.lun = (uint8_t *)&req_hdr->lun;
2400                 params.lun_len = sizeof(req_hdr->lun);
2401                 params.lun_set = 1;
2402                 params.cmd_sn = req_hdr->cmd_sn;
2403                 params.cmd_sn_set = 1;
2404                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2405                         &params);
2406                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2407                 break;
2408         case ISCSI_FUNCTION_CLEAR_ACA:
2409                 params.fn = SCST_CLEAR_ACA;
2410                 params.lun = (uint8_t *)&req_hdr->lun;
2411                 params.lun_len = sizeof(req_hdr->lun);
2412                 params.lun_set = 1;
2413                 params.cmd_sn = req_hdr->cmd_sn;
2414                 params.cmd_sn_set = 1;
2415                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2416                         &params);
2417                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2418                 break;
2419         case ISCSI_FUNCTION_TARGET_COLD_RESET:
2420         case ISCSI_FUNCTION_TARGET_WARM_RESET:
2421                 target_abort(req, 1);
2422                 params.fn = SCST_TARGET_RESET;
2423                 params.cmd_sn = req_hdr->cmd_sn;
2424                 params.cmd_sn_set = 1;
2425                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2426                         &params);
2427                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2428                 break;
2429         case ISCSI_FUNCTION_LOGICAL_UNIT_RESET:
2430                 target_abort(req, 0);
2431                 params.fn = SCST_LUN_RESET;
2432                 params.lun = (uint8_t *)&req_hdr->lun;
2433                 params.lun_len = sizeof(req_hdr->lun);
2434                 params.lun_set = 1;
2435                 params.cmd_sn = req_hdr->cmd_sn;
2436                 params.cmd_sn_set = 1;
2437                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2438                         &params);
2439                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2440                 break;
2441         case ISCSI_FUNCTION_TASK_REASSIGN:
2442                 rc = -1;
2443                 status = ISCSI_RESPONSE_ALLEGIANCE_REASSIGNMENT_UNSUPPORTED;
2444                 break;
2445         default:
2446                 PRINT_ERROR("Unknown TM function %d", function);
2447                 rc = -1;
2448                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2449                 break;
2450         }
2451
2452 reject:
2453         if (rc != 0)
2454                 iscsi_send_task_mgmt_resp(req, status);
2455
2456         return;
2457 }
2458
2459 static void noop_out_exec(struct iscsi_cmnd *req)
2460 {
2461         struct iscsi_cmnd *rsp;
2462         struct iscsi_nop_in_hdr *rsp_hdr;
2463
2464         TRACE_ENTRY();
2465
2466         TRACE_DBG("%p", req);
2467
2468         if (cmnd_itt(req) != cpu_to_be32(ISCSI_RESERVED_TAG)) {
2469                 rsp = iscsi_alloc_main_rsp(req);
2470
2471                 rsp_hdr = (struct iscsi_nop_in_hdr *)&rsp->pdu.bhs;
2472                 rsp_hdr->opcode = ISCSI_OP_NOOP_IN;
2473                 rsp_hdr->flags = ISCSI_FLG_FINAL;
2474                 rsp_hdr->itt = req->pdu.bhs.itt;
2475                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
2476
2477                 if (req->pdu.datasize)
2478                         sBUG_ON(req->sg == NULL);
2479                 else
2480                         sBUG_ON(req->sg != NULL);
2481
2482                 if (req->sg) {
2483                         rsp->sg = req->sg;
2484                         rsp->sg_cnt = req->sg_cnt;
2485                         rsp->bufflen = req->bufflen;
2486                 }
2487
2488                 /* We already checked it in check_segment_length() */
2489                 sBUG_ON(get_pgcnt(req->pdu.datasize, 0) > ISCSI_CONN_IOV_MAX);
2490
2491                 rsp->pdu.datasize = req->pdu.datasize;
2492         }
2493
2494         req_cmnd_release(req);
2495
2496         TRACE_EXIT();
2497         return;
2498 }
2499
2500 static void logout_exec(struct iscsi_cmnd *req)
2501 {
2502         struct iscsi_logout_req_hdr *req_hdr;
2503         struct iscsi_cmnd *rsp;
2504         struct iscsi_logout_rsp_hdr *rsp_hdr;
2505
2506         PRINT_INFO("Logout received from initiator %s",
2507                 req->conn->session->initiator_name);
2508         TRACE_DBG("%p", req);
2509
2510         req_hdr = (struct iscsi_logout_req_hdr *)&req->pdu.bhs;
2511         rsp = iscsi_alloc_main_rsp(req);
2512         rsp_hdr = (struct iscsi_logout_rsp_hdr *)&rsp->pdu.bhs;
2513         rsp_hdr->opcode = ISCSI_OP_LOGOUT_RSP;
2514         rsp_hdr->flags = ISCSI_FLG_FINAL;
2515         rsp_hdr->itt = req_hdr->itt;
2516         rsp->should_close_conn = 1;
2517
2518         req_cmnd_release(req);
2519
2520         return;
2521 }
2522
2523 static void iscsi_cmnd_exec(struct iscsi_cmnd *cmnd)
2524 {
2525         TRACE_ENTRY();
2526
2527         TRACE_DBG("cmnd %p, op %x, SN %u", cmnd, cmnd_opcode(cmnd),
2528                 cmnd->pdu.bhs.sn);
2529
2530         iscsi_extracheck_is_rd_thread(cmnd->conn);
2531
2532         if (cmnd_opcode(cmnd) == ISCSI_OP_SCSI_CMD) {
2533                 if (cmnd->r2t_len_to_receive == 0)
2534                         iscsi_restart_cmnd(cmnd);
2535                 else if (cmnd->r2t_len_to_send != 0)
2536                         send_r2t(cmnd);
2537                 goto out;
2538         }
2539
2540         if (cmnd->prelim_compl_flags != 0) {
2541                 TRACE_MGMT_DBG("Terminating prelim completed non-SCSI cmnd %p "
2542                         "(op %x)", cmnd, cmnd_opcode(cmnd));
2543                 req_cmnd_release(cmnd);
2544                 goto out;
2545         }
2546
2547         switch (cmnd_opcode(cmnd)) {
2548         case ISCSI_OP_NOOP_OUT:
2549                 noop_out_exec(cmnd);
2550                 break;
2551         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2552                 execute_task_management(cmnd);
2553                 break;
2554         case ISCSI_OP_LOGOUT_CMD:
2555                 logout_exec(cmnd);
2556                 break;
2557         default:
2558                 PRINT_CRIT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
2559                 sBUG();
2560                 break;
2561         }
2562
2563 out:
2564         TRACE_EXIT();
2565         return;
2566 }
2567
2568 static void set_cork(struct socket *sock, int on)
2569 {
2570         int opt = on;
2571         mm_segment_t oldfs;
2572
2573         oldfs = get_fs();
2574         set_fs(get_ds());
2575         sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK,
2576                               (void __force __user *)&opt, sizeof(opt));
2577         set_fs(oldfs);
2578         return;
2579 }
2580
2581 void cmnd_tx_start(struct iscsi_cmnd *cmnd)
2582 {
2583         struct iscsi_conn *conn = cmnd->conn;
2584
2585         TRACE_DBG("conn %p, cmnd %p, opcode %x", conn, cmnd, cmnd_opcode(cmnd));
2586         iscsi_cmnd_set_length(&cmnd->pdu);
2587
2588         iscsi_extracheck_is_wr_thread(conn);
2589
2590         set_cork(conn->sock, 1);
2591
2592         conn->write_iop = conn->write_iov;
2593         conn->write_iop->iov_base = (void __force __user *)(&cmnd->pdu.bhs);
2594         conn->write_iop->iov_len = sizeof(cmnd->pdu.bhs);
2595         conn->write_iop_used = 1;
2596         conn->write_size = sizeof(cmnd->pdu.bhs) + cmnd->pdu.datasize;
2597         conn->write_offset = 0;
2598
2599         switch (cmnd_opcode(cmnd)) {
2600         case ISCSI_OP_NOOP_IN:
2601                 cmnd_set_sn(cmnd, 1);
2602                 break;
2603         case ISCSI_OP_SCSI_RSP:
2604                 cmnd_set_sn(cmnd, 1);
2605                 break;
2606         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2607                 cmnd_set_sn(cmnd, 1);
2608                 break;
2609         case ISCSI_OP_TEXT_RSP:
2610                 cmnd_set_sn(cmnd, 1);
2611                 break;
2612         case ISCSI_OP_SCSI_DATA_IN:
2613         {
2614                 struct iscsi_data_in_hdr *rsp =
2615                         (struct iscsi_data_in_hdr *)&cmnd->pdu.bhs;
2616                 u32 offset = cpu_to_be32(rsp->buffer_offset);
2617
2618                 TRACE_DBG("cmnd %p, offset %u, datasize %u, bufflen %u", cmnd,
2619                         offset, cmnd->pdu.datasize, cmnd->bufflen);
2620
2621                 sBUG_ON(offset > cmnd->bufflen);
2622                 sBUG_ON(offset + cmnd->pdu.datasize > cmnd->bufflen);
2623
2624                 conn->write_offset = offset;
2625
2626                 cmnd_set_sn(cmnd, (rsp->flags & ISCSI_FLG_FINAL) ? 1 : 0);
2627                 break;
2628         }
2629         case ISCSI_OP_LOGOUT_RSP:
2630                 cmnd_set_sn(cmnd, 1);
2631                 break;
2632         case ISCSI_OP_R2T:
2633                 cmnd->pdu.bhs.sn = cmnd_set_sn(cmnd, 0);
2634                 break;
2635         case ISCSI_OP_ASYNC_MSG:
2636                 cmnd_set_sn(cmnd, 1);
2637                 break;
2638         case ISCSI_OP_REJECT:
2639                 cmnd_set_sn(cmnd, 1);
2640                 break;
2641         default:
2642                 PRINT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
2643                 break;
2644         }
2645
2646         iscsi_dump_pdu(&cmnd->pdu);
2647         return;
2648 }
2649
2650 void cmnd_tx_end(struct iscsi_cmnd *cmnd)
2651 {
2652         struct iscsi_conn *conn = cmnd->conn;
2653
2654         TRACE_DBG("%p:%x (should_close_conn %d, should_close_all_conn %d)",
2655                 cmnd, cmnd_opcode(cmnd), cmnd->should_close_conn,
2656                 cmnd->should_close_all_conn);
2657
2658 #ifdef CONFIG_SCST_EXTRACHECKS
2659         switch (cmnd_opcode(cmnd)) {
2660         case ISCSI_OP_NOOP_IN:
2661         case ISCSI_OP_SCSI_RSP:
2662         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2663         case ISCSI_OP_TEXT_RSP:
2664         case ISCSI_OP_R2T:
2665         case ISCSI_OP_ASYNC_MSG:
2666         case ISCSI_OP_REJECT:
2667         case ISCSI_OP_SCSI_DATA_IN:
2668         case ISCSI_OP_LOGOUT_RSP:
2669                 break;
2670         default:
2671                 PRINT_CRIT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2672                 sBUG();
2673                 break;
2674         }
2675 #endif
2676
2677         if (unlikely(cmnd->should_close_conn)) {
2678                 if (cmnd->should_close_all_conn) {
2679                         PRINT_INFO("Closing all connections for target %x at "
2680                                 "initiator's %s request",
2681                                 cmnd->conn->session->target->tid,
2682                                 conn->session->initiator_name);
2683                         target_del_all_sess(cmnd->conn->session->target, 0);
2684                 } else {
2685                         PRINT_INFO("Closing connection at initiator's %s "
2686                                 "request", conn->session->initiator_name);
2687                         mark_conn_closed(conn);
2688                 }
2689         }
2690
2691         set_cork(cmnd->conn->sock, 0);
2692         return;
2693 }
2694
2695 /*
2696  * Push the command for execution. This functions reorders the commands.
2697  * Called from the read thread.
2698  *
2699  * Basically, since we don't support MC/S and TCP guarantees data delivery
2700  * order, all that SN's stuff isn't needed at all (commands delivery order is
2701  * a natural commands execution order), but insane iSCSI spec requires
2702  * us to check it and we have to, because some crazy initiators can rely
2703  * on the SN's based order and reorder requests during sending. For all other
2704  * normal initiators all that code is a NOP.
2705  */
2706 static void iscsi_push_cmnd(struct iscsi_cmnd *cmnd)
2707 {
2708         struct iscsi_session *session = cmnd->conn->session;
2709         struct list_head *entry;
2710         u32 cmd_sn;
2711
2712         TRACE_DBG("cmnd %p, iSCSI opcode %x, sn %u, exp sn %u", cmnd,
2713                 cmnd_opcode(cmnd), cmnd->pdu.bhs.sn, session->exp_cmd_sn);
2714
2715         iscsi_extracheck_is_rd_thread(cmnd->conn);
2716
2717         sBUG_ON(cmnd->parent_req != NULL);
2718
2719         if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
2720                 TRACE_DBG("Immediate cmd %p (cmd_sn %u)", cmnd,
2721                         cmnd->pdu.bhs.sn);
2722                 iscsi_cmnd_exec(cmnd);
2723                 goto out;
2724         }
2725
2726         spin_lock(&session->sn_lock);
2727
2728         cmd_sn = cmnd->pdu.bhs.sn;
2729         if (cmd_sn == session->exp_cmd_sn) {
2730                 while (1) {
2731                         session->exp_cmd_sn = ++cmd_sn;
2732
2733                         if (unlikely(session->tm_active > 0)) {
2734                                 if (before(cmd_sn, session->tm_sn)) {
2735                                         struct iscsi_conn *conn = cmnd->conn;
2736
2737                                         spin_unlock(&session->sn_lock);
2738
2739                                         spin_lock_bh(&conn->cmd_list_lock);
2740                                         __cmnd_abort(cmnd);
2741                                         spin_unlock_bh(&conn->cmd_list_lock);
2742
2743                                         spin_lock(&session->sn_lock);
2744                                 }
2745                                 iscsi_check_send_delayed_tm_resp(session);
2746                         }
2747
2748                         spin_unlock(&session->sn_lock);
2749
2750                         iscsi_cmnd_exec(cmnd);
2751
2752                         spin_lock(&session->sn_lock);
2753
2754                         if (list_empty(&session->pending_list))
2755                                 break;
2756                         cmnd = list_entry(session->pending_list.next,
2757                                           struct iscsi_cmnd,
2758                                           pending_list_entry);
2759                         if (cmnd->pdu.bhs.sn != cmd_sn)
2760                                 break;
2761
2762                         list_del(&cmnd->pending_list_entry);
2763                         cmnd->pending = 0;
2764
2765                         TRACE_MGMT_DBG("Processing pending cmd %p (cmd_sn %u)",
2766                                 cmnd, cmd_sn);
2767                 }
2768         } else {
2769                 int drop = 0;
2770
2771                 TRACE_DBG("Pending cmd %p (cmd_sn %u, exp_cmd_sn %u)",
2772                         cmnd, cmd_sn, session->exp_cmd_sn);
2773
2774                 /*
2775                  * iSCSI RFC 3720: "The target MUST silently ignore any
2776                  * non-immediate command outside of [from ExpCmdSN to MaxCmdSN
2777                  * inclusive] range". But we won't honor the MaxCmdSN
2778                  * requirement, because, since we adjust MaxCmdSN from the
2779                  * separate write thread, rarely it is possible that initiator
2780                  * can legally send command with CmdSN>MaxSN. But it won't
2781                  * hurt anything, in the worst case it will lead to
2782                  * additional QUEUE FULL status.
2783                  */
2784
2785                 if (unlikely(before(cmd_sn, session->exp_cmd_sn))) {
2786                         PRINT_ERROR("Unexpected cmd_sn (%u,%u)", cmd_sn,
2787                                 session->exp_cmd_sn);
2788                         drop = 1;
2789                 }
2790
2791 #if 0
2792                 if (unlikely(after(cmd_sn, session->exp_cmd_sn +
2793                                         iscsi_get_allowed_cmds(session)))) {
2794                         TRACE_MGMT_DBG("Too large cmd_sn %u (exp_cmd_sn %u, "
2795                                 "max_sn %u)", cmd_sn, session->exp_cmd_sn,
2796                                 iscsi_get_allowed_cmds(session));
2797                 }
2798 #endif
2799
2800                 spin_unlock(&session->sn_lock);
2801
2802                 if (unlikely(drop)) {
2803                         req_cmnd_release_force(cmnd);
2804                         goto out;
2805                 }
2806
2807                 if (unlikely(test_bit(ISCSI_CMD_ABORTED,
2808                                         &cmnd->prelim_compl_flags))) {
2809                         struct iscsi_cmnd *tm_clone;
2810
2811                         TRACE_MGMT_DBG("Pending aborted cmnd %p, creating TM "
2812                                 "clone (scst cmd %p, state %d)", cmnd,
2813                                 cmnd->scst_cmd, cmnd->scst_state);
2814
2815                         tm_clone = cmnd_alloc(cmnd->conn, NULL);
2816                         if (tm_clone != NULL) {
2817                                 set_bit(ISCSI_CMD_ABORTED,
2818                                         &tm_clone->prelim_compl_flags);
2819                                 tm_clone->pdu = cmnd->pdu;
2820
2821                                 TRACE_MGMT_DBG("TM clone %p created",
2822                                                tm_clone);
2823
2824                                 iscsi_cmnd_exec(cmnd);
2825                                 cmnd = tm_clone;
2826                         } else
2827                                 PRINT_ERROR("%s", "Unable to create TM clone");
2828                 }
2829
2830                 TRACE_MGMT_DBG("Pending cmnd %p (op %x, sn %u, exp sn %u)",
2831                         cmnd, cmnd_opcode(cmnd), cmd_sn, session->exp_cmd_sn);
2832
2833                 spin_lock(&session->sn_lock);
2834                 list_for_each(entry, &session->pending_list) {
2835                         struct iscsi_cmnd *tmp =
2836                                 list_entry(entry, struct iscsi_cmnd,
2837                                            pending_list_entry);
2838                         if (before(cmd_sn, tmp->pdu.bhs.sn))
2839                                 break;
2840                 }
2841                 list_add_tail(&cmnd->pending_list_entry, entry);
2842                 cmnd->pending = 1;
2843         }
2844
2845         spin_unlock(&session->sn_lock);
2846 out:
2847         return;
2848 }
2849
2850 static int check_segment_length(struct iscsi_cmnd *cmnd)
2851 {
2852         struct iscsi_conn *conn = cmnd->conn;
2853         struct iscsi_session *session = conn->session;
2854
2855         if (unlikely(cmnd->pdu.datasize > session->sess_params.max_recv_data_length)) {
2856                 PRINT_ERROR("Initiator %s violated negotiated parameters: "
2857                         "data too long (ITT %x, datasize %u, "
2858                         "max_recv_data_length %u", session->initiator_name,
2859                         cmnd_itt(cmnd), cmnd->pdu.datasize,
2860                         session->sess_params.max_recv_data_length);
2861                 mark_conn_closed(conn);
2862                 return -EINVAL;
2863         }
2864         return 0;
2865 }
2866
2867 int cmnd_rx_start(struct iscsi_cmnd *cmnd)
2868 {
2869         int res, rc;
2870
2871         iscsi_dump_pdu(&cmnd->pdu);
2872
2873         res = check_segment_length(cmnd);
2874         if (res != 0)
2875                 goto out;
2876
2877         switch (cmnd_opcode(cmnd)) {
2878         case ISCSI_OP_SCSI_CMD:
2879                 res = scsi_cmnd_start(cmnd);
2880                 if (unlikely(res < 0))
2881                         goto out;
2882                 spin_lock(&cmnd->conn->session->sn_lock);
2883                 __update_stat_sn(cmnd);
2884                 rc = check_cmd_sn(cmnd);
2885                 spin_unlock(&cmnd->conn->session->sn_lock);
2886                 break;
2887         case ISCSI_OP_SCSI_DATA_OUT:
2888                 res = data_out_start(cmnd);
2889                 goto out;
2890         case ISCSI_OP_NOOP_OUT:
2891                 rc = noop_out_start(cmnd);
2892                 break;
2893         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2894         case ISCSI_OP_LOGOUT_CMD:
2895                 spin_lock(&cmnd->conn->session->sn_lock);
2896                 __update_stat_sn(cmnd);
2897                 rc = check_cmd_sn(cmnd);
2898                 spin_unlock(&cmnd->conn->session->sn_lock);
2899                 break;
2900         case ISCSI_OP_TEXT_CMD:
2901         case ISCSI_OP_SNACK_CMD:
2902         default:
2903                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2904                 break;
2905         }
2906
2907         if (unlikely(rc < 0)) {
2908                 PRINT_ERROR("Error %d (iSCSI opcode %x, ITT %x)", rc,
2909                         cmnd_opcode(cmnd), cmnd_itt(cmnd));
2910                 res = create_reject_rsp(cmnd, -rc, true);
2911         }
2912
2913 out:
2914         TRACE_EXIT_RES(res);
2915         return res;
2916 }
2917
2918 void cmnd_rx_end(struct iscsi_cmnd *cmnd)
2919 {
2920         TRACE_ENTRY();
2921
2922         TRACE_DBG("cmnd %p, opcode %x", cmnd, cmnd_opcode(cmnd));
2923
2924         switch (cmnd_opcode(cmnd)) {
2925         case ISCSI_OP_SCSI_CMD:
2926         case ISCSI_OP_NOOP_OUT:
2927         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2928         case ISCSI_OP_LOGOUT_CMD:
2929                 iscsi_push_cmnd(cmnd);
2930                 goto out;
2931         case ISCSI_OP_SCSI_DATA_OUT:
2932                 data_out_end(cmnd);
2933                 break;
2934         default:
2935                 PRINT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
2936                 break;
2937         }
2938
2939         req_cmnd_release(cmnd);
2940
2941 out:
2942         TRACE_EXIT();
2943         return;
2944 }
2945
2946 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
2947 static int iscsi_alloc_data_buf(struct scst_cmd *cmd)
2948 {
2949         /*
2950          * sock->ops->sendpage() is async zero copy operation,
2951          * so we must be sure not to free and reuse
2952          * the command's buffer before the sending was completed
2953          * by the network layers. It is possible only if we
2954          * don't use SGV cache.
2955          */
2956         EXTRACHECKS_BUG_ON(!(scst_cmd_get_data_direction(cmd) & SCST_DATA_READ));
2957         scst_cmd_set_no_sgv(cmd);
2958         return 1;
2959 }
2960 #endif
2961
2962 static void iscsi_preprocessing_done(struct scst_cmd *scst_cmd)
2963 {
2964         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
2965                                 scst_cmd_get_tgt_priv(scst_cmd);
2966
2967         TRACE_DBG("req %p", req);
2968
2969         if (req->conn->rx_task == current)
2970                 req->scst_state = ISCSI_CMD_STATE_AFTER_PREPROC;
2971         else {
2972                 /*
2973                  * We wait for the state change without any protection, so
2974                  * without cmnd_get() it is possible that req will die
2975                  * "immediately" after the state assignment and
2976                  * iscsi_make_conn_rd_active() will operate on dead data.
2977                  * We use the ordered version of cmnd_get(), because "get"
2978                  * must be done before the state assignment.
2979                  *
2980                  * We protected from the race on calling cmnd_rx_continue(),
2981                  * because there can be only one read thread processing
2982                  * connection.
2983                  */
2984                 cmnd_get_ordered(req);
2985                 req->scst_state = ISCSI_CMD_STATE_AFTER_PREPROC;
2986                 iscsi_make_conn_rd_active(req->conn);
2987                 if (unlikely(req->conn->closing)) {
2988                         TRACE_DBG("Waking up closing conn %p", req->conn);
2989                         wake_up(&req->conn->read_state_waitQ);
2990                 }
2991                 cmnd_put(req);
2992         }
2993
2994         return;
2995 }
2996
2997 /*
2998  * No locks.
2999  *
3000  * IMPORTANT! Connection conn must be protected by additional conn_get()
3001  * upon entrance in this function, because otherwise it could be destroyed
3002  * inside as a result of iscsi_send(), which releases sent commands.
3003  */
3004 static void iscsi_try_local_processing(struct iscsi_conn *conn)
3005 {
3006         int local;
3007
3008         TRACE_ENTRY();
3009
3010         spin_lock_bh(&iscsi_wr_lock);
3011         switch (conn->wr_state) {
3012         case ISCSI_CONN_WR_STATE_IN_LIST:
3013                 list_del(&conn->wr_list_entry);
3014                 /* go through */
3015         case ISCSI_CONN_WR_STATE_IDLE:
3016 #ifdef CONFIG_SCST_EXTRACHECKS
3017                 conn->wr_task = current;
3018 #endif
3019                 conn->wr_state = ISCSI_CONN_WR_STATE_PROCESSING;
3020                 conn->wr_space_ready = 0;
3021                 local = 1;
3022                 break;
3023         default:
3024                 local = 0;
3025                 break;
3026         }
3027         spin_unlock_bh(&iscsi_wr_lock);
3028
3029         if (local) {
3030                 int rc = 1;
3031
3032                 if (test_write_ready(conn))
3033                         rc = iscsi_send(conn);
3034
3035                 spin_lock_bh(&iscsi_wr_lock);
3036 #ifdef CONFIG_SCST_EXTRACHECKS
3037                 conn->wr_task = NULL;
3038 #endif
3039                 if ((rc <= 0) || test_write_ready(conn)) {
3040                         list_add_tail(&conn->wr_list_entry, &iscsi_wr_list);
3041                         conn->wr_state = ISCSI_CONN_WR_STATE_IN_LIST;
3042                         wake_up(&iscsi_wr_waitQ);
3043                 } else
3044                         conn->wr_state = ISCSI_CONN_WR_STATE_IDLE;
3045                 spin_unlock_bh(&iscsi_wr_lock);
3046         }
3047
3048         TRACE_EXIT();
3049         return;
3050 }
3051
3052 static int iscsi_xmit_response(struct scst_cmd *scst_cmd)
3053 {
3054         int is_send_status = scst_cmd_get_is_send_status(scst_cmd);
3055         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
3056                                         scst_cmd_get_tgt_priv(scst_cmd);
3057         struct iscsi_conn *conn = req->conn;
3058         int status = scst_cmd_get_status(scst_cmd);
3059         u8 *sense = scst_cmd_get_sense_buffer(scst_cmd);
3060         int sense_len = scst_cmd_get_sense_buffer_len(scst_cmd);
3061
3062         if (unlikely(scst_cmd_atomic(scst_cmd)))
3063                 return SCST_TGT_RES_NEED_THREAD_CTX;
3064
3065         scst_cmd_set_tgt_priv(scst_cmd, NULL);
3066
3067         EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_RESTARTED);
3068
3069         if (unlikely(scst_cmd_aborted(scst_cmd)))
3070                 set_bit(ISCSI_CMD_ABORTED, &req->prelim_compl_flags);
3071
3072         if (unlikely(req->prelim_compl_flags != 0)) {
3073                 if (test_bit(ISCSI_CMD_ABORTED, &req->prelim_compl_flags)) {
3074                         TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
3075                                 req->scst_cmd);
3076                         scst_set_delivery_status(req->scst_cmd,
3077                                 SCST_CMD_DELIVERY_ABORTED);
3078                         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
3079                         req_cmnd_release_force(req);
3080                         goto out;
3081                 }
3082
3083                 TRACE_DBG("Prelim completed req %p", req);
3084
3085                 /*
3086                  * We could preliminary have finished req before we
3087                  * knew its device, so check if we return correct sense
3088                  * format.
3089                  */
3090                 scst_check_convert_sense(scst_cmd);
3091
3092                 if (!req->own_sg) {
3093                         req->sg = scst_cmd_get_sg(scst_cmd);
3094                         req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
3095                 }
3096         } else {
3097                 EXTRACHECKS_BUG_ON(req->own_sg);
3098                 req->sg = scst_cmd_get_sg(scst_cmd);
3099                 req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
3100         }
3101
3102         req->bufflen = scst_cmd_get_resp_data_len(scst_cmd);
3103
3104         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
3105
3106         TRACE_DBG("req %p, is_send_status=%x, req->bufflen=%d, req->sg=%p, "
3107                 "req->sg_cnt %d", req, is_send_status, req->bufflen, req->sg,
3108                 req->sg_cnt);
3109
3110         EXTRACHECKS_BUG_ON(req->hashed);
3111         if (req->main_rsp != NULL)
3112                 EXTRACHECKS_BUG_ON(cmnd_opcode(req->main_rsp) != ISCSI_OP_REJECT);
3113
3114         if (unlikely((req->bufflen != 0) && !is_send_status)) {
3115                 PRINT_CRIT_ERROR("%s", "Sending DATA without STATUS is "
3116                         "unsupported");
3117                 scst_set_cmd_error(scst_cmd,
3118                         SCST_LOAD_SENSE(scst_sense_hardw_error));
3119                 sBUG(); /* ToDo */
3120         }
3121
3122         if (req->bufflen != 0) {
3123                 /*
3124                  * Check above makes sure that is_send_status is set,
3125                  * so status is valid here, but in future that could change.
3126                  * ToDo
3127                  */
3128                 if ((status != SAM_STAT_CHECK_CONDITION) &&
3129                     ((cmnd_hdr(req)->flags & (ISCSI_CMD_WRITE|ISCSI_CMD_READ)) !=
3130                                 (ISCSI_CMD_WRITE|ISCSI_CMD_READ))) {
3131                         send_data_rsp(req, status, is_send_status);
3132                 } else {
3133                         struct iscsi_cmnd *rsp;
3134                         send_data_rsp(req, 0, 0);
3135                         if (is_send_status) {
3136                                 rsp = create_status_rsp(req, status, sense,
3137                                         sense_len, true);
3138                                 iscsi_cmnd_init_write(rsp, 0);
3139                         }
3140                 }
3141         } else if (is_send_status) {
3142                 struct iscsi_cmnd *rsp;
3143                 rsp = create_status_rsp(req, status, sense, sense_len, false);
3144                 iscsi_cmnd_init_write(rsp, 0);
3145         }
3146 #ifdef CONFIG_SCST_EXTRACHECKS
3147         else
3148                 sBUG();
3149 #endif
3150
3151         /*
3152          * "_ordered" here to protect from reorder, which can lead to
3153          * preliminary connection destroy in req_cmnd_release(). Just in
3154          * case, actually, because reordering shouldn't go so far, but who
3155          * knows..
3156          */
3157         conn_get_ordered(conn);
3158         req_cmnd_release(req);
3159         iscsi_try_local_processing(conn);
3160         conn_put(conn);
3161
3162 out:
3163         return SCST_TGT_RES_SUCCESS;
3164 }
3165
3166 /* Called under sn_lock */
3167 static bool iscsi_is_delay_tm_resp(struct iscsi_cmnd *rsp)
3168 {
3169         bool res = 0;
3170         struct iscsi_task_mgt_hdr *req_hdr =
3171                 (struct iscsi_task_mgt_hdr *)&rsp->parent_req->pdu.bhs;
3172         int function = req_hdr->function & ISCSI_FUNCTION_MASK;
3173         struct iscsi_session *sess = rsp->conn->session;
3174
3175         TRACE_ENTRY();
3176
3177         /* This should be checked for immediate TM commands as well */
3178
3179         switch (function) {
3180         default:
3181                 if (before(sess->exp_cmd_sn, req_hdr->cmd_sn))
3182                         res = 1;
3183                 break;
3184         }
3185
3186         TRACE_EXIT_RES(res);
3187         return res;
3188 }
3189
3190 /* Called under sn_lock, but might drop it inside, then reaquire */
3191 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess)
3192         __acquires(&sn_lock)
3193         __releases(&sn_lock)
3194 {
3195         struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
3196
3197         TRACE_ENTRY();
3198
3199         if (tm_rsp == NULL)
3200                 goto out;
3201
3202         if (iscsi_is_delay_tm_resp(tm_rsp))
3203                 goto out;
3204
3205         TRACE_MGMT_DBG("Sending delayed rsp %p", tm_rsp);
3206
3207         sess->tm_rsp = NULL;
3208         sess->tm_active--;
3209
3210         spin_unlock(&sess->sn_lock);
3211
3212         sBUG_ON(sess->tm_active < 0);
3213
3214         iscsi_cmnd_init_write(tm_rsp, ISCSI_INIT_WRITE_WAKE);
3215
3216         spin_lock(&sess->sn_lock);
3217
3218 out:
3219         TRACE_EXIT();
3220         return;
3221 }
3222
3223 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status)
3224 {
3225         struct iscsi_cmnd *rsp;
3226         struct iscsi_task_mgt_hdr *req_hdr =
3227                                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
3228         struct iscsi_task_rsp_hdr *rsp_hdr;
3229         struct iscsi_session *sess = req->conn->session;
3230         int fn = req_hdr->function & ISCSI_FUNCTION_MASK;
3231
3232         TRACE_ENTRY();
3233
3234         TRACE_MGMT_DBG("TM req %p finished", req);
3235         TRACE(TRACE_MGMT, "TM fn %d finished, status %d", fn, status);
3236
3237         rsp = iscsi_alloc_rsp(req);
3238         rsp_hdr = (struct iscsi_task_rsp_hdr *)&rsp->pdu.bhs;
3239
3240         rsp_hdr->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
3241         rsp_hdr->flags = ISCSI_FLG_FINAL;
3242         rsp_hdr->itt = req_hdr->itt;
3243         rsp_hdr->response = status;
3244
3245         if (fn == ISCSI_FUNCTION_TARGET_COLD_RESET) {
3246                 rsp->should_close_conn = 1;
3247                 rsp->should_close_all_conn = 1;
3248         }
3249
3250         sBUG_ON(sess->tm_rsp != NULL);
3251
3252         spin_lock(&sess->sn_lock);
3253         if (iscsi_is_delay_tm_resp(rsp)) {
3254                 TRACE_MGMT_DBG("Delaying TM fn %d response %p "
3255                         "(req %p), because not all affected commands "
3256                         "received (TM cmd sn %u, exp sn %u)",
3257                         req_hdr->function & ISCSI_FUNCTION_MASK, rsp, req,
3258                         req_hdr->cmd_sn, sess->exp_cmd_sn);
3259                 sess->tm_rsp = rsp;
3260                 spin_unlock(&sess->sn_lock);
3261                 goto out_release;
3262         }
3263         sess->tm_active--;
3264         spin_unlock(&sess->sn_lock);
3265
3266         sBUG_ON(sess->tm_active < 0);
3267
3268         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_WAKE);
3269
3270 out_release:
3271         req_cmnd_release(req);
3272
3273         TRACE_EXIT();
3274         return;
3275 }
3276
3277 static inline int iscsi_get_mgmt_response(int status)
3278 {
3279         switch (status) {
3280         case SCST_MGMT_STATUS_SUCCESS:
3281                 return ISCSI_RESPONSE_FUNCTION_COMPLETE;
3282
3283         case SCST_MGMT_STATUS_TASK_NOT_EXIST:
3284                 return ISCSI_RESPONSE_UNKNOWN_TASK;
3285
3286         case SCST_MGMT_STATUS_LUN_NOT_EXIST:
3287                 return ISCSI_RESPONSE_UNKNOWN_LUN;
3288
3289         case SCST_MGMT_STATUS_FN_NOT_SUPPORTED:
3290                 return ISCSI_RESPONSE_FUNCTION_UNSUPPORTED;
3291
3292         case SCST_MGMT_STATUS_REJECTED:
3293         case SCST_MGMT_STATUS_FAILED:
3294         default:
3295                 return ISCSI_RESPONSE_FUNCTION_REJECTED;
3296         }
3297 }
3298
3299 static void iscsi_task_mgmt_fn_done(struct scst_mgmt_cmd *scst_mcmd)
3300 {
3301         int fn = scst_mgmt_cmd_get_fn(scst_mcmd);
3302         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
3303                                 scst_mgmt_cmd_get_tgt_priv(scst_mcmd);
3304         int status =
3305                 iscsi_get_mgmt_response(scst_mgmt_cmd_get_status(scst_mcmd));
3306
3307         if ((status == ISCSI_RESPONSE_UNKNOWN_TASK) &&
3308             (fn == SCST_ABORT_TASK)) {
3309                 /* If we are here, we found the task, so must succeed */
3310                 status = ISCSI_RESPONSE_FUNCTION_COMPLETE;
3311         }
3312
3313         TRACE_MGMT_DBG("req %p, scst_mcmd %p, fn %d, scst status %d, status %d",
3314                 req, scst_mcmd, fn, scst_mgmt_cmd_get_status(scst_mcmd),
3315                 status);
3316
3317         switch (fn) {
3318         case SCST_NEXUS_LOSS_SESS:
3319         case SCST_ABORT_ALL_TASKS_SESS:
3320                 /* They are internal */
3321                 break;
3322         default:
3323                 iscsi_send_task_mgmt_resp(req, status);
3324                 scst_mgmt_cmd_set_tgt_priv(scst_mcmd, NULL);
3325                 break;
3326         }
3327         return;
3328 }
3329
3330 static int iscsi_scsi_aen(struct scst_aen *aen)
3331 {
3332         int res = SCST_AEN_RES_SUCCESS;
3333         uint64_t lun = scst_aen_get_lun(aen);
3334         const uint8_t *sense = scst_aen_get_sense(aen);
3335         int sense_len = scst_aen_get_sense_len(aen);
3336         struct iscsi_session *sess = scst_sess_get_tgt_priv(
3337                                         scst_aen_get_sess(aen));
3338         struct iscsi_conn *conn;
3339         bool found;
3340         struct iscsi_cmnd *fake_req, *rsp;
3341         struct iscsi_async_msg_hdr *rsp_hdr;
3342         struct scatterlist *sg;
3343
3344         TRACE_ENTRY();
3345
3346         TRACE_MGMT_DBG("SCSI AEN to sess %p (initiator %s)", sess,
3347                 sess->initiator_name);
3348
3349         mutex_lock(&sess->target->target_mutex);
3350
3351         found = false;
3352         list_for_each_entry_reverse(conn, &sess->conn_list, conn_list_entry) {
3353                 if (!test_bit(ISCSI_CONN_SHUTTINGDOWN, &conn->conn_aflags) &&
3354                     (conn->conn_reinst_successor == NULL)) {
3355                         found = true;
3356                         break;
3357                 }
3358         }
3359         if (!found) {
3360                 TRACE_MGMT_DBG("Unable to find alive conn for sess %p", sess);
3361                 goto out_err;
3362         }
3363
3364         /* Create a fake request */
3365         fake_req = cmnd_alloc(conn, NULL);
3366         if (fake_req == NULL) {
3367                 PRINT_ERROR("%s", "Unable to alloc fake AEN request");
3368                 goto out_err;
3369         }
3370
3371         mutex_unlock(&sess->target->target_mutex);
3372
3373         rsp = iscsi_alloc_main_rsp(fake_req);
3374         if (rsp == NULL) {
3375                 PRINT_ERROR("%s", "Unable to alloc AEN rsp");
3376                 goto out_err_free_req;
3377         }
3378
3379         fake_req->scst_state = ISCSI_CMD_STATE_AEN;
3380         fake_req->scst_aen = aen;
3381
3382         rsp_hdr = (struct iscsi_async_msg_hdr *)&rsp->pdu.bhs;
3383
3384         rsp_hdr->opcode = ISCSI_OP_ASYNC_MSG;
3385         rsp_hdr->flags = ISCSI_FLG_FINAL;
3386         rsp_hdr->lun = lun; /* it's already in SCSI form */
3387         rsp_hdr->ffffffff = 0xffffffff;
3388         rsp_hdr->async_event = ISCSI_ASYNC_SCSI;
3389
3390         sg = rsp->sg = rsp->rsp_sg;
3391         rsp->sg_cnt = 2;
3392         rsp->own_sg = 1;
3393
3394         sg_init_table(sg, 2);
3395         sg_set_buf(&sg[0], &rsp->sense_hdr, sizeof(rsp->sense_hdr));
3396         sg_set_buf(&sg[1], sense, sense_len);
3397
3398         rsp->sense_hdr.length = cpu_to_be16(sense_len);
3399         rsp->pdu.datasize = sizeof(rsp->sense_hdr) + sense_len;
3400         rsp->bufflen = rsp->pdu.datasize;
3401
3402         req_cmnd_release(fake_req);
3403
3404 out:
3405         TRACE_EXIT_RES(res);
3406         return res;
3407
3408 out_err_free_req:
3409         req_cmnd_release(fake_req);
3410
3411 out_err:
3412         mutex_unlock(&sess->target->target_mutex);
3413         res = SCST_AEN_RES_FAILED;
3414         goto out;
3415 }
3416
3417 static int iscsi_report_aen(struct scst_aen *aen)
3418 {
3419         int res;
3420         int event_fn = scst_aen_get_event_fn(aen);
3421
3422         TRACE_ENTRY();
3423
3424         switch (event_fn) {
3425         case SCST_AEN_SCSI:
3426                 res = iscsi_scsi_aen(aen);
3427                 break;
3428         default:
3429                 TRACE_MGMT_DBG("Unsupported AEN %d", event_fn);
3430                 res = SCST_AEN_RES_NOT_SUPPORTED;
3431                 break;
3432         }
3433
3434         TRACE_EXIT_RES(res);
3435         return res;
3436 }
3437
3438 static int iscsi_target_detect(struct scst_tgt_template *templ)
3439 {
3440         /* Nothing to do */
3441         return 0;
3442 }
3443
3444 static int iscsi_target_release(struct scst_tgt *scst_tgt)
3445 {
3446         /* Nothing to do */
3447         return 0;
3448 }
3449
3450 #if !defined(CONFIG_SCST_PROC) && \
3451         (defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING))
3452 static struct scst_trace_log iscsi_local_trace_tbl[] = {
3453     { TRACE_D_WRITE,            "d_write" },
3454     { TRACE_CONN_OC,            "conn" },
3455     { TRACE_CONN_OC_DBG,        "conn_dbg" },
3456     { TRACE_D_IOV,              "iov" },
3457     { TRACE_D_DUMP_PDU,         "pdu" },
3458     { TRACE_NET_PG,             "net_page" },
3459     { 0,                        NULL }
3460 };
3461
3462 #define ISCSI_TRACE_TLB_HELP    ", d_read, d_write, conn, conn_dbg, iov, pdu, net_page"
3463 #endif
3464
3465 #define ISCSI_MGMT_CMD_HELP     \
3466         "       echo \"add_attribute IncomingUser name password\" >mgmt\n" \
3467         "       echo \"del_attribute IncomingUser name\" >mgmt\n" \
3468         "       echo \"add_attribute OutgoingUser name password\" >mgmt\n" \
3469         "       echo \"del_attribute OutgoingUser name\" >mgmt\n" \
3470         "       echo \"add_target_attribute target_name IncomingUser name password\" >mgmt\n" \
3471         "       echo \"del_target_attribute target_name IncomingUser name\" >mgmt\n" \
3472         "       echo \"add_target_attribute target_name OutgoingUser name password\" >mgmt\n" \
3473         "       echo \"del_target_attribute target_name OutgoingUser name\" >mgmt\n"
3474
3475 struct scst_tgt_template iscsi_template = {
3476         .name = "iscsi",
3477         .sg_tablesize = 0xFFFF /* no limit */,
3478         .threads_num = 0,
3479         .no_clustering = 1,
3480         .xmit_response_atomic = 0,
3481 #ifndef CONFIG_SCST_PROC
3482         .tgtt_attrs = iscsi_attrs,
3483         .tgt_attrs = iscsi_tgt_attrs,
3484         .sess_attrs = iscsi_sess_attrs,
3485         .enable_target = iscsi_enable_target,
3486         .is_target_enabled = iscsi_is_target_enabled,
3487         .add_target = iscsi_sysfs_add_target,
3488         .del_target = iscsi_sysfs_del_target,
3489         .mgmt_cmd = iscsi_sysfs_mgmt_cmd,
3490         .mgmt_cmd_help = ISCSI_MGMT_CMD_HELP,
3491 #endif
3492 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
3493         .default_trace_flags = ISCSI_DEFAULT_LOG_FLAGS,
3494         .trace_flags = &trace_flag,
3495 #if !defined(CONFIG_SCST_PROC) && \
3496         (defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING))
3497         .trace_tbl = iscsi_local_trace_tbl,
3498         .trace_tbl_help = ISCSI_TRACE_TLB_HELP,
3499 #endif
3500 #endif
3501         .detect = iscsi_target_detect,
3502         .release = iscsi_target_release,
3503         .xmit_response = iscsi_xmit_response,
3504 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3505         .alloc_data_buf = iscsi_alloc_data_buf,
3506 #endif
3507         .preprocessing_done = iscsi_preprocessing_done,
3508         .pre_exec = iscsi_pre_exec,
3509         .task_mgmt_affected_cmds_done = iscsi_task_mgmt_affected_cmds_done,
3510         .task_mgmt_fn_done = iscsi_task_mgmt_fn_done,
3511         .report_aen = iscsi_report_aen,
3512 };
3513
3514 static __init int iscsi_run_threads(int count, char *name, int (*fn)(void *))
3515 {
3516         int res = 0;
3517         int i;
3518         struct iscsi_thread_t *thr;
3519
3520         for (i = 0; i < count; i++) {
3521                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
3522                 if (!thr) {
3523                         res = -ENOMEM;
3524                         PRINT_ERROR("Failed to allocate thr %d", res);
3525                         goto out;
3526                 }
3527                 thr->thr = kthread_run(fn, NULL, "%s%d", name, i);
3528                 if (IS_ERR(thr->thr)) {
3529                         res = PTR_ERR(thr->thr);
3530                         PRINT_ERROR("kthread_create() failed: %d", res);
3531                         kfree(thr);
3532                         goto out;
3533                 }
3534                 list_add_tail(&thr->threads_list_entry, &iscsi_threads_list);
3535         }
3536
3537 out:
3538         return res;
3539 }
3540
3541 static void iscsi_stop_threads(void)
3542 {
3543         struct iscsi_thread_t *t, *tmp;
3544
3545         list_for_each_entry_safe(t, tmp, &iscsi_threads_list,
3546                                 threads_list_entry) {
3547                 int rc = kthread_stop(t->thr);
3548                 if (rc < 0)
3549                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
3550                 list_del(&t->threads_list_entry);
3551                 kfree(t);
3552         }
3553         return;
3554 }
3555
3556 static int __init iscsi_init(void)
3557 {
3558         int err = 0;
3559         int num;
3560
3561         PRINT_INFO("iSCSI SCST Target - version %s", ISCSI_VERSION_STRING);
3562
3563         dummy_page = alloc_pages(GFP_KERNEL, 0);
3564         if (dummy_page == NULL) {
3565                 PRINT_ERROR("%s", "Dummy page allocation failed");
3566                 goto out;
3567         }
3568
3569         sg_init_table(&dummy_sg, 1);
3570         sg_set_page(&dummy_sg, dummy_page, PAGE_SIZE, 0);
3571
3572 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3573         err = net_set_get_put_page_callbacks(iscsi_get_page_callback,
3574                         iscsi_put_page_callback);
3575         if (err != 0) {
3576                 PRINT_INFO("Unable to set page callbackes: %d", err);
3577                 goto out_free_dummy;
3578         }
3579 #else
3580 #ifndef GENERATING_UPSTREAM_PATCH
3581         PRINT_WARNING("%s",
3582                 "CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION "
3583                 "not enabled in your kernel. ISCSI-SCST will be working with "
3584                 "not the best performance. Refer README file for details.");
3585 #endif
3586 #endif
3587
3588         ctr_major = register_chrdev(0, ctr_name, &ctr_fops);
3589         if (ctr_major < 0) {
3590                 PRINT_ERROR("failed to register the control device %d",
3591                             ctr_major);
3592                 err = ctr_major;
3593                 goto out_callb;
3594         }
3595
3596         err = event_init();
3597         if (err < 0)
3598                 goto out_reg;
3599
3600         iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS);
3601         if (!iscsi_cmnd_cache) {
3602                 err = -ENOMEM;
3603                 goto out_event;
3604         }
3605
3606         err = scst_register_target_template(&iscsi_template);
3607         if (err < 0)
3608                 goto out_kmem;
3609
3610 #ifdef CONFIG_SCST_PROC
3611         err = iscsi_procfs_init();
3612         if (err < 0)
3613                 goto out_reg_tmpl;
3614 #endif
3615
3616         num = max((int)num_online_cpus(), 2);
3617
3618         err = iscsi_run_threads(num, "iscsird", istrd);
3619         if (err != 0)
3620                 goto out_thr;
3621