0680d3730527640b36a7d12306e5e32a5acc5b39
[mirror/scst/.git] / iscsi-scst / kernel / iscsi.c
1 /*
2  *  Copyright (C) 2002 - 2003 Ardis Technolgies <roman@ardistech.com>
3  *  Copyright (C) 2007 - 2009 Vladislav Bolkhovitin
4  *  Copyright (C) 2007 - 2009 ID7 Ltd.
5  *
6  *  This program is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU General Public License
8  *  as published by the Free Software Foundation, version 2
9  *  of the License.
10  *
11  *  This program is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  *  GNU General Public License for more details.
15  */
16
17 #include <linux/module.h>
18 #include <linux/hash.h>
19 #include <linux/kthread.h>
20 #include <linux/scatterlist.h>
21 #include <net/tcp.h>
22 #include <scsi/scsi.h>
23
24 #include "iscsi.h"
25 #include "digest.h"
26
27 #ifndef GENERATING_UPSTREAM_PATCH
28 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
29 #warning "Patch put_page_callback-<kernel-version>.patch not applied on your\
30  kernel or CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION\
31  config option not set. ISCSI-SCST will be working with not the best\
32  performance. Refer README file for details."
33 #endif
34 #endif
35
36 #define ISCSI_INIT_WRITE_WAKE           0x1
37 #define ISCSI_INIT_WRITE_REMOVE_HASH    0x2
38
39 static int ctr_major;
40 static char ctr_name[] = "iscsi-scst-ctl";
41 static int iscsi_template_registered;
42
43 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
44 unsigned long iscsi_trace_flag = ISCSI_DEFAULT_LOG_FLAGS;
45 #endif
46
47 static struct kmem_cache *iscsi_cmnd_cache;
48
49 DEFINE_SPINLOCK(iscsi_rd_lock);
50 LIST_HEAD(iscsi_rd_list);
51 DECLARE_WAIT_QUEUE_HEAD(iscsi_rd_waitQ);
52
53 DEFINE_SPINLOCK(iscsi_wr_lock);
54 LIST_HEAD(iscsi_wr_list);
55 DECLARE_WAIT_QUEUE_HEAD(iscsi_wr_waitQ);
56
57 static struct page *dummy_page;
58 static struct scatterlist dummy_sg;
59
60 struct iscsi_thread_t {
61         struct task_struct *thr;
62         struct list_head threads_list_entry;
63 };
64
65 static LIST_HEAD(iscsi_threads_list);
66
67 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd);
68 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status);
69 static void cmnd_prepare_get_rejected_cmd_data(struct iscsi_cmnd *cmnd);
70 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess);
71 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd);
72 static void req_cmnd_release(struct iscsi_cmnd *req);
73
74 static inline u32 cmnd_write_size(struct iscsi_cmnd *cmnd)
75 {
76         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
77
78         if (hdr->flags & ISCSI_CMD_WRITE)
79                 return be32_to_cpu(hdr->data_length);
80         return 0;
81 }
82
83 static inline int cmnd_read_size(struct iscsi_cmnd *cmnd)
84 {
85         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
86
87         if (hdr->flags & ISCSI_CMD_READ) {
88                 struct iscsi_ahs_hdr *ahdr;
89
90                 if (!(hdr->flags & ISCSI_CMD_WRITE))
91                         return be32_to_cpu(hdr->data_length);
92
93                 ahdr = (struct iscsi_ahs_hdr *)cmnd->pdu.ahs;
94                 if (ahdr != NULL) {
95                         uint8_t *p = (uint8_t *)ahdr;
96                         int size = 0;
97                         do {
98                                 int s;
99
100                                 ahdr = (struct iscsi_ahs_hdr *)p;
101
102                                 if (ahdr->ahstype == ISCSI_AHSTYPE_RLENGTH) {
103                                         struct iscsi_rlength_ahdr *rh =
104                                               (struct iscsi_rlength_ahdr *)ahdr;
105                                         return be32_to_cpu(rh->read_length);
106                                 }
107
108                                 s = 3 + be16_to_cpu(ahdr->ahslength);
109                                 s = (s + 3) & -4;
110                                 size += s;
111                                 p += s;
112                         } while (size < cmnd->pdu.ahssize);
113                 }
114                 return -1;
115         }
116         return 0;
117 }
118
119 void iscsi_restart_cmnd(struct iscsi_cmnd *cmnd)
120 {
121         EXTRACHECKS_BUG_ON(cmnd->data_waiting);
122
123         if (unlikely(test_bit(ISCSI_CONN_REINSTATING,
124                         &cmnd->conn->conn_aflags))) {
125                 struct iscsi_target *target = cmnd->conn->session->target;
126                 bool get_out;
127
128                 mutex_lock(&target->target_mutex);
129
130                 get_out = test_bit(ISCSI_CONN_REINSTATING,
131                                 &cmnd->conn->conn_aflags);
132                 /* Let's don't look dead */
133                 if (scst_cmd_get_cdb(cmnd->scst_cmd)[0] == TEST_UNIT_READY)
134                         get_out = false;
135
136                 if (!get_out)
137                         goto unlock_cont;
138
139                 TRACE_MGMT_DBG("Pending cmnd %p, because conn %p is "
140                         "reinstated", cmnd, cmnd->conn);
141
142                 cmnd->scst_state = ISCSI_CMD_STATE_REINST_PENDING;
143                 list_add_tail(&cmnd->reinst_pending_cmd_list_entry,
144                         &cmnd->conn->reinst_pending_cmd_list);
145
146 unlock_cont:
147                 mutex_unlock(&target->target_mutex);
148
149                 if (get_out)
150                         goto out;
151         }
152
153         cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
154         scst_restart_cmd(cmnd->scst_cmd, SCST_PREPROCESS_STATUS_SUCCESS,
155                 SCST_CONTEXT_THREAD);
156
157 out:
158         return;
159 }
160
161 static inline void iscsi_restart_waiting_cmnd(struct iscsi_cmnd *cmnd)
162 {
163         /*
164          * There is no race with conn_abort(), since all functions
165          * called from single read thread
166          */
167         iscsi_extracheck_is_rd_thread(cmnd->conn);
168         cmnd->data_waiting = 0;
169
170         iscsi_restart_cmnd(cmnd);
171         return;
172 }
173
174 static inline void iscsi_fail_waiting_cmnd(struct iscsi_cmnd *cmnd)
175 {
176         TRACE_MGMT_DBG("Failing data waiting cmd %p", cmnd);
177
178         /*
179          * There is no race with conn_abort(), since all functions
180          * called from single read thread
181          */
182         iscsi_extracheck_is_rd_thread(cmnd->conn);
183         cmnd->data_waiting = 0;
184
185         req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
186 }
187
188 struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn,
189                               struct iscsi_cmnd *parent)
190 {
191         struct iscsi_cmnd *cmnd;
192
193         /* ToDo: __GFP_NOFAIL?? */
194 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
195         cmnd = kmem_cache_alloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
196         memset(cmnd, 0, sizeof(*cmnd));
197 #else
198         cmnd = kmem_cache_zalloc(iscsi_cmnd_cache, GFP_KERNEL|__GFP_NOFAIL);
199 #endif
200
201         atomic_set(&cmnd->ref_cnt, 1);
202         cmnd->scst_state = ISCSI_CMD_STATE_NEW;
203         cmnd->conn = conn;
204         cmnd->parent_req = parent;
205
206         if (parent == NULL) {
207                 conn_get(conn);
208
209 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
210                 atomic_set(&cmnd->net_ref_cnt, 0);
211 #endif
212                 spin_lock_init(&cmnd->rsp_cmd_lock);
213                 INIT_LIST_HEAD(&cmnd->rsp_cmd_list);
214                 INIT_LIST_HEAD(&cmnd->rx_ddigest_cmd_list);
215
216                 spin_lock_bh(&conn->cmd_list_lock);
217                 list_add_tail(&cmnd->cmd_list_entry, &conn->cmd_list);
218                 spin_unlock_bh(&conn->cmd_list_lock);
219         }
220
221         TRACE_DBG("conn %p, parent %p, cmnd %p", conn, parent, cmnd);
222         return cmnd;
223 }
224
225 /* Frees a command. Also frees the additional header. */
226 static void cmnd_free(struct iscsi_cmnd *cmnd)
227 {
228         TRACE_DBG("%p", cmnd);
229
230         if (unlikely(cmnd->tm_aborted)) {
231                 TRACE_MGMT_DBG("Free aborted cmd %p (scst cmd %p, state %d, "
232                         "parent_req %p)", cmnd, cmnd->scst_cmd,
233                         cmnd->scst_state, cmnd->parent_req);
234         }
235
236         /* Catch users from cmd_list or rsp_cmd_list */
237         EXTRACHECKS_BUG_ON(atomic_read(&cmnd->ref_cnt) != 0);
238
239         kfree(cmnd->pdu.ahs);
240
241         if (unlikely(cmnd->on_write_list || cmnd->on_written_list)) {
242                 struct iscsi_scsi_cmd_hdr *req = cmnd_hdr(cmnd);
243
244                 PRINT_CRIT_ERROR("cmnd %p still on some list?, %x, %x, %x, "
245                         "%x, %x, %x, %x", cmnd, req->opcode, req->scb[0],
246                         req->flags, req->itt, be32_to_cpu(req->data_length),
247                         req->cmd_sn, be32_to_cpu(cmnd->pdu.datasize));
248
249                 if (unlikely(cmnd->parent_req)) {
250                         struct iscsi_scsi_cmd_hdr *preq =
251                                         cmnd_hdr(cmnd->parent_req);
252                         PRINT_CRIT_ERROR("%p %x %u", preq, preq->opcode,
253                                 preq->scb[0]);
254                 }
255                 sBUG();
256         }
257
258         kmem_cache_free(iscsi_cmnd_cache, cmnd);
259         return;
260 }
261
262 /* Might be called unded some lock and on SIRQ */
263 void cmnd_done(struct iscsi_cmnd *cmnd)
264 {
265         TRACE_DBG("%p", cmnd);
266
267         if (unlikely(cmnd->tm_aborted)) {
268                 TRACE_MGMT_DBG("Done aborted cmd %p (scst cmd %p, state %d, "
269                         "parent_req %p)", cmnd, cmnd->scst_cmd,
270                         cmnd->scst_state, cmnd->parent_req);
271         }
272
273         EXTRACHECKS_BUG_ON(cmnd->on_rx_digest_list);
274
275         if (cmnd->on_written_list) {
276                 struct iscsi_conn *conn = cmnd->conn;
277                 TRACE_DBG("Deleting cmd %p from conn %p written_list", cmnd,
278                         conn);
279                 spin_lock_bh(&conn->write_list_lock);
280                 list_del(&cmnd->written_list_entry);
281                 cmnd->on_written_list = 0;
282                 spin_unlock_bh(&conn->write_list_lock);
283         }
284
285         if (cmnd->parent_req == NULL) {
286                 struct iscsi_conn *conn = cmnd->conn;
287                 TRACE_DBG("Deleting req %p from conn %p", cmnd, conn);
288
289                 spin_lock_bh(&conn->cmd_list_lock);
290                 list_del(&cmnd->cmd_list_entry);
291                 spin_unlock_bh(&conn->cmd_list_lock);
292
293                 conn_put(conn);
294
295                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rsp_cmd_list));
296                 EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rx_ddigest_cmd_list));
297
298                 /* Order between above and below code is important! */
299
300                 if ((cmnd->scst_cmd != NULL) || (cmnd->scst_aen != NULL)) {
301                         switch (cmnd->scst_state) {
302                         case ISCSI_CMD_STATE_PROCESSED:
303                                 TRACE_DBG("cmd %p PROCESSED", cmnd);
304                                 scst_tgt_cmd_done(cmnd->scst_cmd,
305                                         SCST_CONTEXT_DIRECT_ATOMIC);
306                                 break;
307
308                         case ISCSI_CMD_STATE_AFTER_PREPROC:
309                         {
310                                 struct scst_cmd *scst_cmd = cmnd->scst_cmd;
311                                 TRACE_DBG("cmd %p AFTER_PREPROC", cmnd);
312                                 cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
313                                 cmnd->scst_cmd = NULL;
314                                 scst_restart_cmd(scst_cmd,
315                                         SCST_PREPROCESS_STATUS_ERROR_FATAL,
316                                         SCST_CONTEXT_THREAD);
317                                 break;
318                         }
319
320                         case ISCSI_CMD_STATE_AEN:
321                                 TRACE_DBG("cmd %p AEN PROCESSED", cmnd);
322                                 scst_aen_done(cmnd->scst_aen);
323                                 break;
324
325                         default:
326                                 PRINT_CRIT_ERROR("Unexpected cmnd scst state "
327                                         "%d", cmnd->scst_state);
328                                 sBUG();
329                                 break;
330                         }
331                 }
332         } else {
333                 TRACE_DBG("Deleting rsp %p from parent %p", cmnd,
334                         cmnd->parent_req);
335
336                 spin_lock_bh(&cmnd->parent_req->rsp_cmd_lock);
337                 list_del(&cmnd->rsp_cmd_list_entry);
338                 spin_unlock_bh(&cmnd->parent_req->rsp_cmd_lock);
339
340                 cmnd_put(cmnd->parent_req);
341         }
342
343         /* Order between above and below code is important! */
344
345         if (cmnd->own_sg) {
346                 TRACE_DBG("%s", "own_sg");
347                 if ((cmnd->sg != &dummy_sg) && (cmnd->sg != cmnd->rsp_sg))
348                         scst_free(cmnd->sg, cmnd->sg_cnt);
349 #ifdef CONFIG_SCST_DEBUG
350                 cmnd->own_sg = 0;
351                 cmnd->sg = NULL;
352                 cmnd->sg_cnt = -1;
353 #endif
354         }
355
356         if (cmnd->dec_active_cmnds) {
357                 struct iscsi_session *sess = cmnd->conn->session;
358                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
359                         "new value %d)", cmnd, sess,
360                         atomic_read(&sess->active_cmds)-1);
361                 atomic_dec(&sess->active_cmds);
362 #ifdef CONFIG_SCST_EXTRACHECKS
363                 if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
364                         PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
365                                 atomic_read(&sess->active_cmds));
366                         sBUG();
367                 }
368 #endif
369         }
370
371         cmnd_free(cmnd);
372         return;
373 }
374
375 /*
376  * Corresponding conn may also get destroyed after this function, except only
377  * if it's called from the read thread!
378  *
379  * It can't be called in parallel with iscsi_cmnds_init_write()!
380  */
381 void req_cmnd_release_force(struct iscsi_cmnd *req, int flags)
382 {
383         struct iscsi_cmnd *rsp, *t;
384         struct iscsi_conn *conn = req->conn;
385         LIST_HEAD(cmds_list);
386
387         TRACE_ENTRY();
388
389         TRACE_MGMT_DBG("%p", req);
390
391         sBUG_ON(req == conn->read_cmnd);
392
393         if (flags & ISCSI_FORCE_RELEASE_WRITE) {
394                 spin_lock_bh(&conn->write_list_lock);
395                 list_for_each_entry_safe(rsp, t, &conn->write_list,
396                                                 write_list_entry) {
397                         if (rsp->parent_req != req)
398                                 continue;
399
400                         cmd_del_from_write_list(rsp);
401
402                         list_add_tail(&rsp->write_list_entry, &cmds_list);
403                 }
404                 spin_unlock_bh(&conn->write_list_lock);
405
406                 list_for_each_entry_safe(rsp, t, &cmds_list,
407                                                 write_list_entry) {
408                         TRACE_MGMT_DBG("Putting write rsp %p", rsp);
409                         list_del(&rsp->write_list_entry);
410                         cmnd_put(rsp);
411                 }
412         }
413
414 again_rsp:
415         spin_lock_bh(&req->rsp_cmd_lock);
416         list_for_each_entry_reverse(rsp, &req->rsp_cmd_list,
417                         rsp_cmd_list_entry) {
418                 bool r;
419
420                 if (rsp->force_cleanup_done)
421                         continue;
422
423                 rsp->force_cleanup_done = 1;
424
425                 if (cmnd_get_check(rsp))
426                         continue;
427
428                 spin_unlock_bh(&req->rsp_cmd_lock);
429
430                 spin_lock_bh(&conn->write_list_lock);
431                 r = rsp->on_write_list || rsp->write_processing_started;
432                 spin_unlock_bh(&conn->write_list_lock);
433
434                 cmnd_put(rsp);
435
436                 if (r)
437                         goto again_rsp;
438
439                 /*
440                  * If both on_write_list and write_processing_started not set,
441                  * we can safely put() rsp.
442                  */
443                 TRACE_MGMT_DBG("Putting rsp %p", rsp);
444                 cmnd_put(rsp);
445                 goto again_rsp;
446         }
447         spin_unlock_bh(&req->rsp_cmd_lock);
448
449         req_cmnd_release(req);
450
451         TRACE_EXIT();
452         return;
453 }
454
455 /*
456  * Corresponding conn may also get destroyed after this function, except only
457  * if it's called from the read thread!
458  */
459 static void req_cmnd_release(struct iscsi_cmnd *req)
460 {
461         struct iscsi_cmnd *c, *t;
462
463         TRACE_ENTRY();
464
465         TRACE_DBG("%p", req);
466
467 #ifdef CONFIG_SCST_EXTRACHECKS
468         sBUG_ON(req->release_called);
469         req->release_called = 1;
470 #endif
471
472         if (unlikely(req->tm_aborted)) {
473                 TRACE_MGMT_DBG("Release aborted req cmd %p (scst cmd %p, "
474                         "state %d)", req, req->scst_cmd, req->scst_state);
475         }
476
477         sBUG_ON(req->parent_req != NULL);
478
479         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
480                                 rx_ddigest_cmd_list_entry) {
481                 cmd_del_from_rx_ddigest_list(c);
482                 cmnd_put(c);
483         }
484
485         if (req->hashed)
486                 cmnd_remove_hash(req);
487
488         if (req->dec_active_cmnds) {
489                 struct iscsi_session *sess = req->conn->session;
490                 TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
491                         "new value %d)", req, sess,
492                         atomic_read(&sess->active_cmds)-1);
493                 atomic_dec(&sess->active_cmds);
494                 req->dec_active_cmnds = 0;
495 #ifdef CONFIG_SCST_EXTRACHECKS
496                 if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
497                         PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
498                                 atomic_read(&sess->active_cmds));
499                         sBUG();
500                 }
501 #endif
502         }
503
504         cmnd_put(req);
505
506         TRACE_EXIT();
507         return;
508 }
509
510 /*
511  * Corresponding conn may also get destroyed after this function, except only
512  * if it's called from the read thread!
513  */
514 void rsp_cmnd_release(struct iscsi_cmnd *cmnd)
515 {
516         TRACE_DBG("%p", cmnd);
517
518 #ifdef CONFIG_SCST_EXTRACHECKS
519         sBUG_ON(cmnd->release_called);
520         cmnd->release_called = 1;
521 #endif
522
523         sBUG_ON(cmnd->hashed);
524         sBUG_ON(cmnd->parent_req == NULL);
525
526         cmnd_put(cmnd);
527         return;
528 }
529
530 /**
531  * create a new command used as response.
532  *
533  * iscsi_cmnd_create_rsp_cmnd -
534  * @cmnd: ptr to request command
535  *
536  * @return    ptr to response command or NULL
537  */
538 static struct iscsi_cmnd *iscsi_cmnd_create_rsp_cmnd(struct iscsi_cmnd *parent)
539 {
540         struct iscsi_cmnd *rsp;
541
542         rsp = cmnd_alloc(parent->conn, parent);
543
544         spin_lock_bh(&parent->rsp_cmd_lock);
545         TRACE_DBG("Adding rsp %p to parent %p", rsp, parent);
546         list_add_tail(&rsp->rsp_cmd_list_entry, &parent->rsp_cmd_list);
547         spin_unlock_bh(&parent->rsp_cmd_lock);
548         cmnd_get(parent);
549         return rsp;
550 }
551
552 static inline struct iscsi_cmnd *get_rsp_cmnd(struct iscsi_cmnd *req)
553 {
554         struct iscsi_cmnd *res = NULL;
555
556         /* Currently this lock isn't needed, but just in case.. */
557         spin_lock_bh(&req->rsp_cmd_lock);
558         if (!list_empty(&req->rsp_cmd_list)) {
559                 res = list_entry(req->rsp_cmd_list.prev, struct iscsi_cmnd,
560                         rsp_cmd_list_entry);
561         }
562         spin_unlock_bh(&req->rsp_cmd_lock);
563
564         return res;
565 }
566
567 static void iscsi_cmnds_init_write(struct list_head *send, int flags)
568 {
569         struct iscsi_cmnd *rsp = list_entry(send->next, struct iscsi_cmnd,
570                                                 write_list_entry);
571         struct iscsi_conn *conn = rsp->conn;
572         struct list_head *pos, *next;
573
574         sBUG_ON(list_empty(send));
575
576         /*
577          * If we don't remove hashed req cmd from the hash list here, before
578          * submitting it for transmission, we will have a race, when for
579          * some reason cmd's release is delayed after transmission and
580          * initiator sends cmd with the same ITT => this command will be
581          * erroneously rejected as a duplicate.
582          */
583         if ((flags & ISCSI_INIT_WRITE_REMOVE_HASH) &&
584             rsp->parent_req->hashed &&
585             (rsp->parent_req->r2t_length == 0) &&
586             (rsp->parent_req->outstanding_r2t == 0))
587                 cmnd_remove_hash(rsp->parent_req);
588
589         if (!(conn->ddigest_type & DIGEST_NONE)) {
590                 list_for_each(pos, send) {
591                         rsp = list_entry(pos, struct iscsi_cmnd,
592                                                 write_list_entry);
593
594                         if (rsp->pdu.datasize != 0) {
595                                 TRACE_DBG("Doing data digest (%p:%x)", rsp,
596                                         cmnd_opcode(rsp));
597                                 digest_tx_data(rsp);
598                         }
599                 }
600         }
601
602         spin_lock_bh(&conn->write_list_lock);
603         list_for_each_safe(pos, next, send) {
604                 rsp = list_entry(pos, struct iscsi_cmnd, write_list_entry);
605
606                 TRACE_DBG("%p:%x", rsp, cmnd_opcode(rsp));
607
608                 sBUG_ON(conn != rsp->conn);
609
610                 list_del(&rsp->write_list_entry);
611                 cmd_add_on_write_list(conn, rsp);
612         }
613         spin_unlock_bh(&conn->write_list_lock);
614
615         if (flags & ISCSI_INIT_WRITE_WAKE)
616                 iscsi_make_conn_wr_active(conn);
617
618         return;
619 }
620
621 static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags)
622 {
623         LIST_HEAD(head);
624
625         if (unlikely(rsp->on_write_list)) {
626                 PRINT_CRIT_ERROR("cmd already on write list (%x %x %x %x %u "
627                         "%u %u %u %u %u %u %d %d",
628                         cmnd_itt(rsp), cmnd_ttt(rsp), cmnd_opcode(rsp),
629                         cmnd_scsicode(rsp), rsp->r2t_sn,
630                         rsp->r2t_length, rsp->is_unsolicited_data,
631                         rsp->target_task_tag, rsp->outstanding_r2t,
632                         rsp->hdigest, rsp->ddigest,
633                         list_empty(&rsp->rsp_cmd_list), rsp->hashed);
634                 sBUG();
635         }
636         list_add_tail(&rsp->write_list_entry, &head);
637         iscsi_cmnds_init_write(&head, flags);
638         return;
639 }
640
641 static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
642 {
643         struct iscsi_cmnd *rsp;
644         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
645         struct iscsi_data_in_hdr *rsp_hdr;
646         u32 pdusize, expsize, size, offset, sn;
647         LIST_HEAD(send);
648
649         TRACE_DBG("req %p", req);
650
651         pdusize = req->conn->session->sess_param.max_xmit_data_length;
652         expsize = req->read_size;
653         size = min(expsize, (u32)req->bufflen);
654         offset = 0;
655         sn = 0;
656
657         while (1) {
658                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
659                 TRACE_DBG("rsp %p", rsp);
660                 rsp->sg = req->sg;
661                 rsp->sg_cnt = req->sg_cnt;
662                 rsp->bufflen = req->bufflen;
663                 rsp_hdr = (struct iscsi_data_in_hdr *)&rsp->pdu.bhs;
664
665                 rsp_hdr->opcode = ISCSI_OP_SCSI_DATA_IN;
666                 rsp_hdr->itt = req_hdr->itt;
667                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
668                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
669                 rsp_hdr->data_sn = cpu_to_be32(sn);
670
671                 if (size <= pdusize) {
672                         TRACE_DBG("offset %d, size %d", offset, size);
673                         rsp->pdu.datasize = size;
674                         if (send_status) {
675                                 int scsisize;
676
677                                 TRACE_DBG("status %x", status);
678
679                                 EXTRACHECKS_BUG_ON((cmnd_hdr(req)->flags & ISCSI_CMD_WRITE) != 0);
680
681                                 rsp_hdr->flags = ISCSI_FLG_FINAL | ISCSI_FLG_STATUS;
682                                 rsp_hdr->cmd_status = status;
683
684                                 scsisize = req->bufflen;
685                                 if (scsisize < expsize) {
686                                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
687                                         size = expsize - scsisize;
688                                 } else if (scsisize > expsize) {
689                                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
690                                         size = scsisize - expsize;
691                                 } else
692                                         size = 0;
693                                 rsp_hdr->residual_count = cpu_to_be32(size);
694                         }
695                         list_add_tail(&rsp->write_list_entry, &send);
696                         break;
697                 }
698
699                 TRACE_DBG("pdusize %d, offset %d, size %d", pdusize, offset,
700                         size);
701
702                 rsp->pdu.datasize = pdusize;
703
704                 size -= pdusize;
705                 offset += pdusize;
706                 sn++;
707
708                 list_add_tail(&rsp->write_list_entry, &send);
709         }
710         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_REMOVE_HASH);
711         return;
712 }
713
714 static struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req, int status,
715         const u8 *sense_buf, int sense_len)
716 {
717         struct iscsi_cmnd *rsp;
718         struct iscsi_scsi_rsp_hdr *rsp_hdr;
719         struct scatterlist *sg;
720
721         rsp = iscsi_cmnd_create_rsp_cmnd(req);
722         TRACE_DBG("%p", rsp);
723
724         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
725         rsp_hdr->opcode = ISCSI_OP_SCSI_RSP;
726         rsp_hdr->flags = ISCSI_FLG_FINAL;
727         rsp_hdr->response = ISCSI_RESPONSE_COMMAND_COMPLETED;
728         rsp_hdr->cmd_status = status;
729         rsp_hdr->itt = cmnd_hdr(req)->itt;
730
731         if (SCST_SENSE_VALID(sense_buf)) {
732                 TRACE_DBG("%s", "SENSE VALID");
733
734                 sg = rsp->sg = rsp->rsp_sg;
735                 rsp->sg_cnt = 2;
736                 rsp->own_sg = 1;
737
738                 sg_init_table(sg, 2);
739                 sg_set_buf(&sg[0], &rsp->sense_hdr, sizeof(rsp->sense_hdr));
740                 sg_set_buf(&sg[1], sense_buf, sense_len);
741
742                 rsp->sense_hdr.length = cpu_to_be16(sense_len);
743
744                 rsp->pdu.datasize = sizeof(rsp->sense_hdr) + sense_len;
745                 rsp->bufflen = rsp->pdu.datasize;
746         } else {
747                 rsp->pdu.datasize = 0;
748                 rsp->bufflen = 0;
749         }
750
751         return rsp;
752 }
753
754 static void iscsi_cmnd_reject(struct iscsi_cmnd *req, int reason)
755 {
756         struct iscsi_cmnd *rsp;
757         struct iscsi_reject_hdr *rsp_hdr;
758         struct scatterlist *sg;
759
760         TRACE_MGMT_DBG("Reject: req %p, reason %x", req, reason);
761
762         sBUG_ON(req->rejected);
763         req->rejected = 1;
764         req->reject_reason = ISCSI_REJECT_CMD;
765
766         rsp = iscsi_cmnd_create_rsp_cmnd(req);
767         rsp_hdr = (struct iscsi_reject_hdr *)&rsp->pdu.bhs;
768
769         rsp_hdr->opcode = ISCSI_OP_REJECT;
770         rsp_hdr->ffffffff = ISCSI_RESERVED_TAG;
771         rsp_hdr->reason = reason;
772
773         sg = rsp->sg = rsp->rsp_sg;
774         rsp->sg_cnt = 1;
775         rsp->own_sg = 1;
776         sg_init_one(sg, &req->pdu.bhs, sizeof(struct iscsi_hdr));
777         rsp->bufflen = rsp->pdu.datasize = sizeof(struct iscsi_hdr);
778
779         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
780                                          ISCSI_INIT_WRITE_WAKE);
781
782         cmnd_prepare_get_rejected_cmd_data(req);
783         return;
784 }
785
786 static inline int iscsi_get_allowed_cmds(struct iscsi_session *sess)
787 {
788         int res = max(-1, (int)sess->max_queued_cmnds -
789                                 atomic_read(&sess->active_cmds)-1);
790         TRACE_DBG("allowed cmds %d (sess %p, active_cmds %d)", res,
791                 sess, atomic_read(&sess->active_cmds));
792         return res;
793 }
794
795 static u32 cmnd_set_sn(struct iscsi_cmnd *cmnd, int set_stat_sn)
796 {
797         struct iscsi_conn *conn = cmnd->conn;
798         struct iscsi_session *sess = conn->session;
799         u32 res;
800
801         spin_lock(&sess->sn_lock);
802
803         if (set_stat_sn)
804                 cmnd->pdu.bhs.sn = cpu_to_be32(conn->stat_sn++);
805         cmnd->pdu.bhs.exp_sn = cpu_to_be32(sess->exp_cmd_sn);
806         cmnd->pdu.bhs.max_sn = cpu_to_be32(sess->exp_cmd_sn +
807                                  iscsi_get_allowed_cmds(sess));
808
809         res = cpu_to_be32(conn->stat_sn);
810
811         spin_unlock(&sess->sn_lock);
812         return res;
813 }
814
815 /* Called under sn_lock */
816 static void __update_stat_sn(struct iscsi_cmnd *cmnd)
817 {
818         struct iscsi_conn *conn = cmnd->conn;
819         u32 exp_stat_sn;
820
821         cmnd->pdu.bhs.exp_sn = exp_stat_sn = be32_to_cpu(cmnd->pdu.bhs.exp_sn);
822         TRACE_DBG("%x,%x", cmnd_opcode(cmnd), exp_stat_sn);
823         if ((int)(exp_stat_sn - conn->exp_stat_sn) > 0 &&
824             (int)(exp_stat_sn - conn->stat_sn) <= 0) {
825                 /* free pdu resources */
826                 cmnd->conn->exp_stat_sn = exp_stat_sn;
827         }
828         return;
829 }
830
831 static inline void update_stat_sn(struct iscsi_cmnd *cmnd)
832 {
833         spin_lock(&cmnd->conn->session->sn_lock);
834         __update_stat_sn(cmnd);
835         spin_unlock(&cmnd->conn->session->sn_lock);
836         return;
837 }
838
839 /* Called under sn_lock */
840 static int check_cmd_sn(struct iscsi_cmnd *cmnd)
841 {
842         struct iscsi_session *session = cmnd->conn->session;
843         u32 cmd_sn;
844
845         cmnd->pdu.bhs.sn = cmd_sn = be32_to_cpu(cmnd->pdu.bhs.sn);
846         TRACE_DBG("%d(%d)", cmd_sn, session->exp_cmd_sn);
847         if (likely((s32)(cmd_sn - session->exp_cmd_sn) >= 0))
848                 return 0;
849         PRINT_ERROR("sequence error (%x,%x)", cmd_sn, session->exp_cmd_sn);
850         return -ISCSI_REASON_PROTOCOL_ERROR;
851 }
852
853 static inline struct iscsi_cmnd *__cmnd_find_hash(
854         struct iscsi_session *session, u32 itt, u32 ttt)
855 {
856         struct list_head *head;
857         struct iscsi_cmnd *cmnd;
858
859         head = &session->cmnd_hash[cmnd_hashfn(itt)];
860
861         list_for_each_entry(cmnd, head, hash_list_entry) {
862                 if (cmnd->pdu.bhs.itt == itt) {
863                         if (ttt != ISCSI_RESERVED_TAG &&
864                             ttt != cmnd->target_task_tag)
865                                 continue;
866                         return cmnd;
867                 }
868         }
869         return NULL;
870 }
871
872 static struct iscsi_cmnd *cmnd_find_hash(struct iscsi_session *session,
873         u32 itt, u32 ttt)
874 {
875         struct iscsi_cmnd *cmnd;
876
877         spin_lock(&session->cmnd_hash_lock);
878         cmnd = __cmnd_find_hash(session, itt, ttt);
879         spin_unlock(&session->cmnd_hash_lock);
880
881         return cmnd;
882 }
883
884 static struct iscsi_cmnd *cmnd_find_hash_get(struct iscsi_session *session,
885         u32 itt, u32 ttt)
886 {
887         struct iscsi_cmnd *cmnd;
888
889         spin_lock(&session->cmnd_hash_lock);
890         cmnd = __cmnd_find_hash(session, itt, ttt);
891         if (cmnd != NULL) {
892                 if (unlikely(cmnd_get_check(cmnd)))
893                         cmnd = NULL;
894         }
895         spin_unlock(&session->cmnd_hash_lock);
896
897         return cmnd;
898 }
899
900 static int cmnd_insert_hash(struct iscsi_cmnd *cmnd)
901 {
902         struct iscsi_session *session = cmnd->conn->session;
903         struct iscsi_cmnd *tmp;
904         struct list_head *head;
905         int err = 0;
906         u32 itt = cmnd->pdu.bhs.itt;
907
908         TRACE_DBG("%p:%x", cmnd, itt);
909         if (unlikely(itt == ISCSI_RESERVED_TAG)) {
910                 PRINT_ERROR("%s", "ITT is RESERVED_TAG");
911                 PRINT_BUFFER("Incorrect BHS", &cmnd->pdu.bhs,
912                         sizeof(cmnd->pdu.bhs));
913                 err = -ISCSI_REASON_PROTOCOL_ERROR;
914                 goto out;
915         }
916
917         spin_lock(&session->cmnd_hash_lock);
918
919         head = &session->cmnd_hash[cmnd_hashfn(cmnd->pdu.bhs.itt)];
920
921         tmp = __cmnd_find_hash(session, itt, ISCSI_RESERVED_TAG);
922         if (likely(!tmp)) {
923                 list_add_tail(&cmnd->hash_list_entry, head);
924                 cmnd->hashed = 1;
925         } else {
926                 PRINT_ERROR("Task %x in progress, cmnd %p", itt, cmnd);
927                 err = -ISCSI_REASON_TASK_IN_PROGRESS;
928         }
929
930         spin_unlock(&session->cmnd_hash_lock);
931
932         if (likely(!err)) {
933                 spin_lock(&session->sn_lock);
934                 __update_stat_sn(cmnd);
935                 err = check_cmd_sn(cmnd);
936                 spin_unlock(&session->sn_lock);
937         }
938
939 out:
940         return err;
941 }
942
943 static void cmnd_remove_hash(struct iscsi_cmnd *cmnd)
944 {
945         struct iscsi_session *session = cmnd->conn->session;
946         struct iscsi_cmnd *tmp;
947
948         spin_lock(&session->cmnd_hash_lock);
949
950         tmp = __cmnd_find_hash(session, cmnd->pdu.bhs.itt, ISCSI_RESERVED_TAG);
951
952         if (likely(tmp && tmp == cmnd)) {
953                 list_del(&cmnd->hash_list_entry);
954                 cmnd->hashed = 0;
955         } else {
956                 PRINT_ERROR("%p:%x not found", cmnd, cmnd_itt(cmnd));
957         }
958
959         spin_unlock(&session->cmnd_hash_lock);
960         return;
961 }
962
963 static void cmnd_prepare_get_rejected_cmd_data(struct iscsi_cmnd *cmnd)
964 {
965         struct iscsi_conn *conn = cmnd->conn;
966         struct scatterlist *sg = cmnd->sg;
967         char __user *addr;
968         u32 size;
969         unsigned int i;
970
971         TRACE_MGMT_DBG("Skipping (%p, %x %x %x %u, %p, scst state %d)", cmnd,
972                 cmnd_itt(cmnd), cmnd_opcode(cmnd), cmnd_hdr(cmnd)->scb[0],
973                 cmnd->pdu.datasize, cmnd->scst_cmd, cmnd->scst_state);
974
975         iscsi_extracheck_is_rd_thread(conn);
976
977         size = cmnd->pdu.datasize;
978         if (!size)
979                 return;
980
981         if (sg == NULL) {
982                 /*
983                  * There are no problems with the safety from concurrent
984                  * accesses to dummy_page in dummy_sg, since data only
985                  * will be read and then discarded.
986                  */
987                 sg = cmnd->sg = &dummy_sg;
988                 cmnd->bufflen = PAGE_SIZE;
989                 cmnd->own_sg = 1;
990         }
991
992         addr = (char __force __user *)(page_address(sg_page(&sg[0])));
993         sBUG_ON(addr == NULL);
994         conn->read_size = size;
995         for (i = 0; size > PAGE_SIZE; i++, size -= cmnd->bufflen) {
996                 /* We already checked pdu.datasize in check_segment_length() */
997                 sBUG_ON(i >= ISCSI_CONN_IOV_MAX);
998                 conn->read_iov[i].iov_base = addr;
999                 conn->read_iov[i].iov_len = cmnd->bufflen;
1000         }
1001         conn->read_iov[i].iov_base = addr;
1002         conn->read_iov[i].iov_len = size;
1003         conn->read_msg.msg_iov = conn->read_iov;
1004         conn->read_msg.msg_iovlen = ++i;
1005
1006         return;
1007 }
1008
1009 static void iscsi_set_resid(struct iscsi_cmnd *req, struct iscsi_cmnd *rsp,
1010         bool data_used)
1011 {
1012         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1013         struct iscsi_scsi_rsp_hdr *rsp_hdr;
1014         int resid, resp_len, in_resp_len;
1015
1016         if ((req_hdr->flags & ISCSI_CMD_READ) &&
1017             (req_hdr->flags & ISCSI_CMD_WRITE)) {
1018                 rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
1019
1020                 if (data_used) {
1021                         resp_len = req->bufflen;
1022                         if (req->scst_cmd != NULL)
1023                                 in_resp_len = scst_cmd_get_in_bufflen(req->scst_cmd);
1024                         else
1025                                 in_resp_len = 0;
1026                 } else {
1027                         resp_len = 0;
1028                         in_resp_len = 0;
1029                 }
1030
1031                 resid = be32_to_cpu(req_hdr->data_length) - in_resp_len;
1032                 if (resid > 0) {
1033                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
1034                         rsp_hdr->residual_count = cpu_to_be32(resid);
1035                 } else if (resid < 0) {
1036                         resid = -resid;
1037                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
1038                         rsp_hdr->residual_count = cpu_to_be32(resid);
1039                 }
1040
1041                 resid = req->read_size - resp_len;
1042                 if (resid > 0) {
1043                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_UNDERFLOW;
1044                         rsp_hdr->bi_residual_count = cpu_to_be32(resid);
1045                 } else if (resid < 0) {
1046                         resid = -resid;
1047                         rsp_hdr->flags |= ISCSI_FLG_BIRESIDUAL_OVERFLOW;
1048                         rsp_hdr->bi_residual_count = cpu_to_be32(resid);
1049                 }
1050         } else {
1051                 if (data_used)
1052                         resp_len = req->bufflen;
1053                 else
1054                         resp_len = 0;
1055
1056                 resid = req->read_size - resp_len;
1057                 if (resid > 0) {
1058                         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
1059                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_UNDERFLOW;
1060                         rsp_hdr->residual_count = cpu_to_be32(resid);
1061                 } else if (resid < 0) {
1062                         rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
1063                         resid = -resid;
1064                         rsp_hdr->flags |= ISCSI_FLG_RESIDUAL_OVERFLOW;
1065                         rsp_hdr->residual_count = cpu_to_be32(resid);
1066                 }
1067         }
1068         return;
1069 }
1070
1071 static void cmnd_reject_scsi_cmd(struct iscsi_cmnd *req)
1072 {
1073         struct iscsi_cmnd *rsp;
1074
1075         TRACE_DBG("%p", req);
1076
1077         sBUG_ON(req->rejected);
1078         req->rejected = 1;
1079         req->reject_reason = ISCSI_REJECT_SCSI_CMD;
1080
1081         rsp = get_rsp_cmnd(req);
1082         if (rsp == NULL) {
1083                 /* That can be true for aborted commands */
1084                 goto out_reject;
1085         }
1086
1087         sBUG_ON(cmnd_opcode(rsp) != ISCSI_OP_SCSI_RSP);
1088
1089         iscsi_set_resid(req, rsp, false);
1090
1091         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
1092                                          ISCSI_INIT_WRITE_WAKE);
1093
1094 out_reject:
1095         cmnd_prepare_get_rejected_cmd_data(req);
1096         return;
1097 }
1098
1099 static int cmnd_prepare_recv_pdu(struct iscsi_conn *conn,
1100         struct iscsi_cmnd *cmd, u32 offset, u32 size)
1101 {
1102         struct scatterlist *sg = cmd->sg;
1103         unsigned int bufflen = cmd->bufflen;
1104         unsigned int idx, i;
1105         char __user *addr;
1106         int res = 0;
1107
1108         TRACE_DBG("%p %u,%u", cmd->sg, offset, size);
1109
1110         iscsi_extracheck_is_rd_thread(conn);
1111
1112         if (unlikely((offset >= bufflen) ||
1113                      (offset + size > bufflen))) {
1114                 PRINT_ERROR("Wrong ltn (%u %u %u)", offset, size, bufflen);
1115                 mark_conn_closed(conn);
1116                 res = -EIO;
1117                 goto out;
1118         }
1119
1120         offset += sg[0].offset;
1121         idx = offset >> PAGE_SHIFT;
1122         offset &= ~PAGE_MASK;
1123
1124         conn->read_msg.msg_iov = conn->read_iov;
1125         conn->read_size = size;
1126
1127         i = 0;
1128         while (1) {
1129                 addr = (char __force __user *)(page_address(sg_page(&sg[idx])));
1130                 sBUG_ON(addr == NULL);
1131                 conn->read_iov[i].iov_base = addr + offset;
1132                 if (offset + size <= PAGE_SIZE) {
1133                         TRACE_DBG("idx=%d, offset=%u, size=%d, addr=%p",
1134                                 idx, offset, size, addr);
1135                         conn->read_iov[i].iov_len = size;
1136                         conn->read_msg.msg_iovlen = ++i;
1137                         break;
1138                 }
1139                 conn->read_iov[i].iov_len = PAGE_SIZE - offset;
1140                 TRACE_DBG("idx=%d, offset=%u, size=%d, iov_len=%zd, addr=%p",
1141                         idx, offset, size, conn->read_iov[i].iov_len, addr);
1142                 size -= conn->read_iov[i].iov_len;
1143                 if (unlikely(++i >= ISCSI_CONN_IOV_MAX)) {
1144                         PRINT_ERROR("Initiator %s violated negotiated "
1145                                 "parameters by sending too much data (size "
1146                                 "left %d)", conn->session->initiator_name,
1147                                 size);
1148                         mark_conn_closed(conn);
1149                         res = -EINVAL;
1150                         break;
1151                 }
1152                 idx++;
1153                 offset = sg[idx].offset;
1154         }
1155         TRACE_DBG("msg_iov=%p, msg_iovlen=%zd",
1156                 conn->read_msg.msg_iov, conn->read_msg.msg_iovlen);
1157
1158 out:
1159         return res;
1160 }
1161
1162 static void send_r2t(struct iscsi_cmnd *req)
1163 {
1164         struct iscsi_session *session = req->conn->session;
1165         struct iscsi_cmnd *rsp;
1166         struct iscsi_r2t_hdr *rsp_hdr;
1167         u32 offset, burst;
1168         LIST_HEAD(send);
1169
1170         if (unlikely(req->tm_aborted)) {
1171                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted on R2T "
1172                         "(r2t_length %d, outstanding_r2t %d)", req,
1173                         req->scst_cmd, req->r2t_length, req->outstanding_r2t);
1174                 if (req->outstanding_r2t == 0)
1175                         iscsi_fail_waiting_cmnd(req);
1176                 goto out;
1177         }
1178
1179         /*
1180          * There is no race with data_out_start() and conn_abort(), since
1181          * all functions called from single read thread
1182          */
1183         iscsi_extracheck_is_rd_thread(req->conn);
1184
1185         burst = session->sess_param.max_burst_length;
1186         offset = be32_to_cpu(cmnd_hdr(req)->data_length) - req->r2t_length;
1187
1188         do {
1189                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
1190                 rsp->pdu.bhs.ttt = req->target_task_tag;
1191                 rsp_hdr = (struct iscsi_r2t_hdr *)&rsp->pdu.bhs;
1192                 rsp_hdr->opcode = ISCSI_OP_R2T;
1193                 rsp_hdr->flags = ISCSI_FLG_FINAL;
1194                 rsp_hdr->lun = cmnd_hdr(req)->lun;
1195                 rsp_hdr->itt = cmnd_hdr(req)->itt;
1196                 rsp_hdr->r2t_sn = cpu_to_be32(req->r2t_sn++);
1197                 rsp_hdr->buffer_offset = cpu_to_be32(offset);
1198                 if (req->r2t_length > burst) {
1199                         rsp_hdr->data_length = cpu_to_be32(burst);
1200                         req->r2t_length -= burst;
1201                         offset += burst;
1202                 } else {
1203                         rsp_hdr->data_length = cpu_to_be32(req->r2t_length);
1204                         req->r2t_length = 0;
1205                 }
1206
1207                 TRACE_WRITE("%x %u %u %u %u", cmnd_itt(req),
1208                         be32_to_cpu(rsp_hdr->data_length),
1209                         be32_to_cpu(rsp_hdr->buffer_offset),
1210                         be32_to_cpu(rsp_hdr->r2t_sn), req->outstanding_r2t);
1211
1212                 list_add_tail(&rsp->write_list_entry, &send);
1213
1214                 if (++req->outstanding_r2t >= session->sess_param.max_outstanding_r2t)
1215                         break;
1216
1217         } while (req->r2t_length != 0);
1218
1219         iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_WAKE);
1220
1221 out:
1222         return;
1223 }
1224
1225 static int iscsi_pre_exec(struct scst_cmd *scst_cmd)
1226 {
1227         int res = SCST_PREPROCESS_STATUS_SUCCESS;
1228         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
1229                 scst_cmd_get_tgt_priv(scst_cmd);
1230         struct iscsi_cmnd *c, *t;
1231
1232         TRACE_ENTRY();
1233
1234         EXTRACHECKS_BUG_ON(scst_cmd_atomic(scst_cmd));
1235
1236         /* If data digest isn't used this list will be empty */
1237         list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
1238                                 rx_ddigest_cmd_list_entry) {
1239                 TRACE_DBG("Checking digest of RX ddigest cmd %p", c);
1240                 if (digest_rx_data(c) != 0) {
1241                         scst_set_cmd_error(scst_cmd,
1242                                 SCST_LOAD_SENSE(iscsi_sense_crc_error));
1243                         res = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
1244                         /*
1245                          * The rest of rx_ddigest_cmd_list will be freed
1246                          * in req_cmnd_release()
1247                          */
1248                         goto out;
1249                 }
1250                 cmd_del_from_rx_ddigest_list(c);
1251                 cmnd_put(c);
1252         }
1253
1254 out:
1255         TRACE_EXIT_RES(res);
1256         return res;
1257 }
1258
1259 static int noop_out_start(struct iscsi_cmnd *cmnd)
1260 {
1261         struct iscsi_conn *conn = cmnd->conn;
1262         u32 size, tmp;
1263         int i, err = 0;
1264
1265         TRACE_DBG("%p", cmnd);
1266
1267         iscsi_extracheck_is_rd_thread(conn);
1268
1269         if (unlikely(cmnd_ttt(cmnd) != cpu_to_be32(ISCSI_RESERVED_TAG))) {
1270                 /*
1271                  * We don't request a NOP-Out by sending a NOP-In.
1272                  * See 10.18.2 in the draft 20.
1273                  */
1274                 PRINT_ERROR("Initiator sent command with not RESERVED tag and "
1275                         "TTT %x", cmnd_itt(cmnd));
1276                 err = -ISCSI_REASON_PROTOCOL_ERROR;
1277                 goto out;
1278         }
1279
1280         if (cmnd_itt(cmnd) == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1281                 if (unlikely(!(cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE)))
1282                         PRINT_ERROR("%s", "Initiator sent RESERVED tag for "
1283                                 "non-immediate command");
1284                 spin_lock(&conn->session->sn_lock);
1285                 __update_stat_sn(cmnd);
1286                 err = check_cmd_sn(cmnd);
1287                 spin_unlock(&conn->session->sn_lock);
1288                 if (unlikely(err))
1289                         goto out;
1290         } else {
1291                 err = cmnd_insert_hash(cmnd);
1292                 if (unlikely(err < 0)) {
1293                         PRINT_ERROR("Can't insert in hash: ignore this "
1294                                 "request %x", cmnd_itt(cmnd));
1295                         goto out;
1296                 }
1297         }
1298
1299         size = cmnd->pdu.datasize;
1300
1301         if (size) {
1302                 conn->read_msg.msg_iov = conn->read_iov;
1303                 if (cmnd->pdu.bhs.itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
1304                         struct scatterlist *sg;
1305
1306                         cmnd->sg = sg = scst_alloc(size, GFP_KERNEL,
1307                                                 &cmnd->sg_cnt);
1308                         if (sg == NULL) {
1309                                 TRACE(TRACE_OUT_OF_MEM, "Allocating buffer for"
1310                                       " %d NOP-Out payload failed", size);
1311                                 err = -ISCSI_REASON_OUT_OF_RESOURCES;
1312                                 goto out;
1313                         }
1314
1315                         /* We already checked it in check_segment_length() */
1316                         sBUG_ON(cmnd->sg_cnt > (signed)ISCSI_CONN_IOV_MAX);
1317
1318                         cmnd->own_sg = 1;
1319                         cmnd->bufflen = size;
1320
1321                         for (i = 0; i < cmnd->sg_cnt; i++) {
1322                                 conn->read_iov[i].iov_base =
1323                                         (void __force __user *)(page_address(sg_page(&sg[i])));
1324                                 tmp = min_t(u32, size, PAGE_SIZE);
1325                                 conn->read_iov[i].iov_len = tmp;
1326                                 conn->read_size += tmp;
1327                                 size -= tmp;
1328                         }
1329                         sBUG_ON(size != 0);
1330                 } else {
1331                         /*
1332                          * There are no problems with the safety from concurrent
1333                          * accesses to dummy_page, since for ISCSI_RESERVED_TAG
1334                          * the data only read and then discarded.
1335                          */
1336                         for (i = 0; i < (signed)ISCSI_CONN_IOV_MAX; i++) {
1337                                 conn->read_iov[i].iov_base =
1338                                         (void __force __user *)(page_address(dummy_page));
1339                                 tmp = min_t(u32, size, PAGE_SIZE);
1340                                 conn->read_iov[i].iov_len = tmp;
1341                                 conn->read_size += tmp;
1342                                 size -= tmp;
1343                         }
1344
1345                         /* We already checked size in check_segment_length() */
1346                         sBUG_ON(size != 0);
1347                 }
1348
1349                 conn->read_msg.msg_iovlen = i;
1350                 TRACE_DBG("msg_iov=%p, msg_iovlen=%zd", conn->read_msg.msg_iov,
1351                         conn->read_msg.msg_iovlen);
1352         }
1353
1354 out:
1355         return err;
1356 }
1357
1358 static inline u32 get_next_ttt(struct iscsi_conn *conn)
1359 {
1360         u32 ttt;
1361         struct iscsi_session *session = conn->session;
1362
1363         iscsi_extracheck_is_rd_thread(conn);
1364
1365         if (session->next_ttt == ISCSI_RESERVED_TAG)
1366                 session->next_ttt++;
1367         ttt = session->next_ttt++;
1368
1369         return cpu_to_be32(ttt);
1370 }
1371
1372 int cmnd_rx_continue(struct iscsi_cmnd *req)
1373 {
1374         struct iscsi_conn *conn = req->conn;
1375         struct iscsi_session *session = conn->session;
1376         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1377         struct scst_cmd *scst_cmd = req->scst_cmd;
1378         scst_data_direction dir;
1379         int res = 0;
1380
1381         TRACE_ENTRY();
1382
1383         TRACE_DBG("scsi command: %02x", req_hdr->scb[0]);
1384
1385         if (unlikely(req->scst_state != ISCSI_CMD_STATE_AFTER_PREPROC)) {
1386                 TRACE_DBG("req %p is in %x state", req, req->scst_state);
1387                 if (req->scst_state == ISCSI_CMD_STATE_PROCESSED) {
1388                         cmnd_reject_scsi_cmd(req);
1389                         goto out;
1390                 }
1391                 if (unlikely(req->tm_aborted)) {
1392                         TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
1393                                 req->scst_cmd);
1394                         cmnd_prepare_get_rejected_cmd_data(req);
1395                         goto out;
1396                 }
1397                 sBUG();
1398         }
1399
1400         dir = scst_cmd_get_data_direction(scst_cmd);
1401         if (dir & SCST_DATA_WRITE) {
1402                 req->is_unsolicited_data = !(req_hdr->flags & ISCSI_CMD_FINAL);
1403                 req->r2t_length = be32_to_cpu(req_hdr->data_length) -
1404                                         req->pdu.datasize;
1405                 if (req->r2t_length > 0)
1406                         req->data_waiting = 1;
1407         } else {
1408                 if (unlikely(!(req_hdr->flags & ISCSI_CMD_FINAL) ||
1409                              req->pdu.datasize)) {
1410                         PRINT_ERROR("Unexpected unsolicited data (ITT %x "
1411                                 "CDB %x", cmnd_itt(req), req_hdr->scb[0]);
1412                         scst_set_cmd_error(scst_cmd,
1413                                 SCST_LOAD_SENSE(iscsi_sense_unexpected_unsolicited_data));
1414                         if (scst_cmd_get_sense_buffer(scst_cmd) != NULL)
1415                                 create_status_rsp(req, SAM_STAT_CHECK_CONDITION,
1416                                     scst_cmd_get_sense_buffer(scst_cmd),
1417                                     scst_cmd_get_sense_buffer_len(scst_cmd));
1418                         else
1419                                 create_status_rsp(req, SAM_STAT_BUSY, NULL, 0);
1420                         cmnd_reject_scsi_cmd(req);
1421                         goto out;
1422                 }
1423         }
1424
1425         req->target_task_tag = get_next_ttt(conn);
1426         if (dir != SCST_DATA_BIDI) {
1427                 req->sg = scst_cmd_get_sg(scst_cmd);
1428                 req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
1429                 req->bufflen = scst_cmd_get_bufflen(scst_cmd);
1430         } else {
1431                 req->sg = scst_cmd_get_in_sg(scst_cmd);
1432                 req->sg_cnt = scst_cmd_get_in_sg_cnt(scst_cmd);
1433                 req->bufflen = scst_cmd_get_in_bufflen(scst_cmd);
1434         }
1435         if (unlikely(req->r2t_length > req->bufflen)) {
1436                 PRINT_ERROR("req->r2t_length %d > req->bufflen %d",
1437                         req->r2t_length, req->bufflen);
1438                 req->r2t_length = req->bufflen;
1439         }
1440
1441         TRACE_DBG("req=%p, dir=%d, is_unsolicited_data=%d, "
1442                 "r2t_length=%d, bufflen=%d", req, dir,
1443                 req->is_unsolicited_data, req->r2t_length, req->bufflen);
1444
1445         if (unlikely(!session->sess_param.immediate_data &&
1446                      req->pdu.datasize)) {
1447                 PRINT_ERROR("Initiator %s violated negotiated parameters: "
1448                         "forbidden immediate data sent (ITT %x, op  %x)",
1449                         session->initiator_name, cmnd_itt(req),
1450                         req_hdr->scb[0]);
1451                 res = -EINVAL;
1452                 goto out;
1453         }
1454
1455         if (unlikely(session->sess_param.initial_r2t &&
1456                      !(req_hdr->flags & ISCSI_CMD_FINAL))) {
1457                 PRINT_ERROR("Initiator %s violated negotiated parameters: "
1458                         "initial R2T is required (ITT %x, op  %x)",
1459                         session->initiator_name, cmnd_itt(req),
1460                         req_hdr->scb[0]);
1461                 res = -EINVAL;
1462                 goto out;
1463         }
1464
1465         if (req->pdu.datasize)
1466                 res = cmnd_prepare_recv_pdu(conn, req, 0, req->pdu.datasize);
1467
1468 out:
1469         /* Aborted commands will be freed in cmnd_rx_end() */
1470         TRACE_EXIT_RES(res);
1471         return res;
1472 }
1473
1474 static int scsi_cmnd_start(struct iscsi_cmnd *req)
1475 {
1476         struct iscsi_conn *conn = req->conn;
1477         struct iscsi_session *session = conn->session;
1478         struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
1479         struct scst_cmd *scst_cmd;
1480         scst_data_direction dir;
1481         struct iscsi_ahs_hdr *ahdr;
1482         int res = 0;
1483
1484         TRACE_ENTRY();
1485
1486         TRACE_DBG("scsi command: %02x", req_hdr->scb[0]);
1487
1488         TRACE_DBG("Incrementing active_cmds (cmd %p, sess %p, "
1489                 "new value %d)", req, session,
1490                 atomic_read(&session->active_cmds)+1);
1491         atomic_inc(&session->active_cmds);
1492         req->dec_active_cmnds = 1;
1493
1494         scst_cmd = scst_rx_cmd(session->scst_sess,
1495                 (uint8_t *)&req_hdr->lun, sizeof(req_hdr->lun),
1496                 req_hdr->scb, sizeof(req_hdr->scb), SCST_NON_ATOMIC);
1497         if (scst_cmd == NULL) {
1498                 create_status_rsp(req, SAM_STAT_BUSY, NULL, 0);
1499                 cmnd_reject_scsi_cmd(req);
1500                 goto out;
1501         }
1502
1503         req->scst_cmd = scst_cmd;
1504         scst_cmd_set_tag(scst_cmd, req_hdr->itt);
1505         scst_cmd_set_tgt_priv(scst_cmd, req);
1506
1507         if ((req_hdr->flags & ISCSI_CMD_READ) &&
1508             (req_hdr->flags & ISCSI_CMD_WRITE)) {
1509                 int sz = cmnd_read_size(req);
1510                 if (unlikely(sz < 0)) {
1511                         PRINT_ERROR("%s", "BIDI data transfer, but initiator "
1512                                 "not supplied Bidirectional Read Expected Data "
1513                                 "Transfer Length AHS");
1514                         scst_set_cmd_error(scst_cmd,
1515                            SCST_LOAD_SENSE(scst_sense_parameter_value_invalid));
1516                         /*
1517                          * scst_cmd_init_done() will handle commands with
1518                          * set status as preliminary completed
1519                          */
1520                 } else {
1521                         req->read_size = sz;
1522                         dir = SCST_DATA_BIDI;
1523                         scst_cmd_set_expected(scst_cmd, dir, sz);
1524                         scst_cmd_set_expected_in_transfer_len(scst_cmd,
1525                                 be32_to_cpu(req_hdr->data_length));
1526 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
1527                         scst_cmd_set_tgt_need_alloc_data_buf(scst_cmd);
1528 #endif
1529                 }
1530         } else if (req_hdr->flags & ISCSI_CMD_READ) {
1531                 req->read_size = be32_to_cpu(req_hdr->data_length);
1532                 dir = SCST_DATA_READ;
1533                 scst_cmd_set_expected(scst_cmd, dir, req->read_size);
1534 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
1535                 scst_cmd_set_tgt_need_alloc_data_buf(scst_cmd);
1536 #endif
1537         } else if (req_hdr->flags & ISCSI_CMD_WRITE) {
1538                 dir = SCST_DATA_WRITE;
1539                 scst_cmd_set_expected(scst_cmd, dir,
1540                         be32_to_cpu(req_hdr->data_length));
1541         } else {
1542                 dir = SCST_DATA_NONE;
1543                 scst_cmd_set_expected(scst_cmd, dir, 0);
1544         }
1545
1546         switch (req_hdr->flags & ISCSI_CMD_ATTR_MASK) {
1547         case ISCSI_CMD_SIMPLE:
1548                 scst_cmd->queue_type = SCST_CMD_QUEUE_SIMPLE;
1549                 break;
1550         case ISCSI_CMD_HEAD_OF_QUEUE:
1551                 scst_cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
1552                 break;
1553         case ISCSI_CMD_ORDERED:
1554                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1555                 break;
1556         case ISCSI_CMD_ACA:
1557                 scst_cmd->queue_type = SCST_CMD_QUEUE_ACA;
1558                 break;
1559         case ISCSI_CMD_UNTAGGED:
1560                 scst_cmd->queue_type = SCST_CMD_QUEUE_UNTAGGED;
1561                 break;
1562         default:
1563                 PRINT_ERROR("Unknown task code %x, use ORDERED instead",
1564                         req_hdr->flags & ISCSI_CMD_ATTR_MASK);
1565                 scst_cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
1566                 break;
1567         }
1568
1569         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
1570         scst_cmd_set_tgt_sn(scst_cmd, req_hdr->cmd_sn);
1571
1572         ahdr = (struct iscsi_ahs_hdr *)req->pdu.ahs;
1573         if (ahdr != NULL) {
1574                 uint8_t *p = (uint8_t *)ahdr;
1575                 int size = 0;
1576                 do {
1577                         int s;
1578
1579                         ahdr = (struct iscsi_ahs_hdr *)p;
1580
1581                         if (ahdr->ahstype == ISCSI_AHSTYPE_CDB) {
1582                                 struct iscsi_cdb_ahdr *eca =
1583                                         (struct iscsi_cdb_ahdr *)ahdr;
1584                                 scst_cmd_set_ext_cdb(scst_cmd, eca->cdb,
1585                                         be16_to_cpu(ahdr->ahslength) - 1);
1586                                 break;
1587                         }
1588                         s = 3 + be16_to_cpu(ahdr->ahslength);
1589                         s = (s + 3) & -4;
1590                         size += s;
1591                         p += s;
1592                 } while (size < req->pdu.ahssize);
1593         }
1594
1595         TRACE_DBG("START Command (tag %d, queue_type %d)",
1596                 req_hdr->itt, scst_cmd->queue_type);
1597         req->scst_state = ISCSI_CMD_STATE_RX_CMD;
1598         conn->rx_task = current;
1599         scst_cmd_init_stage1_done(scst_cmd, SCST_CONTEXT_DIRECT, 0);
1600
1601         if (req->scst_state != ISCSI_CMD_STATE_RX_CMD)
1602                 res = cmnd_rx_continue(req);
1603         else {
1604                 TRACE_DBG("Delaying req %p post processing (scst_state %d)",
1605                         req, req->scst_state);
1606                 res = 1;
1607         }
1608
1609 out:
1610         /* Aborted commands will be freed in cmnd_rx_end() */
1611         TRACE_EXIT_RES(res);
1612         return res;
1613 }
1614
1615 static int data_out_start(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
1616 {
1617         struct iscsi_data_out_hdr *req_hdr =
1618                 (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1619         struct iscsi_cmnd *orig_req = NULL;
1620         u32 offset = be32_to_cpu(req_hdr->buffer_offset);
1621         int res = 0;
1622
1623         TRACE_ENTRY();
1624
1625         /*
1626          * There is no race with send_r2t() and conn_abort(), since
1627          * all functions called from single read thread
1628          */
1629         iscsi_extracheck_is_rd_thread(cmnd->conn);
1630
1631         update_stat_sn(cmnd);
1632
1633         cmnd->cmd_req = orig_req = cmnd_find_hash(conn->session, req_hdr->itt,
1634                                         req_hdr->ttt);
1635         if (unlikely(orig_req == NULL)) {
1636                 /* It might happen if req was aborted and then freed */
1637                 TRACE(TRACE_MGMT_MINOR, "Unable to find scsi task %x %x",
1638                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1639                 goto out_reject;
1640         }
1641
1642         if (orig_req->is_unsolicited_data) {
1643                 if (unlikely(orig_req->r2t_length < cmnd->pdu.datasize)) {
1644                         PRINT_ERROR("Data size (%d) > R2T length (%d)",
1645                                 cmnd->pdu.datasize, orig_req->r2t_length);
1646                         mark_conn_closed(conn);
1647                         res = -EINVAL;
1648                         goto out;
1649                 }
1650                 orig_req->r2t_length -= cmnd->pdu.datasize;
1651         }
1652
1653         /* Check unsolicited burst data */
1654         if (unlikely((req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) &&
1655                      (orig_req->pdu.bhs.flags & ISCSI_FLG_FINAL))) {
1656                 PRINT_ERROR("Unexpected data from %x %x",
1657                         cmnd_itt(cmnd), cmnd_ttt(cmnd));
1658                 mark_conn_closed(conn);
1659                 res = -EINVAL;
1660                 goto out;
1661         }
1662
1663         TRACE_WRITE("%u %p %p %u %u", req_hdr->ttt, cmnd, orig_req,
1664                 offset, cmnd->pdu.datasize);
1665
1666         res = cmnd_prepare_recv_pdu(conn, orig_req, offset, cmnd->pdu.datasize);
1667
1668 out:
1669         TRACE_EXIT_RES(res);
1670         return res;
1671
1672 out_reject:
1673         sBUG_ON(cmnd->rejected);
1674         cmnd->rejected = 1;
1675         cmnd->reject_reason = ISCSI_REJECT_DATA;
1676         cmnd_prepare_get_rejected_cmd_data(cmnd);
1677         goto out;
1678 }
1679
1680 static void data_out_end(struct iscsi_cmnd *cmnd)
1681 {
1682         struct iscsi_data_out_hdr *req_hdr =
1683                 (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
1684         struct iscsi_cmnd *req;
1685
1686         sBUG_ON(cmnd == NULL);
1687         req = cmnd->cmd_req;
1688         sBUG_ON(req == NULL);
1689
1690         TRACE_DBG("cmnd %p, req %p", cmnd, req);
1691
1692         iscsi_extracheck_is_rd_thread(cmnd->conn);
1693
1694         if (!(cmnd->conn->ddigest_type & DIGEST_NONE) &&
1695             !cmnd->ddigest_checked) {
1696                 cmd_add_on_rx_ddigest_list(req, cmnd);
1697                 cmnd_get(cmnd);
1698         }
1699
1700         if (req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
1701                 TRACE_DBG("ISCSI_RESERVED_TAG, FINAL %x",
1702                         req_hdr->flags & ISCSI_FLG_FINAL);
1703
1704                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1705                         req->is_unsolicited_data = 0;
1706                         if (req->pending)
1707                                 goto out_put;
1708                 } else
1709                         goto out_put;
1710         } else {
1711                 TRACE_DBG("FINAL %x, outstanding_r2t %d, r2t_length %d",
1712                         req_hdr->flags & ISCSI_FLG_FINAL,
1713                         req->outstanding_r2t, req->r2t_length);
1714
1715                 if (req_hdr->flags & ISCSI_FLG_FINAL) {
1716                         if (unlikely(req->is_unsolicited_data)) {
1717                                 PRINT_ERROR("Unexpected unsolicited data "
1718                                         "(r2t_length %u, outstanding_r2t %d)",
1719                                         req->r2t_length,
1720                                         req->is_unsolicited_data);
1721                                 mark_conn_closed(req->conn);
1722                                 goto out_put;
1723                         }
1724                         req->outstanding_r2t--;
1725                 } else
1726                         goto out_put;
1727         }
1728
1729         if (req->r2t_length != 0) {
1730                 if (!req->is_unsolicited_data)
1731                         send_r2t(req);
1732         } else
1733                 iscsi_restart_waiting_cmnd(req);
1734
1735 out_put:
1736         cmnd_put(cmnd);
1737         return;
1738 }
1739
1740 static void __cmnd_abort(struct iscsi_cmnd *cmnd)
1741 {
1742         /*
1743          * Here, if cmnd is data_waiting, we should iscsi_fail_waiting_cmnd()
1744          * it. But, since this function can be called from any thread, not only
1745          * from the read one, we at the moment can't do that, because of
1746          * absence of appropriate locking protection. But this isn't a stuff
1747          * for 1.0.0. So, currently a misbehaving initiator, not sending
1748          * data in R2T state for a sharing between targets device, for which
1749          * for some reason an aborting TM command, e.g. TARGET RESET, from
1750          * another initiator is issued, can block response for this TM command
1751          * virtually forever and by this make the issuing initiator eventually
1752          * put the device offline.
1753          *
1754          * ToDo in the next version, possibly a simple connection mutex, taken
1755          * by the read thread before starting any processing and by this
1756          * function, should be sufficient.
1757          */
1758
1759         TRACE_MGMT_DBG("Aborting cmd %p, scst_cmd %p (scst state %x, "
1760                 "ref_cnt %d, itt %x, sn %u, op %x, r2t_len %x, CDB op %x, "
1761                 "size to write %u, is_unsolicited_data %d, "
1762                 "outstanding_r2t %d, data_waiting %d, sess->exp_cmd_sn %u, "
1763                 "conn %p, rd_task %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
1764                 atomic_read(&cmnd->ref_cnt), cmnd_itt(cmnd), cmnd->pdu.bhs.sn,
1765                 cmnd_opcode(cmnd), cmnd->r2t_length, cmnd_scsicode(cmnd),
1766                 cmnd_write_size(cmnd), cmnd->is_unsolicited_data,
1767                 cmnd->outstanding_r2t, cmnd->data_waiting,
1768                 cmnd->conn->session->exp_cmd_sn, cmnd->conn,
1769                 cmnd->conn->rd_task);
1770
1771 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
1772         TRACE_MGMT_DBG("net_ref_cnt %d", atomic_read(&cmnd->net_ref_cnt));
1773 #endif
1774
1775         cmnd->tm_aborted = 1;
1776
1777         return;
1778 }
1779
1780 /* Must be called from the read or conn close thread */
1781 static int cmnd_abort(struct iscsi_cmnd *req)
1782 {
1783         struct iscsi_session *session = req->conn->session;
1784         struct iscsi_task_mgt_hdr *req_hdr =
1785                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1786         struct iscsi_cmnd *cmnd;
1787         int err;
1788
1789         req_hdr->ref_cmd_sn = be32_to_cpu(req_hdr->ref_cmd_sn);
1790
1791         if (after(req_hdr->ref_cmd_sn, req_hdr->cmd_sn)) {
1792                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) > CmdSN(%u)",
1793                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn);
1794                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1795                 goto out;
1796         }
1797
1798         cmnd = cmnd_find_hash_get(session, req_hdr->rtt, ISCSI_RESERVED_TAG);
1799         if (cmnd) {
1800                 struct iscsi_conn *conn = cmnd->conn;
1801                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1802
1803                 if (req_hdr->lun != hdr->lun) {
1804                         PRINT_ERROR("ABORT TASK: LUN mismatch: req LUN "
1805                                     "%llx, cmd LUN %llx, rtt %u",
1806                                     (long long unsigned int)req_hdr->lun,
1807                                     (long long unsigned int)hdr->lun,
1808                                     req_hdr->rtt);
1809                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1810                         goto out_put;
1811                 }
1812
1813                 if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
1814                         if (req_hdr->ref_cmd_sn != req_hdr->cmd_sn) {
1815                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != TM "
1816                                         "cmd CmdSN(%u) for immediate command "
1817                                         "%p", req_hdr->ref_cmd_sn,
1818                                         req_hdr->cmd_sn, cmnd);
1819                                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1820                                 goto out_put;
1821                         }
1822                 } else {
1823                         if (req_hdr->ref_cmd_sn != hdr->cmd_sn) {
1824                                 PRINT_ERROR("ABORT TASK: RefCmdSN(%u) != "
1825                                         "CmdSN(%u) for command %p",
1826                                         req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
1827                                         cmnd);
1828                                 err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1829                                 goto out_put;
1830                         }
1831                 }
1832
1833                 if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1834                     (req_hdr->cmd_sn == hdr->cmd_sn)) {
1835                         PRINT_ERROR("ABORT TASK: SN mismatch: req SN %x, "
1836                                 "cmd SN %x, rtt %u", req_hdr->cmd_sn,
1837                                 hdr->cmd_sn, req_hdr->rtt);
1838                         err = ISCSI_RESPONSE_FUNCTION_REJECTED;
1839                         goto out_put;
1840                 }
1841
1842                 spin_lock_bh(&conn->cmd_list_lock);
1843                 __cmnd_abort(cmnd);
1844                 spin_unlock_bh(&conn->cmd_list_lock);
1845
1846                 cmnd_put(cmnd);
1847                 err = 0;
1848         } else {
1849                 TRACE_MGMT_DBG("cmd RTT %x not found", req_hdr->rtt);
1850                 err = ISCSI_RESPONSE_UNKNOWN_TASK;
1851         }
1852
1853 out:
1854         return err;
1855
1856 out_put:
1857         cmnd_put(cmnd);
1858         goto out;
1859 }
1860
1861 /* Must be called from the read or conn close thread */
1862 static int target_abort(struct iscsi_cmnd *req, int all)
1863 {
1864         struct iscsi_target *target = req->conn->session->target;
1865         struct iscsi_task_mgt_hdr *req_hdr =
1866                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1867         struct iscsi_session *session;
1868         struct iscsi_conn *conn;
1869         struct iscsi_cmnd *cmnd;
1870
1871         mutex_lock(&target->target_mutex);
1872
1873         list_for_each_entry(session, &target->session_list,
1874                             session_list_entry) {
1875                 list_for_each_entry(conn, &session->conn_list,
1876                                     conn_list_entry) {
1877                         spin_lock_bh(&conn->cmd_list_lock);
1878                         list_for_each_entry(cmnd, &conn->cmd_list,
1879                                             cmd_list_entry) {
1880                                 if (cmnd == req)
1881                                         continue;
1882                                 if (all)
1883                                         __cmnd_abort(cmnd);
1884                                 else if (req_hdr->lun == cmnd_hdr(cmnd)->lun)
1885                                         __cmnd_abort(cmnd);
1886                         }
1887                         spin_unlock_bh(&conn->cmd_list_lock);
1888                 }
1889         }
1890
1891         mutex_unlock(&target->target_mutex);
1892         return 0;
1893 }
1894
1895 /* Must be called from the read or conn close thread */
1896 static void task_set_abort(struct iscsi_cmnd *req)
1897 {
1898         struct iscsi_session *session = req->conn->session;
1899         struct iscsi_task_mgt_hdr *req_hdr =
1900                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1901         struct iscsi_target *target = session->target;
1902         struct iscsi_conn *conn;
1903         struct iscsi_cmnd *cmnd;
1904
1905         mutex_lock(&target->target_mutex);
1906
1907         list_for_each_entry(conn, &session->conn_list, conn_list_entry) {
1908                 spin_lock_bh(&conn->cmd_list_lock);
1909                 list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1910                         struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
1911                         if (cmnd == req)
1912                                 continue;
1913                         if (req_hdr->lun != hdr->lun)
1914                                 continue;
1915                         if (before(req_hdr->cmd_sn, hdr->cmd_sn) ||
1916                             req_hdr->cmd_sn == hdr->cmd_sn)
1917                                 continue;
1918                         __cmnd_abort(cmnd);
1919                 }
1920                 spin_unlock_bh(&conn->cmd_list_lock);
1921         }
1922
1923         mutex_unlock(&target->target_mutex);
1924         return;
1925 }
1926
1927 /* Must be called from the read or conn close thread */
1928 void conn_abort(struct iscsi_conn *conn)
1929 {
1930         struct iscsi_cmnd *cmnd;
1931
1932         TRACE_MGMT_DBG("Aborting conn %p", conn);
1933
1934         iscsi_extracheck_is_rd_thread(conn);
1935
1936         spin_lock_bh(&conn->cmd_list_lock);
1937 again:
1938         list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
1939                 __cmnd_abort(cmnd);
1940                 if (cmnd->data_waiting) {
1941                         if (!cmnd_get_check(cmnd)) {
1942                                 spin_unlock_bh(&conn->cmd_list_lock);
1943
1944                                 /* ToDo: this is racy for MC/S */
1945                                 TRACE_MGMT_DBG("Restarting data waiting cmd "
1946                                         "%p", cmnd);
1947                                 iscsi_fail_waiting_cmnd(cmnd);
1948
1949                                 cmnd_put(cmnd);
1950
1951                                 /*
1952                                  * We are in the read thread, so we may not
1953                                  * worry that after cmnd release conn gets
1954                                  * released as well.
1955                                  */
1956                                 spin_lock_bh(&conn->cmd_list_lock);
1957                                 goto again;
1958                         }
1959                 }
1960         }
1961         spin_unlock_bh(&conn->cmd_list_lock);
1962
1963         return;
1964 }
1965
1966 static void execute_task_management(struct iscsi_cmnd *req)
1967 {
1968         struct iscsi_conn *conn = req->conn;
1969         struct iscsi_session *sess = conn->session;
1970         struct iscsi_task_mgt_hdr *req_hdr =
1971                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
1972         int rc, status, function = req_hdr->function & ISCSI_FUNCTION_MASK;
1973         struct scst_rx_mgmt_params params;
1974
1975         TRACE((function == ISCSI_FUNCTION_ABORT_TASK) ?
1976                         TRACE_MGMT_MINOR : TRACE_MGMT,
1977                 "TM fn %d", function);
1978
1979         TRACE_MGMT_DBG("TM req %p, itt %x, rtt %x, sn %u, con %p", req,
1980                 cmnd_itt(req), req_hdr->rtt, req_hdr->cmd_sn, conn);
1981
1982         iscsi_extracheck_is_rd_thread(conn);
1983
1984         spin_lock(&sess->sn_lock);
1985         sess->tm_active++;
1986         sess->tm_sn = req_hdr->cmd_sn;
1987         if (sess->tm_rsp != NULL) {
1988                 struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
1989
1990                 TRACE(TRACE_MGMT_MINOR, "Dropping delayed TM rsp %p", tm_rsp);
1991
1992                 sess->tm_rsp = NULL;
1993                 sess->tm_active--;
1994
1995                 spin_unlock(&sess->sn_lock);
1996
1997                 sBUG_ON(sess->tm_active < 0);
1998
1999                 rsp_cmnd_release(tm_rsp);
2000         } else
2001                 spin_unlock(&sess->sn_lock);
2002
2003         memset(&params, 0, sizeof(params));
2004         params.atomic = SCST_NON_ATOMIC;
2005         params.tgt_priv = req;
2006
2007         if ((function != ISCSI_FUNCTION_ABORT_TASK) &&
2008             (req_hdr->rtt != ISCSI_RESERVED_TAG)) {
2009                 PRINT_ERROR("Invalid RTT %x (TM fn %d)", req_hdr->rtt,
2010                         function);
2011                 rc = -1;
2012                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2013                 goto reject;
2014         }
2015
2016         /* cmd_sn is already in CPU format converted in check_cmd_sn() */
2017
2018         switch (function) {
2019         case ISCSI_FUNCTION_ABORT_TASK:
2020                 rc = -1;
2021                 status = cmnd_abort(req);
2022                 if (status == 0) {
2023                         params.fn = SCST_ABORT_TASK;
2024                         params.tag = req_hdr->rtt;
2025                         params.tag_set = 1;
2026                         params.lun = (uint8_t *)&req_hdr->lun;
2027                         params.lun_len = sizeof(req_hdr->lun);
2028                         params.lun_set = 1;
2029                         params.cmd_sn = req_hdr->cmd_sn;
2030                         params.cmd_sn_set = 1;
2031                         rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2032                                 &params);
2033                         status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2034                 }
2035                 break;
2036         case ISCSI_FUNCTION_ABORT_TASK_SET:
2037                 task_set_abort(req);
2038                 params.fn = SCST_ABORT_TASK_SET;
2039                 params.lun = (uint8_t *)&req_hdr->lun;
2040                 params.lun_len = sizeof(req_hdr->lun);
2041                 params.lun_set = 1;
2042                 params.cmd_sn = req_hdr->cmd_sn;
2043                 params.cmd_sn_set = 1;
2044                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2045                         &params);
2046                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2047                 break;
2048         case ISCSI_FUNCTION_CLEAR_TASK_SET:
2049                 task_set_abort(req);
2050                 params.fn = SCST_CLEAR_TASK_SET;
2051                 params.lun = (uint8_t *)&req_hdr->lun;
2052                 params.lun_len = sizeof(req_hdr->lun);
2053                 params.lun_set = 1;
2054                 params.cmd_sn = req_hdr->cmd_sn;
2055                 params.cmd_sn_set = 1;
2056                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2057                         &params);
2058                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2059                 break;
2060         case ISCSI_FUNCTION_CLEAR_ACA:
2061                 params.fn = SCST_CLEAR_ACA;
2062                 params.lun = (uint8_t *)&req_hdr->lun;
2063                 params.lun_len = sizeof(req_hdr->lun);
2064                 params.lun_set = 1;
2065                 params.cmd_sn = req_hdr->cmd_sn;
2066                 params.cmd_sn_set = 1;
2067                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2068                         &params);
2069                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2070                 break;
2071         case ISCSI_FUNCTION_TARGET_COLD_RESET:
2072         case ISCSI_FUNCTION_TARGET_WARM_RESET:
2073                 target_abort(req, 1);
2074                 params.fn = SCST_TARGET_RESET;
2075                 params.cmd_sn = req_hdr->cmd_sn;
2076                 params.cmd_sn_set = 1;
2077                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2078                         &params);
2079                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2080                 break;
2081         case ISCSI_FUNCTION_LOGICAL_UNIT_RESET:
2082                 target_abort(req, 0);
2083                 params.fn = SCST_LUN_RESET;
2084                 params.lun = (uint8_t *)&req_hdr->lun;
2085                 params.lun_len = sizeof(req_hdr->lun);
2086                 params.lun_set = 1;
2087                 params.cmd_sn = req_hdr->cmd_sn;
2088                 params.cmd_sn_set = 1;
2089                 rc = scst_rx_mgmt_fn(conn->session->scst_sess,
2090                         &params);
2091                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2092                 break;
2093         case ISCSI_FUNCTION_TASK_REASSIGN:
2094                 rc = -1;
2095                 status = ISCSI_RESPONSE_ALLEGIANCE_REASSIGNMENT_UNSUPPORTED;
2096                 break;
2097         default:
2098                 PRINT_ERROR("Unknown TM function %d", function);
2099                 rc = -1;
2100                 status = ISCSI_RESPONSE_FUNCTION_REJECTED;
2101                 break;
2102         }
2103
2104 reject:
2105         if (rc != 0)
2106                 iscsi_send_task_mgmt_resp(req, status);
2107
2108         return;
2109 }
2110
2111 static void noop_out_exec(struct iscsi_cmnd *req)
2112 {
2113         struct iscsi_cmnd *rsp;
2114         struct iscsi_nop_in_hdr *rsp_hdr;
2115
2116         TRACE_DBG("%p", req);
2117
2118         if (cmnd_itt(req) != cpu_to_be32(ISCSI_RESERVED_TAG)) {
2119                 rsp = iscsi_cmnd_create_rsp_cmnd(req);
2120
2121                 rsp_hdr = (struct iscsi_nop_in_hdr *)&rsp->pdu.bhs;
2122                 rsp_hdr->opcode = ISCSI_OP_NOOP_IN;
2123                 rsp_hdr->flags = ISCSI_FLG_FINAL;
2124                 rsp_hdr->itt = req->pdu.bhs.itt;
2125                 rsp_hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
2126
2127                 if (req->pdu.datasize)
2128                         sBUG_ON(req->sg == NULL);
2129                 else
2130                         sBUG_ON(req->sg != NULL);
2131
2132                 if (req->sg) {
2133                         rsp->sg = req->sg;
2134                         rsp->sg_cnt = req->sg_cnt;
2135                         rsp->bufflen = req->bufflen;
2136                 }
2137
2138                 /* We already checked it in check_segment_length() */
2139                 sBUG_ON(get_pgcnt(req->pdu.datasize, 0) > ISCSI_CONN_IOV_MAX);
2140
2141                 rsp->pdu.datasize = req->pdu.datasize;
2142                 iscsi_cmnd_init_write(rsp,
2143                         ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2144                 req_cmnd_release(req);
2145         } else
2146                 cmnd_put(req);
2147         return;
2148 }
2149
2150 static void logout_exec(struct iscsi_cmnd *req)
2151 {
2152         struct iscsi_logout_req_hdr *req_hdr;
2153         struct iscsi_cmnd *rsp;
2154         struct iscsi_logout_rsp_hdr *rsp_hdr;
2155
2156         PRINT_INFO("Logout received from initiator %s",
2157                 req->conn->session->initiator_name);
2158         TRACE_DBG("%p", req);
2159
2160         req_hdr = (struct iscsi_logout_req_hdr *)&req->pdu.bhs;
2161         rsp = iscsi_cmnd_create_rsp_cmnd(req);
2162         rsp_hdr = (struct iscsi_logout_rsp_hdr *)&rsp->pdu.bhs;
2163         rsp_hdr->opcode = ISCSI_OP_LOGOUT_RSP;
2164         rsp_hdr->flags = ISCSI_FLG_FINAL;
2165         rsp_hdr->itt = req_hdr->itt;
2166         rsp->should_close_conn = 1;
2167         iscsi_cmnd_init_write(rsp,
2168                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2169         req_cmnd_release(req);
2170         return;
2171 }
2172
2173 static void iscsi_cmnd_exec(struct iscsi_cmnd *cmnd)
2174 {
2175         TRACE_ENTRY();
2176
2177         TRACE_DBG("%p,%x,%u", cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn);
2178
2179         iscsi_extracheck_is_rd_thread(cmnd->conn);
2180
2181         if (unlikely(cmnd->tm_aborted)) {
2182                 TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted", cmnd,
2183                         cmnd->scst_cmd);
2184                 req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
2185                 goto out;
2186         }
2187
2188         if (unlikely(cmnd->rejected))
2189                 goto out_rejected;
2190
2191         switch (cmnd_opcode(cmnd)) {
2192         case ISCSI_OP_SCSI_CMD:
2193                 if (cmnd->r2t_length != 0) {
2194                         if (!cmnd->is_unsolicited_data) {
2195                                 send_r2t(cmnd);
2196                                 break;
2197                         }
2198                 } else
2199                         iscsi_restart_cmnd(cmnd);
2200                 break;
2201         case ISCSI_OP_NOOP_OUT:
2202                 noop_out_exec(cmnd);
2203                 break;
2204         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2205                 execute_task_management(cmnd);
2206                 break;
2207         case ISCSI_OP_LOGOUT_CMD:
2208                 logout_exec(cmnd);
2209                 break;
2210         default:
2211                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2212                 req_cmnd_release(cmnd);
2213                 break;
2214         }
2215 out:
2216         TRACE_EXIT();
2217         return;
2218
2219 out_rejected:
2220         TRACE_MGMT_DBG("Rejected cmd %p (reason %d)", cmnd,
2221                 cmnd->reject_reason);
2222         switch (cmnd->reject_reason) {
2223         default:
2224                 PRINT_ERROR("Unexpected reject reason %d",
2225                             cmnd->reject_reason);
2226                 /* go through */
2227         case ISCSI_REJECT_SCSI_CMD:
2228                 req_cmnd_release(cmnd);
2229                 break;
2230         }
2231         goto out;
2232 }
2233
2234 /*
2235  * Note: the code belows passes a kernel space pointer (&opt) to setsockopt()
2236  * while the declaration of setsockopt specifies that it expects a user space
2237  * pointer. This seems to work fine, and this approach is also used in some
2238  * other parts of the Linux kernel (see e.g. fs/ocfs2/cluster/tcp.c).
2239  */
2240 static void set_cork(struct socket *sock, int on)
2241 {
2242         int opt = on;
2243         mm_segment_t oldfs;
2244
2245         oldfs = get_fs();
2246         set_fs(get_ds());
2247         sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK,
2248                               (void __force __user *)&opt, sizeof(opt));
2249         set_fs(oldfs);
2250         return;
2251 }
2252
2253 void cmnd_tx_start(struct iscsi_cmnd *cmnd)
2254 {
2255         struct iscsi_conn *conn = cmnd->conn;
2256
2257         TRACE_DBG("conn %p, cmnd %p, opcode %x", conn, cmnd, cmnd_opcode(cmnd));
2258         iscsi_cmnd_set_length(&cmnd->pdu);
2259
2260         iscsi_extracheck_is_wr_thread(conn);
2261
2262         set_cork(conn->sock, 1);
2263
2264         conn->write_iop = conn->write_iov;
2265         conn->write_iop->iov_base = (void __force __user *)(&cmnd->pdu.bhs);
2266         conn->write_iop->iov_len = sizeof(cmnd->pdu.bhs);
2267         conn->write_iop_used = 1;
2268         conn->write_size = sizeof(cmnd->pdu.bhs) + cmnd->pdu.datasize;
2269         conn->write_offset = 0;
2270
2271         switch (cmnd_opcode(cmnd)) {
2272         case ISCSI_OP_NOOP_IN:
2273                 cmnd_set_sn(cmnd, 1);
2274                 break;
2275         case ISCSI_OP_SCSI_RSP:
2276                 cmnd_set_sn(cmnd, 1);
2277                 break;
2278         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2279                 cmnd_set_sn(cmnd, 1);
2280                 break;
2281         case ISCSI_OP_TEXT_RSP:
2282                 cmnd_set_sn(cmnd, 1);
2283                 break;
2284         case ISCSI_OP_SCSI_DATA_IN:
2285         {
2286                 struct iscsi_data_in_hdr *rsp =
2287                         (struct iscsi_data_in_hdr *)&cmnd->pdu.bhs;
2288                 u32 offset = cpu_to_be32(rsp->buffer_offset);
2289
2290                 TRACE_DBG("cmnd %p, offset %u, datasize %u, bufflen %u", cmnd,
2291                         offset, cmnd->pdu.datasize, cmnd->bufflen);
2292
2293                 sBUG_ON(offset > cmnd->bufflen);
2294                 sBUG_ON(offset + cmnd->pdu.datasize > cmnd->bufflen);
2295
2296                 conn->write_offset = offset;
2297
2298                 cmnd_set_sn(cmnd, (rsp->flags & ISCSI_FLG_FINAL) ? 1 : 0);
2299                 break;
2300         }
2301         case ISCSI_OP_LOGOUT_RSP:
2302                 cmnd_set_sn(cmnd, 1);
2303                 break;
2304         case ISCSI_OP_R2T:
2305                 cmnd->pdu.bhs.sn = cmnd_set_sn(cmnd, 0);
2306                 break;
2307         case ISCSI_OP_ASYNC_MSG:
2308                 cmnd_set_sn(cmnd, 1);
2309                 break;
2310         case ISCSI_OP_REJECT:
2311                 cmnd_set_sn(cmnd, 1);
2312                 break;
2313         default:
2314                 PRINT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
2315                 break;
2316         }
2317
2318         iscsi_dump_pdu(&cmnd->pdu);
2319         return;
2320 }
2321
2322 void cmnd_tx_end(struct iscsi_cmnd *cmnd)
2323 {
2324         struct iscsi_conn *conn = cmnd->conn;
2325
2326         TRACE_DBG("%p:%x (should_close_conn %d, should_close_all_conn %d)",
2327                 cmnd, cmnd_opcode(cmnd), cmnd->should_close_conn,
2328                 cmnd->should_close_all_conn);
2329
2330         switch (cmnd_opcode(cmnd)) {
2331         case ISCSI_OP_NOOP_IN:
2332         case ISCSI_OP_SCSI_RSP:
2333         case ISCSI_OP_SCSI_TASK_MGT_RSP:
2334         case ISCSI_OP_TEXT_RSP:
2335         case ISCSI_OP_R2T:
2336         case ISCSI_OP_ASYNC_MSG:
2337         case ISCSI_OP_REJECT:
2338         case ISCSI_OP_SCSI_DATA_IN:
2339         case ISCSI_OP_LOGOUT_RSP:
2340                 break;
2341         default:
2342                 PRINT_CRIT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2343                 sBUG();
2344                 break;
2345         }
2346
2347         if (unlikely(cmnd->should_close_conn)) {
2348                 if (cmnd->should_close_all_conn) {
2349                         PRINT_INFO("Closing all connections for target %x at "
2350                                 "initiator's %s request",
2351                                 cmnd->conn->session->target->tid,
2352                                 conn->session->initiator_name);
2353                         target_del_all_sess(cmnd->conn->session->target, 0);
2354                 } else {
2355                         PRINT_INFO("Closing connection at initiator's %s "
2356                                 "request", conn->session->initiator_name);
2357                         mark_conn_closed(conn);
2358                 }
2359         }
2360
2361         set_cork(cmnd->conn->sock, 0);
2362         return;
2363 }
2364
2365 /*
2366  * Push the command for execution. This functions reorders the commands.
2367  * Called from the read thread.
2368  */
2369 static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd)
2370 {
2371         struct iscsi_session *session = cmnd->conn->session;
2372         struct list_head *entry;
2373         u32 cmd_sn;
2374
2375         TRACE_DBG("%p:%x %u,%u",
2376                 cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn,
2377                 session->exp_cmd_sn);
2378
2379         iscsi_extracheck_is_rd_thread(cmnd->conn);
2380
2381         sBUG_ON(cmnd->parent_req != NULL);
2382
2383         if (cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE) {
2384                 TRACE_DBG("Immediate cmd %p (cmd_sn %u)", cmnd,
2385                         cmnd->pdu.bhs.sn);
2386                 iscsi_cmnd_exec(cmnd);
2387                 goto out;
2388         }
2389
2390         spin_lock(&session->sn_lock);
2391
2392         cmd_sn = cmnd->pdu.bhs.sn;
2393         if (cmd_sn == session->exp_cmd_sn) {
2394                 while (1) {
2395                         session->exp_cmd_sn = ++cmd_sn;
2396
2397                         if (unlikely(session->tm_active > 0)) {
2398                                 if (before(cmd_sn, session->tm_sn)) {
2399                                         struct iscsi_conn *conn = cmnd->conn;
2400
2401                                         spin_unlock(&session->sn_lock);
2402
2403                                         spin_lock_bh(&conn->cmd_list_lock);
2404                                         __cmnd_abort(cmnd);
2405                                         spin_unlock_bh(&conn->cmd_list_lock);
2406
2407                                         spin_lock(&session->sn_lock);
2408                                 }
2409                                 iscsi_check_send_delayed_tm_resp(session);
2410                         }
2411
2412                         spin_unlock(&session->sn_lock);
2413
2414                         iscsi_cmnd_exec(cmnd);
2415
2416                         spin_lock(&session->sn_lock);
2417
2418                         if (list_empty(&session->pending_list))
2419                                 break;
2420                         cmnd = list_entry(session->pending_list.next,
2421                                           struct iscsi_cmnd,
2422                                           pending_list_entry);
2423                         if (cmnd->pdu.bhs.sn != cmd_sn)
2424                                 break;
2425
2426                         list_del(&cmnd->pending_list_entry);
2427                         cmnd->pending = 0;
2428
2429                         TRACE_DBG("Processing pending cmd %p (cmd_sn %u)",
2430                                 cmnd, cmd_sn);
2431                 }
2432         } else {
2433                 int drop = 0;
2434
2435                 TRACE_DBG("Pending cmd %p (cmd_sn %u, exp_cmd_sn %u)",
2436                         cmnd, cmd_sn, session->exp_cmd_sn);
2437
2438                 /*
2439                  * iSCSI RFC 3720: "The target MUST silently ignore any
2440                  * non-immediate command outside of [from ExpCmdSN to MaxCmdSN
2441                  * inclusive] range". But we won't honor the MaxCmdSN
2442                  * requirement, because, since we adjust MaxCmdSN from the
2443                  * separate write thread, rarely it is possible that initiator
2444                  * can legally send command with CmdSN>MaxSN. But it won't
2445                  * hurt anything, in the worst case it will lead to
2446                  * additional QUEUE FULL status.
2447                  */
2448
2449                 if (unlikely(before(cmd_sn, session->exp_cmd_sn))) {
2450                         PRINT_ERROR("Unexpected cmd_sn (%u,%u)", cmd_sn,
2451                                 session->exp_cmd_sn);
2452                         drop = 1;
2453                 }
2454
2455 #if 0
2456                 if (unlikely(after(cmd_sn, session->exp_cmd_sn +
2457                                         iscsi_get_allowed_cmds(session)))) {
2458                         TRACE_MGMT_DBG("Too large cmd_sn %u (exp_cmd_sn %u, "
2459                                 "max_sn %u)", cmd_sn, session->exp_cmd_sn,
2460                                 iscsi_get_allowed_cmds(session));
2461                 }
2462 #endif
2463
2464                 spin_unlock(&session->sn_lock);
2465
2466                 if (unlikely(drop)) {
2467                         req_cmnd_release_force(cmnd,
2468                                                ISCSI_FORCE_RELEASE_WRITE);
2469                         goto out;
2470                 }
2471
2472                 if (unlikely(cmnd->tm_aborted)) {
2473                         struct iscsi_cmnd *tm_clone;
2474
2475                         TRACE_MGMT_DBG("Pending aborted cmnd %p, creating TM "
2476                                 "clone (scst cmd %p, state %d)", cmnd,
2477                                 cmnd->scst_cmd, cmnd->scst_state);
2478
2479                         tm_clone = cmnd_alloc(cmnd->conn, NULL);
2480                         if (tm_clone != NULL) {
2481                                 tm_clone->tm_aborted = 1;
2482                                 tm_clone->pdu = cmnd->pdu;
2483
2484                                 TRACE_MGMT_DBG("TM clone %p created",
2485                                                tm_clone);
2486
2487                                 iscsi_cmnd_exec(cmnd);
2488                                 cmnd = tm_clone;
2489                         } else
2490                                 PRINT_ERROR("%s", "Unable to create TM clone");
2491                 }
2492
2493                 spin_lock(&session->sn_lock);
2494                 list_for_each(entry, &session->pending_list) {
2495                         struct iscsi_cmnd *tmp =
2496                                 list_entry(entry, struct iscsi_cmnd,
2497                                            pending_list_entry);
2498                         if (before(cmd_sn, tmp->pdu.bhs.sn))
2499                                 break;
2500                 }
2501                 list_add_tail(&cmnd->pending_list_entry, entry);
2502                 cmnd->pending = 1;
2503         }
2504
2505         spin_unlock(&session->sn_lock);
2506 out:
2507         return;
2508 }
2509
2510 static int check_segment_length(struct iscsi_cmnd *cmnd)
2511 {
2512         struct iscsi_conn *conn = cmnd->conn;
2513         struct iscsi_session *session = conn->session;
2514
2515         if (unlikely(cmnd->pdu.datasize > session->sess_param.max_recv_data_length)) {
2516                 PRINT_ERROR("Initiator %s violated negotiated parameters: "
2517                         "data too long (ITT %x, datasize %u, "
2518                         "max_recv_data_length %u", session->initiator_name,
2519                         cmnd_itt(cmnd), cmnd->pdu.datasize,
2520                         session->sess_param.max_recv_data_length);
2521                 mark_conn_closed(conn);
2522                 return -EINVAL;
2523         }
2524         return 0;
2525 }
2526
2527 int cmnd_rx_start(struct iscsi_cmnd *cmnd)
2528 {
2529         struct iscsi_conn *conn = cmnd->conn;
2530         int res, rc;
2531
2532         iscsi_dump_pdu(&cmnd->pdu);
2533
2534         res = check_segment_length(cmnd);
2535         if (res != 0)
2536                 goto out;
2537
2538         switch (cmnd_opcode(cmnd)) {
2539         case ISCSI_OP_NOOP_OUT:
2540                 rc = noop_out_start(cmnd);
2541                 break;
2542         case ISCSI_OP_SCSI_CMD:
2543                 rc = cmnd_insert_hash(cmnd);
2544                 if (likely(rc == 0)) {
2545                         res = scsi_cmnd_start(cmnd);
2546                         if (unlikely(res != 0))
2547                                 goto out;
2548                 }
2549                 break;
2550         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2551                 rc = cmnd_insert_hash(cmnd);
2552                 break;
2553         case ISCSI_OP_SCSI_DATA_OUT:
2554                 res = data_out_start(conn, cmnd);
2555                 rc = 0; /* to avoid compiler warning */
2556                 if (unlikely(res != 0))
2557                         goto out;
2558                 break;
2559         case ISCSI_OP_LOGOUT_CMD:
2560                 rc = cmnd_insert_hash(cmnd);
2561                 break;
2562         case ISCSI_OP_TEXT_CMD:
2563         case ISCSI_OP_SNACK_CMD:
2564                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2565                 break;
2566         default:
2567                 rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
2568                 break;
2569         }
2570
2571         if (unlikely(rc < 0)) {
2572                 struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
2573                 PRINT_ERROR("Error %d (iSCSI opcode %x, ITT %x, op %x)", rc,
2574                         cmnd_opcode(cmnd), cmnd_itt(cmnd),
2575                         (cmnd_opcode(cmnd) == ISCSI_OP_SCSI_CMD ?
2576                                 hdr->scb[0] : -1));
2577                 iscsi_cmnd_reject(cmnd, -rc);
2578         }
2579
2580 out:
2581         TRACE_EXIT_RES(res);
2582         return res;
2583 }
2584
2585 void cmnd_rx_end(struct iscsi_cmnd *cmnd)
2586 {
2587         TRACE_ENTRY();
2588
2589         TRACE_DBG("%p:%x", cmnd, cmnd_opcode(cmnd));
2590
2591         if (unlikely(cmnd->rejected))
2592                 goto out_rejected;
2593
2594 cont:
2595         switch (cmnd_opcode(cmnd)) {
2596         case ISCSI_OP_SCSI_CMD:
2597         case ISCSI_OP_NOOP_OUT:
2598         case ISCSI_OP_SCSI_TASK_MGT_MSG:
2599         case ISCSI_OP_LOGOUT_CMD:
2600                 iscsi_session_push_cmnd(cmnd);
2601                 break;
2602         case ISCSI_OP_SCSI_DATA_OUT:
2603                 data_out_end(cmnd);
2604                 break;
2605         default:
2606                 PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
2607                 req_cmnd_release(cmnd);
2608                 break;
2609         }
2610
2611 out:
2612         TRACE_EXIT();
2613         return;
2614
2615 out_rejected:
2616         switch (cmnd->reject_reason) {
2617         default:
2618                 PRINT_ERROR("Unexpected reject reason %d",
2619                             cmnd->reject_reason);
2620                 /* go through */
2621         case ISCSI_REJECT_CMD:
2622         case ISCSI_REJECT_DATA:
2623                 req_cmnd_release(cmnd);
2624                 break;
2625         case ISCSI_REJECT_SCSI_CMD:
2626                 goto cont;
2627         }
2628         goto out;
2629 }
2630
2631 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
2632 static int iscsi_alloc_data_buf(struct scst_cmd *cmd)
2633 {
2634         /*
2635          * sock->ops->sendpage() is async zero copy operation,
2636          * so we must be sure not to free and reuse
2637          * the command's buffer before the sending was completed
2638          * by the network layers. It is possible only if we
2639          * don't use SGV cache.
2640          */
2641         EXTRACHECKS_BUG_ON(!(scst_cmd_get_data_direction(cmd) & SCST_DATA_READ));
2642         scst_cmd_set_no_sgv(cmd);
2643         return 1;
2644 }
2645 #endif
2646
2647 static inline void iscsi_set_state_wake_up(struct iscsi_cmnd *req,
2648         int new_state)
2649 {
2650         if (req->conn->rx_task == current)
2651                 req->scst_state = new_state;
2652         else {
2653                 /*
2654                  * We wait for the state change without any protection, so
2655                  * without cmnd_get() it is possible that req will die
2656                  * "immediately" after the state assignment and
2657                  * iscsi_make_conn_rd_active() will operate on dead data.
2658                  * We use the ordered version of cmnd_get(), because "get"
2659                  * must be done before the state assignment.
2660                  */
2661                 cmnd_get_ordered(req);
2662                 req->scst_state = new_state;
2663                 iscsi_make_conn_rd_active(req->conn);
2664                 if (unlikely(req->conn->closing)) {
2665                         TRACE_DBG("Waking up closing conn %p", req->conn);
2666                         wake_up(&req->conn->read_state_waitQ);
2667                 }
2668                 cmnd_put(req);
2669         }
2670         return;
2671 }
2672
2673 static void iscsi_preprocessing_done(struct scst_cmd *scst_cmd)
2674 {
2675         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
2676                                 scst_cmd_get_tgt_priv(scst_cmd);
2677
2678         TRACE_DBG("req %p", req);
2679
2680         EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_RX_CMD);
2681
2682         iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_AFTER_PREPROC);
2683         return;
2684 }
2685
2686 /*
2687  * No locks.
2688  *
2689  * IMPORTANT! Connection conn must be protected by additional conn_get()
2690  * upon entrance in this function, because otherwise it could be destroyed
2691  * inside as a result of iscsi_send(), which releases sent commands.
2692  */
2693 static void iscsi_try_local_processing(struct iscsi_conn *conn)
2694 {
2695         int local;
2696
2697         TRACE_ENTRY();
2698
2699         spin_lock_bh(&iscsi_wr_lock);
2700         switch (conn->wr_state) {
2701         case ISCSI_CONN_WR_STATE_IN_LIST:
2702                 list_del(&conn->wr_list_entry);
2703                 /* go through */
2704         case ISCSI_CONN_WR_STATE_IDLE:
2705 #ifdef CONFIG_SCST_EXTRACHECKS
2706                 conn->wr_task = current;
2707 #endif
2708                 conn->wr_state = ISCSI_CONN_WR_STATE_PROCESSING;
2709                 conn->wr_space_ready = 0;
2710                 local = 1;
2711                 break;
2712         default:
2713                 local = 0;
2714                 break;
2715         }
2716         spin_unlock_bh(&iscsi_wr_lock);
2717
2718         if (local) {
2719                 int rc = 1;
2720
2721                 if (test_write_ready(conn))
2722                         rc = iscsi_send(conn);
2723
2724                 spin_lock_bh(&iscsi_wr_lock);
2725 #ifdef CONFIG_SCST_EXTRACHECKS
2726                 conn->wr_task = NULL;
2727 #endif
2728                 if ((rc <= 0) || test_write_ready(conn)) {
2729                         list_add_tail(&conn->wr_list_entry, &iscsi_wr_list);
2730                         conn->wr_state = ISCSI_CONN_WR_STATE_IN_LIST;
2731                         wake_up(&iscsi_wr_waitQ);
2732                 } else
2733                         conn->wr_state = ISCSI_CONN_WR_STATE_IDLE;
2734                 spin_unlock_bh(&iscsi_wr_lock);
2735         }
2736
2737         TRACE_EXIT();
2738         return;
2739 }
2740
2741 static int iscsi_xmit_response(struct scst_cmd *scst_cmd)
2742 {
2743         int is_send_status = scst_cmd_get_is_send_status(scst_cmd);
2744         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
2745                                         scst_cmd_get_tgt_priv(scst_cmd);
2746         struct iscsi_conn *conn = req->conn;
2747         int status = scst_cmd_get_status(scst_cmd);
2748         u8 *sense = scst_cmd_get_sense_buffer(scst_cmd);
2749         int sense_len = scst_cmd_get_sense_buffer_len(scst_cmd);
2750         int old_state = req->scst_state;
2751
2752         if (scst_cmd_atomic(scst_cmd))
2753                 return SCST_TGT_RES_NEED_THREAD_CTX;
2754
2755         scst_cmd_set_tgt_priv(scst_cmd, NULL);
2756
2757         req->tm_aborted |= scst_cmd_aborted(scst_cmd) ? 1 : 0;
2758         if (unlikely(req->tm_aborted)) {
2759                 TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
2760                         req->scst_cmd);
2761
2762                 scst_set_delivery_status(req->scst_cmd,
2763                         SCST_CMD_DELIVERY_ABORTED);
2764
2765                 if (old_state == ISCSI_CMD_STATE_RESTARTED) {
2766                         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2767                         req_cmnd_release_force(req, ISCSI_FORCE_RELEASE_WRITE);
2768                 } else
2769                         iscsi_set_state_wake_up(req,
2770                                                 ISCSI_CMD_STATE_PROCESSED);
2771
2772                 goto out;
2773         }
2774
2775         if (unlikely(old_state != ISCSI_CMD_STATE_RESTARTED)) {
2776                 TRACE_DBG("req %p on %d state", req, old_state);
2777
2778                 /*
2779                  * We could preliminary have finished req before we knew its
2780                  * device, so check if we return correct sense format.
2781                  */
2782                 scst_check_convert_sense(scst_cmd);
2783
2784                 create_status_rsp(req, status, sense, sense_len);
2785
2786                 switch (old_state) {
2787                 case ISCSI_CMD_STATE_RX_CMD:
2788                 case ISCSI_CMD_STATE_AFTER_PREPROC:
2789                         break;
2790                 default:
2791                         sBUG();
2792                 }
2793
2794                 iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
2795                 goto out;
2796         }
2797
2798         req->scst_state = ISCSI_CMD_STATE_PROCESSED;
2799
2800         req->bufflen = scst_cmd_get_resp_data_len(scst_cmd);
2801         req->sg = scst_cmd_get_sg(scst_cmd);
2802         req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
2803
2804         TRACE_DBG("req %p, is_send_status=%x, req->bufflen=%d, req->sg=%p, "
2805                 "req->sg_cnt %d", req, is_send_status, req->bufflen, req->sg,
2806                 req->sg_cnt);
2807
2808         if (unlikely((req->bufflen != 0) && !is_send_status)) {
2809                 PRINT_CRIT_ERROR("%s", "Sending DATA without STATUS is "
2810                         "unsupported");
2811                 scst_set_cmd_error(scst_cmd,
2812                         SCST_LOAD_SENSE(scst_sense_hardw_error));
2813                 sBUG();
2814         }
2815
2816         if (req->bufflen != 0) {
2817                 /*
2818                  * Check above makes sure that is_send_status is set,
2819                  * so status is valid here, but in future that could change.
2820                  * ToDo
2821                  */
2822                 if ((status != SAM_STAT_CHECK_CONDITION) &&
2823                     ((cmnd_hdr(req)->flags & (ISCSI_CMD_WRITE|ISCSI_CMD_READ)) !=
2824                                 (ISCSI_CMD_WRITE|ISCSI_CMD_READ))) {
2825                         send_data_rsp(req, status, is_send_status);
2826                 } else {
2827                         struct iscsi_cmnd *rsp;
2828                         send_data_rsp(req, 0, 0);
2829                         if (is_send_status) {
2830                                 rsp = create_status_rsp(req, status, sense,
2831                                         sense_len);
2832                                 iscsi_set_resid(req, rsp, true);
2833                                 iscsi_cmnd_init_write(rsp,
2834                                         ISCSI_INIT_WRITE_REMOVE_HASH);
2835                         }
2836                 }
2837         } else if (is_send_status) {
2838                 struct iscsi_cmnd *rsp;
2839                 rsp = create_status_rsp(req, status, sense, sense_len);
2840                 iscsi_set_resid(req, rsp, false);
2841                 iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH);
2842         }
2843 #ifdef CONFIG_SCST_EXTRACHECKS
2844         else
2845                 sBUG();
2846 #endif
2847
2848         /*
2849          * "_ordered" here to protect from reorder, which can lead to
2850          * preliminary connection destroy in req_cmnd_release(). Just in
2851          * case, actually, because reordering shouldn't go so far, but who
2852          * knows..
2853          */
2854         conn_get_ordered(conn);
2855         req_cmnd_release(req);
2856         iscsi_try_local_processing(conn);
2857         conn_put(conn);
2858
2859 out:
2860         return SCST_TGT_RES_SUCCESS;
2861 }
2862
2863 /* Called under sn_lock */
2864 static bool iscsi_is_delay_tm_resp(struct iscsi_cmnd *rsp)
2865 {
2866         bool res = 0;
2867         struct iscsi_task_mgt_hdr *req_hdr =
2868                 (struct iscsi_task_mgt_hdr *)&rsp->parent_req->pdu.bhs;
2869         int function = req_hdr->function & ISCSI_FUNCTION_MASK;
2870         struct iscsi_session *sess = rsp->conn->session;
2871
2872         TRACE_ENTRY();
2873
2874         /* This should be checked for immediate TM commands as well */
2875
2876         switch (function) {
2877         default:
2878                 if (before(sess->exp_cmd_sn, req_hdr->cmd_sn))
2879                         res = 1;
2880                 break;
2881         }
2882
2883         TRACE_EXIT_RES(res);
2884         return res;
2885 }
2886
2887 /* Called under sn_lock, but might drop it inside, then reaquire */
2888 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess)
2889         __acquires(&sn_lock)
2890         __releases(&sn_lock)
2891 {
2892         struct iscsi_cmnd *tm_rsp = sess->tm_rsp;
2893
2894         TRACE_ENTRY();
2895
2896         if (tm_rsp == NULL)
2897                 goto out;
2898
2899         if (iscsi_is_delay_tm_resp(tm_rsp))
2900                 goto out;
2901
2902         TRACE(TRACE_MGMT_MINOR, "Sending delayed rsp %p", tm_rsp);
2903
2904         sess->tm_rsp = NULL;
2905         sess->tm_active--;
2906
2907         spin_unlock(&sess->sn_lock);
2908
2909         sBUG_ON(sess->tm_active < 0);
2910
2911         iscsi_cmnd_init_write(tm_rsp,
2912                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2913
2914         spin_lock(&sess->sn_lock);
2915
2916 out:
2917         TRACE_EXIT();
2918         return;
2919 }
2920
2921 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status)
2922 {
2923         struct iscsi_cmnd *rsp;
2924         struct iscsi_task_mgt_hdr *req_hdr =
2925                                 (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
2926         struct iscsi_task_rsp_hdr *rsp_hdr;
2927         struct iscsi_session *sess = req->conn->session;
2928         int fn = req_hdr->function & ISCSI_FUNCTION_MASK;
2929
2930         TRACE_ENTRY();
2931
2932         TRACE_MGMT_DBG("TM req %p finished", req);
2933         TRACE((req_hdr->function == ISCSI_FUNCTION_ABORT_TASK) ?
2934                          TRACE_MGMT_MINOR : TRACE_MGMT,
2935                 "TM fn %d finished, status %d", fn, status);
2936
2937         rsp = iscsi_cmnd_create_rsp_cmnd(req);
2938         rsp_hdr = (struct iscsi_task_rsp_hdr *)&rsp->pdu.bhs;
2939
2940         rsp_hdr->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
2941         rsp_hdr->flags = ISCSI_FLG_FINAL;
2942         rsp_hdr->itt = req_hdr->itt;
2943         rsp_hdr->response = status;
2944
2945         if (fn == ISCSI_FUNCTION_TARGET_COLD_RESET) {
2946                 rsp->should_close_conn = 1;
2947                 rsp->should_close_all_conn = 1;
2948         }
2949
2950         sBUG_ON(sess->tm_rsp != NULL);
2951
2952         spin_lock(&sess->sn_lock);
2953         if (iscsi_is_delay_tm_resp(rsp)) {
2954                 TRACE(TRACE_MGMT_MINOR, "Delaying TM fn %d response %p "
2955                         "(req %p), because not all affected commands received "
2956                         "(TM cmd sn %u, exp sn %u)",
2957                         req_hdr->function & ISCSI_FUNCTION_MASK, rsp, req,
2958                         req_hdr->cmd_sn, sess->exp_cmd_sn);
2959                 sess->tm_rsp = rsp;
2960                 spin_unlock(&sess->sn_lock);
2961                 goto out_release;
2962         }
2963         sess->tm_active--;
2964         spin_unlock(&sess->sn_lock);
2965
2966         sBUG_ON(sess->tm_active < 0);
2967
2968         iscsi_cmnd_init_write(rsp,
2969                 ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
2970
2971 out_release:
2972         req_cmnd_release(req);
2973
2974         TRACE_EXIT();
2975         return;
2976 }
2977
2978 static inline int iscsi_get_mgmt_response(int status)
2979 {
2980         switch (status) {
2981         case SCST_MGMT_STATUS_SUCCESS:
2982                 return ISCSI_RESPONSE_FUNCTION_COMPLETE;
2983
2984         case SCST_MGMT_STATUS_TASK_NOT_EXIST:
2985                 return ISCSI_RESPONSE_UNKNOWN_TASK;
2986
2987         case SCST_MGMT_STATUS_LUN_NOT_EXIST:
2988                 return ISCSI_RESPONSE_UNKNOWN_LUN;
2989
2990         case SCST_MGMT_STATUS_FN_NOT_SUPPORTED:
2991                 return ISCSI_RESPONSE_FUNCTION_UNSUPPORTED;
2992
2993         case SCST_MGMT_STATUS_REJECTED:
2994         case SCST_MGMT_STATUS_FAILED:
2995         default:
2996                 return ISCSI_RESPONSE_FUNCTION_REJECTED;
2997         }
2998 }
2999
3000 static void iscsi_task_mgmt_fn_done(struct scst_mgmt_cmd *scst_mcmd)
3001 {
3002         int fn = scst_mgmt_cmd_get_fn(scst_mcmd);
3003         struct iscsi_cmnd *req = (struct iscsi_cmnd *)
3004                                 scst_mgmt_cmd_get_tgt_priv(scst_mcmd);
3005         int status =
3006                 iscsi_get_mgmt_response(scst_mgmt_cmd_get_status(scst_mcmd));
3007
3008         TRACE_MGMT_DBG("req %p, scst_mcmd %p, fn %d, scst status %d",
3009                 req, scst_mcmd, fn, scst_mgmt_cmd_get_status(scst_mcmd));
3010
3011         switch (fn) {
3012         case SCST_NEXUS_LOSS_SESS:
3013         case SCST_ABORT_ALL_TASKS_SESS:
3014                 /* They are internal */
3015                 break;
3016         default:
3017                 iscsi_send_task_mgmt_resp(req, status);
3018                 scst_mgmt_cmd_set_tgt_priv(scst_mcmd, NULL);
3019                 break;
3020         }
3021         return;
3022 }
3023
3024 static int iscsi_scsi_aen(struct scst_aen *aen)
3025 {
3026         int res = SCST_AEN_RES_SUCCESS;
3027         uint64_t lun = scst_aen_get_lun(aen);
3028         const uint8_t *sense = scst_aen_get_sense(aen);
3029         int sense_len = scst_aen_get_sense_len(aen);
3030         struct iscsi_session *sess = scst_sess_get_tgt_priv(
3031                                         scst_aen_get_sess(aen));
3032         struct iscsi_conn *conn;
3033         bool found;
3034         struct iscsi_cmnd *fake_req, *rsp;
3035         struct iscsi_async_msg_hdr *rsp_hdr;
3036         struct scatterlist *sg;
3037
3038         TRACE_ENTRY();
3039
3040         TRACE_MGMT_DBG("SCSI AEN to sess %p (initiator %s)", sess,
3041                 sess->initiator_name);
3042
3043         mutex_lock(&sess->target->target_mutex);
3044
3045         found = false;
3046         list_for_each_entry_reverse(conn, &sess->conn_list, conn_list_entry) {
3047                 if (!test_bit(ISCSI_CONN_SHUTTINGDOWN, &conn->conn_aflags) &&
3048                     (conn->conn_reinst_successor == NULL)) {
3049                         found = true;
3050                         break;
3051                 }
3052         }
3053         if (!found) {
3054                 TRACE_MGMT_DBG("Unable to find alive conn for sess %p", sess);
3055                 goto out_err;
3056         }
3057
3058         /* Create a fake request */
3059         fake_req = cmnd_alloc(conn, NULL);
3060         if (fake_req == NULL) {
3061                 PRINT_ERROR("%s", "Unable to alloc fake AEN request");
3062                 goto out_err;
3063         }
3064
3065         mutex_unlock(&sess->target->target_mutex);
3066
3067         rsp = iscsi_cmnd_create_rsp_cmnd(fake_req);
3068         if (rsp == NULL) {
3069                 PRINT_ERROR("%s", "Unable to alloc AEN rsp");
3070                 goto out_err_free_req;
3071         }
3072
3073         fake_req->scst_state = ISCSI_CMD_STATE_AEN;
3074         fake_req->scst_aen = aen;
3075
3076         rsp_hdr = (struct iscsi_async_msg_hdr *)&rsp->pdu.bhs;
3077
3078         rsp_hdr->opcode = ISCSI_OP_ASYNC_MSG;
3079         rsp_hdr->flags = ISCSI_FLG_FINAL;
3080         rsp_hdr->lun = lun; /* it's already in SCSI form */
3081         rsp_hdr->ffffffff = 0xffffffff;
3082         rsp_hdr->async_event = ISCSI_ASYNC_SCSI;
3083
3084         sg = rsp->sg = rsp->rsp_sg;
3085         rsp->sg_cnt = 2;
3086         rsp->own_sg = 1;
3087
3088         sg_init_table(sg, 2);
3089         sg_set_buf(&sg[0], &rsp->sense_hdr, sizeof(rsp->sense_hdr));
3090         sg_set_buf(&sg[1], sense, sense_len);
3091
3092         rsp->sense_hdr.length = cpu_to_be16(sense_len);
3093         rsp->pdu.datasize = sizeof(rsp->sense_hdr) + sense_len;
3094         rsp->bufflen = rsp->pdu.datasize;
3095
3096         iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_WAKE);
3097
3098         req_cmnd_release(fake_req);
3099
3100 out:
3101         TRACE_EXIT_RES(res);
3102         return res;
3103
3104 out_err_free_req:
3105         req_cmnd_release(fake_req);
3106
3107 out_err:
3108         mutex_unlock(&sess->target->target_mutex);
3109         res = SCST_AEN_RES_FAILED;
3110         goto out;
3111 }
3112
3113 static int iscsi_report_aen(struct scst_aen *aen)
3114 {
3115         int res;
3116         int event_fn = scst_aen_get_event_fn(aen);
3117
3118         TRACE_ENTRY();
3119
3120         switch (event_fn) {
3121         case SCST_AEN_SCSI:
3122                 res = iscsi_scsi_aen(aen);
3123                 break;
3124         default:
3125                 TRACE_MGMT_DBG("Unsupported AEN %d", event_fn);
3126                 res = SCST_AEN_RES_NOT_SUPPORTED;
3127                 break;
3128         }
3129
3130         TRACE_EXIT_RES(res);
3131         return res;
3132 }
3133
3134 static int iscsi_target_detect(struct scst_tgt_template *templ)
3135 {
3136         /* Nothing to do */
3137         return 0;
3138 }
3139
3140 static int iscsi_target_release(struct scst_tgt *scst_tgt)
3141 {
3142         /* Nothing to do */
3143         return 0;
3144 }
3145
3146 #ifndef CONFIG_SCST_PROC
3147 static struct scst_trace_log iscsi_local_trace_tbl[] = {
3148     { TRACE_D_READ,             "d_read" },
3149     { TRACE_D_WRITE,            "d_write" },
3150     { TRACE_CONN_OC,            "conn" },
3151     { TRACE_CONN_OC_DBG,        "conn_dbg" },
3152     { TRACE_D_IOV,              "iov" },
3153     { TRACE_D_DUMP_PDU,         "pdu" },
3154     { TRACE_NET_PG,             "net_page" },
3155     { 0,                        NULL }
3156 };
3157
3158 #define ISCSI_TRACE_TLB_HELP    ", d_read, d_write, conn, conn_dbg, iov, pdu, net_page"
3159 #endif
3160
3161 struct scst_tgt_template iscsi_template = {
3162         .name = "iscsi",
3163         .sg_tablesize = 0xFFFF /* no limit */,
3164         .threads_num = 0,
3165         .no_clustering = 1,
3166         .xmit_response_atomic = 0,
3167 #ifndef CONFIG_SCST_PROC
3168         .tgtt_attrs = iscsi_attrs,
3169         .tgt_attrs = iscsi_tgt_attrs,
3170         .sess_attrs = iscsi_sess_attrs,
3171         .enable_tgt = iscsi_enable_target,
3172         .is_tgt_enabled = iscsi_is_target_enabled,
3173 #endif
3174 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
3175         .default_trace_flags = ISCSI_DEFAULT_LOG_FLAGS,
3176         .trace_flags = &trace_flag,
3177 #ifndef CONFIG_SCST_PROC
3178         .trace_tbl = iscsi_local_trace_tbl,
3179         .trace_tbl_help = ISCSI_TRACE_TLB_HELP,
3180 #endif
3181 #endif
3182         .detect = iscsi_target_detect,
3183         .release = iscsi_target_release,
3184         .xmit_response = iscsi_xmit_response,
3185 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3186         .alloc_data_buf = iscsi_alloc_data_buf,
3187 #endif
3188         .preprocessing_done = iscsi_preprocessing_done,
3189         .pre_exec = iscsi_pre_exec,
3190         .task_mgmt_affected_cmds_done = iscsi_task_mgmt_affected_cmds_done,
3191         .task_mgmt_fn_done = iscsi_task_mgmt_fn_done,
3192         .report_aen = iscsi_report_aen,
3193 };
3194
3195 static __init int iscsi_run_threads(int count, char *name, int (*fn)(void *))
3196 {
3197         int res = 0;
3198         int i;
3199         struct iscsi_thread_t *thr;
3200
3201         for (i = 0; i < count; i++) {
3202                 thr = kmalloc(sizeof(*thr), GFP_KERNEL);
3203                 if (!thr) {
3204                         res = -ENOMEM;
3205                         PRINT_ERROR("Failed to allocate thr %d", res);
3206                         goto out;
3207                 }
3208                 thr->thr = kthread_run(fn, NULL, "%s%d", name, i);
3209                 if (IS_ERR(thr->thr)) {
3210                         res = PTR_ERR(thr->thr);
3211                         PRINT_ERROR("kthread_create() failed: %d", res);
3212                         kfree(thr);
3213                         goto out;
3214                 }
3215                 list_add_tail(&thr->threads_list_entry, &iscsi_threads_list);
3216         }
3217
3218 out:
3219         return res;
3220 }
3221
3222 static void iscsi_stop_threads(void)
3223 {
3224         struct iscsi_thread_t *t, *tmp;
3225
3226         list_for_each_entry_safe(t, tmp, &iscsi_threads_list,
3227                                 threads_list_entry) {
3228                 int rc = kthread_stop(t->thr);
3229                 if (rc < 0)
3230                         TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
3231                 list_del(&t->threads_list_entry);
3232                 kfree(t);
3233         }
3234         return;
3235 }
3236
3237 static int __init iscsi_init(void)
3238 {
3239         int err = 0;
3240         int num;
3241
3242         PRINT_INFO("iSCSI SCST Target - version %s", ISCSI_VERSION_STRING);
3243
3244         dummy_page = alloc_pages(GFP_KERNEL, 0);
3245         if (dummy_page == NULL) {
3246                 PRINT_ERROR("%s", "Dummy page allocation failed");
3247                 goto out;
3248         }
3249
3250         sg_init_table(&dummy_sg, 1);
3251         sg_set_page(&dummy_sg, dummy_page, PAGE_SIZE, 0);
3252
3253 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3254         err = net_set_get_put_page_callbacks(iscsi_get_page_callback,
3255                         iscsi_put_page_callback);
3256         if (err != 0) {
3257                 PRINT_INFO("Unable to set page callbackes: %d", err);
3258                 goto out_free_dummy;
3259         }
3260 #else
3261 #ifndef GENERATING_UPSTREAM_PATCH
3262         PRINT_WARNING("%s",
3263                 "CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION "
3264                 "not enabled in your kernel. ISCSI-SCST will be working with "
3265                 "not the best performance. Refer README file for details.");
3266 #endif
3267 #endif
3268
3269         ctr_major = register_chrdev(0, ctr_name, &ctr_fops);
3270         if (ctr_major < 0) {
3271                 PRINT_ERROR("failed to register the control device %d",
3272                             ctr_major);
3273                 err = ctr_major;
3274                 goto out_callb;
3275         }
3276
3277         err = event_init();
3278         if (err < 0)
3279                 goto out_reg;
3280
3281         iscsi_cmnd_cache = KMEM_CACHE(iscsi_cmnd, SCST_SLAB_FLAGS);
3282         if (!iscsi_cmnd_cache) {
3283                 err = -ENOMEM;
3284                 goto out_event;
3285         }
3286
3287         err = scst_register_target_template(&iscsi_template);
3288         if (err < 0)
3289                 goto out_kmem;
3290
3291         iscsi_template_registered = 1;
3292
3293 #ifdef CONFIG_SCST_PROC
3294         err = iscsi_procfs_init();
3295         if (err < 0)
3296                 goto out_reg_tmpl;
3297 #endif
3298
3299         num = max((int)num_online_cpus(), 2);
3300
3301         err = iscsi_run_threads(num, "iscsird", istrd);
3302         if (err != 0)
3303                 goto out_thr;
3304
3305         err = iscsi_run_threads(num, "iscsiwr", istwr);
3306         if (err != 0)
3307                 goto out_thr;
3308
3309 out:
3310         return err;
3311
3312 out_thr:
3313 #ifdef CONFIG_SCST_PROC
3314         iscsi_procfs_exit();
3315 #endif
3316         iscsi_stop_threads();
3317
3318 #ifdef CONFIG_SCST_PROC
3319 out_reg_tmpl:
3320 #endif
3321         scst_unregister_target_template(&iscsi_template);
3322
3323 out_kmem:
3324         kmem_cache_destroy(iscsi_cmnd_cache);
3325
3326 out_event:
3327         event_exit();
3328
3329 out_reg:
3330         unregister_chrdev(ctr_major, ctr_name);
3331
3332 out_callb:
3333 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3334         net_set_get_put_page_callbacks(NULL, NULL);
3335
3336 out_free_dummy:
3337 #endif
3338         __free_pages(dummy_page, 0);
3339         goto out;
3340 }
3341
3342 static void __exit iscsi_exit(void)
3343 {
3344         iscsi_stop_threads();
3345
3346         unregister_chrdev(ctr_major, ctr_name);
3347
3348 #ifdef CONFIG_SCST_PROC
3349         iscsi_procfs_exit();
3350 #endif
3351         event_exit();
3352
3353         kmem_cache_destroy(iscsi_cmnd_cache);
3354
3355         scst_unregister_target_template(&iscsi_template);
3356
3357 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
3358         net_set_get_put_page_callbacks(NULL, NULL);
3359 #endif
3360
3361         __free_pages(dummy_page, 0);
3362         return;
3363 }
3364
3365 module_init(iscsi_init);
3366 module_exit(iscsi_exit);
3367
3368 MODULE_LICENSE("GPL");