A huge chunk of related to each other changes, which had to be tested together.
authorvlnb <vlnb@d57e44dd-8a1f-0410-8b47-8ef2f437770f>
Wed, 6 Jan 2010 13:02:22 +0000 (13:02 +0000)
committervlnb <vlnb@d57e44dd-8a1f-0410-8b47-8ef2f437770f>
Wed, 6 Jan 2010 13:02:22 +0000 (13:02 +0000)
iSCSI-SCST: A huge improvements in errors recovery and iSCSI RFC complaince as well as performance. Also:

 - Fixes and improvements for MaxOutstandingR2T>1

 - Flow control tracing added.

 - Cleanups

SCST core:

 - Now for scst_cmd_init_stage1_done() commands preprocessing_done() is always called before xmit_response(), even in case of abort or error.

 - Fixed recently introduced bug, which can lead to sending responses for aborted commands after reply on the corresponding TM command already sent.

 - Flow control tracing added.

 - Now it is possible to call functions setting commands execution status (e.g., scst_set_cmd_error_status()) several times for the same command. Only the first call will be completed, other calls - ignored.

 - All commands are counted and shown in proc/sysfs now. Before only active, i.e. not yet executed commands, were counted and shown there.

git-svn-id: https://scst.svn.sourceforge.net/svnroot/scst/trunk@1431 d57e44dd-8a1f-0410-8b47-8ef2f437770f

17 files changed:
iscsi-scst/ToDo
iscsi-scst/kernel/config.c
iscsi-scst/kernel/conn.c
iscsi-scst/kernel/digest.c
iscsi-scst/kernel/iscsi.c
iscsi-scst/kernel/iscsi.h
iscsi-scst/kernel/iscsi_dbg.h
iscsi-scst/kernel/iscsi_hdr.h
iscsi-scst/kernel/nthread.c
iscsi-scst/kernel/session.c
iscsi-scst/usr/param.c
scst/include/scst.h
scst/include/scst_debug.h
scst/src/scst_lib.c
scst/src/scst_proc.c
scst/src/scst_sysfs.c
scst/src/scst_targ.c

index 091f97c..5754d42 100644 (file)
@@ -1,5 +1,3 @@
  - Fix support of ranges in parameters negotiation.
 
- - Fix SNACK command handling. Currently it violates iSCSI RFC.
-
  - Minor "ToDo"'s spread in the code.
index 15d4bc4..c7f7144 100644 (file)
@@ -26,7 +26,6 @@
 #define ISCSI_PROC_LOG_ENTRY_NAME      "trace_level"
 
 static struct scst_trace_log iscsi_local_trace_tbl[] = {
-    { TRACE_D_READ,            "d_read" },
     { TRACE_D_WRITE,           "d_write" },
     { TRACE_CONN_OC,           "conn" },
     { TRACE_CONN_OC_DBG,       "conn_dbg" },
@@ -605,7 +604,7 @@ void iscsi_dump_pdu(struct iscsi_pdu *pdu)
 
                buf = (void *)&pdu->bhs;
                printk(KERN_DEBUG "BHS: (%p,%zd)\n", buf, sizeof(pdu->bhs));
-               for (i = 0; i < sizeof(pdu->bhs); i++)
+               for (i = 0; i < (int)sizeof(pdu->bhs); i++)
                        iscsi_dump_char(*buf++, text, &pos);
                iscsi_dump_char(-1, text, &pos);
 
@@ -618,4 +617,25 @@ void iscsi_dump_pdu(struct iscsi_pdu *pdu)
                printk(KERN_DEBUG "Data: (%d)\n", pdu->datasize);
        }
 }
+
+unsigned long iscsi_get_flow_ctrl_or_mgmt_dbg_log_flag(struct iscsi_cmnd *cmnd)
+{
+       unsigned long flag;
+
+       if (cmnd->cmd_req != NULL)
+               cmnd = cmnd->cmd_req;
+
+       if (cmnd->scst_cmd == NULL)
+               flag = TRACE_MGMT_DEBUG;
+       else {
+               int status = scst_cmd_get_status(cmnd->scst_cmd);
+               if ((status == SAM_STAT_TASK_SET_FULL) ||
+                   (status == SAM_STAT_BUSY))
+                       flag = TRACE_FLOW_CONTROL;
+               else
+                       flag = TRACE_MGMT_DEBUG;
+       }
+       return flag;
+}
+
 #endif /* CONFIG_SCST_DEBUG */
index 1ef6f7b..ca2f0ae 100644 (file)
@@ -372,6 +372,9 @@ static void iscsi_write_space_ready(struct sock *sk)
 static void conn_rsp_timer_fn(unsigned long arg)
 {
        struct iscsi_conn *conn = (struct iscsi_conn *)arg;
+       struct iscsi_cmnd *cmnd;
+       unsigned long j = jiffies;
+       unsigned long timeout_time = j + ISCSI_RSP_SCHED_TIMEOUT;
 
        TRACE_ENTRY();
 
@@ -379,32 +382,107 @@ static void conn_rsp_timer_fn(unsigned long arg)
 
        spin_lock_bh(&conn->write_list_lock);
 
-       if (!list_empty(&conn->written_list)) {
-               struct iscsi_cmnd *wr_cmd = list_entry(conn->written_list.next,
-                               struct iscsi_cmnd, written_list_entry);
+       if (!list_empty(&conn->write_timeout_list)) {
+               cmnd = list_entry(conn->write_timeout_list.next,
+                               struct iscsi_cmnd, write_timeout_list_entry);
 
-               if (unlikely(time_after_eq(jiffies, wr_cmd->write_timeout))) {
+               if (unlikely(time_after_eq(j,
+                               cmnd->write_start + ISCSI_RSP_TIMEOUT))) {
                        if (!conn->closing) {
-                               PRINT_ERROR("Timeout sending data to initiator"
-                                       " %s (SID %llx), closing connection",
+                               PRINT_ERROR("Timeout sending data/waiting "
+                                       "for reply to/from initiator "
+                                       "%s (SID %llx), closing connection",
                                        conn->session->initiator_name,
                                        (long long unsigned int)
                                                conn->session->sid);
+                               /*
+                                * We must call mark_conn_closed() outside of
+                                * write_list_lock or we will have a circular
+                                * locking dependency with iscsi_rd_lock.
+                                */
+                               spin_unlock_bh(&conn->write_list_lock);
                                mark_conn_closed(conn);
+                               goto out;
                        }
-               } else {
+               } else if (!timer_pending(&conn->rsp_timer) ||
+                          time_after(conn->rsp_timer.expires, timeout_time)) {
                        TRACE_DBG("Restarting timer on %ld (conn %p)",
-                               wr_cmd->write_timeout, conn);
+                               timeout_time, conn);
                        /*
                         * Timer might have been restarted while we were
                         * entering here.
                         */
-                       mod_timer(&conn->rsp_timer, wr_cmd->write_timeout);
+                       mod_timer(&conn->rsp_timer, timeout_time);
                }
        }
 
        spin_unlock_bh(&conn->write_list_lock);
 
+       if (unlikely(conn->conn_tm_active)) {
+               TRACE_MGMT_DBG("TM active: making conn %p RD active", conn);
+               iscsi_make_conn_rd_active(conn);
+       }
+
+out:
+       TRACE_EXIT();
+       return;
+}
+
+void iscsi_check_tm_data_wait_timeouts(struct iscsi_conn *conn, bool force)
+{
+       struct iscsi_cmnd *cmnd;
+       unsigned long j = jiffies;
+       bool aborted_cmds_pending;
+       unsigned long timeout_time = j + ISCSI_TM_DATA_WAIT_SCHED_TIMEOUT;
+
+       TRACE_ENTRY();
+
+       TRACE_DBG_FLAG(force ? TRACE_CONN_OC_DBG : TRACE_MGMT_DEBUG,
+               "j %ld (TIMEOUT %d, force %d)", j,
+               ISCSI_TM_DATA_WAIT_SCHED_TIMEOUT, force);
+
+again:
+       spin_lock_bh(&iscsi_rd_lock);
+       spin_lock(&conn->write_list_lock);
+
+       aborted_cmds_pending = false;
+       list_for_each_entry(cmnd, &conn->write_timeout_list,
+                               write_timeout_list_entry) {
+               if (test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags)) {
+                       TRACE_DBG_FLAG(force ? TRACE_CONN_OC_DBG : TRACE_MGMT_DEBUG,
+                               "Checking aborted cmnd %p (scst_state %d, "
+                               "on_write_timeout_list %d, write_start %ld, "
+                               "r2t_len_to_receive %d)", cmnd,
+                               cmnd->scst_state, cmnd->on_write_timeout_list,
+                               cmnd->write_start, cmnd->r2t_len_to_receive);
+                       if ((cmnd->r2t_len_to_receive != 0) &&
+                           (time_after_eq(j, cmnd->write_start + ISCSI_TM_DATA_WAIT_TIMEOUT) ||
+                            force)) {
+                               spin_unlock(&conn->write_list_lock);
+                               spin_unlock_bh(&iscsi_rd_lock);
+                               iscsi_fail_data_waiting_cmnd(cmnd);
+                               goto again;
+                       }
+                       aborted_cmds_pending = true;
+               }
+       }
+
+       if (aborted_cmds_pending) {
+               if (!force &&
+                   (!timer_pending(&conn->rsp_timer) ||
+                    time_after(conn->rsp_timer.expires, timeout_time))) {
+                       TRACE_MGMT_DBG("Mod timer on %ld (conn %p)",
+                               timeout_time, conn);
+                       mod_timer(&conn->rsp_timer, timeout_time);
+               }
+       } else {
+               TRACE_MGMT_DBG("Clearing conn_tm_active for conn %p", conn);
+               conn->conn_tm_active = 0;
+       }
+
+       spin_unlock(&conn->write_list_lock);
+       spin_unlock_bh(&iscsi_rd_lock);
+
        TRACE_EXIT();
        return;
 }
@@ -515,7 +593,7 @@ static int conn_free(struct iscsi_conn *conn)
        sBUG_ON(atomic_read(&conn->conn_ref_cnt) != 0);
        sBUG_ON(!list_empty(&conn->cmd_list));
        sBUG_ON(!list_empty(&conn->write_list));
-       sBUG_ON(!list_empty(&conn->written_list));
+       sBUG_ON(!list_empty(&conn->write_timeout_list));
        sBUG_ON(conn->conn_reinst_successor != NULL);
        sBUG_ON(!test_bit(ISCSI_CONN_SHUTTINGDOWN, &conn->conn_aflags));
 
@@ -599,7 +677,7 @@ static int iscsi_conn_alloc(struct iscsi_session *session,
        INIT_LIST_HEAD(&conn->cmd_list);
        spin_lock_init(&conn->write_list_lock);
        INIT_LIST_HEAD(&conn->write_list);
-       INIT_LIST_HEAD(&conn->written_list);
+       INIT_LIST_HEAD(&conn->write_timeout_list);
        setup_timer(&conn->rsp_timer, conn_rsp_timer_fn, (unsigned long)conn);
        init_waitqueue_head(&conn->read_state_waitQ);
        init_completion(&conn->ready_to_free);
index 00cbfe2..bc56f6e 100644 (file)
@@ -157,12 +157,14 @@ int digest_rx_data(struct iscsi_cmnd *cmnd)
        u32 offset, crc;
        int res = 0;
 
-       if (unlikely(cmnd->rejected))
-               goto out;
-
        switch (cmnd_opcode(cmnd)) {
        case ISCSI_OP_SCSI_DATA_OUT:
                req = cmnd->cmd_req;
+               if (unlikely(req == NULL)) {
+                       /* It can be for prelim completed commands */
+                       req = cmnd;
+                       goto out;
+               }
                req_hdr = (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
                offset = be32_to_cpu(req_hdr->buffer_offset);
                break;
@@ -172,6 +174,16 @@ int digest_rx_data(struct iscsi_cmnd *cmnd)
                offset = 0;
        }
 
+       /*
+        * We need to skip the digest check for prelim completed commands,
+        * because we use shared data buffer for them, so, most likely, the
+        * check will fail. Plus, for such commands we sometimes don't have
+        * sg_cnt set correctly (cmnd_prepare_get_rejected_cmd_data() doesn't
+        * do it).
+        */
+       if (unlikely(req->prelim_compl_flags != 0))
+               goto out;
+
        crc = digest_data(req, cmnd->pdu.datasize, offset,
                cmnd->conn->rpadding);
 
index 0680d37..9ad9f0f 100644 (file)
@@ -34,7 +34,6 @@
 #endif
 
 #define ISCSI_INIT_WRITE_WAKE          0x1
-#define ISCSI_INIT_WRITE_REMOVE_HASH   0x2
 
 static int ctr_major;
 static char ctr_name[] = "iscsi-scst-ctl";
@@ -64,12 +63,40 @@ struct iscsi_thread_t {
 
 static LIST_HEAD(iscsi_threads_list);
 
-static void cmnd_remove_hash(struct iscsi_cmnd *cmnd);
+static void cmnd_remove_data_wait_hash(struct iscsi_cmnd *cmnd);
 static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status);
-static void cmnd_prepare_get_rejected_cmd_data(struct iscsi_cmnd *cmnd);
 static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess);
-static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd);
 static void req_cmnd_release(struct iscsi_cmnd *req);
+static int iscsi_preliminary_complete(struct iscsi_cmnd *req,
+       struct iscsi_cmnd *orig_req, bool get_data);
+static int cmnd_insert_data_wait_hash(struct iscsi_cmnd *cmnd);
+static void __cmnd_abort(struct iscsi_cmnd *cmnd);
+static void iscsi_set_resid(struct iscsi_cmnd *rsp, bool bufflen_set);
+static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags);
+
+static void req_del_from_write_timeout_list(struct iscsi_cmnd *req)
+{
+       struct iscsi_conn *conn;
+
+       TRACE_ENTRY();
+
+       if (!req->on_write_timeout_list)
+               goto out;
+
+       conn = req->conn;
+
+       TRACE_DBG("Deleting cmd %p from conn %p write_timeout_list",
+               req, conn);
+
+       spin_lock_bh(&conn->write_list_lock);
+       list_del(&req->write_timeout_list_entry);
+       req->on_write_timeout_list = 0;
+       spin_unlock_bh(&conn->write_list_lock);
+
+out:
+       TRACE_EXIT();
+       return;
+}
 
 static inline u32 cmnd_write_size(struct iscsi_cmnd *cmnd)
 {
@@ -93,7 +120,7 @@ static inline int cmnd_read_size(struct iscsi_cmnd *cmnd)
                ahdr = (struct iscsi_ahs_hdr *)cmnd->pdu.ahs;
                if (ahdr != NULL) {
                        uint8_t *p = (uint8_t *)ahdr;
-                       int size = 0;
+                       unsigned int size = 0;
                        do {
                                int s;
 
@@ -118,7 +145,23 @@ static inline int cmnd_read_size(struct iscsi_cmnd *cmnd)
 
 void iscsi_restart_cmnd(struct iscsi_cmnd *cmnd)
 {
-       EXTRACHECKS_BUG_ON(cmnd->data_waiting);
+       int status;
+
+       TRACE_ENTRY();
+
+       EXTRACHECKS_BUG_ON(cmnd->r2t_len_to_receive != 0);
+       EXTRACHECKS_BUG_ON(cmnd->r2t_len_to_send != 0);
+
+       iscsi_extracheck_is_rd_thread(cmnd->conn);
+
+       req_del_from_write_timeout_list(cmnd);
+
+       /*
+        * Let's remove cmnd from the hash earlier to keep it smaller.
+        * See also corresponding comment in req_cmnd_release().
+        */
+       if (cmnd->hashed)
+               cmnd_remove_data_wait_hash(cmnd);
 
        if (unlikely(test_bit(ISCSI_CONN_REINSTATING,
                        &cmnd->conn->conn_aflags))) {
@@ -150,39 +193,53 @@ unlock_cont:
                        goto out;
        }
 
+       if (unlikely(cmnd->prelim_compl_flags != 0)) {
+               if (test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags)) {
+                       TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted", cmnd,
+                               cmnd->scst_cmd);
+                       req_cmnd_release_force(cmnd);
+                       goto out;
+               }
+
+               if (cmnd->scst_cmd == NULL) {
+                       TRACE_MGMT_DBG("Finishing preliminary completed cmd %p "
+                               "with NULL scst_cmd", cmnd);
+                       req_cmnd_release(cmnd);
+                       goto out;
+               }
+               EXTRACHECKS_BUG_ON(scst_cmd_get_status(cmnd->scst_cmd) == 0);
+
+               status = SCST_PREPROCESS_STATUS_ERROR_SENSE_SET;
+       } else
+               status = SCST_PREPROCESS_STATUS_SUCCESS;
+
        cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
-       scst_restart_cmd(cmnd->scst_cmd, SCST_PREPROCESS_STATUS_SUCCESS,
-               SCST_CONTEXT_THREAD);
+
+       scst_restart_cmd(cmnd->scst_cmd, status, SCST_CONTEXT_THREAD);
 
 out:
+       TRACE_EXIT();
        return;
 }
 
-static inline void iscsi_restart_waiting_cmnd(struct iscsi_cmnd *cmnd)
+void iscsi_fail_data_waiting_cmnd(struct iscsi_cmnd *cmnd)
 {
-       /*
-        * There is no race with conn_abort(), since all functions
-        * called from single read thread
-        */
-       iscsi_extracheck_is_rd_thread(cmnd->conn);
-       cmnd->data_waiting = 0;
-
-       iscsi_restart_cmnd(cmnd);
-       return;
-}
+       TRACE_ENTRY();
 
-static inline void iscsi_fail_waiting_cmnd(struct iscsi_cmnd *cmnd)
-{
-       TRACE_MGMT_DBG("Failing data waiting cmd %p", cmnd);
+       TRACE_MGMT_DBG("Failing data waiting cmnd %p", cmnd);
 
        /*
         * There is no race with conn_abort(), since all functions
         * called from single read thread
         */
        iscsi_extracheck_is_rd_thread(cmnd->conn);
-       cmnd->data_waiting = 0;
+       cmnd->r2t_len_to_receive = 0;
+       cmnd->r2t_len_to_send = 0;
+
+       req_cmnd_release_force(cmnd);
 
-       req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
+       TRACE_EXIT();
+       return;
 }
 
 struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn,
@@ -209,9 +266,9 @@ struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn,
 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
                atomic_set(&cmnd->net_ref_cnt, 0);
 #endif
-               spin_lock_init(&cmnd->rsp_cmd_lock);
                INIT_LIST_HEAD(&cmnd->rsp_cmd_list);
                INIT_LIST_HEAD(&cmnd->rx_ddigest_cmd_list);
+               cmnd->target_task_tag = cpu_to_be32(ISCSI_RESERVED_TAG);
 
                spin_lock_bh(&conn->cmd_list_lock);
                list_add_tail(&cmnd->cmd_list_entry, &conn->cmd_list);
@@ -225,9 +282,11 @@ struct iscsi_cmnd *cmnd_alloc(struct iscsi_conn *conn,
 /* Frees a command. Also frees the additional header. */
 static void cmnd_free(struct iscsi_cmnd *cmnd)
 {
-       TRACE_DBG("%p", cmnd);
+       TRACE_ENTRY();
+
+       TRACE_DBG("cmnd %p", cmnd);
 
-       if (unlikely(cmnd->tm_aborted)) {
+       if (unlikely(test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags))) {
                TRACE_MGMT_DBG("Free aborted cmd %p (scst cmd %p, state %d, "
                        "parent_req %p)", cmnd, cmnd->scst_cmd,
                        cmnd->scst_state, cmnd->parent_req);
@@ -238,7 +297,8 @@ static void cmnd_free(struct iscsi_cmnd *cmnd)
 
        kfree(cmnd->pdu.ahs);
 
-       if (unlikely(cmnd->on_write_list || cmnd->on_written_list)) {
+#ifdef CONFIG_SCST_EXTRACHECKS
+       if (unlikely(cmnd->on_write_list || cmnd->on_write_timeout_list)) {
                struct iscsi_scsi_cmd_hdr *req = cmnd_hdr(cmnd);
 
                PRINT_CRIT_ERROR("cmnd %p still on some list?, %x, %x, %x, "
@@ -254,36 +314,36 @@ static void cmnd_free(struct iscsi_cmnd *cmnd)
                }
                sBUG();
        }
+#endif
 
        kmem_cache_free(iscsi_cmnd_cache, cmnd);
+
+       TRACE_EXIT();
        return;
 }
 
-/* Might be called unded some lock and on SIRQ */
+/* Might be called under some lock and on SIRQ */
 void cmnd_done(struct iscsi_cmnd *cmnd)
 {
-       TRACE_DBG("%p", cmnd);
+       TRACE_ENTRY();
 
-       if (unlikely(cmnd->tm_aborted)) {
+       TRACE_DBG("cmnd %p", cmnd);
+
+       if (unlikely(test_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags))) {
                TRACE_MGMT_DBG("Done aborted cmd %p (scst cmd %p, state %d, "
                        "parent_req %p)", cmnd, cmnd->scst_cmd,
                        cmnd->scst_state, cmnd->parent_req);
        }
 
        EXTRACHECKS_BUG_ON(cmnd->on_rx_digest_list);
+       EXTRACHECKS_BUG_ON(cmnd->hashed);
 
-       if (cmnd->on_written_list) {
-               struct iscsi_conn *conn = cmnd->conn;
-               TRACE_DBG("Deleting cmd %p from conn %p written_list", cmnd,
-                       conn);
-               spin_lock_bh(&conn->write_list_lock);
-               list_del(&cmnd->written_list_entry);
-               cmnd->on_written_list = 0;
-               spin_unlock_bh(&conn->write_list_lock);
-       }
+       req_del_from_write_timeout_list(cmnd);
 
        if (cmnd->parent_req == NULL) {
                struct iscsi_conn *conn = cmnd->conn;
+               struct iscsi_cmnd *rsp, *t;
+
                TRACE_DBG("Deleting req %p from conn %p", cmnd, conn);
 
                spin_lock_bh(&conn->cmd_list_lock);
@@ -292,7 +352,6 @@ void cmnd_done(struct iscsi_cmnd *cmnd)
 
                conn_put(conn);
 
-               EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rsp_cmd_list));
                EXTRACHECKS_BUG_ON(!list_empty(&cmnd->rx_ddigest_cmd_list));
 
                /* Order between above and below code is important! */
@@ -307,6 +366,7 @@ void cmnd_done(struct iscsi_cmnd *cmnd)
 
                        case ISCSI_CMD_STATE_AFTER_PREPROC:
                        {
+                               /* It can be for some aborted commands */
                                struct scst_cmd *scst_cmd = cmnd->scst_cmd;
                                TRACE_DBG("cmd %p AFTER_PREPROC", cmnd);
                                cmnd->scst_state = ISCSI_CMD_STATE_RESTARTED;
@@ -322,6 +382,9 @@ void cmnd_done(struct iscsi_cmnd *cmnd)
                                scst_aen_done(cmnd->scst_aen);
                                break;
 
+                       case ISCSI_CMD_STATE_OUT_OF_SCST_PRELIM_COMPL:
+                               break;
+
                        default:
                                PRINT_CRIT_ERROR("Unexpected cmnd scst state "
                                        "%d", cmnd->scst_state);
@@ -329,46 +392,67 @@ void cmnd_done(struct iscsi_cmnd *cmnd)
                                break;
                        }
                }
-       } else {
-               TRACE_DBG("Deleting rsp %p from parent %p", cmnd,
-                       cmnd->parent_req);
 
-               spin_lock_bh(&cmnd->parent_req->rsp_cmd_lock);
-               list_del(&cmnd->rsp_cmd_list_entry);
-               spin_unlock_bh(&cmnd->parent_req->rsp_cmd_lock);
+               if (cmnd->own_sg) {
+                       TRACE_DBG("own_sg for req %p", cmnd);
+                       if (cmnd->sg != &dummy_sg)
+                               scst_free(cmnd->sg, cmnd->sg_cnt);
+#ifdef CONFIG_SCST_DEBUG
+                       cmnd->own_sg = 0;
+                       cmnd->sg = NULL;
+                       cmnd->sg_cnt = -1;
+#endif
+               }
 
-               cmnd_put(cmnd->parent_req);
-       }
+               if (cmnd->dec_active_cmnds) {
+                       struct iscsi_session *sess = cmnd->conn->session;
+                       TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
+                               "new value %d)", cmnd, sess,
+                               atomic_read(&sess->active_cmds)-1);
+                       atomic_dec(&sess->active_cmds);
+#ifdef CONFIG_SCST_EXTRACHECKS
+                       if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
+                               PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
+                                       atomic_read(&sess->active_cmds));
+                               sBUG();
+                       }
+#endif
+               }
 
-       /* Order between above and below code is important! */
+               list_for_each_entry_safe(rsp, t, &cmnd->rsp_cmd_list,
+                                       rsp_cmd_list_entry) {
+                       cmnd_free(rsp);
+               }
 
-       if (cmnd->own_sg) {
-               TRACE_DBG("%s", "own_sg");
-               if ((cmnd->sg != &dummy_sg) && (cmnd->sg != cmnd->rsp_sg))
-                       scst_free(cmnd->sg, cmnd->sg_cnt);
+               cmnd_free(cmnd);
+       } else {
+               if (cmnd->own_sg) {
+                       TRACE_DBG("own_sg for rsp %p", cmnd);
+                       if ((cmnd->sg != &dummy_sg) && (cmnd->sg != cmnd->rsp_sg))
+                               scst_free(cmnd->sg, cmnd->sg_cnt);
 #ifdef CONFIG_SCST_DEBUG
-               cmnd->own_sg = 0;
-               cmnd->sg = NULL;
-               cmnd->sg_cnt = -1;
+                       cmnd->own_sg = 0;
+                       cmnd->sg = NULL;
+                       cmnd->sg_cnt = -1;
 #endif
-       }
+               }
 
-       if (cmnd->dec_active_cmnds) {
-               struct iscsi_session *sess = cmnd->conn->session;
-               TRACE_DBG("Decrementing active_cmds (cmd %p, sess %p, "
-                       "new value %d)", cmnd, sess,
-                       atomic_read(&sess->active_cmds)-1);
-               atomic_dec(&sess->active_cmds);
-#ifdef CONFIG_SCST_EXTRACHECKS
-               if (unlikely(atomic_read(&sess->active_cmds) < 0)) {
-                       PRINT_CRIT_ERROR("active_cmds < 0 (%d)!!",
-                               atomic_read(&sess->active_cmds));
-                       sBUG();
+               EXTRACHECKS_BUG_ON(cmnd->dec_active_cmnds);
+
+               if (cmnd == cmnd->parent_req->main_rsp) {
+                       TRACE_DBG("Finishing main rsp %p (req %p)", cmnd,
+                               cmnd->parent_req);
+                       cmnd->parent_req->main_rsp = NULL;
                }
-#endif
+
+               cmnd_put(cmnd->parent_req);
+               /*
+                * rsp will be freed on the last parent's put and can already
+                * be freed!!
+                */
        }
 
-       cmnd_free(cmnd);
+       TRACE_EXIT();
        return;
 }
 
@@ -378,7 +462,7 @@ void cmnd_done(struct iscsi_cmnd *cmnd)
  *
  * It can't be called in parallel with iscsi_cmnds_init_write()!
  */
-void req_cmnd_release_force(struct iscsi_cmnd *req, int flags)
+void req_cmnd_release_force(struct iscsi_cmnd *req)
 {
        struct iscsi_cmnd *rsp, *t;
        struct iscsi_conn *conn = req->conn;
@@ -386,33 +470,28 @@ void req_cmnd_release_force(struct iscsi_cmnd *req, int flags)
 
        TRACE_ENTRY();
 
-       TRACE_MGMT_DBG("%p", req);
+       TRACE_MGMT_DBG("req %p", req);
 
        sBUG_ON(req == conn->read_cmnd);
 
-       if (flags & ISCSI_FORCE_RELEASE_WRITE) {
-               spin_lock_bh(&conn->write_list_lock);
-               list_for_each_entry_safe(rsp, t, &conn->write_list,
-                                               write_list_entry) {
-                       if (rsp->parent_req != req)
-                               continue;
+       spin_lock_bh(&conn->write_list_lock);
+       list_for_each_entry_safe(rsp, t, &conn->write_list, write_list_entry) {
+               if (rsp->parent_req != req)
+                       continue;
 
-                       cmd_del_from_write_list(rsp);
+               cmd_del_from_write_list(rsp);
 
-                       list_add_tail(&rsp->write_list_entry, &cmds_list);
-               }
-               spin_unlock_bh(&conn->write_list_lock);
+               list_add_tail(&rsp->write_list_entry, &cmds_list);
+       }
+       spin_unlock_bh(&conn->write_list_lock);
 
-               list_for_each_entry_safe(rsp, t, &cmds_list,
-                                               write_list_entry) {
-                       TRACE_MGMT_DBG("Putting write rsp %p", rsp);
-                       list_del(&rsp->write_list_entry);
-                       cmnd_put(rsp);
-               }
+       list_for_each_entry_safe(rsp, t, &cmds_list, write_list_entry) {
+               TRACE_MGMT_DBG("Putting write rsp %p", rsp);
+               list_del(&rsp->write_list_entry);
+               cmnd_put(rsp);
        }
 
-again_rsp:
-       spin_lock_bh(&req->rsp_cmd_lock);
+       /* Supposed nobody can add responses in the list anymore */
        list_for_each_entry_reverse(rsp, &req->rsp_cmd_list,
                        rsp_cmd_list_entry) {
                bool r;
@@ -425,8 +504,6 @@ again_rsp:
                if (cmnd_get_check(rsp))
                        continue;
 
-               spin_unlock_bh(&req->rsp_cmd_lock);
-
                spin_lock_bh(&conn->write_list_lock);
                r = rsp->on_write_list || rsp->write_processing_started;
                spin_unlock_bh(&conn->write_list_lock);
@@ -434,7 +511,7 @@ again_rsp:
                cmnd_put(rsp);
 
                if (r)
-                       goto again_rsp;
+                       continue;
 
                /*
                 * If both on_write_list and write_processing_started not set,
@@ -442,9 +519,13 @@ again_rsp:
                 */
                TRACE_MGMT_DBG("Putting rsp %p", rsp);
                cmnd_put(rsp);
-               goto again_rsp;
        }
-       spin_unlock_bh(&req->rsp_cmd_lock);
+
+       if (req->main_rsp != NULL) {
+               TRACE_MGMT_DBG("Putting main rsp %p", req->main_rsp);
+               cmnd_put(req->main_rsp);
+               req->main_rsp = NULL;
+       }
 
        req_cmnd_release(req);
 
@@ -462,28 +543,45 @@ static void req_cmnd_release(struct iscsi_cmnd *req)
 
        TRACE_ENTRY();
 
-       TRACE_DBG("%p", req);
+       TRACE_DBG("req %p", req);
 
 #ifdef CONFIG_SCST_EXTRACHECKS
        sBUG_ON(req->release_called);
        req->release_called = 1;
 #endif
 
-       if (unlikely(req->tm_aborted)) {
+       if (unlikely(test_bit(ISCSI_CMD_ABORTED, &req->prelim_compl_flags))) {
                TRACE_MGMT_DBG("Release aborted req cmd %p (scst cmd %p, "
                        "state %d)", req, req->scst_cmd, req->scst_state);
        }
 
        sBUG_ON(req->parent_req != NULL);
 
+       /*
+        * We have to remove hashed req from the hash list before sending
+        * response. Otherwise we can have a race, when for some reason cmd's
+        * release (and, hence, removal from the hash) is delayed after the
+        * transmission and initiator sends cmd with the same ITT, hence
+        * the new command will be erroneously rejected as a duplicate.
+        */
+       if (unlikely(req->hashed)) {
+               /* It sometimes can happen during errors recovery */
+               cmnd_remove_data_wait_hash(req);
+       }
+
+       if (unlikely(req->main_rsp != NULL)) {
+               TRACE_DBG("Sending main rsp %p", req->main_rsp);
+               iscsi_cmnd_init_write(req->main_rsp, ISCSI_INIT_WRITE_WAKE);
+               req->main_rsp = NULL;
+       }
+
        list_for_each_entry_safe(c, t, &req->rx_ddigest_cmd_list,
                                rx_ddigest_cmd_list_entry) {
                cmd_del_from_rx_ddigest_list(c);
                cmnd_put(c);
        }
 
-       if (req->hashed)
-               cmnd_remove_hash(req);
+       EXTRACHECKS_BUG_ON(req->pending);
 
        if (req->dec_active_cmnds) {
                struct iscsi_session *sess = req->conn->session;
@@ -520,48 +618,42 @@ void rsp_cmnd_release(struct iscsi_cmnd *cmnd)
        cmnd->release_called = 1;
 #endif
 
-       sBUG_ON(cmnd->hashed);
-       sBUG_ON(cmnd->parent_req == NULL);
+       EXTRACHECKS_BUG_ON(cmnd->parent_req == NULL);
 
        cmnd_put(cmnd);
        return;
 }
 
-/**
- * create a new command used as response.
- *
- * iscsi_cmnd_create_rsp_cmnd -
- * @cmnd: ptr to request command
- *
- * @return    ptr to response command or NULL
- */
-static struct iscsi_cmnd *iscsi_cmnd_create_rsp_cmnd(struct iscsi_cmnd *parent)
+static struct iscsi_cmnd *iscsi_alloc_rsp(struct iscsi_cmnd *parent)
 {
        struct iscsi_cmnd *rsp;
 
+       TRACE_ENTRY();
+
        rsp = cmnd_alloc(parent->conn, parent);
 
-       spin_lock_bh(&parent->rsp_cmd_lock);
        TRACE_DBG("Adding rsp %p to parent %p", rsp, parent);
        list_add_tail(&rsp->rsp_cmd_list_entry, &parent->rsp_cmd_list);
-       spin_unlock_bh(&parent->rsp_cmd_lock);
+
        cmnd_get(parent);
+
+       TRACE_EXIT_HRES((unsigned long)rsp);
        return rsp;
 }
 
-static inline struct iscsi_cmnd *get_rsp_cmnd(struct iscsi_cmnd *req)
+static inline struct iscsi_cmnd *iscsi_alloc_main_rsp(struct iscsi_cmnd *parent)
 {
-       struct iscsi_cmnd *res = NULL;
+       struct iscsi_cmnd *rsp;
 
-       /* Currently this lock isn't needed, but just in case.. */
-       spin_lock_bh(&req->rsp_cmd_lock);
-       if (!list_empty(&req->rsp_cmd_list)) {
-               res = list_entry(req->rsp_cmd_list.prev, struct iscsi_cmnd,
-                       rsp_cmd_list_entry);
-       }
-       spin_unlock_bh(&req->rsp_cmd_lock);
+       TRACE_ENTRY();
 
-       return res;
+       EXTRACHECKS_BUG_ON(parent->main_rsp != 0);
+
+       rsp = iscsi_alloc_rsp(parent);
+       parent->main_rsp = rsp;
+
+       TRACE_EXIT_HRES((unsigned long)rsp);
+       return rsp;
 }
 
 static void iscsi_cmnds_init_write(struct list_head *send, int flags)
@@ -573,19 +665,6 @@ static void iscsi_cmnds_init_write(struct list_head *send, int flags)
 
        sBUG_ON(list_empty(send));
 
-       /*
-        * If we don't remove hashed req cmd from the hash list here, before
-        * submitting it for transmission, we will have a race, when for
-        * some reason cmd's release is delayed after transmission and
-        * initiator sends cmd with the same ITT => this command will be
-        * erroneously rejected as a duplicate.
-        */
-       if ((flags & ISCSI_INIT_WRITE_REMOVE_HASH) &&
-           rsp->parent_req->hashed &&
-           (rsp->parent_req->r2t_length == 0) &&
-           (rsp->parent_req->outstanding_r2t == 0))
-               cmnd_remove_hash(rsp->parent_req);
-
        if (!(conn->ddigest_type & DIGEST_NONE)) {
                list_for_each(pos, send) {
                        rsp = list_entry(pos, struct iscsi_cmnd,
@@ -622,17 +701,16 @@ static void iscsi_cmnd_init_write(struct iscsi_cmnd *rsp, int flags)
 {
        LIST_HEAD(head);
 
+#ifdef CONFIG_SCST_EXTRACHECKS
        if (unlikely(rsp->on_write_list)) {
-               PRINT_CRIT_ERROR("cmd already on write list (%x %x %x %x %u "
-                       "%u %u %u %u %u %u %d %d",
-                       cmnd_itt(rsp), cmnd_ttt(rsp), cmnd_opcode(rsp),
-                       cmnd_scsicode(rsp), rsp->r2t_sn,
-                       rsp->r2t_length, rsp->is_unsolicited_data,
-                       rsp->target_task_tag, rsp->outstanding_r2t,
+               PRINT_CRIT_ERROR("cmd already on write list (%x %x %x "
+                       "%u %u %d %d", cmnd_itt(rsp),
+                       cmnd_opcode(rsp), cmnd_scsicode(rsp),
                        rsp->hdigest, rsp->ddigest,
                        list_empty(&rsp->rsp_cmd_list), rsp->hashed);
                sBUG();
        }
+#endif
        list_add_tail(&rsp->write_list_entry, &head);
        iscsi_cmnds_init_write(&head, flags);
        return;
@@ -655,7 +733,7 @@ static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
        sn = 0;
 
        while (1) {
-               rsp = iscsi_cmnd_create_rsp_cmnd(req);
+               rsp = iscsi_alloc_rsp(req);
                TRACE_DBG("rsp %p", rsp);
                rsp->sg = req->sg;
                rsp->sg_cnt = req->sg_cnt;
@@ -672,7 +750,7 @@ static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
                        TRACE_DBG("offset %d, size %d", offset, size);
                        rsp->pdu.datasize = size;
                        if (send_status) {
-                               int scsisize;
+                               unsigned int scsisize;
 
                                TRACE_DBG("status %x", status);
 
@@ -707,19 +785,18 @@ static void send_data_rsp(struct iscsi_cmnd *req, u8 status, int send_status)
 
                list_add_tail(&rsp->write_list_entry, &send);
        }
-       iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_REMOVE_HASH);
+       iscsi_cmnds_init_write(&send, 0);
        return;
 }
 
-static struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req, int status,
-       const u8 *sense_buf, int sense_len)
+static void iscsi_init_status_rsp(struct iscsi_cmnd *rsp,
+       int status, const u8 *sense_buf, int sense_len, bool bufflen_set)
 {
-       struct iscsi_cmnd *rsp;
+       struct iscsi_cmnd *req = rsp->parent_req;
        struct iscsi_scsi_rsp_hdr *rsp_hdr;
        struct scatterlist *sg;
 
-       rsp = iscsi_cmnd_create_rsp_cmnd(req);
-       TRACE_DBG("%p", rsp);
+       TRACE_ENTRY();
 
        rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
        rsp_hdr->opcode = ISCSI_OP_SCSI_RSP;
@@ -748,22 +825,168 @@ static struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req, int status,
                rsp->bufflen = 0;
        }
 
+       iscsi_set_resid(rsp, bufflen_set);
+
+       TRACE_EXIT();
+       return;
+}
+
+static inline struct iscsi_cmnd *create_status_rsp(struct iscsi_cmnd *req,
+       int status, const u8 *sense_buf, int sense_len, bool bufflen_set)
+{
+       struct iscsi_cmnd *rsp;
+
+       TRACE_ENTRY();
+
+       rsp = iscsi_alloc_rsp(req);
+       TRACE_DBG("rsp %p", rsp);
+
+       iscsi_init_status_rsp(rsp, status, sense_buf, sense_len, bufflen_set);
+
+       TRACE_EXIT_HRES((unsigned long)rsp);
        return rsp;
 }
 
-static void iscsi_cmnd_reject(struct iscsi_cmnd *req, int reason)
+static struct iscsi_cmnd *create_prelim_status_rsp(struct iscsi_cmnd *req,
+       int status, const u8 *sense_buf, int sense_len)
 {
        struct iscsi_cmnd *rsp;
+
+       TRACE_ENTRY();
+
+       rsp = iscsi_alloc_main_rsp(req);
+       TRACE_DBG("main rsp %p", rsp);
+
+       iscsi_init_status_rsp(rsp, status, sense_buf, sense_len, false);
+
+       TRACE_EXIT_HRES((unsigned long)rsp);
+       return rsp;
+}
+
+static int iscsi_set_prelim_r2t_len_to_receive(struct iscsi_cmnd *req)
+{
+       struct iscsi_hdr *req_hdr = &req->pdu.bhs;
+       int res = 0;
+
+       TRACE_ENTRY();
+
+       if (req_hdr->flags & ISCSI_CMD_FINAL)
+               goto out;
+
+       res = cmnd_insert_data_wait_hash(req);
+       if (res != 0) {
+               /*
+                * We have to close connection, because otherwise a data
+                * corruption is possible if we allow to receive data
+                * for this request in another request with dublicated ITT.
+                */
+               mark_conn_closed(req->conn);
+               goto out;
+       }
+
+       /*
+        * We need to wait for one or more PDUs. Let's simplify
+        * other code and pretend we need to receive 1 byte.
+        * In data_out_start() we will correct it.
+        */
+       if (req->outstanding_r2t == 0) {
+               req->outstanding_r2t = 1;
+               req_add_to_write_timeout_list(req);
+       }
+       req->r2t_len_to_receive = 1;
+       req->r2t_len_to_send = 0;
+
+       TRACE_DBG("req %p, op %x, outstanding_r2t %d, r2t_len_to_receive %d, "
+               "r2t_len_to_send %d", req, cmnd_opcode(req),
+               req->outstanding_r2t, req->r2t_len_to_receive,
+               req->r2t_len_to_send);
+
+out:
+       TRACE_EXIT_RES(res);
+       return res;
+}
+
+static int create_preliminary_status_rsp(struct iscsi_cmnd *req,
+       int status, const u8 *sense_buf, int sense_len)
+{
+       int res = 0;
+       struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
+
+       TRACE_ENTRY();
+
+       if (req->prelim_compl_flags != 0) {
+               TRACE_MGMT_DBG("req %p already prelim completed", req);
+               goto out;
+       }
+
+       req->scst_state = ISCSI_CMD_STATE_OUT_OF_SCST_PRELIM_COMPL;
+
+       if ((req_hdr->flags & ISCSI_CMD_READ) &&
+           (req_hdr->flags & ISCSI_CMD_WRITE)) {
+               int sz = cmnd_read_size(req);
+               if (sz > 0)
+                       req->read_size = sz;
+       } else if (req_hdr->flags & ISCSI_CMD_READ)
+               req->read_size = be32_to_cpu(req_hdr->data_length);
+
+       create_prelim_status_rsp(req, status, sense_buf, sense_len);
+       res = iscsi_preliminary_complete(req, req, true);
+
+out:
+       TRACE_EXIT_RES(res);
+       return res;
+}
+
+static int set_scst_preliminary_status_rsp(struct iscsi_cmnd *req,
+       bool get_data, int key, int asc, int ascq)
+{
+       int res = 0;
+
+       TRACE_ENTRY();
+
+       if (req->scst_cmd == NULL) {
+               /* There must be already error set */
+               goto complete;
+       }
+
+       scst_set_cmd_error(req->scst_cmd, key, asc, ascq);
+
+complete:
+       res = iscsi_preliminary_complete(req, req, get_data);
+
+       TRACE_EXIT_RES(res);
+       return res;
+}
+
+static int create_reject_rsp(struct iscsi_cmnd *req, int reason, bool get_data)
+{
+       int res = 0;
+       struct iscsi_cmnd *rsp;
        struct iscsi_reject_hdr *rsp_hdr;
        struct scatterlist *sg;
 
+       TRACE_ENTRY();
+
        TRACE_MGMT_DBG("Reject: req %p, reason %x", req, reason);
 
-       sBUG_ON(req->rejected);
-       req->rejected = 1;
-       req->reject_reason = ISCSI_REJECT_CMD;
+       if (cmnd_opcode(req) == ISCSI_OP_SCSI_CMD) {
+               if (req->scst_cmd == NULL) {
+                       /* BUSY status must be already set */
+                       struct iscsi_scsi_rsp_hdr *rsp_hdr;
+                       rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&req->main_rsp->pdu.bhs;
+                       sBUG_ON(rsp_hdr->cmd_status == 0);
+                       /*
+                        * Let's not send REJECT here. The initiator will retry
+                        * and, hopefully, next time we will not fail allocating
+                        * scst_cmd, so we will then send the REJECT.
+                        */
+                       goto out;
+               } else
+                       set_scst_preliminary_status_rsp(req, get_data,
+                               SCST_LOAD_SENSE(scst_sense_invalid_message));
+       }
 
-       rsp = iscsi_cmnd_create_rsp_cmnd(req);
+       rsp = iscsi_alloc_main_rsp(req);
        rsp_hdr = (struct iscsi_reject_hdr *)&rsp->pdu.bhs;
 
        rsp_hdr->opcode = ISCSI_OP_REJECT;
@@ -776,11 +999,11 @@ static void iscsi_cmnd_reject(struct iscsi_cmnd *req, int reason)
        sg_init_one(sg, &req->pdu.bhs, sizeof(struct iscsi_hdr));
        rsp->bufflen = rsp->pdu.datasize = sizeof(struct iscsi_hdr);
 
-       iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
-                                        ISCSI_INIT_WRITE_WAKE);
+       res = iscsi_preliminary_complete(req, req, true);
 
-       cmnd_prepare_get_rejected_cmd_data(req);
-       return;
+out:
+       TRACE_EXIT_RES(res);
+       return res;
 }
 
 static inline int iscsi_get_allowed_cmds(struct iscsi_session *sess)
@@ -850,54 +1073,73 @@ static int check_cmd_sn(struct iscsi_cmnd *cmnd)
        return -ISCSI_REASON_PROTOCOL_ERROR;
 }
 
-static inline struct iscsi_cmnd *__cmnd_find_hash(
-       struct iscsi_session *session, u32 itt, u32 ttt)
+static struct iscsi_cmnd *cmnd_find_itt_get(struct iscsi_conn *conn, u32 itt)
+{
+       struct iscsi_cmnd *cmnd, *found_cmnd = NULL;
+
+       spin_lock_bh(&conn->cmd_list_lock);
+       list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
+               if ((cmnd->pdu.bhs.itt == itt) && !cmnd_get_check(cmnd)) {
+                       found_cmnd = cmnd;
+                       break;
+               }
+       }
+       spin_unlock_bh(&conn->cmd_list_lock);
+
+       return found_cmnd;
+}
+
+/**
+ ** We use the ITT hash only to find original request PDU for subsequent
+ ** Data-Out PDUs.
+ **/
+
+/* Must be called under cmnd_data_wait_hash_lock */
+static struct iscsi_cmnd *__cmnd_find_data_wait_hash(struct iscsi_conn *conn,
+       u32 itt)
 {
        struct list_head *head;
        struct iscsi_cmnd *cmnd;
 
-       head = &session->cmnd_hash[cmnd_hashfn(itt)];
+       head = &conn->session->cmnd_data_wait_hash[cmnd_hashfn(itt)];
 
        list_for_each_entry(cmnd, head, hash_list_entry) {
-               if (cmnd->pdu.bhs.itt == itt) {
-                       if (ttt != ISCSI_RESERVED_TAG &&
-                           ttt != cmnd->target_task_tag)
-                               continue;
+               if (cmnd->pdu.bhs.itt == itt)
                        return cmnd;
-               }
        }
        return NULL;
 }
 
-static struct iscsi_cmnd *cmnd_find_hash(struct iscsi_session *session,
-       u32 itt, u32 ttt)
+static struct iscsi_cmnd *cmnd_find_data_wait_hash(struct iscsi_conn *conn,
+       u32 itt)
 {
-       struct iscsi_cmnd *cmnd;
+       struct iscsi_cmnd *res;
+       struct iscsi_session *session = conn->session;
 
-       spin_lock(&session->cmnd_hash_lock);
-       cmnd = __cmnd_find_hash(session, itt, ttt);
-       spin_unlock(&session->cmnd_hash_lock);
+       spin_lock(&session->cmnd_data_wait_hash_lock);
+       res = __cmnd_find_data_wait_hash(conn, itt);
+       spin_unlock(&session->cmnd_data_wait_hash_lock);
 
-       return cmnd;
+       return res;
 }
 
-static struct iscsi_cmnd *cmnd_find_hash_get(struct iscsi_session *session,
-       u32 itt, u32 ttt)
+static inline u32 get_next_ttt(struct iscsi_conn *conn)
 {
-       struct iscsi_cmnd *cmnd;
+       u32 ttt;
+       struct iscsi_session *session = conn->session;
 
-       spin_lock(&session->cmnd_hash_lock);
-       cmnd = __cmnd_find_hash(session, itt, ttt);
-       if (cmnd != NULL) {
-               if (unlikely(cmnd_get_check(cmnd)))
-                       cmnd = NULL;
-       }
-       spin_unlock(&session->cmnd_hash_lock);
+       /* Not compatible with MC/S! */
 
-       return cmnd;
+       iscsi_extracheck_is_rd_thread(conn);
+
+       if (unlikely(session->next_ttt == ISCSI_RESERVED_TAG))
+               session->next_ttt++;
+       ttt = session->next_ttt++;
+
+       return ttt;
 }
 
-static int cmnd_insert_hash(struct iscsi_cmnd *cmnd)
+static int cmnd_insert_data_wait_hash(struct iscsi_cmnd *cmnd)
 {
        struct iscsi_session *session = cmnd->conn->session;
        struct iscsi_cmnd *tmp;
@@ -905,6 +1147,19 @@ static int cmnd_insert_hash(struct iscsi_cmnd *cmnd)
        int err = 0;
        u32 itt = cmnd->pdu.bhs.itt;
 
+       if (unlikely(cmnd->hashed)) {
+               /* It can be for preliminary completed commands */
+               goto out;
+       }
+
+       /*
+        * We don't need TTT, because ITT/buffer_offset pair is sufficient
+        * to find out the original request and buffer for Data-Out PDUs, but
+        * crazy iSCSI spec requires us to send this superfluous field in
+        * R2T PDUs and some initiators may rely on it.
+        */
+       cmnd->target_task_tag = get_next_ttt(cmnd->conn);
+
        TRACE_DBG("%p:%x", cmnd, itt);
        if (unlikely(itt == ISCSI_RESERVED_TAG)) {
                PRINT_ERROR("%s", "ITT is RESERVED_TAG");
@@ -914,12 +1169,14 @@ static int cmnd_insert_hash(struct iscsi_cmnd *cmnd)
                goto out;
        }
 
-       spin_lock(&session->cmnd_hash_lock);
+       spin_lock(&session->cmnd_data_wait_hash_lock);
 
-       head = &session->cmnd_hash[cmnd_hashfn(cmnd->pdu.bhs.itt)];
+       head = &session->cmnd_data_wait_hash[cmnd_hashfn(itt)];
 
-       tmp = __cmnd_find_hash(session, itt, ISCSI_RESERVED_TAG);
+       tmp = __cmnd_find_data_wait_hash(cmnd->conn, itt);
        if (likely(!tmp)) {
+               TRACE_DBG("Adding cmnd %p to the hash (ITT %x)", cmnd,
+                       cmnd_itt(cmnd));
                list_add_tail(&cmnd->hash_list_entry, head);
                cmnd->hashed = 1;
        } else {
@@ -927,40 +1184,35 @@ static int cmnd_insert_hash(struct iscsi_cmnd *cmnd)
                err = -ISCSI_REASON_TASK_IN_PROGRESS;
        }
 
-       spin_unlock(&session->cmnd_hash_lock);
-
-       if (likely(!err)) {
-               spin_lock(&session->sn_lock);
-               __update_stat_sn(cmnd);
-               err = check_cmd_sn(cmnd);
-               spin_unlock(&session->sn_lock);
-       }
+       spin_unlock(&session->cmnd_data_wait_hash_lock);
 
 out:
        return err;
 }
 
-static void cmnd_remove_hash(struct iscsi_cmnd *cmnd)
+static void cmnd_remove_data_wait_hash(struct iscsi_cmnd *cmnd)
 {
        struct iscsi_session *session = cmnd->conn->session;
        struct iscsi_cmnd *tmp;
 
-       spin_lock(&session->cmnd_hash_lock);
+       spin_lock(&session->cmnd_data_wait_hash_lock);
 
-       tmp = __cmnd_find_hash(session, cmnd->pdu.bhs.itt, ISCSI_RESERVED_TAG);
+       tmp = __cmnd_find_data_wait_hash(cmnd->conn, cmnd->pdu.bhs.itt);
 
        if (likely(tmp && tmp == cmnd)) {
+               TRACE_DBG("Deleting cmnd %p from the hash (ITT %x)", cmnd,
+                       cmnd_itt(cmnd));
                list_del(&cmnd->hash_list_entry);
                cmnd->hashed = 0;
-       } else {
+       } else
                PRINT_ERROR("%p:%x not found", cmnd, cmnd_itt(cmnd));
-       }
 
-       spin_unlock(&session->cmnd_hash_lock);
+       spin_unlock(&session->cmnd_data_wait_hash_lock);
+
        return;
 }
 
-static void cmnd_prepare_get_rejected_cmd_data(struct iscsi_cmnd *cmnd)
+static void cmnd_prepare_get_rejected_immed_data(struct iscsi_cmnd *cmnd)
 {
        struct iscsi_conn *conn = cmnd->conn;
        struct scatterlist *sg = cmnd->sg;
@@ -968,7 +1220,11 @@ static void cmnd_prepare_get_rejected_cmd_data(struct iscsi_cmnd *cmnd)
        u32 size;
        unsigned int i;
 
-       TRACE_MGMT_DBG("Skipping (%p, %x %x %x %u, %p, scst state %d)", cmnd,
+       TRACE_ENTRY();
+
+       TRACE_DBG_FLAG(iscsi_get_flow_ctrl_or_mgmt_dbg_log_flag(cmnd),
+               "Skipping (cmnd %p, ITT %x, op %x, cmd op %x, "
+               "datasize %u, scst_cmd %p, scst state %d)", cmnd,
                cmnd_itt(cmnd), cmnd_opcode(cmnd), cmnd_hdr(cmnd)->scb[0],
                cmnd->pdu.datasize, cmnd->scst_cmd, cmnd->scst_state);
 
@@ -976,7 +1232,9 @@ static void cmnd_prepare_get_rejected_cmd_data(struct iscsi_cmnd *cmnd)
 
        size = cmnd->pdu.datasize;
        if (!size)
-               return;
+               goto out;
+
+       /* We already checked pdu.datasize in check_segment_length() */
 
        if (sg == NULL) {
                /*
@@ -1003,12 +1261,14 @@ static void cmnd_prepare_get_rejected_cmd_data(struct iscsi_cmnd *cmnd)
        conn->read_msg.msg_iov = conn->read_iov;
        conn->read_msg.msg_iovlen = ++i;
 
+out:
+       TRACE_EXIT();
        return;
 }
 
-static void iscsi_set_resid(struct iscsi_cmnd *req, struct iscsi_cmnd *rsp,
-       bool data_used)
+static void iscsi_set_resid(struct iscsi_cmnd *rsp, bool bufflen_set)
 {
+       struct iscsi_cmnd *req = rsp->parent_req;
        struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
        struct iscsi_scsi_rsp_hdr *rsp_hdr;
        int resid, resp_len, in_resp_len;
@@ -1017,7 +1277,7 @@ static void iscsi_set_resid(struct iscsi_cmnd *req, struct iscsi_cmnd *rsp,
            (req_hdr->flags & ISCSI_CMD_WRITE)) {
                rsp_hdr = (struct iscsi_scsi_rsp_hdr *)&rsp->pdu.bhs;
 
-               if (data_used) {
+               if (bufflen_set) {
                        resp_len = req->bufflen;
                        if (req->scst_cmd != NULL)
                                in_resp_len = scst_cmd_get_in_bufflen(req->scst_cmd);
@@ -1048,7 +1308,7 @@ static void iscsi_set_resid(struct iscsi_cmnd *req, struct iscsi_cmnd *rsp,
                        rsp_hdr->bi_residual_count = cpu_to_be32(resid);
                }
        } else {
-               if (data_used)
+               if (bufflen_set)
                        resp_len = req->bufflen;
                else
                        resp_len = 0;
@@ -1068,32 +1328,45 @@ static void iscsi_set_resid(struct iscsi_cmnd *req, struct iscsi_cmnd *rsp,
        return;
 }
 
-static void cmnd_reject_scsi_cmd(struct iscsi_cmnd *req)
+static int iscsi_preliminary_complete(struct iscsi_cmnd *req,
+       struct iscsi_cmnd *orig_req, bool get_data)
 {
-       struct iscsi_cmnd *rsp;
+       struct iscsi_hdr *req_hdr = &req->pdu.bhs;
+       int res = 0;
+       bool set_r2t_len;
 
-       TRACE_DBG("%p", req);
+       TRACE_ENTRY();
 
-       sBUG_ON(req->rejected);
-       req->rejected = 1;
-       req->reject_reason = ISCSI_REJECT_SCSI_CMD;
+       TRACE_DBG_FLAG(iscsi_get_flow_ctrl_or_mgmt_dbg_log_flag(orig_req),
+               "Prelim completed req %p, orig_req %p (FINAL %x, "
+               "outstanding_r2t %d)", req, orig_req,
+               (req_hdr->flags & ISCSI_CMD_FINAL), orig_req->outstanding_r2t);
 
-       rsp = get_rsp_cmnd(req);
-       if (rsp == NULL) {
-               /* That can be true for aborted commands */
-               goto out_reject;
+       iscsi_extracheck_is_rd_thread(req->conn);
+       sBUG_ON(req->parent_req != NULL);
+
+       if (test_bit(ISCSI_CMD_PRELIM_COMPLETED, &req->prelim_compl_flags)) {
+               TRACE_MGMT_DBG("req %p already prelim completed", req);
+               /* To not try to get data twice */
+               get_data = false;
        }
 
-       sBUG_ON(cmnd_opcode(rsp) != ISCSI_OP_SCSI_RSP);
+       set_r2t_len = !req->hashed &&
+                     (cmnd_opcode(req) == ISCSI_OP_SCSI_CMD) &&
+                     !test_bit(ISCSI_CMD_PRELIM_COMPLETED,
+                               &orig_req->prelim_compl_flags);
+       set_bit(ISCSI_CMD_PRELIM_COMPLETED, &orig_req->prelim_compl_flags);
 
-       iscsi_set_resid(req, rsp, false);
+       TRACE_DBG("get_data %d, set_r2t_len %d", get_data, set_r2t_len);
 
-       iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH |
-                                        ISCSI_INIT_WRITE_WAKE);
+       if (get_data)
+               cmnd_prepare_get_rejected_immed_data(req);
 
-out_reject:
-       cmnd_prepare_get_rejected_cmd_data(req);
-       return;
+       if (set_r2t_len)
+               res = iscsi_set_prelim_r2t_len_to_receive(orig_req);
+
+       TRACE_EXIT_RES(res);
+       return res;
 }
 
 static int cmnd_prepare_recv_pdu(struct iscsi_conn *conn,
@@ -1161,20 +1434,15 @@ out:
 
 static void send_r2t(struct iscsi_cmnd *req)
 {
-       struct iscsi_session *session = req->conn->session;
+       struct iscsi_session *sess = req->conn->session;
        struct iscsi_cmnd *rsp;
        struct iscsi_r2t_hdr *rsp_hdr;
        u32 offset, burst;
        LIST_HEAD(send);
 
-       if (unlikely(req->tm_aborted)) {
-               TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted on R2T "
-                       "(r2t_length %d, outstanding_r2t %d)", req,
-                       req->scst_cmd, req->r2t_length, req->outstanding_r2t);
-               if (req->outstanding_r2t == 0)
-                       iscsi_fail_waiting_cmnd(req);
-               goto out;
-       }
+       TRACE_ENTRY();
+
+       EXTRACHECKS_BUG_ON(req->r2t_len_to_send == 0);
 
        /*
         * There is no race with data_out_start() and conn_abort(), since
@@ -1182,11 +1450,24 @@ static void send_r2t(struct iscsi_cmnd *req)
         */
        iscsi_extracheck_is_rd_thread(req->conn);
 
-       burst = session->sess_param.max_burst_length;
-       offset = be32_to_cpu(cmnd_hdr(req)->data_length) - req->r2t_length;
+       /*
+        * We don't need to check for PRELIM_COMPLETED here, because for such
+        * commands we set r2t_len_to_send = 0, hence made sure we won't
+        * called here.
+        */
+
+       EXTRACHECKS_BUG_ON(req->outstanding_r2t >
+                          sess->sess_param.max_outstanding_r2t);
+
+       if (req->outstanding_r2t == sess->sess_param.max_outstanding_r2t)
+               goto out;
+
+       burst = sess->sess_param.max_burst_length;
+       offset = be32_to_cpu(cmnd_hdr(req)->data_length) -
+                       req->r2t_len_to_send;
 
        do {
-               rsp = iscsi_cmnd_create_rsp_cmnd(req);
+               rsp = iscsi_alloc_rsp(req);
                rsp->pdu.bhs.ttt = req->target_task_tag;
                rsp_hdr = (struct iscsi_r2t_hdr *)&rsp->pdu.bhs;
                rsp_hdr->opcode = ISCSI_OP_R2T;
@@ -1195,30 +1476,31 @@ static void send_r2t(struct iscsi_cmnd *req)
                rsp_hdr->itt = cmnd_hdr(req)->itt;
                rsp_hdr->r2t_sn = cpu_to_be32(req->r2t_sn++);
                rsp_hdr->buffer_offset = cpu_to_be32(offset);
-               if (req->r2t_length > burst) {
+               if (req->r2t_len_to_send > burst) {
                        rsp_hdr->data_length = cpu_to_be32(burst);
-                       req->r2t_length -= burst;
+                       req->r2t_len_to_send -= burst;
                        offset += burst;
                } else {
-                       rsp_hdr->data_length = cpu_to_be32(req->r2t_length);
-                       req->r2t_length = 0;
+                       rsp_hdr->data_length = cpu_to_be32(req->r2t_len_to_send);
+                       req->r2t_len_to_send = 0;
                }
 
-               TRACE_WRITE("%x %u %u %u %u", cmnd_itt(req),
+               TRACE_WRITE("req %p, data_length %u, buffer_offset %u, "
+                       "r2t_sn %u, outstanding_r2t %u", req,
                        be32_to_cpu(rsp_hdr->data_length),
                        be32_to_cpu(rsp_hdr->buffer_offset),
                        be32_to_cpu(rsp_hdr->r2t_sn), req->outstanding_r2t);
 
                list_add_tail(&rsp->write_list_entry, &send);
+               req->outstanding_r2t++;
 
-               if (++req->outstanding_r2t >= session->sess_param.max_outstanding_r2t)
-                       break;
-
-       } while (req->r2t_length != 0);
+       } while ((req->outstanding_r2t < sess->sess_param.max_outstanding_r2t) &&
+                (req->r2t_len_to_send != 0));
 
        iscsi_cmnds_init_write(&send, ISCSI_INIT_WRITE_WAKE);
 
 out:
+       TRACE_EXIT();
        return;
 }
 
@@ -1259,6 +1541,7 @@ out:
 static int noop_out_start(struct iscsi_cmnd *cmnd)
 {
        struct iscsi_conn *conn = cmnd->conn;
+       struct iscsi_hdr *req_hdr = &cmnd->pdu.bhs;
        u32 size, tmp;
        int i, err = 0;
 
@@ -1266,13 +1549,9 @@ static int noop_out_start(struct iscsi_cmnd *cmnd)
 
        iscsi_extracheck_is_rd_thread(conn);
 
-       if (unlikely(cmnd_ttt(cmnd) != cpu_to_be32(ISCSI_RESERVED_TAG))) {
-               /*
-                * We don't request a NOP-Out by sending a NOP-In.
-                * See 10.18.2 in the draft 20.
-                */
-               PRINT_ERROR("Initiator sent command with not RESERVED tag and "
-                       "TTT %x", cmnd_itt(cmnd));
+       if (!(req_hdr->flags & ISCSI_FLG_FINAL)) {
+               PRINT_ERROR("%s", "Initiator sent NOP-Out with not a single "
+                       "PDU");
                err = -ISCSI_REASON_PROTOCOL_ERROR;
                goto out;
        }
@@ -1280,22 +1559,16 @@ static int noop_out_start(struct iscsi_cmnd *cmnd)
        if (cmnd_itt(cmnd) == cpu_to_be32(ISCSI_RESERVED_TAG)) {
                if (unlikely(!(cmnd->pdu.bhs.opcode & ISCSI_OP_IMMEDIATE)))
                        PRINT_ERROR("%s", "Initiator sent RESERVED tag for "
-                               "non-immediate command");
-               spin_lock(&conn->session->sn_lock);
-               __update_stat_sn(cmnd);
-               err = check_cmd_sn(cmnd);
-               spin_unlock(&conn->session->sn_lock);
-               if (unlikely(err))
-                       goto out;
-       } else {
-               err = cmnd_insert_hash(cmnd);
-               if (unlikely(err < 0)) {
-                       PRINT_ERROR("Can't insert in hash: ignore this "
-                               "request %x", cmnd_itt(cmnd));
-                       goto out;
-               }
+                               "non-immediate NOP-Out command");
        }
 
+       spin_lock(&conn->session->sn_lock);
+       __update_stat_sn(cmnd);
+       err = check_cmd_sn(cmnd);
+       spin_unlock(&conn->session->sn_lock);
+       if (unlikely(err))
+               goto out;
+
        size = cmnd->pdu.datasize;
 
        if (size) {
@@ -1355,20 +1628,6 @@ out:
        return err;
 }
 
-static inline u32 get_next_ttt(struct iscsi_conn *conn)
-{
-       u32 ttt;
-       struct iscsi_session *session = conn->session;
-
-       iscsi_extracheck_is_rd_thread(conn);
-
-       if (session->next_ttt == ISCSI_RESERVED_TAG)
-               session->next_ttt++;
-       ttt = session->next_ttt++;
-
-       return cpu_to_be32(ttt);
-}
-
 int cmnd_rx_continue(struct iscsi_cmnd *req)
 {
        struct iscsi_conn *conn = req->conn;
@@ -1376,53 +1635,26 @@ int cmnd_rx_continue(struct iscsi_cmnd *req)
        struct iscsi_scsi_cmd_hdr *req_hdr = cmnd_hdr(req);
        struct scst_cmd *scst_cmd = req->scst_cmd;
        scst_data_direction dir;
+       bool unsolicited_data_expected = false;
        int res = 0;
 
        TRACE_ENTRY();
 
-       TRACE_DBG("scsi command: %02x", req_hdr->scb[0]);
+       TRACE_DBG("scsi command: %x", req_hdr->scb[0]);
 
-       if (unlikely(req->scst_state != ISCSI_CMD_STATE_AFTER_PREPROC)) {
-               TRACE_DBG("req %p is in %x state", req, req->scst_state);
-               if (req->scst_state == ISCSI_CMD_STATE_PROCESSED) {
-                       cmnd_reject_scsi_cmd(req);
-                       goto out;
-               }
-               if (unlikely(req->tm_aborted)) {
-                       TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
-                               req->scst_cmd);
-                       cmnd_prepare_get_rejected_cmd_data(req);
-                       goto out;
-               }
-               sBUG();
-       }
+       EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_AFTER_PREPROC);
 
        dir = scst_cmd_get_data_direction(scst_cmd);
-       if (dir & SCST_DATA_WRITE) {
-               req->is_unsolicited_data = !(req_hdr->flags & ISCSI_CMD_FINAL);
-               req->r2t_length = be32_to_cpu(req_hdr->data_length) -
-                                       req->pdu.datasize;
-               if (req->r2t_length > 0)
-                       req->data_waiting = 1;
-       } else {
-               if (unlikely(!(req_hdr->flags & ISCSI_CMD_FINAL) ||
-                            req->pdu.datasize)) {
-                       PRINT_ERROR("Unexpected unsolicited data (ITT %x "
-                               "CDB %x", cmnd_itt(req), req_hdr->scb[0]);
-                       scst_set_cmd_error(scst_cmd,
-                               SCST_LOAD_SENSE(iscsi_sense_unexpected_unsolicited_data));
-                       if (scst_cmd_get_sense_buffer(scst_cmd) != NULL)
-                               create_status_rsp(req, SAM_STAT_CHECK_CONDITION,
-                                   scst_cmd_get_sense_buffer(scst_cmd),
-                                   scst_cmd_get_sense_buffer_len(scst_cmd));
-                       else
-                               create_status_rsp(req, SAM_STAT_BUSY, NULL, 0);
-                       cmnd_reject_scsi_cmd(req);
-                       goto out;
-               }
+
+       /* Check prelim_compl_flags here to save R2Ts */
+       if (unlikely(scst_cmd_completed(scst_cmd) ||
+           unlikely(req->prelim_compl_flags != 0))) {
+               res = iscsi_preliminary_complete(req, req, true);
+               goto trace;
        }
 
-       req->target_task_tag = get_next_ttt(conn);
+       /* For prelim completed commands sg&K can be already set! */
+
        if (dir != SCST_DATA_BIDI) {
                req->sg = scst_cmd_get_sg(scst_cmd);
                req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
@@ -1432,43 +1664,102 @@ int cmnd_rx_continue(struct iscsi_cmnd *req)
                req->sg_cnt = scst_cmd_get_in_sg_cnt(scst_cmd);
                req->bufflen = scst_cmd_get_in_bufflen(scst_cmd);
        }
-       if (unlikely(req->r2t_length > req->bufflen)) {
-               PRINT_ERROR("req->r2t_length %d > req->bufflen %d",
-                       req->r2t_length, req->bufflen);
-               req->r2t_length = req->bufflen;
-       }
 
-       TRACE_DBG("req=%p, dir=%d, is_unsolicited_data=%d, "
-               "r2t_length=%d, bufflen=%d", req, dir,
-               req->is_unsolicited_data, req->r2t_length, req->bufflen);
+       if (dir & SCST_DATA_WRITE) {
+               unsolicited_data_expected = !(req_hdr->flags & ISCSI_CMD_FINAL);
 
-       if (unlikely(!session->sess_param.immediate_data &&
-                    req->pdu.datasize)) {
-               PRINT_ERROR("Initiator %s violated negotiated parameters: "
-                       "forbidden immediate data sent (ITT %x, op  %x)",
-                       session->initiator_name, cmnd_itt(req),
-                       req_hdr->scb[0]);
-               res = -EINVAL;
-               goto out;
-       }
+               if (unlikely(session->sess_param.initial_r2t &&
+                   unsolicited_data_expected)) {
+                       PRINT_ERROR("Initiator %s violated negotiated "
+                               "parameters: initial R2T is required (ITT %x, "
+                               "op  %x)", session->initiator_name,
+                               cmnd_itt(req), req_hdr->scb[0]);
+                       goto out_close;
+               }
 
-       if (unlikely(session->sess_param.initial_r2t &&
-                    !(req_hdr->flags & ISCSI_CMD_FINAL))) {
-               PRINT_ERROR("Initiator %s violated negotiated parameters: "
-                       "initial R2T is required (ITT %x, op  %x)",
-                       session->initiator_name, cmnd_itt(req),
-                       req_hdr->scb[0]);
-               res = -EINVAL;
-               goto out;
+               if (unlikely(!session->sess_param.immediate_data &&
+                   req->pdu.datasize)) {
+                       PRINT_ERROR("Initiator %s violated negotiated "
+                               "parameters: forbidden immediate data sent "
+                               "(ITT %x, op  %x)", session->initiator_name,
+                               cmnd_itt(req), req_hdr->scb[0]);
+                       goto out_close;
+               }
+
+               if (unlikely(session->sess_param.first_burst_length < req->pdu.datasize)) {
+                       PRINT_ERROR("Initiator %s violated negotiated "
+                               "parameters: immediate data len (%d) > "
+                               "first_burst_length (%d) (ITT %x, op  %x)",
+                               session->initiator_name,
+                               req->pdu.datasize,
+                               session->sess_param.first_burst_length,
+                               cmnd_itt(req), req_hdr->scb[0]);
+                       goto out_close;
+               }
+
+               req->r2t_len_to_receive = be32_to_cpu(req_hdr->data_length) -
+                                         req->pdu.datasize;
+
+               if (unlikely(req->r2t_len_to_receive > req->bufflen)) {
+                       PRINT_ERROR("req->r2t_len_to_receive %d > req->bufflen "
+                               "%d", req->r2t_len_to_receive, req->bufflen);
+                       goto out_close;
+               }
+
+               res = cmnd_insert_data_wait_hash(req);
+               if (unlikely(res != 0)) {
+                       /*
+                        * We have to close connection, because otherwise a data
+                        * corruption is possible if we allow to receive data
+                        * for this request in another request with dublicated
+                        * ITT.
+                        */
+                       goto out_close;
+               }
+
+               if (unsolicited_data_expected) {
+                       req->outstanding_r2t = 1;
+                       req->r2t_len_to_send = req->r2t_len_to_receive -
+                               min_t(unsigned int,
+                                     session->sess_param.first_burst_length -
+                                               req->pdu.datasize,
+                                     req->r2t_len_to_receive);
+               } else
+                       req->r2t_len_to_send = req->r2t_len_to_receive;
+
+               req_add_to_write_timeout_list(req);
+
+               if (req->pdu.datasize) {
+                       res = cmnd_prepare_recv_pdu(conn, req, 0, req->pdu.datasize);
+                       /* For performance better to send R2Ts ASAP */
+                       if (likely(res == 0) && (req->r2t_len_to_send != 0))
+                               send_r2t(req);
+               }
+       } else {
+               if (unlikely(!(req_hdr->flags & ISCSI_CMD_FINAL) ||
+                            req->pdu.datasize)) {
+                       PRINT_ERROR("Unexpected unsolicited data (ITT %x "
+                               "CDB %x", cmnd_itt(req), req_hdr->scb[0]);
+                       set_scst_preliminary_status_rsp(req, true,
+                               SCST_LOAD_SENSE(iscsi_sense_unexpected_unsolicited_data));
+               }
        }
 
-       if (req->pdu.datasize)
-               res = cmnd_prepare_recv_pdu(conn, req, 0, req->pdu.datasize);
+trace: 
+       TRACE_DBG("req=%p, dir=%d, unsolicited_data_expected=%d, "
+               "r2t_len_to_receive=%d, r2t_len_to_send=%d, bufflen=%d, "
+               "own_sg %d", req, dir, unsolicited_data_expected,
+               req->r2t_len_to_receive, req->r2t_len_to_send, req->bufflen,
+               req->own_sg);
 
 out:
-       /* Aborted commands will be freed in cmnd_rx_end() */
        TRACE_EXIT_RES(res);
        return res;
+
+out_close:
+       mark_conn_closed(conn);
+       res = -EINVAL;
+       goto out;
 }
 
 static int scsi_cmnd_start(struct iscsi_cmnd *req)
@@ -1483,7 +1774,7 @@ static int scsi_cmnd_start(struct iscsi_cmnd *req)
 
        TRACE_ENTRY();
 
-       TRACE_DBG("scsi command: %02x", req_hdr->scb[0]);
+       TRACE_DBG("scsi command: %x", req_hdr->scb[0]);
 
        TRACE_DBG("Incrementing active_cmds (cmd %p, sess %p, "
                "new value %d)", req, session,
@@ -1495,8 +1786,8 @@ static int scsi_cmnd_start(struct iscsi_cmnd *req)
                (uint8_t *)&req_hdr->lun, sizeof(req_hdr->lun),
                req_hdr->scb, sizeof(req_hdr->scb), SCST_NON_ATOMIC);
        if (scst_cmd == NULL) {
-               create_status_rsp(req, SAM_STAT_BUSY, NULL, 0);
-               cmnd_reject_scsi_cmd(req);
+               res = create_preliminary_status_rsp(req, SAM_STAT_BUSY,
+                       NULL, 0);
                goto out;
        }
 
@@ -1511,12 +1802,8 @@ static int scsi_cmnd_start(struct iscsi_cmnd *req)
                        PRINT_ERROR("%s", "BIDI data transfer, but initiator "
                                "not supplied Bidirectional Read Expected Data "
                                "Transfer Length AHS");
-                       scst_set_cmd_error(scst_cmd,
+                       set_scst_preliminary_status_rsp(req, true,
                           SCST_LOAD_SENSE(scst_sense_parameter_value_invalid));
-                       /*
-                        * scst_cmd_init_done() will handle commands with
-                        * set status as preliminary completed
-                        */
                } else {
                        req->read_size = sz;
                        dir = SCST_DATA_BIDI;
@@ -1572,7 +1859,7 @@ static int scsi_cmnd_start(struct iscsi_cmnd *req)
        ahdr = (struct iscsi_ahs_hdr *)req->pdu.ahs;
        if (ahdr != NULL) {
                uint8_t *p = (uint8_t *)ahdr;
-               int size = 0;
+               unsigned int size = 0;
                do {
                        int s;
 
@@ -1592,7 +1879,7 @@ static int scsi_cmnd_start(struct iscsi_cmnd *req)
                } while (size < req->pdu.ahssize);
        }
 
-       TRACE_DBG("START Command (tag %d, queue_type %d)",
+       TRACE_DBG("START Command (itt %x, queue_type %d)",
                req_hdr->itt, scst_cmd->queue_type);
        req->scst_state = ISCSI_CMD_STATE_RX_CMD;
        conn->rx_task = current;
@@ -1607,16 +1894,17 @@ static int scsi_cmnd_start(struct iscsi_cmnd *req)
        }
 
 out:
-       /* Aborted commands will be freed in cmnd_rx_end() */
        TRACE_EXIT_RES(res);
        return res;
 }
 
-static int data_out_start(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
+static int data_out_start(struct iscsi_cmnd *cmnd)
 {
+       struct iscsi_conn *conn = cmnd->conn;
        struct iscsi_data_out_hdr *req_hdr =
                (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
-       struct iscsi_cmnd *orig_req = NULL;
+       struct iscsi_cmnd *orig_req;
+       struct iscsi_hdr *orig_req_hdr;
        u32 offset = be32_to_cpu(req_hdr->buffer_offset);
        int res = 0;
 
@@ -1630,51 +1918,59 @@ static int data_out_start(struct iscsi_conn *conn, struct iscsi_cmnd *cmnd)
 
        update_stat_sn(cmnd);
 
-       cmnd->cmd_req = orig_req = cmnd_find_hash(conn->session, req_hdr->itt,
-                                       req_hdr->ttt);
+       orig_req = cmnd_find_data_wait_hash(conn, req_hdr->itt);
+       cmnd->cmd_req = orig_req;
        if (unlikely(orig_req == NULL)) {
-               /* It might happen if req was aborted and then freed */
-               TRACE(TRACE_MGMT_MINOR, "Unable to find scsi task %x %x",
-                       cmnd_itt(cmnd), cmnd_ttt(cmnd));
-               goto out_reject;
+               /*
+                * It shouldn't happen, since we don't abort any request until
+                * we received all related PDUs from the initiator or timeout
+                * them. Let's quietly drop such PDUs.
+                */
+               TRACE(TRACE_MGMT_MINOR, "Unable to find scsi task ITT %x",
+                       cmnd_itt(cmnd));
+               res = iscsi_preliminary_complete(cmnd, cmnd, true);
+               goto out;
        }
 
-       if (orig_req->is_unsolicited_data) {
-               if (unlikely(orig_req->r2t_length < cmnd->pdu.datasize)) {
-                       PRINT_ERROR("Data size (%d) > R2T length (%d)",
-                               cmnd->pdu.datasize, orig_req->r2t_length);
-                       mark_conn_closed(conn);
-                       res = -EINVAL;
-                       goto out;
+       if (unlikely(orig_req->r2t_len_to_receive < cmnd->pdu.datasize)) {
+               if (orig_req->prelim_compl_flags != 0) {
+                       /* We can have fake r2t_len_to_receive */
+                       goto go;
                }
-               orig_req->r2t_length -= cmnd->pdu.datasize;
+               PRINT_ERROR("Data size (%d) > R2T length to receive (%d)",
+                       cmnd->pdu.datasize, orig_req->r2t_len_to_receive);
+               set_scst_preliminary_status_rsp(orig_req, false,
+                       SCST_LOAD_SENSE(iscsi_sense_incorrect_amount_of_data));
+               goto go;
        }
 
-       /* Check unsolicited burst data */
-       if (unlikely((req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) &&
-                    (orig_req->pdu.bhs.flags & ISCSI_FLG_FINAL))) {
-               PRINT_ERROR("Unexpected data from %x %x",
-                       cmnd_itt(cmnd), cmnd_ttt(cmnd));
-               mark_conn_closed(conn);
-               res = -EINVAL;
+       /* Crazy iSCSI spec requires us to make this unneeded check */
+       orig_req_hdr = &orig_req->pdu.bhs;
+       if (unlikely(orig_req_hdr->lun != req_hdr->lun)) {
+               PRINT_ERROR("Wrong LUN (%lld) in Data-Out PDU (expected %lld), "
+                       "orig_req %p, cmnd %p", (unsigned long long)req_hdr->lun,
+                       (unsigned long long)orig_req_hdr->lun, orig_req, cmnd);
+               create_reject_rsp(orig_req, ISCSI_REASON_PROTOCOL_ERROR, false);
+               goto go;
+       }
+
+go:
+       if (req_hdr->flags & ISCSI_FLG_FINAL)
+               orig_req->outstanding_r2t--;
+
+       if (unlikely(orig_req->prelim_compl_flags != 0)) {
+               res = iscsi_preliminary_complete(cmnd, orig_req, true);
                goto out;
        }
 
-       TRACE_WRITE("%u %p %p %u %u", req_hdr->ttt, cmnd, orig_req,
-               offset, cmnd->pdu.datasize);
+       TRACE_WRITE("cmnd %p, orig_req %p, offset %u, datasize %u", cmnd,
+               orig_req, offset, cmnd->pdu.datasize);
 
        res = cmnd_prepare_recv_pdu(conn, orig_req, offset, cmnd->pdu.datasize);
 
 out:
        TRACE_EXIT_RES(res);
        return res;
-
-out_reject:
-       sBUG_ON(cmnd->rejected);
-       cmnd->rejected = 1;
-       cmnd->reject_reason = ISCSI_REJECT_DATA;
-       cmnd_prepare_get_rejected_cmd_data(cmnd);
-       goto out;
 }
 
 static void data_out_end(struct iscsi_cmnd *cmnd)
@@ -1683,9 +1979,12 @@ static void data_out_end(struct iscsi_cmnd *cmnd)
                (struct iscsi_data_out_hdr *)&cmnd->pdu.bhs;
        struct iscsi_cmnd *req;
 
-       sBUG_ON(cmnd == NULL);
+       TRACE_ENTRY();
+
+       EXTRACHECKS_BUG_ON(cmnd == NULL);
        req = cmnd->cmd_req;
-       sBUG_ON(req == NULL);
+       if (unlikely(req == NULL))
+               goto out;
 
        TRACE_DBG("cmnd %p, req %p", cmnd, req);
 
@@ -1697,74 +1996,58 @@ static void data_out_end(struct iscsi_cmnd *cmnd)
                cmnd_get(cmnd);
        }
 
-       if (req_hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
-               TRACE_DBG("ISCSI_RESERVED_TAG, FINAL %x",
-                       req_hdr->flags & ISCSI_FLG_FINAL);
+       /*
+        * Now we received the data and can adjust r2t_len_to_receive of the
+        * orig req. We couldn't do it earlier, because it will break data
+        * receiving errors recovery (calls of iscsi_fail_data_waiting_cmnd()).
+        */
+       req->r2t_len_to_receive -= cmnd->pdu.datasize;
 
-               if (req_hdr->flags & ISCSI_FLG_FINAL) {
-                       req->is_unsolicited_data = 0;
-                       if (req->pending)
-                               goto out_put;
-               } else
-                       goto out_put;
-       } else {
-               TRACE_DBG("FINAL %x, outstanding_r2t %d, r2t_length %d",
-                       req_hdr->flags & ISCSI_FLG_FINAL,
-                       req->outstanding_r2t, req->r2t_length);
-
-               if (req_hdr->flags & ISCSI_FLG_FINAL) {
-                       if (unlikely(req->is_unsolicited_data)) {
-                               PRINT_ERROR("Unexpected unsolicited data "
-                                       "(r2t_length %u, outstanding_r2t %d)",
-                                       req->r2t_length,
-                                       req->is_unsolicited_data);
-                               mark_conn_closed(req->conn);
-                               goto out_put;
-                       }
-                       req->outstanding_r2t--;
-               } else
-                       goto out_put;
+       if (unlikely(req->prelim_compl_flags != 0)) {
+               /*
+                * We might need to wait for one or more PDUs. Let's simplify
+                * other code.
+                */
+               req->r2t_len_to_receive = req->outstanding_r2t;
+               req->r2t_len_to_send = 0;
        }
 
-       if (req->r2t_length != 0) {
-               if (!req->is_unsolicited_data)
-                       send_r2t(req);
-       } else
-               iscsi_restart_waiting_cmnd(req);
+       TRACE_DBG("req %p, FINAL %x, outstanding_r2t %d, r2t_len_to_receive %d,"
+               " r2t_len_to_send %d", req, req_hdr->flags & ISCSI_FLG_FINAL,
+               req->outstanding_r2t, req->r2t_len_to_receive,
+               req->r2t_len_to_send);
 
-out_put:
-       cmnd_put(cmnd);
+       if (!(req_hdr->flags & ISCSI_FLG_FINAL))
+               goto out;
+
+       if (req->r2t_len_to_receive == 0) {
+               if (!req->pending)
+                       iscsi_restart_cmnd(req);
+       } else if (req->r2t_len_to_send != 0)
+               send_r2t(req);
+
+out:
+       TRACE_EXIT();
        return;
 }
 
+/* Might be called under target_mutex and cmd_list_lock */
 static void __cmnd_abort(struct iscsi_cmnd *cmnd)
 {
-       /*
-        * Here, if cmnd is data_waiting, we should iscsi_fail_waiting_cmnd()
-        * it. But, since this function can be called from any thread, not only
-        * from the read one, we at the moment can't do that, because of
-        * absence of appropriate locking protection. But this isn't a stuff
-        * for 1.0.0. So, currently a misbehaving initiator, not sending
-        * data in R2T state for a sharing between targets device, for which
-        * for some reason an aborting TM command, e.g. TARGET RESET, from
-        * another initiator is issued, can block response for this TM command
-        * virtually forever and by this make the issuing initiator eventually
-        * put the device offline.
-        *
-        * ToDo in the next version, possibly a simple connection mutex, taken
-        * by the read thread before starting any processing and by this
-        * function, should be sufficient.
-        */
+       unsigned long timeout_time = jiffies + ISCSI_TM_DATA_WAIT_SCHED_TIMEOUT;
+       struct iscsi_conn *conn = cmnd->conn;
 
        TRACE_MGMT_DBG("Aborting cmd %p, scst_cmd %p (scst state %x, "
-               "ref_cnt %d, itt %x, sn %u, op %x, r2t_len %x, CDB op %x, "
-               "size to write %u, is_unsolicited_data %d, "
-               "outstanding_r2t %d, data_waiting %d, sess->exp_cmd_sn %u, "
-               "conn %p, rd_task %p)", cmnd, cmnd->scst_cmd, cmnd->scst_state,
-               atomic_read(&cmnd->ref_cnt), cmnd_itt(cmnd), cmnd->pdu.bhs.sn,
-               cmnd_opcode(cmnd), cmnd->r2t_length, cmnd_scsicode(cmnd),
-               cmnd_write_size(cmnd), cmnd->is_unsolicited_data,
-               cmnd->outstanding_r2t, cmnd->data_waiting,
+               "ref_cnt %d, on_write_timeout_list %d, write_start %ld, ITT %x, "
+               "sn %u, op %x, r2t_len_to_receive %d, r2t_len_to_send %d, "
+               "CDB op %x, size to write %u, outstanding_r2t %d, "
+               "sess->exp_cmd_sn %u, conn %p, rd_task %p)",
+               cmnd, cmnd->scst_cmd, cmnd->scst_state,
+               atomic_read(&cmnd->ref_cnt), cmnd->on_write_timeout_list,
+               cmnd->write_start, cmnd_itt(cmnd), cmnd->pdu.bhs.sn,
+               cmnd_opcode(cmnd), cmnd->r2t_len_to_receive,
+               cmnd->r2t_len_to_send, cmnd_scsicode(cmnd),
+               cmnd_write_size(cmnd), cmnd->outstanding_r2t,
                cmnd->conn->session->exp_cmd_sn, cmnd->conn,
                cmnd->conn->rd_task);
 
@@ -1772,30 +2055,63 @@ static void __cmnd_abort(struct iscsi_cmnd *cmnd)
        TRACE_MGMT_DBG("net_ref_cnt %d", atomic_read(&cmnd->net_ref_cnt));
 #endif
 
-       cmnd->tm_aborted = 1;
+       /*
+        * Lock to sync with iscsi_check_tm_data_wait_timeouts(), including
+        * CMD_ABORTED bit set.
+        */
+       spin_lock_bh(&iscsi_rd_lock);
+
+       /*
+        * We suppose that preliminary commands completion is tested by
+        * comparing prelim_compl_flags with 0. Otherwise a race is possible,
+        * like sending command in SCST core as PRELIM_COMPLETED, while it
+        * wasn't aborted in it yet and have as the result a wrong success
+        * status sent to the initiator.
+        */
+       set_bit(ISCSI_CMD_ABORTED, &cmnd->prelim_compl_flags);
+
+       TRACE_MGMT_DBG("Setting conn_tm_active for conn %p", conn);
+       conn->conn_tm_active = 1;
+
+       spin_unlock_bh(&iscsi_rd_lock);
+
+       /*
+        * We need the lock to sync with req_add_to_write_timeout_list() and
+        * close races for rsp_timer.expires.
+        */
+       spin_lock_bh(&conn->write_list_lock);
+       if (!timer_pending(&conn->rsp_timer) ||
+           time_after(conn->rsp_timer.expires, timeout_time)) {
+               TRACE_MGMT_DBG("Mod timer on %ld (conn %p)", timeout_time,
+                       conn);
+               mod_timer(&conn->rsp_timer, timeout_time);
+       } else
+               TRACE_MGMT_DBG("Timer for conn %p is going to fire on %ld "
+                       "(timeout time %ld)", conn, conn->rsp_timer.expires,
+                       timeout_time);
+       spin_unlock_bh(&conn->write_list_lock);
 
        return;
 }
 
 /* Must be called from the read or conn close thread */
-static int cmnd_abort(struct iscsi_cmnd *req)
+static int cmnd_abort(struct iscsi_cmnd *req, int *status)
 {
-       struct iscsi_session *session = req->conn->session;
        struct iscsi_task_mgt_hdr *req_hdr =
                (struct iscsi_task_mgt_hdr *)&req->pdu.bhs;
        struct iscsi_cmnd *cmnd;
-       int err;
+       int res = -1;
 
        req_hdr->ref_cmd_sn = be32_to_cpu(req_hdr->ref_cmd_sn);
 
-       if (after(req_hdr->ref_cmd_sn, req_hdr->cmd_sn)) {
-               PRINT_ERROR("ABORT TASK: RefCmdSN(%u) > CmdSN(%u)",
+       if (!before(req_hdr->ref_cmd_sn, req_hdr->cmd_sn)) {
+               TRACE(TRACE_MGMT, "ABORT TASK: RefCmdSN(%u) > CmdSN(%u)",
                        req_hdr->ref_cmd_sn, req_hdr->cmd_sn);
-               err = ISCSI_RESPONSE_FUNCTION_REJECTED;
+               *status = ISCSI_RESPONSE_UNKNOWN_TASK;
                goto out;
        }
 
-       cmnd = cmnd_find_hash_get(session, req_hdr->rtt, ISCSI_RESERVED_TAG);
+       cmnd = cmnd_find_itt_get(req->conn, req_hdr->rtt);
        if (cmnd) {
                struct iscsi_conn *conn = cmnd->conn;
                struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
@@ -1806,7 +2122,7 @@ static int cmnd_abort(struct iscsi_cmnd *req)
                                    (long long unsigned int)req_hdr->lun,
                                    (long long unsigned int)hdr->lun,
                                    req_hdr->rtt);
-                       err = ISCSI_RESPONSE_FUNCTION_REJECTED;
+                       *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
                        goto out_put;
                }
 
@@ -1816,7 +2132,7 @@ static int cmnd_abort(struct iscsi_cmnd *req)
                                        "cmd CmdSN(%u) for immediate command "
                                        "%p", req_hdr->ref_cmd_sn,
                                        req_hdr->cmd_sn, cmnd);
-                               err = ISCSI_RESPONSE_FUNCTION_REJECTED;
+                               *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
                                goto out_put;
                        }
                } else {
@@ -1825,7 +2141,7 @@ static int cmnd_abort(struct iscsi_cmnd *req)
                                        "CmdSN(%u) for command %p",
                                        req_hdr->ref_cmd_sn, req_hdr->cmd_sn,
                                        cmnd);
-                               err = ISCSI_RESPONSE_FUNCTION_REJECTED;
+                               *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
                                goto out_put;
                        }
                }
@@ -1835,7 +2151,7 @@ static int cmnd_abort(struct iscsi_cmnd *req)
                        PRINT_ERROR("ABORT TASK: SN mismatch: req SN %x, "
                                "cmd SN %x, rtt %u", req_hdr->cmd_sn,
                                hdr->cmd_sn, req_hdr->rtt);
-                       err = ISCSI_RESPONSE_FUNCTION_REJECTED;
+                       *status = ISCSI_RESPONSE_FUNCTION_REJECTED;
                        goto out_put;
                }
 
@@ -1844,14 +2160,36 @@ static int cmnd_abort(struct iscsi_cmnd *req)
                spin_unlock_bh(&conn->cmd_list_lock);
 
                cmnd_put(cmnd);
-               err = 0;
+               res = 0;
        } else {
                TRACE_MGMT_DBG("cmd RTT %x not found", req_hdr->rtt);
-               err = ISCSI_RESPONSE_UNKNOWN_TASK;
+               /*
+                * iSCSI RFC:
+                *
+                * b)  If the Referenced Task Tag does not identify an existing task,
+                * but if the CmdSN indicated by the RefCmdSN field in the Task
+                * Management function request is within the valid CmdSN window
+                * and less than the CmdSN of the Task Management function
+                * request itself, then targets must consider the CmdSN received
+                * and return the "Function complete" response.
+                *
+                * c)  If the Referenced Task Tag does not identify an existing task
+                * and if the CmdSN indicated by the RefCmdSN field in the Task
+                * Management function request is outside the valid CmdSN window,
+                * then targets must return the "Task does not exist" response.
+                *
+                * 128 seems to be a good "window".
+                */
+               if (between(req_hdr->ref_cmd_sn, req_hdr->cmd_sn - 128,
+                           req_hdr->cmd_sn)) {
+                       *status = ISCSI_RESPONSE_FUNCTION_COMPLETE;
+                       res = 0;
+               } else
+                       *status = ISCSI_RESPONSE_UNKNOWN_TASK;
        }
 
 out:
-       return err;
+       return res;
 
 out_put:
        cmnd_put(cmnd);
@@ -1937,14 +2275,12 @@ void conn_abort(struct iscsi_conn *conn)
 again:
        list_for_each_entry(cmnd, &conn->cmd_list, cmd_list_entry) {
                __cmnd_abort(cmnd);
-               if (cmnd->data_waiting) {
+               if (cmnd->r2t_len_to_receive != 0) {
                        if (!cmnd_get_check(cmnd)) {
                                spin_unlock_bh(&conn->cmd_list_lock);
 
                                /* ToDo: this is racy for MC/S */
-                               TRACE_MGMT_DBG("Restarting data waiting cmd "
-                                       "%p", cmnd);
-                               iscsi_fail_waiting_cmnd(cmnd);
+                               iscsi_fail_data_waiting_cmnd(cmnd);
 
                                cmnd_put(cmnd);
 
@@ -1976,7 +2312,7 @@ static void execute_task_management(struct iscsi_cmnd *req)
                        TRACE_MGMT_MINOR : TRACE_MGMT,
                "TM fn %d", function);
 
-       TRACE_MGMT_DBG("TM req %p, itt %x, rtt %x, sn %u, con %p", req,
+       TRACE_MGMT_DBG("TM req %p, ITT %x, RTT %x, sn %u, con %p", req,
                cmnd_itt(req), req_hdr->rtt, req_hdr->cmd_sn, conn);
 
        iscsi_extracheck_is_rd_thread(conn);
@@ -2017,9 +2353,8 @@ static void execute_task_management(struct iscsi_cmnd *req)
 
        switch (function) {
        case ISCSI_FUNCTION_ABORT_TASK:
-               rc = -1;
-               status = cmnd_abort(req);
-               if (status == 0) {
+               rc = cmnd_abort(req, &status);
+               if (rc == 0) {
                        params.fn = SCST_ABORT_TASK;
                        params.tag = req_hdr->rtt;
                        params.tag_set = 1;
@@ -2113,10 +2448,12 @@ static void noop_out_exec(struct iscsi_cmnd *req)
        struct iscsi_cmnd *rsp;
        struct iscsi_nop_in_hdr *rsp_hdr;
 
+       TRACE_ENTRY();
+
        TRACE_DBG("%p", req);
 
        if (cmnd_itt(req) != cpu_to_be32(ISCSI_RESERVED_TAG)) {
-               rsp = iscsi_cmnd_create_rsp_cmnd(req);
+               rsp = iscsi_alloc_main_rsp(req);
 
                rsp_hdr = (struct iscsi_nop_in_hdr *)&rsp->pdu.bhs;
                rsp_hdr->opcode = ISCSI_OP_NOOP_IN;
@@ -2139,11 +2476,11 @@ static void noop_out_exec(struct iscsi_cmnd *req)
                sBUG_ON(get_pgcnt(req->pdu.datasize, 0) > ISCSI_CONN_IOV_MAX);
 
                rsp->pdu.datasize = req->pdu.datasize;
-               iscsi_cmnd_init_write(rsp,
-                       ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
-               req_cmnd_release(req);
-       } else
-               cmnd_put(req);
+       }
+
+       req_cmnd_release(req);
+
+       TRACE_EXIT();
        return;
 }
 
@@ -2158,15 +2495,15 @@ static void logout_exec(struct iscsi_cmnd *req)
        TRACE_DBG("%p", req);
 
        req_hdr = (struct iscsi_logout_req_hdr *)&req->pdu.bhs;
-       rsp = iscsi_cmnd_create_rsp_cmnd(req);
+       rsp = iscsi_alloc_main_rsp(req);
        rsp_hdr = (struct iscsi_logout_rsp_hdr *)&rsp->pdu.bhs;
        rsp_hdr->opcode = ISCSI_OP_LOGOUT_RSP;
        rsp_hdr->flags = ISCSI_FLG_FINAL;
        rsp_hdr->itt = req_hdr->itt;
        rsp->should_close_conn = 1;
-       iscsi_cmnd_init_write(rsp,
-               ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
+
        req_cmnd_release(req);
+
        return;
 }
 
@@ -2174,30 +2511,27 @@ static void iscsi_cmnd_exec(struct iscsi_cmnd *cmnd)
 {
        TRACE_ENTRY();
 
-       TRACE_DBG("%p,%x,%u", cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn);
+       TRACE_DBG("cmnd %p, op %x, SN %u", cmnd, cmnd_opcode(cmnd),
+               cmnd->pdu.bhs.sn);
 
        iscsi_extracheck_is_rd_thread(cmnd->conn);
 
-       if (unlikely(cmnd->tm_aborted)) {
-               TRACE_MGMT_DBG("cmnd %p (scst_cmd %p) aborted", cmnd,
-                       cmnd->scst_cmd);
-               req_cmnd_release_force(cmnd, ISCSI_FORCE_RELEASE_WRITE);
+       if (cmnd_opcode(cmnd) == ISCSI_OP_SCSI_CMD) {
+               if (cmnd->r2t_len_to_receive == 0)
+                       iscsi_restart_cmnd(cmnd);
+               else if (cmnd->r2t_len_to_send != 0)
+                       send_r2t(cmnd);
                goto out;
        }
 
-       if (unlikely(cmnd->rejected))
-               goto out_rejected;
+       if (cmnd->prelim_compl_flags != 0) {
+               TRACE_MGMT_DBG("Terminating prelim completed non-SCSI cmnd %p "
+                       "(op %x)", cmnd, cmnd_opcode(cmnd));
+               req_cmnd_release(cmnd);
+               goto out;
+       }
 
        switch (cmnd_opcode(cmnd)) {
-       case ISCSI_OP_SCSI_CMD:
-               if (cmnd->r2t_length != 0) {
-                       if (!cmnd->is_unsolicited_data) {
-                               send_r2t(cmnd);
-                               break;
-                       }
-               } else
-                       iscsi_restart_cmnd(cmnd);
-               break;
        case ISCSI_OP_NOOP_OUT:
                noop_out_exec(cmnd);
                break;
@@ -2208,35 +2542,16 @@ static void iscsi_cmnd_exec(struct iscsi_cmnd *cmnd)
                logout_exec(cmnd);
                break;
        default:
-               PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
-               req_cmnd_release(cmnd);
+               PRINT_CRIT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
+               sBUG();
                break;
        }
+
 out:
        TRACE_EXIT();
        return;
-
-out_rejected:
-       TRACE_MGMT_DBG("Rejected cmd %p (reason %d)", cmnd,
-               cmnd->reject_reason);
-       switch (cmnd->reject_reason) {
-       default:
-               PRINT_ERROR("Unexpected reject reason %d",
-                           cmnd->reject_reason);
-               /* go through */
-       case ISCSI_REJECT_SCSI_CMD:
-               req_cmnd_release(cmnd);
-               break;
-       }
-       goto out;
 }
 
-/*
- * Note: the code belows passes a kernel space pointer (&opt) to setsockopt()
- * while the declaration of setsockopt specifies that it expects a user space
- * pointer. This seems to work fine, and this approach is also used in some
- * other parts of the Linux kernel (see e.g. fs/ocfs2/cluster/tcp.c).
- */
 static void set_cork(struct socket *sock, int on)
 {
        int opt = on;
@@ -2327,6 +2642,7 @@ void cmnd_tx_end(struct iscsi_cmnd *cmnd)
                cmnd, cmnd_opcode(cmnd), cmnd->should_close_conn,
                cmnd->should_close_all_conn);
 
+#ifdef CONFIG_SCST_EXTRACHECKS
        switch (cmnd_opcode(cmnd)) {
        case ISCSI_OP_NOOP_IN:
        case ISCSI_OP_SCSI_RSP:
@@ -2343,6 +2659,7 @@ void cmnd_tx_end(struct iscsi_cmnd *cmnd)
                sBUG();
                break;
        }
+#endif
 
        if (unlikely(cmnd->should_close_conn)) {
                if (cmnd->should_close_all_conn) {
@@ -2365,16 +2682,22 @@ void cmnd_tx_end(struct iscsi_cmnd *cmnd)
 /*
  * Push the command for execution. This functions reorders the commands.
  * Called from the read thread.
+ *
+ * Basically, since we don't support MC/S and TCP guarantees data delivery
+ * order, all that SN's stuff isn't needed at all (commands delivery order is
+ * a natural commands execution order), but insane iSCSI spec requires
+ * us to check it and we have to, because some crazy initiators can rely
+ * on the SN's based order and reorder requests during sending. For all other
+ * normal initiators all that code is a NOP.
  */
-static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd)
+static void iscsi_push_cmnd(struct iscsi_cmnd *cmnd)
 {
        struct iscsi_session *session = cmnd->conn->session;
        struct list_head *entry;
        u32 cmd_sn;
 
-       TRACE_DBG("%p:%x %u,%u",
-               cmnd, cmnd_opcode(cmnd), cmnd->pdu.bhs.sn,
-               session->exp_cmd_sn);
+       TRACE_DBG("cmnd %p, iSCSI opcode %x, sn %u, exp sn %u", cmnd,
+               cmnd_opcode(cmnd), cmnd->pdu.bhs.sn, session->exp_cmd_sn);
 
        iscsi_extracheck_is_rd_thread(cmnd->conn);
 
@@ -2426,7 +2749,7 @@ static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd)
                        list_del(&cmnd->pending_list_entry);
                        cmnd->pending = 0;
 
-                       TRACE_DBG("Processing pending cmd %p (cmd_sn %u)",
+                       TRACE_MGMT_DBG("Processing pending cmd %p (cmd_sn %u)",
                                cmnd, cmd_sn);
                }
        } else {
@@ -2464,12 +2787,12 @@ static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd)
                spin_unlock(&session->sn_lock);
 
                if (unlikely(drop)) {
-                       req_cmnd_release_force(cmnd,
-                                              ISCSI_FORCE_RELEASE_WRITE);
+                       req_cmnd_release_force(cmnd);
                        goto out;
                }
 
-               if (unlikely(cmnd->tm_aborted)) {
+               if (unlikely(test_bit(ISCSI_CMD_ABORTED,
+                                       &cmnd->prelim_compl_flags))) {
                        struct iscsi_cmnd *tm_clone;
 
                        TRACE_MGMT_DBG("Pending aborted cmnd %p, creating TM "
@@ -2478,7 +2801,8 @@ static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd)
 
                        tm_clone = cmnd_alloc(cmnd->conn, NULL);
                        if (tm_clone != NULL) {
-                               tm_clone->tm_aborted = 1;
+                               set_bit(ISCSI_CMD_ABORTED,
+                                       &tm_clone->prelim_compl_flags);
                                tm_clone->pdu = cmnd->pdu;
 
                                TRACE_MGMT_DBG("TM clone %p created",
@@ -2490,6 +2814,9 @@ static void iscsi_session_push_cmnd(struct iscsi_cmnd *cmnd)
                                PRINT_ERROR("%s", "Unable to create TM clone");
                }
 
+               TRACE_MGMT_DBG("Pending cmnd %p (op %x, sn %u, exp sn %u)",
+                       cmnd, cmnd_opcode(cmnd), cmd_sn, session->exp_cmd_sn);
+
                spin_lock(&session->sn_lock);
                list_for_each(entry, &session->pending_list) {
                        struct iscsi_cmnd *tmp =
@@ -2526,7 +2853,6 @@ static int check_segment_length(struct iscsi_cmnd *cmnd)
 
 int cmnd_rx_start(struct iscsi_cmnd *cmnd)
 {
-       struct iscsi_conn *conn = cmnd->conn;
        int res, rc;
 
        iscsi_dump_pdu(&cmnd->pdu);
@@ -2536,45 +2862,39 @@ int cmnd_rx_start(struct iscsi_cmnd *cmnd)
                goto out;
 
        switch (cmnd_opcode(cmnd)) {
-       case ISCSI_OP_NOOP_OUT:
-               rc = noop_out_start(cmnd);
-               break;
        case ISCSI_OP_SCSI_CMD:
-               rc = cmnd_insert_hash(cmnd);
-               if (likely(rc == 0)) {
-                       res = scsi_cmnd_start(cmnd);
-                       if (unlikely(res != 0))
-                               goto out;
-               }
-               break;
-       case ISCSI_OP_SCSI_TASK_MGT_MSG:
-               rc = cmnd_insert_hash(cmnd);
+               res = scsi_cmnd_start(cmnd);
+               if (unlikely(res < 0))
+                       goto out;
+               spin_lock(&cmnd->conn->session->sn_lock);
+               __update_stat_sn(cmnd);
+               rc = check_cmd_sn(cmnd);
+               spin_unlock(&cmnd->conn->session->sn_lock);
                break;
        case ISCSI_OP_SCSI_DATA_OUT:
-               res = data_out_start(conn, cmnd);
-               rc = 0; /* to avoid compiler warning */
-               if (unlikely(res != 0))
-                       goto out;
+               res = data_out_start(cmnd);
+               goto out;
+       case ISCSI_OP_NOOP_OUT:
+               rc = noop_out_start(cmnd);
                break;
+       case ISCSI_OP_SCSI_TASK_MGT_MSG:
        case ISCSI_OP_LOGOUT_CMD:
-               rc = cmnd_insert_hash(cmnd);
+               spin_lock(&cmnd->conn->session->sn_lock);
+               __update_stat_sn(cmnd);
+               rc = check_cmd_sn(cmnd);
+               spin_unlock(&cmnd->conn->session->sn_lock);
                break;
        case ISCSI_OP_TEXT_CMD:
        case ISCSI_OP_SNACK_CMD:
-               rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
-               break;
        default:
                rc = -ISCSI_REASON_UNSUPPORTED_COMMAND;
                break;
        }
 
        if (unlikely(rc < 0)) {
-               struct iscsi_scsi_cmd_hdr *hdr = cmnd_hdr(cmnd);
-               PRINT_ERROR("Error %d (iSCSI opcode %x, ITT %x, op %x)", rc,
-                       cmnd_opcode(cmnd), cmnd_itt(cmnd),
-                       (cmnd_opcode(cmnd) == ISCSI_OP_SCSI_CMD ?
-                               hdr->scb[0] : -1));
-               iscsi_cmnd_reject(cmnd, -rc);
+               PRINT_ERROR("Error %d (iSCSI opcode %x, ITT %x)", rc,
+                       cmnd_opcode(cmnd), cmnd_itt(cmnd));
+               res = create_reject_rsp(cmnd, -rc, true);
        }
 
 out:
@@ -2586,46 +2906,28 @@ void cmnd_rx_end(struct iscsi_cmnd *cmnd)
 {
        TRACE_ENTRY();
 
-       TRACE_DBG("%p:%x", cmnd, cmnd_opcode(cmnd));
-
-       if (unlikely(cmnd->rejected))
-               goto out_rejected;
+       TRACE_DBG("cmnd %p, opcode %x", cmnd, cmnd_opcode(cmnd));
 
-cont:
        switch (cmnd_opcode(cmnd)) {
        case ISCSI_OP_SCSI_CMD:
        case ISCSI_OP_NOOP_OUT:
        case ISCSI_OP_SCSI_TASK_MGT_MSG:
        case ISCSI_OP_LOGOUT_CMD:
-               iscsi_session_push_cmnd(cmnd);
-               break;
+               iscsi_push_cmnd(cmnd);
+               goto out;
        case ISCSI_OP_SCSI_DATA_OUT:
                data_out_end(cmnd);
                break;
        default:
-               PRINT_ERROR("unexpected cmnd op %x", cmnd_opcode(cmnd));
-               req_cmnd_release(cmnd);
+               PRINT_ERROR("Unexpected cmnd op %x", cmnd_opcode(cmnd));
                break;
        }
 
+       req_cmnd_release(cmnd);
+
 out:
        TRACE_EXIT();
        return;
-
-out_rejected:
-       switch (cmnd->reject_reason) {
-       default:
-               PRINT_ERROR("Unexpected reject reason %d",
-                           cmnd->reject_reason);
-               /* go through */
-       case ISCSI_REJECT_CMD:
-       case ISCSI_REJECT_DATA:
-               req_cmnd_release(cmnd);
-               break;
-       case ISCSI_REJECT_SCSI_CMD:
-               goto cont;
-       }
-       goto out;
 }
 
 #if !defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
@@ -2644,11 +2946,15 @@ static int iscsi_alloc_data_buf(struct scst_cmd *cmd)
 }
 #endif
 
-static inline void iscsi_set_state_wake_up(struct iscsi_cmnd *req,
-       int new_state)
+static void iscsi_preprocessing_done(struct scst_cmd *scst_cmd)
 {
+       struct iscsi_cmnd *req = (struct iscsi_cmnd *)
+                               scst_cmd_get_tgt_priv(scst_cmd);
+
+       TRACE_DBG("req %p", req);
+
        if (req->conn->rx_task == current)
-               req->scst_state = new_state;
+               req->scst_state = ISCSI_CMD_STATE_AFTER_PREPROC;
        else {
                /*
                 * We wait for the state change without any protection, so
@@ -2657,9 +2963,13 @@ static inline void iscsi_set_state_wake_up(struct iscsi_cmnd *req,
                 * iscsi_make_conn_rd_active() will operate on dead data.
                 * We use the ordered version of cmnd_get(), because "get"
                 * must be done before the state assignment.
+                *
+                * We protected from the race on calling cmnd_rx_continue(),
+                * because there can be only one read thread processing
+                * connection.
                 */
                cmnd_get_ordered(req);
-               req->scst_state = new_state;
+               req->scst_state = ISCSI_CMD_STATE_AFTER_PREPROC;
                iscsi_make_conn_rd_active(req->conn);
                if (unlikely(req->conn->closing)) {
                        TRACE_DBG("Waking up closing conn %p", req->conn);
@@ -2667,19 +2977,7 @@ static inline void iscsi_set_state_wake_up(struct iscsi_cmnd *req,
                }
                cmnd_put(req);
        }
-       return;
-}
-
-static void iscsi_preprocessing_done(struct scst_cmd *scst_cmd)
-{
-       struct iscsi_cmnd *req = (struct iscsi_cmnd *)
-                               scst_cmd_get_tgt_priv(scst_cmd);
-
-       TRACE_DBG("req %p", req);
-
-       EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_RX_CMD);
 
-       iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_AFTER_PREPROC);
        return;
 }
 
@@ -2747,70 +3045,68 @@ static int iscsi_xmit_response(struct scst_cmd *scst_cmd)
        int status = scst_cmd_get_status(scst_cmd);
        u8 *sense = scst_cmd_get_sense_buffer(scst_cmd);
        int sense_len = scst_cmd_get_sense_buffer_len(scst_cmd);
-       int old_state = req->scst_state;
 
-       if (scst_cmd_atomic(scst_cmd))
+       if (unlikely(scst_cmd_atomic(scst_cmd)))
                return SCST_TGT_RES_NEED_THREAD_CTX;
 
        scst_cmd_set_tgt_priv(scst_cmd, NULL);
 
-       req->tm_aborted |= scst_cmd_aborted(scst_cmd) ? 1 : 0;
-       if (unlikely(req->tm_aborted)) {
-               TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
-                       req->scst_cmd);
+       EXTRACHECKS_BUG_ON(req->scst_state != ISCSI_CMD_STATE_RESTARTED);
 
-               scst_set_delivery_status(req->scst_cmd,
-                       SCST_CMD_DELIVERY_ABORTED);
+       if (unlikely(scst_cmd_aborted(scst_cmd)))
+               set_bit(ISCSI_CMD_ABORTED, &req->prelim_compl_flags);
 
-               if (old_state == ISCSI_CMD_STATE_RESTARTED) {
+       if (unlikely(req->prelim_compl_flags != 0)) {
+               if (test_bit(ISCSI_CMD_ABORTED, &req->prelim_compl_flags)) {
+                       TRACE_MGMT_DBG("req %p (scst_cmd %p) aborted", req,
+                               req->scst_cmd);
+                       scst_set_delivery_status(req->scst_cmd,
+                               SCST_CMD_DELIVERY_ABORTED);
                        req->scst_state = ISCSI_CMD_STATE_PROCESSED;
-                       req_cmnd_release_force(req, ISCSI_FORCE_RELEASE_WRITE);
-               } else
-                       iscsi_set_state_wake_up(req,
-                                               ISCSI_CMD_STATE_PROCESSED);
+                       req_cmnd_release_force(req);
+                       goto out;
+               }
 
-               goto out;
-       }
+               TRACE_DBG("Prelim completed req %p", req);
 
-       if (unlikely(old_state != ISCSI_CMD_STATE_RESTARTED)) {
-               TRACE_DBG("req %p on %d state", req, old_state);
+               /* To make sure we didn't miss anything above */
+               sBUG_ON(status == 0);
 
                /*
-                * We could preliminary have finished req before we knew its
-                * device, so check if we return correct sense format.
+                * We could preliminary have finished req before we
+                * knew its device, so check if we return correct sense
+                * format.
                 */
                scst_check_convert_sense(scst_cmd);
 
-               create_status_rsp(req, status, sense, sense_len);
-
-               switch (old_state) {
-               case ISCSI_CMD_STATE_RX_CMD:
-               case ISCSI_CMD_STATE_AFTER_PREPROC:
-                       break;
-               default:
-                       sBUG();
+               if (!req->own_sg) {
+                       req->sg = scst_cmd_get_sg(scst_cmd);
+                       req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
                }
-
-               iscsi_set_state_wake_up(req, ISCSI_CMD_STATE_PROCESSED);
-               goto out;
+       } else {
+               EXTRACHECKS_BUG_ON(req->own_sg);
+               req->sg = scst_cmd_get_sg(scst_cmd);
+               req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
        }
 
-       req->scst_state = ISCSI_CMD_STATE_PROCESSED;
-
        req->bufflen = scst_cmd_get_resp_data_len(scst_cmd);
-       req->sg = scst_cmd_get_sg(scst_cmd);
-       req->sg_cnt = scst_cmd_get_sg_cnt(scst_cmd);
+
+       req->scst_state = ISCSI_CMD_STATE_PROCESSED;
 
        TRACE_DBG("req %p, is_send_status=%x, req->bufflen=%d, req->sg=%p, "
                "req->sg_cnt %d", req, is_send_status, req->bufflen, req->sg,
                req->sg_cnt);
 
+       EXTRACHECKS_BUG_ON(req->hashed);
+       if (req->main_rsp != NULL)
+               EXTRACHECKS_BUG_ON(cmnd_opcode(req->main_rsp) != ISCSI_OP_REJECT);
+
        if (unlikely((req->bufflen != 0) && !is_send_status)) {
                PRINT_CRIT_ERROR("%s", "Sending DATA without STATUS is "
                        "unsupported");
                scst_set_cmd_error(scst_cmd,
                        SCST_LOAD_SENSE(scst_sense_hardw_error));
-               sBUG();
+               sBUG(); /* ToDo */
        }
 
        if (req->bufflen != 0) {
@@ -2828,17 +3124,14 @@ static int iscsi_xmit_response(struct scst_cmd *scst_cmd)
                        send_data_rsp(req, 0, 0);
                        if (is_send_status) {
                                rsp = create_status_rsp(req, status, sense,
-                                       sense_len);
-                               iscsi_set_resid(req, rsp, true);
-                               iscsi_cmnd_init_write(rsp,
-                                       ISCSI_INIT_WRITE_REMOVE_HASH);
+                                       sense_len, true);
+                               iscsi_cmnd_init_write(rsp, 0);
                        }
                }
        } else if (is_send_status) {
                struct iscsi_cmnd *rsp;
-               rsp = create_status_rsp(req, status, sense, sense_len);
-               iscsi_set_resid(req, rsp, false);
-               iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_REMOVE_HASH);
+               rsp = create_status_rsp(req, status, sense, sense_len, false);
+               iscsi_cmnd_init_write(rsp, 0);
        }
 #ifdef CONFIG_SCST_EXTRACHECKS
        else
@@ -2908,8 +3201,7 @@ static void iscsi_check_send_delayed_tm_resp(struct iscsi_session *sess)
 
        sBUG_ON(sess->tm_active < 0);
 
-       iscsi_cmnd_init_write(tm_rsp,
-               ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
+       iscsi_cmnd_init_write(tm_rsp, ISCSI_INIT_WRITE_WAKE);
 
        spin_lock(&sess->sn_lock);
 
@@ -2934,7 +3226,7 @@ static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status)
                         TRACE_MGMT_MINOR : TRACE_MGMT,
                "TM fn %d finished, status %d", fn, status);
 
-       rsp = iscsi_cmnd_create_rsp_cmnd(req);
+       rsp = iscsi_alloc_rsp(req);
        rsp_hdr = (struct iscsi_task_rsp_hdr *)&rsp->pdu.bhs;
 
        rsp_hdr->opcode = ISCSI_OP_SCSI_TASK_MGT_RSP;
@@ -2965,8 +3257,7 @@ static void iscsi_send_task_mgmt_resp(struct iscsi_cmnd *req, int status)
 
        sBUG_ON(sess->tm_active < 0);
 
-       iscsi_cmnd_init_write(rsp,
-               ISCSI_INIT_WRITE_REMOVE_HASH | ISCSI_INIT_WRITE_WAKE);
+       iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_WAKE);
 
 out_release:
        req_cmnd_release(req);
@@ -3005,8 +3296,15 @@ static void iscsi_task_mgmt_fn_done(struct scst_mgmt_cmd *scst_mcmd)
        int status =
                iscsi_get_mgmt_response(scst_mgmt_cmd_get_status(scst_mcmd));
 
-       TRACE_MGMT_DBG("req %p, scst_mcmd %p, fn %d, scst status %d",
-               req, scst_mcmd, fn, scst_mgmt_cmd_get_status(scst_mcmd));
+       if ((status == ISCSI_RESPONSE_UNKNOWN_TASK) &&
+           (fn == SCST_ABORT_TASK)) {
+               /* If we are here, we found the task, so must succeed */
+               status = ISCSI_RESPONSE_FUNCTION_COMPLETE;
+       }
+
+       TRACE_MGMT_DBG("req %p, scst_mcmd %p, fn %d, scst status %d, status %d",
+               req, scst_mcmd, fn, scst_mgmt_cmd_get_status(scst_mcmd),
+               status);
 
        switch (fn) {
        case SCST_NEXUS_LOSS_SESS:
@@ -3064,7 +3362,7 @@ static int iscsi_scsi_aen(struct scst_aen *aen)
 
        mutex_unlock(&sess->target->target_mutex);
 
-       rsp = iscsi_cmnd_create_rsp_cmnd(fake_req);
+       rsp = iscsi_alloc_main_rsp(fake_req);
        if (rsp == NULL) {
                PRINT_ERROR("%s", "Unable to alloc AEN rsp");
                goto out_err_free_req;
@@ -3093,8 +3391,6 @@ static int iscsi_scsi_aen(struct scst_aen *aen)
        rsp->pdu.datasize = sizeof(rsp->sense_hdr) + sense_len;
        rsp->bufflen = rsp->pdu.datasize;
 
-       iscsi_cmnd_init_write(rsp, ISCSI_INIT_WRITE_WAKE);
-
        req_cmnd_release(fake_req);
 
 out:
@@ -3145,7 +3441,6 @@ static int iscsi_target_release(struct scst_tgt *scst_tgt)
 
 #ifndef CONFIG_SCST_PROC
 static struct scst_trace_log iscsi_local_trace_tbl[] = {
-    { TRACE_D_READ,            "d_read" },
     { TRACE_D_WRITE,           "d_write" },
     { TRACE_CONN_OC,           "conn" },
     { TRACE_CONN_OC_DBG,       "conn_dbg" },
index c8e73d3..89e9d73 100644 (file)
@@ -29,8 +29,9 @@
 
 #include "iscsi_dbg.h"
 
-#define iscsi_sense_crc_error                  ABORTED_COMMAND, 0x47, 0x5
-#define iscsi_sense_unexpected_unsolicited_data        ABORTED_COMMAND, 0xC, 0xC
+#define iscsi_sense_crc_error                  ABORTED_COMMAND, 0x47, 0x05
+#define iscsi_sense_unexpected_unsolicited_data        ABORTED_COMMAND, 0x0C, 0x0C
+#define iscsi_sense_incorrect_amount_of_data   ABORTED_COMMAND, 0x0C, 0x0D
 
 struct iscsi_sess_param {
        int initial_r2t;
@@ -120,8 +121,13 @@ struct iscsi_session {
        /* Read only, if there are connection(s) */
        struct iscsi_sess_param sess_param;
 
-       spinlock_t cmnd_hash_lock;
-       struct list_head cmnd_hash[1 << ISCSI_HASH_ORDER];
+       /*
+        * In some corner cases commands can be deleted from the hash
+        * not from the corresponding read thread. So, let's simplify
+        * errors recovery and have this lock.
+        */
+       spinlock_t cmnd_data_wait_hash_lock;
+       struct list_head cmnd_data_wait_hash[1 << ISCSI_HASH_ORDER];
 
        struct list_head conn_list; /* protected by target_mutex */
 
@@ -170,7 +176,7 @@ struct iscsi_conn {
        /* List of data pdus to be sent, protected by write_list_lock */
        struct list_head write_list;
        /* List of data pdus being sent, protected by write_list_lock */
-       struct list_head written_list;
+       struct list_head write_timeout_list;
 
        struct timer_list rsp_timer;
 
@@ -208,13 +214,14 @@ struct iscsi_conn {
        int hdigest_type;
        int ddigest_type;
 
-       /* All 5 protected by iscsi_rd_lock */
+       /* All 6 protected by iscsi_rd_lock */
        unsigned short rd_state;
        unsigned short rd_data_ready:1;
        /* Let's save some cache footprint by putting them here */
        unsigned short closing:1;
        unsigned short active_close:1;
        unsigned short deleting:1;
+       unsigned short conn_tm_active:1;
 
        struct list_head rd_list_entry;
 
@@ -264,37 +271,38 @@ struct iscsi_pdu {
 typedef void (iscsi_show_info_t)(struct seq_file *seq,
                                 struct iscsi_target *target);
 
-/** Command's states **/
+/** Commands' states **/
 
 /* New command and SCST processes it */
-#define ISCSI_CMD_STATE_NEW               0
+#define ISCSI_CMD_STATE_NEW            0
 
 /* SCST processes cmd after scst_rx_cmd() */
-#define ISCSI_CMD_STATE_RX_CMD            1
+#define ISCSI_CMD_STATE_RX_CMD         1
 
 /* The command returned from preprocessing_done() */
-#define ISCSI_CMD_STATE_AFTER_PREPROC     2
+#define ISCSI_CMD_STATE_AFTER_PREPROC  2
 
 /* The command is waiting for session or connection reinstatement finished */
-#define ISCSI_CMD_STATE_REINST_PENDING    3
+#define ISCSI_CMD_STATE_REINST_PENDING 3
 
 /* scst_restart_cmd() called and SCST processing it */
-#define ISCSI_CMD_STATE_RESTARTED         4
+#define ISCSI_CMD_STATE_RESTARTED      4
 
 /* SCST done processing */
-#define ISCSI_CMD_STATE_PROCESSED         5
+#define ISCSI_CMD_STATE_PROCESSED      5
 
 /* AEN processing */
-#define ISCSI_CMD_STATE_AEN               6
+#define ISCSI_CMD_STATE_AEN            6
 
-/** Command's reject reasons **/
-#define ISCSI_REJECT_SCSI_CMD             1
-#define ISCSI_REJECT_CMD                  2
-#define ISCSI_REJECT_DATA                 3
+/* Out of SCST core preliminary completed */
+#define ISCSI_CMD_STATE_OUT_OF_SCST_PRELIM_COMPL 7
 
 /*
  * Most of the fields don't need any protection, since accessed from only a
  * single thread, except where noted.
+ *
+ * ToDo: Eventually divide request and response structures in 2 separate
+ * structures and stop this IET-derived garbage.
  */
 struct iscsi_cmnd {
        struct iscsi_conn *conn;
@@ -310,31 +318,41 @@ struct iscsi_cmnd {
        unsigned int own_sg:1;
        unsigned int on_write_list:1;
        unsigned int write_processing_started:1;
-       unsigned int data_waiting:1;
        unsigned int force_cleanup_done:1;
        unsigned int dec_active_cmnds:1;
        unsigned int ddigest_checked:1;
-       unsigned int rejected:1;
-       unsigned int reject_reason:2;
 #ifdef CONFIG_SCST_EXTRACHECKS
        unsigned int on_rx_digest_list:1;
        unsigned int release_called:1;
 #endif
 
-       /* It's async. with the above flags */
-       volatile unsigned int tm_aborted;
+       /*
+        * We suppose that preliminary commands completion is tested by
+        * comparing prelim_compl_flags with 0. Otherwise, because of the
+        * gap between setting different flags a race is possible,
+        * like sending command in SCST core as PRELIM_COMPLETED, while it
+        * wasn't aborted in it yet and have as the result a wrong success
+        * status sent to the initiator.
+        */
+#define ISCSI_CMD_ABORTED              0
+#define ISCSI_CMD_PRELIM_COMPLETED     1
+       unsigned long prelim_compl_flags;
 
        struct list_head hash_list_entry;
 
-       spinlock_t rsp_cmd_lock; /* BH lock */
-
        /*
         * Unions are for readability and grepability and to save some
         * cache footprint.
         */
 
        union {
-               /* Protected by rsp_cmd_lock */
+               /*
+                * Used only to abort not yet sent responses. Usage in
+                * cmnd_done() is only a side effect to have a lockless
+                * accesss to this list from always only a single thread
+                * at any time. So, all responses live in the parent
+                * until it has the last reference put.
+                */
                struct list_head rsp_cmd_list;
                struct list_head rsp_cmd_list_entry;
        };
@@ -346,12 +364,12 @@ struct iscsi_cmnd {
 
        union {
                struct list_head write_list_entry;
-               struct list_head written_list_entry;
+               struct list_head write_timeout_list_entry;
        };
 
        /* Both modified only from single write thread */
-       unsigned int on_written_list:1;
-       unsigned long write_timeout;
+       unsigned int on_write_timeout_list:1;
+       unsigned long write_start;
 
        /*
         * All unprotected, since could be accessed from only a single
@@ -375,7 +393,9 @@ struct iscsi_cmnd {
                                struct scst_cmd *scst_cmd;
                                struct scst_aen *scst_aen;
                        };
-                       int read_size;
+                       unsigned int read_size;
+
+                       struct iscsi_cmnd *main_rsp;
                };
 
                /* Response only fields */
@@ -396,21 +416,29 @@ struct iscsi_cmnd {
        int sg_cnt;
        unsigned int bufflen;
        u32 r2t_sn;
-       u32 r2t_length;
-       u32 is_unsolicited_data;
+       unsigned int r2t_len_to_receive;
+       unsigned int r2t_len_to_send;
+       unsigned int outstanding_r2t;
        u32 target_task_tag;
-       u32 outstanding_r2t;
-
        u32 hdigest;
        u32 ddigest;
 
        struct list_head cmd_list_entry;
 };
 
-/* Flags for req_cmnd_release_force() */
-#define ISCSI_FORCE_RELEASE_WRITE      1
+/**
+ ** Various timeouts. *_SCHED_TIMEOUT is needed to complete a burst of
+ ** commands at once. Otherwise, a part of the burst can be timeouted
+ ** only in double timeout time.
+ **/
 
+/* Max time to wait for our response satisfied */
 #define ISCSI_RSP_TIMEOUT              (30 * HZ)
+#define ISCSI_RSP_SCHED_TIMEOUT                (ISCSI_RSP_TIMEOUT + HZ)
+
+/* Max time to wait for our response satisfied for aborted commands */
+#define ISCSI_TM_DATA_WAIT_TIMEOUT     (10 * HZ)
+#define ISCSI_TM_DATA_WAIT_SCHED_TIMEOUT (ISCSI_TM_DATA_WAIT_TIMEOUT + HZ)
 
 extern struct mutex target_mgmt_mutex;
 
@@ -432,11 +460,12 @@ extern int cmnd_rx_continue(struct iscsi_cmnd *req);
 extern void cmnd_rx_end(struct iscsi_cmnd *);
 extern void cmnd_tx_start(struct iscsi_cmnd *);
 extern void cmnd_tx_end(struct iscsi_cmnd *);
-extern void req_cmnd_release_force(struct iscsi_cmnd *req, int flags);
+extern void req_cmnd_release_force(struct iscsi_cmnd *req);
 extern void rsp_cmnd_release(struct iscsi_cmnd *);
 extern void cmnd_done(struct iscsi_cmnd *cmnd);
 extern void conn_abort(struct iscsi_conn *conn);
 extern void iscsi_restart_cmnd(struct iscsi_cmnd *cmnd);
+extern void iscsi_fail_data_waiting_cmnd(struct iscsi_cmnd *cmnd);
 
 /* conn.c */
 extern struct iscsi_conn *conn_lookup(struct iscsi_session *, u16);
@@ -447,17 +476,16 @@ extern int conn_del(struct iscsi_session *, struct iscsi_kern_conn_info *);
 extern int conn_free(struct iscsi_conn *);
 #endif
 extern void iscsi_make_conn_rd_active(struct iscsi_conn *conn);
-
 #define ISCSI_CONN_ACTIVE_CLOSE                1
 #define ISCSI_CONN_DELETING            2
 extern void __mark_conn_closed(struct iscsi_conn *, int);
-
 extern void mark_conn_closed(struct iscsi_conn *);
 extern void iscsi_make_conn_wr_active(struct iscsi_conn *);
-
 #ifdef CONFIG_SCST_PROC
 extern void conn_info_show(struct seq_file *, struct iscsi_session *);
 #endif
+extern void iscsi_check_tm_data_wait_timeouts(struct iscsi_conn *conn,
+       bool force);
 
 /* nthread.c */
 extern int iscsi_send(struct iscsi_conn *conn);
@@ -468,6 +496,7 @@ extern void iscsi_put_page_callback(struct page *page);
 extern int istrd(void *arg);
 extern int istwr(void *arg);
 extern void iscsi_task_mgmt_affected_cmds_done(struct scst_mgmt_cmd *scst_mcmd);
+extern void req_add_to_write_timeout_list(struct iscsi_cmnd *req);
 
 /* target.c */
 #ifndef CONFIG_SCST_PROC
@@ -598,7 +627,10 @@ static inline void cmnd_put(struct iscsi_cmnd *cmnd)
 static inline void cmd_add_on_write_list(struct iscsi_conn *conn,
        struct iscsi_cmnd *cmnd)
 {
-       TRACE_DBG("%p", cmnd);
+       TRACE_DBG("cmnd %p", cmnd);
+       /* See comment in iscsi_restart_cmnd() */
+       EXTRACHECKS_BUG_ON(cmnd->parent_req->hashed &&
+               (cmnd_opcode(cmnd) != ISCSI_OP_R2T));
        list_add_tail(&cmnd->write_list_entry, &conn->write_list);
        cmnd->on_write_list = 1;
 }
index 10a4ed3..ea5351f 100644 (file)
 
 #include <scst_debug.h>
 
-#define TRACE_D_READ           0x80000000
-#define TRACE_D_WRITE          0x40000000
-#define TRACE_CONN_OC          0x20000000
-#define TRACE_D_IOV            0x10000000
-#define TRACE_D_DUMP_PDU       0x08000000
-#define TRACE_NET_PG           0x04000000
-#define TRACE_CONN_OC_DBG      0x02000000
-
-#define TRACE_D_DATA           (TRACE_D_READ | TRACE_D_WRITE)
-
-#define TRACE_ALL_NO_DATA      \
-       (TRACE_ALL & ~TRACE_D_IOV & ~TRACE_D_DUMP_PDU & ~TRACE_D_DATA)
+#define TRACE_D_WRITE          0x80000000
+#define TRACE_CONN_OC          0x40000000
+#define TRACE_D_IOV            0x20000000
+#define TRACE_D_DUMP_PDU       0x10000000
+#define TRACE_NET_PG           0x08000000
+#define TRACE_CONN_OC_DBG      0x04000000
 
 #ifdef CONFIG_SCST_DEBUG
 #define ISCSI_DEFAULT_LOG_FLAGS (TRACE_FUNCTION | TRACE_LINE | TRACE_PID | \
 
 #ifdef CONFIG_SCST_DEBUG
 struct iscsi_pdu;
+struct iscsi_cmnd;
 extern void iscsi_dump_pdu(struct iscsi_pdu *pdu);
+extern unsigned long iscsi_get_flow_ctrl_or_mgmt_dbg_log_flag(
+       struct iscsi_cmnd *cmnd);
 #else
 #define iscsi_dump_pdu(x) do {} while (0)
+#define iscsi_get_flow_ctrl_or_mgmt_dbg_log_flag(x) do {} while (0)
 #endif
 
 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
@@ -54,20 +52,9 @@ extern unsigned long iscsi_trace_flag;
 #define trace_flag iscsi_trace_flag
 #endif
 
-#ifdef CONFIG_SCST_DEBUG
-
-#define TRACE_CONN_CLOSE(args...)      TRACE(TRACE_CONN_OC, args)
+#define TRACE_CONN_CLOSE(args...)      TRACE_DBG_FLAG(TRACE_DEBUG|TRACE_CONN_OC, args)
 #define TRACE_CONN_CLOSE_DBG(args...)  TRACE(TRACE_CONN_OC_DBG, args)
-#define TRACE_NET_PAGE(args...)                TRACE(TRACE_NET_PG, args)
-#define TRACE_WRITE(args...)           TRACE(TRACE_D_WRITE, args)
-#define TRACE_READ(args...)            TRACE(TRACE_D_READ, args)
-
-#else /* CONFIG_SCST_DEBUG */
-#define TRACE_CONN_CLOSE(format, args...) {}
-#define TRACE_CONN_CLOSE_DBG(format, args...) {}
-#define TRACE_NET_PAGE(format, args...) {}
-#define TRACE_WRITE(args...) {}
-#define TRACE_READ(args...) {}
-#endif
+#define TRACE_NET_PAGE(args...)                TRACE_DBG_FLAG(TRACE_NET_PG, args)
+#define TRACE_WRITE(args...)           TRACE_DBG_FLAG(TRACE_DEBUG|TRACE_D_WRITE, args)
 
 #endif
index 990ee26..d9378be 100644 (file)
@@ -511,7 +511,6 @@ struct iscsi_nop_in_hdr {
 #define ISCSI_RESERVED_TAG     (0xffffffffU)
 
 #define cmnd_hdr(cmnd) ((struct iscsi_scsi_cmd_hdr *) (&((cmnd)->pdu.bhs)))
-#define cmnd_ttt(cmnd) cpu_to_be32((cmnd)->pdu.bhs.ttt)
 #define cmnd_itt(cmnd) cpu_to_be32((cmnd)->pdu.bhs.itt)
 #define cmnd_opcode(cmnd) ((cmnd)->pdu.bhs.opcode & ISCSI_OPCODE_MASK)
 #define cmnd_scsicode(cmnd) (cmnd_hdr((cmnd))->scb[0])
index 290eca8..d97c946 100644 (file)
@@ -78,11 +78,12 @@ again:
                struct iscsi_cmnd *rsp;
                int restart = 0;
 
-               TRACE_CONN_CLOSE_DBG("cmd %p, scst_state %x, data_waiting %d, "
-                       "ref_cnt %d, parent_req %p, net_ref_cnt %d, sg %p",
-                       cmnd, cmnd->scst_state, cmnd->data_waiting,
-                       atomic_read(&cmnd->ref_cnt), cmnd->parent_req,
-                       atomic_read(&cmnd->net_ref_cnt), cmnd->sg);
+               TRACE_CONN_CLOSE_DBG("cmd %p, scst_state %x, "
+                       "r2t_len_to_receive %d, ref_cnt %d, parent_req %p, "
+                       "net_ref_cnt %d, sg %p", cmnd, cmnd->scst_state,
+                       cmnd->r2t_len_to_receive, atomic_read(&cmnd->ref_cnt),
+                       cmnd->parent_req, atomic_read(&cmnd->net_ref_cnt),
+                       cmnd->sg);
 
                sBUG_ON(cmnd->parent_req != NULL);
 
@@ -113,7 +114,6 @@ again:
                                goto again;
                }
 
-               spin_lock_bh(&cmnd->rsp_cmd_lock);
                list_for_each_entry(rsp, &cmnd->rsp_cmd_list,
                                rsp_cmd_list_entry) {
                        TRACE_CONN_CLOSE_DBG("  rsp %p, ref_cnt %d, "
@@ -138,7 +138,6 @@ again:
 
                                        if (page->net_priv != NULL) {
                                                if (restart == 0) {
-                                                       spin_unlock_bh(&cmnd->rsp_cmd_lock);
                                                        spin_unlock_bh(&conn->cmd_list_lock);
                                                        restart = 1;
                                                }
@@ -152,7 +151,6 @@ again:
                                        goto again;
                        }
                }
-               spin_unlock_bh(&cmnd->rsp_cmd_lock);
        }
        spin_unlock_bh(&conn->cmd_list_lock);
 
@@ -191,7 +189,7 @@ static void free_pending_commands(struct iscsi_conn *conn)
 
                                spin_unlock(&session->sn_lock);
 
-                               req_cmnd_release_force(cmnd, 0);
+                               req_cmnd_release_force(cmnd);
 
                                req_freed = 1;
                                spin_lock(&session->sn_lock);
@@ -231,7 +229,7 @@ static void free_orphaned_pending_commands(struct iscsi_conn *conn)
 
                                spin_unlock(&session->sn_lock);
 
-                               req_cmnd_release_force(cmnd, 0);
+                               req_cmnd_release_force(cmnd);
 
                                req_freed = 1;
                                spin_lock(&session->sn_lock);
@@ -261,13 +259,13 @@ static void trace_conn_close(struct iscsi_conn *conn)
        list_for_each_entry(cmnd, &conn->cmd_list,
                        cmd_list_entry) {
                TRACE_CONN_CLOSE_DBG(
-                       "cmd %p, scst_state %x, scst_cmd state %d, "
-                       "data_waiting %d, ref_cnt %d, sn %u, "
+                       "cmd %p, scst_cmd %p, scst_state %x, scst_cmd state "
+                       "%d, r2t_len_to_receive %d, ref_cnt %d, sn %u, "
                        "parent_req %p, pending %d",
-                       cmnd, cmnd->scst_state,
-                       (cmnd->parent_req && cmnd->scst_cmd) ?
+                       cmnd, cmnd->scst_cmd, cmnd->scst_state,
+                       ((cmnd->parent_req == NULL) && cmnd->scst_cmd) ?
                                cmnd->scst_cmd->state : -1,
-                       cmnd->data_waiting, atomic_read(&cmnd->ref_cnt),
+                       cmnd->r2t_len_to_receive, atomic_read(&cmnd->ref_cnt),
                        cmnd->pdu.bhs.sn, cmnd->parent_req, cmnd->pending);
 #if defined(CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION)
                TRACE_CONN_CLOSE_DBG("net_ref_cnt %d, sg %p",
@@ -286,7 +284,6 @@ static void trace_conn_close(struct iscsi_conn *conn)
 
                sBUG_ON(cmnd->parent_req != NULL);
 
-               spin_lock_bh(&cmnd->rsp_cmd_lock);
                list_for_each_entry(rsp, &cmnd->rsp_cmd_list,
                                rsp_cmd_list_entry) {
                        TRACE_CONN_CLOSE_DBG("  rsp %p, "
@@ -305,7 +302,6 @@ static void trace_conn_close(struct iscsi_conn *conn)
                                }
                        }
                }
-               spin_unlock_bh(&cmnd->rsp_cmd_lock);
 #endif /* CONFIG_TCP_ZERO_COPY_TRANSFER_COMPLETION_NOTIFICATION */
        }
        spin_unlock_bh(&conn->cmd_list_lock);
@@ -433,21 +429,27 @@ static void close_conn(struct iscsi_conn *conn)
                struct iscsi_cmnd *cmnd = conn->read_cmnd;
 
                if (cmnd->scst_state == ISCSI_CMD_STATE_RX_CMD) {
-                       TRACE_DBG("Going to wait for cmnd %p to change state "
-                               "from RX_CMD", cmnd);
+                       TRACE_CONN_CLOSE_DBG("Going to wait for cmnd %p to "
+                               "change state from RX_CMD", cmnd);
                }
                wait_event(conn->read_state_waitQ,
                        cmnd->scst_state != ISCSI_CMD_STATE_RX_CMD);
 
+               TRACE_CONN_CLOSE_DBG("Releasing conn->read_cmnd %p (conn %p)",
+                       conn->read_cmnd, conn);
+
                conn->read_cmnd = NULL;
                conn->read_state = RX_INIT_BHS;
-               req_cmnd_release_force(cmnd, 0);
+               req_cmnd_release_force(cmnd);
        }
 
        conn_abort(conn);
 
        /* ToDo: not the best way to wait */
        while (atomic_read(&conn->conn_ref_cnt) != 0) {
+               if (conn->conn_tm_active)
+                       iscsi_check_tm_data_wait_timeouts(conn, true);
+
                mutex_lock(&target->target_mutex);
                spin_lock(&session->sn_lock);
                if (session->tm_rsp && session->tm_rsp->conn == conn) {
@@ -637,6 +639,24 @@ static struct iscsi_cmnd *iscsi_get_send_cmnd(struct iscsi_conn *conn)
        }
        spin_unlock_bh(&conn->write_list_lock);
 
+       if (unlikely(test_bit(ISCSI_CMD_ABORTED,
+                       &cmnd->parent_req->prelim_compl_flags))) {
+               TRACE_MGMT_DBG("Going to send acmd %p (scst cmd %p, "
+                       "state %d, parent_req %p)", cmnd, cmnd->scst_cmd,
+                       cmnd->scst_state, cmnd->parent_req);
+       }
+
+       if (unlikely(cmnd_opcode(cmnd) == ISCSI_OP_SCSI_TASK_MGT_RSP)) {
+               struct iscsi_task_mgt_hdr *req_hdr =
+                       (struct iscsi_task_mgt_hdr *)&cmnd->parent_req->pdu.bhs;
+               struct iscsi_task_rsp_hdr *rsp_hdr =
+                       (struct iscsi_task_rsp_hdr *)&cmnd->pdu.bhs;
+               TRACE_MGMT_DBG("Going to send TM response %p (status %d, "
+                       "fn %d, parent_req %p)", cmnd, rsp_hdr->response,
+                       req_hdr->function & ISCSI_FUNCTION_MASK,
+                       cmnd->parent_req);
+       }
+
        return cmnd;
 }
 
@@ -702,8 +722,10 @@ restart:
                        TRACE_DBG("ERESTARTSYS received for conn %p", conn);
                        goto restart;
                default:
-                       PRINT_ERROR("sock_recvmsg() failed: %d", res);
-                       mark_conn_closed(conn);
+                       if (!conn->closing) {
+                               PRINT_ERROR("sock_recvmsg() failed: %d", res);
+                               mark_conn_closed(conn);
+                       }
                        if (res == 0)
                                res = -EIO;
                        break;
@@ -970,9 +992,15 @@ static void scst_do_job_rd(void)
 
                spin_lock_bh(&iscsi_rd_lock);
 
-               if (closed)
+               if (unlikely(closed))
                        continue;
 
+               if (unlikely(conn->conn_tm_active)) {
+                       spin_unlock_bh(&iscsi_rd_lock);
+                       iscsi_check_tm_data_wait_timeouts(conn, false);
+                       spin_lock_bh(&iscsi_rd_lock);
+               }
+
 #ifdef CONFIG_SCST_EXTRACHECKS
                conn->rd_task = NULL;
 #endif
@@ -1107,7 +1135,76 @@ static inline void __iscsi_get_page_callback(struct iscsi_cmnd *cmd) {}
 static inline void __iscsi_put_page_callback(struct iscsi_cmnd *cmd) {}
 #endif
 
-/* This is partially taken from the Ardis code. */
+void req_add_to_write_timeout_list(struct iscsi_cmnd *req)
+{
+       struct iscsi_conn *conn;
+       unsigned long timeout_time;
+       bool set_conn_tm_active = false;
+
+       TRACE_ENTRY();
+
+       if (req->on_write_timeout_list)
+               goto out;
+
+       conn = req->conn;
+
+       TRACE_DBG("Adding req %p to conn %p write_timeout_list",
+               req, conn);
+
+       spin_lock_bh(&conn->write_list_lock);
+
+       req->on_write_timeout_list = 1;
+       req->write_start = jiffies;
+       list_add_tail(&req->write_timeout_list_entry,
+               &conn->write_timeout_list);
+
+       if (!timer_pending(&conn->rsp_timer)) {
+               if (unlikely(conn->conn_tm_active ||
+                            test_bit(ISCSI_CMD_ABORTED,
+                                       &req->prelim_compl_flags))) {
+                       set_conn_tm_active = true;
+                       timeout_time = req->write_start +
+                               ISCSI_TM_DATA_WAIT_SCHED_TIMEOUT;
+               } else
+                       timeout_time = req->write_start +
+                               ISCSI_RSP_SCHED_TIMEOUT;
+
+               TRACE_DBG("Starting timer on %ld (con %p, write_start %ld)",
+                       timeout_time, conn, req->write_start);
+
+               conn->rsp_timer.expires = timeout_time;
+               add_timer(&conn->rsp_timer);
+       } else if (unlikely(test_bit(ISCSI_CMD_ABORTED,
+                               &req->prelim_compl_flags))) {
+               unsigned long timeout_time = jiffies +
+                                       ISCSI_TM_DATA_WAIT_SCHED_TIMEOUT;
+               set_conn_tm_active = true;
+               if (time_after(conn->rsp_timer.expires, timeout_time)) {
+                       TRACE_MGMT_DBG("Mod timer on %ld (conn %p)",
+                               timeout_time, conn);
+                       mod_timer(&conn->rsp_timer, timeout_time);
+               }
+       }
+
+       spin_unlock_bh(&conn->write_list_lock);
+
+       /*
+        * conn_tm_active can be already cleared by
+        * iscsi_check_tm_data_wait_timeouts(). write_list_lock is an inner
+        * lock for iscsi_rd_lock.
+        */
+       if (unlikely(set_conn_tm_active)) {
+               spin_lock_bh(&iscsi_rd_lock);
+               TRACE_MGMT_DBG("Setting conn_tm_active for conn %p", conn);
+               conn->conn_tm_active = 1;
+               spin_unlock_bh(&iscsi_rd_lock);
+       }
+
+out:
+       TRACE_EXIT();
+       return;
+}
+
 static int write_data(struct iscsi_conn *conn)
 {
        mm_segment_t oldfs;
@@ -1130,7 +1227,7 @@ static int write_data(struct iscsi_conn *conn)
 
        iscsi_extracheck_is_wr_thread(conn);
 
-       if (write_cmnd->own_sg == 0) {
+       if (!write_cmnd->own_sg) {
                ref_cmd = write_cmnd->parent_req;
                ref_cmd_to_parent = true;
        } else {
@@ -1138,28 +1235,7 @@ static int write_data(struct iscsi_conn *conn)
                ref_cmd_to_parent = false;
        }
 
-       if (!ref_cmd->on_written_list) {
-               TRACE_DBG("Adding cmd %p to conn %p written_list", ref_cmd,
-                       conn);
-               spin_lock_bh(&conn->write_list_lock);
-               ref_cmd->on_written_list = 1;
-               ref_cmd->write_timeout = jiffies + ISCSI_RSP_TIMEOUT;
-               list_add_tail(&ref_cmd->written_list_entry,
-                       &conn->written_list);
-               spin_unlock_bh(&conn->write_list_lock);
-       }
-
-       if (!timer_pending(&conn->rsp_timer)) {
-               sBUG_ON(!ref_cmd->on_written_list);
-               spin_lock_bh(&conn->write_list_lock);
-               if (likely(!timer_pending(&conn->rsp_timer))) {
-                       TRACE_DBG("Starting timer on %ld (conn %p)",
-                               ref_cmd->write_timeout, conn);
-                       conn->rsp_timer.expires = ref_cmd->write_timeout;
-                       add_timer(&conn->rsp_timer);
-               }
-               spin_unlock_bh(&conn->write_list_lock);
-       }
+       req_add_to_write_timeout_list(write_cmnd->parent_req);
 
        file = conn->file;
        size = conn->write_size;
index f044545..49c291c 100644 (file)
@@ -91,9 +91,9 @@ static int iscsi_session_alloc(struct iscsi_target *target,
 
        spin_lock_init(&session->sn_lock);
 
-       spin_lock_init(&session->cmnd_hash_lock);
-       for (i = 0; i < ARRAY_SIZE(session->cmnd_hash); i++)
-               INIT_LIST_HEAD(&session->cmnd_hash[i]);
+       spin_lock_init(&session->cmnd_data_wait_hash_lock);
+       for (i = 0; i < ARRAY_SIZE(session->cmnd_data_wait_hash); i++)
+               INIT_LIST_HEAD(&session->cmnd_data_wait_hash[i]);
 
        session->next_ttt = 1;
 
@@ -274,8 +274,8 @@ int session_free(struct iscsi_session *session, bool del)
                sBUG();
        }
 
-       for (i = 0; i < ARRAY_SIZE(session->cmnd_hash); i++)
-               sBUG_ON(!list_empty(&session->cmnd_hash[i]));
+       for (i = 0; i < ARRAY_SIZE(session->cmnd_data_wait_hash); i++)
+               sBUG_ON(!list_empty(&session->cmnd_data_wait_hash[i]));
 
        if (session->sess_reinst_successor != NULL)
                sess_reinst_finished(session->sess_reinst_successor);
index 3fc77f4..b1e1d32 100644 (file)
@@ -302,9 +302,9 @@ struct iscsi_key session_keys[] = {
        {"MaxXmitDataSegmentLength", 8192, -1, 512, -1, &minimum_ops},
        {"MaxBurstLength", 262144, -1, 512, -1, &minimum_ops},
        {"FirstBurstLength", 65536, -1, 512, -1, &minimum_ops},
-       {"DefaultTime2Wait", 2, 2, 0, 3600, &maximum_ops},
-       {"DefaultTime2Retain", 20, 20, 0, 3600, &minimum_ops},
-       {"MaxOutstandingR2T", 1, 20, 1, 65535, &minimum_ops},
+       {"DefaultTime2Wait", 2, 0, 0, 3600, &maximum_ops},
+       {"DefaultTime2Retain", 20, 0, 0, 3600, &minimum_ops},
+       {"MaxOutstandingR2T", 1, 32, 1, 65535, &minimum_ops},
        {"DataPDUInOrder", 1, 0, 0, 1, &or_ops},
        {"DataSequenceInOrder", 1, 0, 0, 1, &or_ops},
        {"ErrorRecoveryLevel", 0, 0, 0, 0, &minimum_ops},
index 5cbe33b..5948e96 100644 (file)
@@ -115,41 +115,44 @@ static inline int list_is_last(const struct list_head *list,
 /* Allocation of the cmd's data buffer */
 #define SCST_CMD_STATE_PREPARE_SPACE 2
 
+/* Calling preprocessing_done() */
+#define SCST_CMD_STATE_PREPROCESSING_DONE 3
+
 /* Target driver's rdy_to_xfer() is going to be called */
-#define SCST_CMD_STATE_RDY_TO_XFER   3
+#define SCST_CMD_STATE_RDY_TO_XFER   4
 
 /* Target driver's pre_exec() is going to be called */
-#define SCST_CMD_STATE_TGT_PRE_EXEC  4
+#define SCST_CMD_STATE_TGT_PRE_EXEC  5
 
 /* Cmd is going to be sent for execution */
-#define SCST_CMD_STATE_SEND_FOR_EXEC 5
+#define SCST_CMD_STATE_SEND_FOR_EXEC 6
 
 /* Cmd is being checked if it should be executed locally */
-#define SCST_CMD_STATE_LOCAL_EXEC    6
+#define SCST_CMD_STATE_LOCAL_EXEC    7
 
 /* Cmd is ready for execution */
-#define SCST_CMD_STATE_REAL_EXEC     7
+#define SCST_CMD_STATE_REAL_EXEC     8
 
 /* Internal post-exec checks */
-#define SCST_CMD_STATE_PRE_DEV_DONE  8
+#define SCST_CMD_STATE_PRE_DEV_DONE  9
 
 /* Internal MODE SELECT pages related checks */
-#define SCST_CMD_STATE_MODE_SELECT_CHECKS 9
+#define SCST_CMD_STATE_MODE_SELECT_CHECKS 10
 
 /* Dev handler's dev_done() is going to be called */
-#define SCST_CMD_STATE_DEV_DONE      10
+#define SCST_CMD_STATE_DEV_DONE      11
 
 /* Target driver's xmit_response() is going to be called */
-#define SCST_CMD_STATE_PRE_XMIT_RESP 11
+#define SCST_CMD_STATE_PRE_XMIT_RESP 12
 
 /* Target driver's xmit_response() is going to be called */
-#define SCST_CMD_STATE_XMIT_RESP     12
+#define SCST_CMD_STATE_XMIT_RESP     13
 
 /* Cmd finished */
-#define SCST_CMD_STATE_FINISHED      13
+#define SCST_CMD_STATE_FINISHED      14
 
 /* Internal cmd finished */
-#define SCST_CMD_STATE_FINISHED_INTERNAL 14
+#define SCST_CMD_STATE_FINISHED_INTERNAL 15
 
 #define SCST_CMD_STATE_LAST_ACTIVE   (SCST_CMD_STATE_FINISHED_INTERNAL+100)
 
@@ -159,8 +162,8 @@ static inline int list_is_last(const struct list_head *list,
 /* LUN translation (cmd->tgt_dev assignment) */
 #define SCST_CMD_STATE_INIT          (SCST_CMD_STATE_LAST_ACTIVE+2)
 
-/* Allocation of the cmd's data buffer */
-#define SCST_CMD_STATE_PREPROCESS_DONE (SCST_CMD_STATE_LAST_ACTIVE+3)
+/* Waiting for scst_restart_cmd() */
+#define SCST_CMD_STATE_PREPROCESSING_DONE_CALLED (SCST_CMD_STATE_LAST_ACTIVE+3)
 
 /* Waiting for data from the initiator (until scst_rx_data() called) */
 #define SCST_CMD_STATE_DATA_WAIT     (SCST_CMD_STATE_LAST_ACTIVE+4)
@@ -704,6 +707,8 @@ struct scst_tgt_template {
         * A target driver could need to do some actions at this stage.
         * After the target driver done the needed actions, it shall call
         * scst_restart_cmd() in order to continue processing this command.
+        * In case of preliminary the command completion, this function will
+        * also be called before xmit_response().
         *
         * Called only if the cmd is queued using scst_cmd_init_stage1_done()
         * instead of scst_cmd_init_done().
@@ -1212,19 +1217,15 @@ struct scst_session {
        struct list_head sess_tgt_dev_list_hash[TGT_DEV_HASH_SIZE];
 
        /*
-        * List of cmds in this session. Used to find a cmd in the
-        * session. Protected by sess_list_lock.
+        * List of cmds in this session. Protected by sess_list_lock.
+        *
+        * We must always keep commands in the sess list from the
+        * very beginning, because otherwise they can be missed during
+        * TM processing. 
         */
-       struct list_head search_cmd_list;
-
-       spinlock_t sess_list_lock; /* protects search_cmd_list, etc */
+       struct list_head sess_cmd_list;
 
-       /*
-        * List of cmds in this in the state after PRE_XMIT_RESP. All the cmds
-        * moved here from search_cmd_list. Needed for hw_pending_work.
-        * Protected by sess_list_lock.
-        */
-       struct list_head after_pre_xmit_cmd_list;
+       spinlock_t sess_list_lock; /* protects sess_cmd_list, etc */
 
        atomic_t refcnt;                /* get/put counter */
 
@@ -1484,7 +1485,7 @@ struct scst_cmd {
        /* The corresponding sn_slot in tgt_dev->sn_slots */
        atomic_t *sn_slot;
 
-       /* List entry for sess's search_cmd_list and after_pre_xmit_cmd_list */
+       /* List entry for sess's sess_cmd_list */
        struct list_head sess_cmd_list_entry;
 
        /*
@@ -2326,14 +2327,18 @@ static inline int scst_rx_mgmt_fn_lun(struct scst_session *sess, int fn,
 int scst_get_cdb_info(struct scst_cmd *cmd);
 
 /*
- * Set error SCSI status in the command and prepares it for returning it
+ * Set error SCSI status in the command and prepares it for returning it.
+ *
+ * Returns 0 on success, error code otherwise.
  */
-void scst_set_cmd_error_status(struct scst_cmd *cmd, int status);
+int scst_set_cmd_error_status(struct scst_cmd *cmd, int status);
 
 /*
- * Set error in the command and fill the sense buffer
+ * Set error in the command and fill the sense buffer.
+ *
+ * Returns 0 on success, error code otherwise.
  */
-void scst_set_cmd_error(struct scst_cmd *cmd, int key, int asc, int ascq);
+int scst_set_cmd_error(struct scst_cmd *cmd, int key, int asc, int ascq);
 
 /*
  * Sets BUSY or TASK QUEUE FULL status
@@ -2490,6 +2495,14 @@ static inline int scst_cmd_atomic(struct scst_cmd *cmd)
        return res;
 }
 
+/*
+ * Returns TRUE if cmd has been completed.
+ */
+static inline int scst_cmd_completed(struct scst_cmd *cmd)
+{
+       return cmd->completed;
+}
+
 static inline enum scst_exec_context __scst_estimate_context(bool direct)
 {
        if (in_irq())
index 0ac2c26..2bd63f8 100644 (file)
@@ -98,6 +98,7 @@
 #define TRACE_MGMT_DEBUG     0x00001000
 #define TRACE_SCSI           0x00002000
 #define TRACE_SPECIAL        0x00004000 /* filtering debug, etc */
+#define TRACE_FLOW_CONTROL   0x00008000 /* flow control in action */
 #define TRACE_ALL            0xffffffff
 /* Flags 0xXXXX0000 are local for users */
 
@@ -308,11 +309,13 @@ do {                                                                      \
 #define TRACE_MEM(format, args...) do {} while (0)
 #define TRACE_SG(format, args...) do {} while (0)
 #define TRACE_DBG(format, args...) do {} while (0)
+#define TRACE_DBG_FLAG(format, args...) do {} while (0)
 #define TRACE_DBG_SPECIAL(format, args...) do {} while (0)
 #define TRACE_MGMT_DBG(format, args...) do {} while (0)
 #define TRACE_MGMT_DBG_SPECIAL(format, args...) do {} while (0)
 #define TRACE_BUFFER(message, buff, len) do {} while (0)
 #define TRACE_BUFF_FLAG(flag, message, buff, len) do {} while (0)
+
 #ifndef GENERATING_UPSTREAM_PATCH
 #define TRACE_ENTRY() do {} while (0)
 #define TRACE_EXIT() do {} while (0)
index 756b1b1..df978af 100644 (file)
@@ -554,7 +554,7 @@ static const struct scst_sdbops scst_scsi_op_table[] = {
         SCST_DATA_NONE, SCST_LONG_TIMEOUT, 0, get_trans_cdb_len_10}
 };
 
-#define SCST_CDB_TBL_SIZE      ARRAY_SIZE(scst_scsi_op_table)
+#define SCST_CDB_TBL_SIZE      ((int)ARRAY_SIZE(scst_scsi_op_table))
 
 static void scst_free_tgt_dev(struct scst_tgt_dev *tgt_dev);
 static void scst_check_internal_sense(struct scst_device *dev, int result,
@@ -620,6 +620,11 @@ int scst_alloc_set_sense(struct scst_cmd *cmd, int atomic,
 
        TRACE_ENTRY();
 
+       /*
+        * We don't check here if the existing sense is valid or not, because
+        * we suppose the caller did it based on cmd->status.
+        */
+
        res = scst_alloc_sense(cmd, atomic);
        if (res != 0) {
                PRINT_BUFFER("Lost sense", sense, len);
@@ -642,10 +647,19 @@ out:
 }
 EXPORT_SYMBOL(scst_alloc_set_sense);
 
-void scst_set_cmd_error_status(struct scst_cmd *cmd, int status)
+int scst_set_cmd_error_status(struct scst_cmd *cmd, int status)
 {
+       int res = 0;
+
        TRACE_ENTRY();
 
+       if (cmd->status != 0) {
+               TRACE_MGMT_DBG("cmd %p already has status %x set", cmd,
+                       cmd->status);
+               res = -EEXIST;
+               goto out;
+       }
+
        cmd->status = status;
        cmd->host_status = DID_OK;
 
@@ -658,21 +672,24 @@ void scst_set_cmd_error_status(struct scst_cmd *cmd, int status)
 
        cmd->completed = 1;
 
-       TRACE_EXIT();
-       return;
+out:
+       TRACE_EXIT_RES(res);
+       return res;
 }
 EXPORT_SYMBOL(scst_set_cmd_error_status);
 
-void scst_set_cmd_error(struct scst_cmd *cmd, int key, int asc, int ascq)
+int scst_set_cmd_error(struct scst_cmd *cmd, int key, int asc, int ascq)
 {
-       int rc;
+       int res;
 
        TRACE_ENTRY();
 
-       scst_set_cmd_error_status(cmd, SAM_STAT_CHECK_CONDITION);
+       res = scst_set_cmd_error_status(cmd, SAM_STAT_CHECK_CONDITION);
+       if (res != 0)
+               goto out;
 
-       rc = scst_alloc_sense(cmd, 1);
-       if (rc != 0) {
+       res = scst_alloc_sense(cmd, 1);
+       if (res != 0) {
                PRINT_ERROR("Lost sense data (key %x, asc %x, ascq %x)",
                        key, asc, ascq);
                goto out;
@@ -683,8 +700,8 @@ void scst_set_cmd_error(struct scst_cmd *cmd, int key, int asc, int ascq)
        TRACE_BUFFER("Sense set", cmd->sense, cmd->sense_valid_len);
 
 out:
-       TRACE_EXIT();
-       return;
+       TRACE_EXIT_RES(res);
+       return res;
 }
 EXPORT_SYMBOL(scst_set_cmd_error);
 
@@ -871,16 +888,22 @@ out:
 }
 EXPORT_SYMBOL(scst_check_convert_sense);
 
-static void scst_set_cmd_error_sense(struct scst_cmd *cmd, uint8_t *sense,
+static int scst_set_cmd_error_sense(struct scst_cmd *cmd, uint8_t *sense,
        unsigned int len)
 {
+       int res;
+
        TRACE_ENTRY();
 
-       scst_set_cmd_error_status(cmd, SAM_STAT_CHECK_CONDITION);
-       scst_alloc_set_sense(cmd, 1, sense, len);
+       res = scst_set_cmd_error_status(cmd, SAM_STAT_CHECK_CONDITION);
+       if (res != 0)
+               goto out;
 
-       TRACE_EXIT();
-       return;
+       res = scst_alloc_set_sense(cmd, 1, sense, len);
+
+out:
+       TRACE_EXIT_RES(res);
+       return res;
 }
 
 void scst_set_busy(struct scst_cmd *cmd)
@@ -891,13 +914,13 @@ void scst_set_busy(struct scst_cmd *cmd)
 
        if ((c <= 1) || (cmd->sess->init_phase != SCST_SESS_IPH_READY)) {
                scst_set_cmd_error_status(cmd, SAM_STAT_BUSY);
-               TRACE(TRACE_MGMT_MINOR, "Sending BUSY status to initiator %s "
+               TRACE(TRACE_FLOW_CONTROL, "Sending BUSY status to initiator %s "
                        "(cmds count %d, queue_type %x, sess->init_phase %d)",
                        cmd->sess->initiator_name, c,
                        cmd->queue_type, cmd->sess->init_phase);
        } else {
                scst_set_cmd_error_status(cmd, SAM_STAT_TASK_SET_FULL);
-               TRACE(TRACE_MGMT_MINOR, "Sending QUEUE_FULL status to "
+               TRACE(TRACE_FLOW_CONTROL, "Sending QUEUE_FULL status to "
                        "initiator %s (cmds count %d, queue_type %x, "
                        "sess->init_phase %d)", cmd->sess->initiator_name, c,
                        cmd->queue_type, cmd->sess->init_phase);
@@ -996,7 +1019,7 @@ static void scst_free_aen(struct scst_aen *aen)
        return;
 };
 
-/* Must be called unded scst_mutex */
+/* Must be called under scst_mutex */
 void scst_gen_aen_or_ua(struct scst_tgt_dev *tgt_dev,
        int key, int asc, int ascq)
 {
@@ -1473,6 +1496,10 @@ int scst_get_cmd_abnormal_done_state(const struct scst_cmd *cmd)
        case SCST_CMD_STATE_INIT:
        case SCST_CMD_STATE_PRE_PARSE:
        case SCST_CMD_STATE_DEV_PARSE:
+               if (cmd->preprocessing_only) {
+                       res = SCST_CMD_STATE_PREPROCESSING_DONE;
+                       break;
+               } /* else go through */
        case SCST_CMD_STATE_DEV_DONE:
                if (cmd->internal)
                        res = SCST_CMD_STATE_FINISHED_INTERNAL;
@@ -1489,8 +1516,19 @@ int scst_get_cmd_abnormal_done_state(const struct scst_cmd *cmd)
                res = SCST_CMD_STATE_XMIT_RESP;
                break;
 
-       case SCST_CMD_STATE_PREPROCESS_DONE:
+       case SCST_CMD_STATE_PREPROCESSING_DONE:
+       case SCST_CMD_STATE_PREPROCESSING_DONE_CALLED:
+               if (cmd->tgt_dev == NULL)
+                       res = SCST_CMD_STATE_PRE_XMIT_RESP;
+               else
+                       res = SCST_CMD_STATE_PRE_DEV_DONE;
+               break;
+
        case SCST_CMD_STATE_PREPARE_SPACE:
+               if (cmd->preprocessing_only) {
+                       res = SCST_CMD_STATE_PREPROCESSING_DONE;
+                       break;
+               } /* else go through */
        case SCST_CMD_STATE_RDY_TO_XFER:
        case SCST_CMD_STATE_DATA_WAIT:
        case SCST_CMD_STATE_TGT_PRE_EXEC:
@@ -1533,7 +1571,8 @@ void scst_set_cmd_abnormal_done_state(struct scst_cmd *cmd)
        cmd->state = scst_get_cmd_abnormal_done_state(cmd);
 
 #ifdef CONFIG_SCST_EXTRACHECKS
-       if ((cmd->state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
+       if (((cmd->state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
+            (cmd->state != SCST_CMD_STATE_PREPROCESSING_DONE)) &&
                   (cmd->tgt_dev == NULL) && !cmd->internal) {
                PRINT_CRIT_ERROR("Wrong not inited cmd state %d (cmd %p, "
                        "op %x)", cmd->state, cmd, cmd->cdb[0]);
@@ -1706,8 +1745,7 @@ static void scst_hw_pending_work_fn(struct delayed_work *work)
        spin_lock_irqsave(&sess->sess_list_lock, flags);
 
 restart:
-       list_for_each_entry(cmd, &sess->search_cmd_list,
-                               sess_cmd_list_entry) {
+       list_for_each_entry(cmd, &sess->sess_cmd_list, sess_cmd_list_entry) {
                int rc;
 
                rc = scst_check_hw_pending_cmd(cmd, cur_time, max_time, sess,
@@ -1720,23 +1758,7 @@ restart:
                        goto restart;
        }
 
-restart1:
-       list_for_each_entry(cmd, &sess->after_pre_xmit_cmd_list,
-                               sess_cmd_list_entry) {
-               int rc;
-
-               rc = scst_check_hw_pending_cmd(cmd, cur_time, max_time, sess,
-                                       &flags, tgtt);
-               if (rc < 0)
-                       break;
-               else if (rc == 0)
-                       continue;
-               else
-                       goto restart1;
-       }
-
-       if (!list_empty(&sess->search_cmd_list) ||
-           !list_empty(&sess->after_pre_xmit_cmd_list)) {
+       if (list_empty(&sess->sess_cmd_list)) {
                /*
                 * For stuck cmds if there is no activity we might need to have
                 * one more run to release them, so reschedule once again.
@@ -2438,7 +2460,7 @@ int scst_acg_add_name(struct scst_acg *acg, const char *name)
                if (strcmp(n->name, name) == 0) {
                        PRINT_ERROR("Name %s already exists in group %s",
                                name, acg->acg_name);
-                       res = -EINVAL;
+                       res = -EEXIST;
                        goto out;
                }
        }
@@ -2834,8 +2856,7 @@ struct scst_session *scst_alloc_session(struct scst_tgt *tgt, gfp_t gfp_mask,
                INIT_LIST_HEAD(sess_tgt_dev_list_head);
        }
        spin_lock_init(&sess->sess_list_lock);
-       INIT_LIST_HEAD(&sess->search_cmd_list);
-       INIT_LIST_HEAD(&sess->after_pre_xmit_cmd_list);
+       INIT_LIST_HEAD(&sess->sess_cmd_list);
        sess->tgt = tgt;
        INIT_LIST_HEAD(&sess->init_deferred_cmd_list);
        INIT_LIST_HEAD(&sess->init_deferred_mcmd_list);
@@ -3867,7 +3888,7 @@ int scst_get_cdb_info(struct scst_cmd *cmd)
 
        op = cmd->cdb[0];       /* get clear opcode */
 
-       TRACE_DBG("opcode=%02x, cdblen=%d bytes, tblsize=%zd, "
+       TRACE_DBG("opcode=%02x, cdblen=%d bytes, tblsize=%d, "
                "dev_type=%d", op, SCST_GET_CDB_LEN(op), SCST_CDB_TBL_SIZE,
                dev_type);
 
@@ -4676,9 +4697,9 @@ void scst_process_reset(struct scst_device *dev,
 
                spin_lock_irq(&sess->sess_list_lock);
 
-               TRACE_DBG("Searching in search cmd list (sess=%p)", sess);
-               list_for_each_entry(cmd, &sess->search_cmd_list,
-                               sess_cmd_list_entry) {
+               TRACE_DBG("Searching in sess cmd list (sess=%p)", sess);
+               list_for_each_entry(cmd, &sess->sess_cmd_list,
+                                       sess_cmd_list_entry) {
                        if (cmd == exclude_cmd)
                                continue;
                        if ((cmd->tgt_dev == tgt_dev) ||
@@ -4736,7 +4757,7 @@ again:
                TRACE_DBG("%s",
                      "SCST_TGT_DEV_UA_PENDING set, but UA_list empty");
                res = -1;
-               goto out_unlock;
+               goto out_unlock_tgt_dev_lock;
        }
 
        UA_entry = list_entry(cmd->tgt_dev->UA_list.next, typeof(*UA_entry),
@@ -4774,8 +4795,9 @@ again:
                goto again;
        }
 
-       scst_set_cmd_error_sense(cmd, UA_entry->UA_sense_buffer,
-               UA_entry->UA_valid_sense_len);
+       if (scst_set_cmd_error_sense(cmd, UA_entry->UA_sense_buffer,
+                       UA_entry->UA_valid_sense_len) != 0)
+               goto out_unlock;
 
        cmd->ua_ignore = 1;
 
@@ -4831,6 +4853,7 @@ out_unlock:
                spin_lock_bh(&cmd->tgt_dev->tgt_dev_lock);
        }
 
+out_unlock_tgt_dev_lock:
        spin_unlock_bh(&cmd->tgt_dev->tgt_dev_lock);
 
        TRACE_EXIT_RES(res);
@@ -5600,7 +5623,6 @@ static void tm_dbg_init_tgt_dev(struct scst_tgt_dev *tgt_dev,
                if (tm_dbg_tgt_dev != NULL)
                        tm_dbg_deinit_tgt_dev(tm_dbg_tgt_dev);
 
-               /* Do TM debugging only for LUN 0 */
                spin_lock_irqsave(&scst_tm_dbg_lock, flags);
                tm_dbg_state = INIT_TM_DBG_STATE;
                tm_dbg_on_state_passes =
index f773bed..e3c2b2b 100644 (file)
@@ -110,6 +110,7 @@ static struct scst_trace_log scst_proc_trace_tbl[] = {
     { TRACE_MGMT,              "mgmt" },
     { TRACE_MGMT_MINOR,                "mgmt_minor" },
     { TRACE_MGMT_DEBUG,                "mgmt_dbg" },
+    { TRACE_FLOW_CONTROL,      "flow_control" },
     { 0,                       NULL }
 };
 
@@ -1975,7 +1976,7 @@ static ssize_t scst_proc_groups_devices_write(struct file *file,
                        if (action == SCST_PROC_ACTION_ADD) {
                                PRINT_ERROR("virt lun %d already exists in "
                                        "group %s", virt_lun, acg->acg_name);
-                               res = -EINVAL;
+                               res = -EEXIST;
                                goto out_free_up;
                        } else {
                                /* Replace */
@@ -2324,15 +2325,26 @@ static int scst_sessions_info_show(struct seq_file *seq, void *v)
 
        seq_printf(seq, "%-20s %-45s %-35s %-15s\n",
                   "Target name", "Initiator name",
-                  "Group name", "Command Count");
+                  "Group name", "Active/All Commands Count");
 
        list_for_each_entry(acg, &scst_acg_list, scst_acg_list_entry) {
                list_for_each_entry(sess, &acg->acg_sess_list,
-                       acg_sess_list_entry) {
-                       seq_printf(seq, "%-20s %-45s %-35s %-15d\n",
+                               acg_sess_list_entry) {
+                       int active_cmds = 0, t;
+                       for (t = TGT_DEV_HASH_SIZE-1; t >= 0; t--) {
+                               struct list_head *sess_tgt_dev_list_head =
+                                       &sess->sess_tgt_dev_list_hash[t];
+                               struct scst_tgt_dev *tgt_dev;
+                               list_for_each_entry(tgt_dev,
+                                               sess_tgt_dev_list_head,
+                                               sess_tgt_dev_list_entry) {
+                                       active_cmds += atomic_read(&tgt_dev->tgt_dev_cmd_count);
+                               }
+                       }
+                       seq_printf(seq, "%-20s %-45s %-35s %d/%d\n",
                                        sess->tgt->tgtt->name,
                                        sess->initiator_name,
-                                       acg->acg_name,
+                                       acg->acg_name, active_cmds,
                                        atomic_read(&sess->sess_cmd_count));
                }
        }
index 03f1468..e3b3779 100644 (file)
@@ -81,6 +81,7 @@ static struct scst_trace_log scst_trace_tbl[] = {
     { TRACE_MGMT,              "mgmt" },
     { TRACE_MGMT_MINOR,                "mgmt_minor" },
     { TRACE_MGMT_DEBUG,                "mgmt_dbg" },
+    { TRACE_FLOW_CONTROL,      "flow_control" },
     { 0,                       NULL }
 };
 
@@ -770,7 +771,7 @@ void scst_device_sysfs_put(struct scst_device *dev)
  * Target sessions directory implementation
  */
 
-ssize_t scst_sess_sysfs_commands_show(struct kobject *kobj,
+static ssize_t scst_sess_sysfs_commands_show(struct kobject *kobj,
                            struct kobj_attribute *attr, char *buf)
 {
        struct scst_session *sess;
@@ -783,7 +784,42 @@ ssize_t scst_sess_sysfs_commands_show(struct kobject *kobj,
 static struct kobj_attribute session_commands_attr =
        __ATTR(commands, S_IRUGO, scst_sess_sysfs_commands_show, NULL);
 
-ssize_t scst_sess_sysfs_initiator_name_show(struct kobject *kobj,
+static ssize_t scst_sess_sysfs_active_commands_show(struct kobject *kobj,
+                           struct kobj_attribute *attr, char *buf)
+{
+       int res;
+       struct scst_session *sess;
+       int active_cmds = 0, t;
+
+       if (mutex_lock_interruptible(&scst_mutex) != 0) {
+               res = -EINTR;
+               goto out;
+       }
+
+       sess = container_of(kobj, struct scst_session, sess_kobj);
+
+       for (t = TGT_DEV_HASH_SIZE-1; t >= 0; t--) {
+               struct list_head *sess_tgt_dev_list_head =
+                       &sess->sess_tgt_dev_list_hash[t];
+               struct scst_tgt_dev *tgt_dev;
+               list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
+                               sess_tgt_dev_list_entry) {
+                       active_cmds += atomic_read(&tgt_dev->tgt_dev_cmd_count);
+               }
+       }
+
+       mutex_unlock(&scst_mutex);
+
+       res = sprintf(buf, "%i\n", active_cmds);
+
+out:
+       return res;
+}
+
+static struct kobj_attribute session_active_commands_attr =
+       __ATTR(commands, S_IRUGO, scst_sess_sysfs_active_commands_show, NULL);
+
+static ssize_t scst_sess_sysfs_initiator_name_show(struct kobject *kobj,
                            struct kobj_attribute *attr, char *buf)
 {
        struct scst_session *sess;
@@ -799,6 +835,7 @@ static struct kobj_attribute session_initiator_name_attr =
 
 static struct attribute *scst_session_attrs[] = {
        &session_commands_attr.attr,
+       &session_active_commands_attr.attr,
        &session_initiator_name_attr.attr,
        NULL,
 };
@@ -1241,7 +1278,7 @@ static ssize_t scst_luns_mgmt_store(struct kobject *kobj,
                        if (action == SCST_LUN_ACTION_ADD) {
                                PRINT_ERROR("virt lun %d already exists in "
                                        "group %s", virt_lun, acg->acg_name);
-                               res = -EINVAL;
+                               res = -EEXIST;
                                goto out_free_up;
                        } else {
                                /* Replace */
index 610f20f..00f2092 100644 (file)
@@ -164,6 +164,8 @@ out_redirect:
                /*
                 * Poor man solution for single threaded targets, where
                 * blocking receiver at least sometimes means blocking all.
+                * For instance, iSCSI target won't be able to receive
+                * Data-Out PDUs.
                 */
                sBUG_ON(*context != SCST_CONTEXT_DIRECT);
                scst_set_busy(cmd);
@@ -222,14 +224,14 @@ void scst_cmd_init_done(struct scst_cmd *cmd,
 
        if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
                /*
-                * We have to always keep command in the search list from the
-                * very beginning, because otherwise it can be missed during
+                * We must always keep commands in the sess list from the
+                * very beginning, because otherwise they can be missed during
                 * TM processing. This check is needed because there might be
                 * old, i.e. deferred, commands and new, i.e. just coming, ones.
                 */
                if (cmd->sess_cmd_list_entry.next == NULL)
                        list_add_tail(&cmd->sess_cmd_list_entry,
-                               &sess->search_cmd_list);
+                               &sess->sess_cmd_list);
                switch (sess->init_phase) {
                case SCST_SESS_IPH_SUCCESS:
                        break;
@@ -250,7 +252,7 @@ void scst_cmd_init_done(struct scst_cmd *cmd,
                }
        } else
                list_add_tail(&cmd->sess_cmd_list_entry,
-                             &sess->search_cmd_list);
+                             &sess->sess_cmd_list);
 
        spin_unlock_irqrestore(&sess->sess_list_lock, flags);
 
@@ -287,12 +289,6 @@ void scst_cmd_init_done(struct scst_cmd *cmd,
        rc = scst_init_cmd(cmd, &pref_context);
        if (unlikely(rc < 0))
                goto out;
-       else if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION)) {
-               if (rc == 0) {
-                       /* Target driver preliminary completed cmd */
-                       scst_set_cmd_abnormal_done_state(cmd);
-               }
-       }
 
 active:
        /* Here cmd must not be in any cmd list, no locks */
@@ -303,12 +299,10 @@ active:
 
        case SCST_CONTEXT_DIRECT:
                scst_process_active_cmd(cmd, false);
-               /* For *NEED_THREAD wake_up() is already done */
                break;
 
        case SCST_CONTEXT_DIRECT_ATOMIC:
                scst_process_active_cmd(cmd, true);
-               /* For *NEED_THREAD wake_up() is already done */
                break;
 
        default:
@@ -639,6 +633,7 @@ static int scst_parse_cmd(struct scst_cmd *cmd)
        }
 
 set_res:
+#ifdef CONFIG_SCST_EXTRACHECKS
        switch (state) {
        case SCST_CMD_STATE_PREPARE_SPACE:
        case SCST_CMD_STATE_PRE_PARSE:
@@ -654,8 +649,10 @@ set_res:
        case SCST_CMD_STATE_XMIT_RESP:
        case SCST_CMD_STATE_FINISHED:
        case SCST_CMD_STATE_FINISHED_INTERNAL:
+#endif
                cmd->state = state;
                res = SCST_CMD_STATE_RES_CONT_SAME;
+#ifdef CONFIG_SCST_EXTRACHECKS
                break;
 
        default:
@@ -670,6 +667,7 @@ set_res:
                }
                goto out_error;
        }
+#endif
 
        if (cmd->resp_data_len == -1) {
                if (cmd->data_direction & SCST_DATA_READ)
@@ -701,7 +699,7 @@ static int scst_prepare_space(struct scst_cmd *cmd)
        TRACE_ENTRY();
 
        if (cmd->data_direction == SCST_DATA_NONE)
-               goto prep_done;
+               goto done;
 
        if (cmd->tgt_need_alloc_data_buf) {
                int orig_bufflen = cmd->bufflen;
@@ -767,30 +765,10 @@ check:
                        goto out_no_space;
        }
 
-prep_done:
-       if (cmd->preprocessing_only) {
-               cmd->preprocessing_only = 0;
-
-               if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
-                       TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
-                               "cmd %p", cmd);
-                       scst_set_cmd_abnormal_done_state(cmd);
-                       res = SCST_CMD_STATE_RES_CONT_SAME;
-                       goto out;
-               }
-
-               res = SCST_CMD_STATE_RES_CONT_NEXT;
-               cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
-
-               TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
-               scst_set_cur_start(cmd);
-               cmd->tgtt->preprocessing_done(cmd);
-               TRACE_DBG("%s", "preprocessing_done() returned");
-               goto out;
-
-       }
-
-       if (cmd->data_direction & SCST_DATA_WRITE)
+done:
+       if (cmd->preprocessing_only)
+               cmd->state = SCST_CMD_STATE_PREPROCESSING_DONE;
+       else if (cmd->data_direction & SCST_DATA_WRITE)
                cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
        else
                cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
@@ -814,6 +792,28 @@ out_error:
        goto out;
 }
 
+static int scst_preprocessing_done(struct scst_cmd *cmd)
+{
+       int res;
+
+       TRACE_ENTRY();
+
+       EXTRACHECKS_BUG_ON(!cmd->preprocessing_only);
+
+       cmd->preprocessing_only = 0;
+
+       res = SCST_CMD_STATE_RES_CONT_NEXT;
+       cmd->state = SCST_CMD_STATE_PREPROCESSING_DONE_CALLED;
+
+       TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
+       scst_set_cur_start(cmd);
+       cmd->tgtt->preprocessing_done(cmd);
+       TRACE_DBG("%s", "preprocessing_done() returned");
+
+       TRACE_EXIT_HRES(res);
+       return res;
+}
+
 void scst_restart_cmd(struct scst_cmd *cmd, int status,
        enum scst_exec_context pref_context)
 {
@@ -2777,6 +2777,7 @@ static int scst_dev_done(struct scst_cmd *cmd)
        }
 
        switch (state) {
+#ifdef CONFIG_SCST_EXTRACHECKS
        case SCST_CMD_STATE_PRE_XMIT_RESP:
        case SCST_CMD_STATE_DEV_PARSE:
        case SCST_CMD_STATE_PRE_PARSE:
@@ -2792,16 +2793,18 @@ static int scst_dev_done(struct scst_cmd *cmd)
        case SCST_CMD_STATE_XMIT_RESP:
        case SCST_CMD_STATE_FINISHED:
        case SCST_CMD_STATE_FINISHED_INTERNAL:
+#else
+       default:
+#endif
                cmd->state = state;
                break;
-
        case SCST_CMD_STATE_NEED_THREAD_CTX:
                TRACE_DBG("Dev handler %s dev_done() requested "
                      "thread context, rescheduling",
                      dev->handler->name);
                res = SCST_CMD_STATE_RES_NEED_THREAD;
                break;
-
+#ifdef CONFIG_SCST_EXTRACHECKS
        default:
                if (state >= 0) {
                        PRINT_ERROR("Dev handler %s dev_done() returned "
@@ -2816,6 +2819,7 @@ static int scst_dev_done(struct scst_cmd *cmd)
                           SCST_LOAD_SENSE(scst_sense_hardw_error));
                scst_set_cmd_abnormal_done_state(cmd);
                break;
+#endif
        }
 
        if (cmd->needs_unblocking)
@@ -2838,7 +2842,6 @@ out:
 static int scst_pre_xmit_response(struct scst_cmd *cmd)
 {
        int res;
-       struct scst_session *sess = cmd->sess;
 
        TRACE_ENTRY();
 
@@ -2860,6 +2863,10 @@ static int scst_pre_xmit_response(struct scst_cmd *cmd)
 #endif
 
        if (likely(cmd->tgt_dev != NULL)) {
+               /*
+                * Those counters protect from not getting too long processing
+                * latency, so we should decrement them after cmd completed.
+                */
                atomic_dec(&cmd->tgt_dev->tgt_dev_cmd_count);
                atomic_dec(&cmd->dev->dev_cmd_count);
 #ifdef CONFIG_SCST_ORDERED_READS
@@ -2879,18 +2886,6 @@ static int scst_pre_xmit_response(struct scst_cmd *cmd)
                }
        }
 
-       /*
-        * If we don't remove cmd from the search list here, before
-        * submitting it for transmittion, we will have a race, when for
-        * some reason cmd's release is delayed after transmittion and
-        * initiator sends cmd with the same tag => it is possible that
-        * a wrong cmd will be found by find() functions.
-        */
-       spin_lock_irq(&sess->sess_list_lock);
-       list_move_tail(&cmd->sess_cmd_list_entry,
-               &sess->after_pre_xmit_cmd_list);
-       spin_unlock_irq(&sess->sess_list_lock);
-
        cmd->done = 1;
        smp_mb(); /* to sync with scst_abort_cmd() */
 
@@ -3318,7 +3313,7 @@ static int __scst_init_cmd(struct scst_cmd *cmd)
 
                cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
                if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
-                       TRACE(TRACE_MGMT_MINOR,
+                       TRACE(TRACE_FLOW_CONTROL,
                                "Too many pending commands (%d) in "
                                "session, returning BUSY to initiator \"%s\"",
                                cnt, (cmd->sess->initiator_name[0] == '\0') ?
@@ -3329,7 +3324,7 @@ static int __scst_init_cmd(struct scst_cmd *cmd)
                cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
                if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
                        if (!failure) {
-                               TRACE(TRACE_MGMT_MINOR,
+                               TRACE(TRACE_FLOW_CONTROL,
                                        "Too many pending device "
                                        "commands (%d), returning BUSY to "
                                        "initiator \"%s\"", cnt,
@@ -3538,6 +3533,10 @@ void scst_process_active_cmd(struct scst_cmd *cmd, bool atomic)
                        res = scst_prepare_space(cmd);
                        break;
 
+               case SCST_CMD_STATE_PREPROCESSING_DONE:
+                       res = scst_preprocessing_done(cmd);
+                       break;
+
                case SCST_CMD_STATE_RDY_TO_XFER:
                        res = scst_rdy_to_xfer(cmd);
                        break;
@@ -3624,8 +3623,8 @@ void scst_process_active_cmd(struct scst_cmd *cmd, bool atomic)
                /* None */
        } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
                spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
+#ifdef CONFIG_SCST_EXTRACHECKS
                switch (cmd->state) {
-               case SCST_CMD_STATE_PRE_PARSE:
                case SCST_CMD_STATE_DEV_PARSE:
                case SCST_CMD_STATE_PREPARE_SPACE:
                case SCST_CMD_STATE_RDY_TO_XFER:
@@ -3633,32 +3632,24 @@ void scst_process_active_cmd(struct scst_cmd *cmd, bool atomic)
                case SCST_CMD_STATE_SEND_FOR_EXEC:
                case SCST_CMD_STATE_LOCAL_EXEC:
                case SCST_CMD_STATE_REAL_EXEC:
-               case SCST_CMD_STATE_PRE_DEV_DONE:
-               case SCST_CMD_STATE_MODE_SELECT_CHECKS:
                case SCST_CMD_STATE_DEV_DONE:
-               case SCST_CMD_STATE_PRE_XMIT_RESP:
                case SCST_CMD_STATE_XMIT_RESP:
-               case SCST_CMD_STATE_FINISHED:
-               case SCST_CMD_STATE_FINISHED_INTERNAL:
+#endif
                        TRACE_DBG("Adding cmd %p to head of active cmd list",
                                  cmd);
                        list_add(&cmd->cmd_list_entry,
                                &cmd->cmd_lists->active_cmd_list);
-                       break;
 #ifdef CONFIG_SCST_EXTRACHECKS
-               /* not very valid commands */
-               case SCST_CMD_STATE_DEFAULT:
-               case SCST_CMD_STATE_NEED_THREAD_CTX:
+                       break;
+               default:
                        PRINT_CRIT_ERROR("cmd %p is in invalid state %d)", cmd,
                                cmd->state);
                        spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
                        sBUG();
                        spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
                        break;
-#endif
-               default:
-                       break;
                }
+#endif
                wake_up(&cmd->cmd_lists->cmd_list_waitQ);
                spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
        } else
@@ -4097,7 +4088,7 @@ void scst_abort_cmd(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd,
 
                /*
                 * cmd can't die here or sess_list_lock already taken and
-                * cmd is in the search list
+                * cmd is in the sess list
                 */
                list_add_tail(&mstb->cmd_mgmt_cmd_list_entry,
                        &cmd->mgmt_cmd_list);
@@ -4250,8 +4241,8 @@ static void __scst_abort_task_set(struct scst_mgmt_cmd *mcmd,
 
        spin_lock_irq(&sess->sess_list_lock);
 
-       TRACE_DBG("Searching in search cmd list (sess=%p)", sess);
-       list_for_each_entry(cmd, &sess->search_cmd_list,
+       TRACE_DBG("Searching in sess cmd list (sess=%p)", sess);
+       list_for_each_entry(cmd, &sess->sess_cmd_list,
                            sess_cmd_list_entry) {
                if ((cmd->tgt_dev == tgt_dev) ||
                    ((cmd->tgt_dev == NULL) &&
@@ -4361,8 +4352,8 @@ static int scst_clear_task_set(struct scst_mgmt_cmd *mcmd)
 
                spin_lock_irq(&sess->sess_list_lock);
 
-               TRACE_DBG("Searching in search cmd list (sess=%p)", sess);
-               list_for_each_entry(cmd, &sess->search_cmd_list,
+               TRACE_DBG("Searching in sess cmd list (sess=%p)", sess);
+               list_for_each_entry(cmd, &sess->sess_cmd_list,
                                    sess_cmd_list_entry) {
                        if ((cmd->dev == dev) ||
                            ((cmd->dev == NULL) &&
@@ -5698,10 +5689,19 @@ static struct scst_cmd *__scst_find_cmd_by_tag(struct scst_session *sess,
 
        /* ToDo: hash list */
 
-       TRACE_DBG("%s (sess=%p, tag=%llu)", "Searching in search cmd list",
+       TRACE_DBG("%s (sess=%p, tag=%llu)", "Searching in sess cmd list",
                  sess, (long long unsigned int)tag);
-       list_for_each_entry(cmd, &sess->search_cmd_list,
+       list_for_each_entry(cmd, &sess->sess_cmd_list,
                        sess_cmd_list_entry) {
+               /*
+                * We must not count done commands, because they were
+                * submitted for transmittion. Otherwise we can have a race,
+                * when for some reason cmd's release delayed after
+                * transmittion and initiator sends cmd with the same tag =>
+                * it can be possible that a wrong cmd will be returned.
+                */
+               if (cmd->done)
+                       continue;
                if (cmd->tag == tag)
                        goto out;
        }
@@ -5725,9 +5725,17 @@ struct scst_cmd *scst_find_cmd(struct scst_session *sess, void *data,
 
        spin_lock_irqsave(&sess->sess_list_lock, flags);
 
-       TRACE_DBG("Searching in search cmd list (sess=%p)", sess);
-       list_for_each_entry(cmd, &sess->search_cmd_list,
-                       sess_cmd_list_entry) {
+       TRACE_DBG("Searching in sess cmd list (sess=%p)", sess);
+       list_for_each_entry(cmd, &sess->sess_cmd_list, sess_cmd_list_entry) {
+               /*
+                * We must not count done commands, because they were
+                * submitted for transmittion. Otherwise we can have a race,
+                * when for some reason cmd's release delayed after
+                * transmittion and initiator sends cmd with the same tag =>
+                * it can be possible that a wrong cmd will be returned.
+                */
+               if (cmd->done)
+                       continue;
                if (cmp_fn(cmd, data))
                        goto out_unlock;
        }