A major cleanup of sending commands for execution code path fixing found problems...
authorvlnb <vlnb@d57e44dd-8a1f-0410-8b47-8ef2f437770f>
Thu, 16 Oct 2008 17:34:33 +0000 (17:34 +0000)
committervlnb <vlnb@d57e44dd-8a1f-0410-8b47-8ef2f437770f>
Thu, 16 Oct 2008 17:34:33 +0000 (17:34 +0000)
git-svn-id: https://scst.svn.sourceforge.net/svnroot/scst/trunk@523 d57e44dd-8a1f-0410-8b47-8ef2f437770f

scst/include/scst.h
scst/src/dev_handlers/scst_vdisk.c
scst/src/scst_lib.c
scst/src/scst_main.c
scst/src/scst_priv.h
scst/src/scst_proc.c
scst/src/scst_targ.c

index e1f16c1..4adb9a5 100644 (file)
 
 #include <scst_const.h>
 
-#ifndef DECLARE_MUTEX_LOCKED
-#define DECLARE_MUTEX_LOCKED(name)     __DECLARE_SEMAPHORE_GENERIC(name, 0)
-#endif
-
-#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19)
-typedef _Bool bool;
-#define true  1
-#define false 0
-#endif
-
 /*
  * Version numbers, the same as for the kernel.
  *
@@ -55,6 +45,18 @@ typedef _Bool bool;
 #define SCST_VERSION_STRING "1.0.1"
 #define SCST_INTERFACE_VERSION SCST_VERSION_STRING "$Revision$" SCST_CONST_VERSION
 
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19)
+typedef _Bool bool;
+#define true  1
+#define false 0
+#endif
+
+#ifndef DECLARE_MUTEX_LOCKED
+#define DECLARE_MUTEX_LOCKED(name)     __DECLARE_SEMAPHORE_GENERIC(name, 0)
+#endif
+
+#define SCST_LOCAL_NAME                        "scst_lcl_drvr"
+
 /*************************************************************
  ** States of command processing state machine. At first,
  ** "active" states, then - "passive" ones. This is to have
@@ -77,26 +79,32 @@ typedef _Bool bool;
 /* Target driver's pre_exec() is going to be called */
 #define SCST_CMD_STATE_TGT_PRE_EXEC  4
 
-/* CDB is going to be sent to SCSI mid-level for execution */
-#define SCST_CMD_STATE_SEND_TO_MIDLEV 5
+/* Cmd is going to be sent for execution */
+#define SCST_CMD_STATE_SEND_FOR_EXEC 5
+
+/* Cmd is being checked if it should be executed locally */
+#define SCST_CMD_STATE_LOCAL_EXEC    6
+
+/* Cmd is ready for execution */
+#define SCST_CMD_STATE_REAL_EXEC     7
 
-/* Internal pos-exec checks */
-#define SCST_CMD_STATE_PRE_DEV_DONE  6
+/* Internal post-exec checks */
+#define SCST_CMD_STATE_PRE_DEV_DONE  8
 
 /* Internal MODE SELECT pages related checks */
-#define SCST_CMD_STATE_MODE_SELECT_CHECKS 7
+#define SCST_CMD_STATE_MODE_SELECT_CHECKS 9
 
 /* Dev handler's dev_done() is going to be called */
-#define SCST_CMD_STATE_DEV_DONE      8
+#define SCST_CMD_STATE_DEV_DONE      10
 
 /* Target driver's xmit_response() is going to be called */
-#define SCST_CMD_STATE_PRE_XMIT_RESP 9
+#define SCST_CMD_STATE_PRE_XMIT_RESP 11
 
 /* Target driver's xmit_response() is going to be called */
-#define SCST_CMD_STATE_XMIT_RESP     10
+#define SCST_CMD_STATE_XMIT_RESP     12
 
 /* The cmd finished */
-#define SCST_CMD_STATE_FINISHED      11
+#define SCST_CMD_STATE_FINISHED      13
 
 #define SCST_CMD_STATE_LAST_ACTIVE   (SCST_CMD_STATE_FINISHED+100)
 
@@ -113,7 +121,7 @@ typedef _Bool bool;
 #define SCST_CMD_STATE_DATA_WAIT     (SCST_CMD_STATE_LAST_ACTIVE+4)
 
 /* Waiting for CDB's execution finish */
-#define SCST_CMD_STATE_EXECUTING     (SCST_CMD_STATE_LAST_ACTIVE+5)
+#define SCST_CMD_STATE_REAL_EXECUTING (SCST_CMD_STATE_LAST_ACTIVE+5)
 
 /* Waiting for response's transmission finish */
 #define SCST_CMD_STATE_XMIT_WAIT     (SCST_CMD_STATE_LAST_ACTIVE+6)
@@ -989,13 +997,10 @@ struct scst_cmd {
         ** Cmd's flags
         *************************************************************/
        /*
-        * Set if expected_sn was incremented, i.e. cmd was sent to
-        * SCSI mid-level for execution
+        * Set if expected_sn should be incremented, i.e. cmd was sent
+        * for execution
         */
-       unsigned int sent_to_midlev:1;
-
-       /* Set if scst_local_exec() was already called for this cmd */
-       unsigned int local_exec_done:1;
+       unsigned int sent_for_exec:1;
 
        /* Set if the cmd's action is completed */
        unsigned int completed:1;
@@ -1018,12 +1023,12 @@ struct scst_cmd {
         */
        unsigned int context_processable:1;
 
-       /* Set if cmd is internally generated */
-       unsigned int internal:1;
-
        /* Set if cmd is being retried */
        unsigned int retry:1;
 
+       /* Set if cmd is internally generated */
+       unsigned int internal:1;
+
        /* Set if the device was blocked by scst_inc_on_dev_cmd() (for debug) */
        unsigned int inc_blocking:1;
 
index a248c22..8e46ad0 100644 (file)
@@ -254,6 +254,11 @@ static int vcdrom_write_proc(char *buffer, char **start, off_t offset,
 static int vdisk_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
        struct scst_tgt_dev *tgt_dev);
 
+/*
+ * Name of FILEIO vdisk can't be changed from "vdisk", since it is the name
+ * of the corresponding /proc/scsi_tgt entry, hence a part of user space ABI.
+ */
+
 #define VDISK_TYPE {                                   \
        .name =                 VDISK_NAME,             \
        .type =                 TYPE_DISK,              \
@@ -290,6 +295,23 @@ static int vdisk_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
        .task_mgmt_fn =         vdisk_task_mgmt_fn,     \
 }
 
+#define VDISK_NULL_TYPE {                              \
+       .name =                 VDISK_NAME "_null",     \
+       .type =                 TYPE_DISK,              \
+       .threads_num =          0,                      \
+       .parse_atomic =         1,                      \
+       .exec_atomic =          1,                      \
+       .dev_done_atomic =      1,                      \
+       .no_proc =              1,                      \
+       .attach =               vdisk_attach,           \
+       .detach =               vdisk_detach,           \
+       .attach_tgt =           vdisk_attach_tgt,       \
+       .detach_tgt =           vdisk_detach_tgt,       \
+       .parse =                vdisk_parse,            \
+       .exec =                 vdisk_do_job,           \
+       .task_mgmt_fn =         vdisk_task_mgmt_fn,     \
+}
+
 #define VCDROM_TYPE {                                  \
        .name =                 VCDROM_NAME,            \
        .type =                 TYPE_ROM,               \
@@ -313,8 +335,9 @@ static DEFINE_MUTEX(scst_vdisk_mutex);
 static LIST_HEAD(vdisk_dev_list);
 static LIST_HEAD(vcdrom_dev_list);
 
-static struct scst_dev_type vdisk_devtype = VDISK_TYPE;
+static struct scst_dev_type vdisk_file_devtype = VDISK_TYPE;
 static struct scst_dev_type vdisk_blk_devtype = VDISK_BLK_TYPE;
+static struct scst_dev_type vdisk_null_devtype = VDISK_NULL_TYPE;
 static struct scst_dev_type vcdrom_devtype = VCDROM_TYPE;
 
 static char *vdisk_proc_help_string =
@@ -539,8 +562,7 @@ static void vdisk_detach(struct scst_device *dev)
 
 static void vdisk_free_thr_data(struct scst_thr_data_hdr *d)
 {
-       struct scst_vdisk_thr *thr = container_of(d, struct scst_vdisk_thr,
-                                               hdr);
+       struct scst_vdisk_thr *thr = container_of(d, struct scst_vdisk_thr, hdr);
 
        TRACE_ENTRY();
 
@@ -556,7 +578,7 @@ static void vdisk_free_thr_data(struct scst_thr_data_hdr *d)
 }
 
 static struct scst_vdisk_thr *vdisk_init_thr_data(
-       struct scst_tgt_dev *tgt_dev)
+       struct scst_tgt_dev *tgt_dev, bool atomic)
 {
        struct scst_vdisk_thr *res;
        struct scst_vdisk_dev *virt_dev =
@@ -565,11 +587,13 @@ static struct scst_vdisk_thr *vdisk_init_thr_data(
        TRACE_ENTRY();
 
 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17)
-       res = kmem_cache_alloc(vdisk_thr_cachep, GFP_KERNEL);
+       res = kmem_cache_alloc(vdisk_thr_cachep,
+                               atomic ? GFP_ATOMIC : GFP_KERNEL);
        if (res != NULL)
                memset(res, 0, sizeof(*res));
 #else
-       res = kmem_cache_zalloc(vdisk_thr_cachep, GFP_KERNEL);
+       res = kmem_cache_zalloc(vdisk_thr_cachep,
+                               atomic ? GFP_ATOMIC : GFP_KERNEL);
 #endif
        if (res == NULL) {
                TRACE(TRACE_OUT_OF_MEM, "%s", "Unable to allocate struct "
@@ -680,13 +704,6 @@ static int vdisk_do_job(struct scst_cmd *cmd)
 
        TRACE_ENTRY();
 
-       if (scst_cmd_atomic(cmd)) {
-               TRACE_DBG("%s", "vdisk exec() can not be done in atomic "
-                       "context, requesting thread context");
-               res = SCST_EXEC_NEED_THREAD;
-               goto out;
-       }
-
        switch (cmd->queue_type) {
        case SCST_CMD_QUEUE_ORDERED:
                TRACE(TRACE_ORDER, "ORDERED cmd %p", cmd);
@@ -709,7 +726,7 @@ static int vdisk_do_job(struct scst_cmd *cmd)
 
        d = scst_find_thr_data(cmd->tgt_dev);
        if (unlikely(d == NULL)) {
-               thr = vdisk_init_thr_data(cmd->tgt_dev);
+               thr = vdisk_init_thr_data(cmd->tgt_dev, scst_cmd_atomic(cmd));
                if (thr == NULL) {
                        scst_set_busy(cmd);
                        goto out_compl;
@@ -965,7 +982,6 @@ out_thr:
 
        res = SCST_EXEC_COMPLETED;
 
-out:
        TRACE_EXIT_RES(res);
        return res;
 }
@@ -1940,7 +1956,8 @@ static struct iovec *vdisk_alloc_iv(struct scst_cmd *cmd,
        iv_count = scst_get_buf_count(cmd);
        if (iv_count > thr->iv_count) {
                kfree(thr->iv);
-               thr->iv = kmalloc(sizeof(*thr->iv) * iv_count, GFP_KERNEL);
+               thr->iv = kmalloc(sizeof(*thr->iv) * iv_count, 
+                       scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
                if (thr->iv == NULL) {
                        PRINT_ERROR("Unable to allocate iv (%d)", iv_count);
                        scst_set_busy(cmd);
@@ -2846,10 +2863,15 @@ static int vdisk_write_proc(char *buffer, char **start, off_t offset,
                        virt_dev->virt_id =
                                scst_register_virtual_device(&vdisk_blk_devtype,
                                                         virt_dev->name);
+               } else if (virt_dev->nullio) {
+                       vdisk_report_registering("NULLIO", virt_dev);
+                       virt_dev->virt_id =
+                               scst_register_virtual_device(&vdisk_null_devtype,
+                                                        virt_dev->name);
                } else {
                        vdisk_report_registering("FILEIO", virt_dev);
                        virt_dev->virt_id =
-                               scst_register_virtual_device(&vdisk_devtype,
+                               scst_register_virtual_device(&vdisk_file_devtype,
                                                         virt_dev->name);
                }
                if (virt_dev->virt_id < 0) {
@@ -3426,10 +3448,10 @@ static int __init init_scst_vdisk_driver(void)
        }
 
        num_threads = num_online_cpus() + 2;
-       vdisk_devtype.threads_num = num_threads;
+       vdisk_file_devtype.threads_num = num_threads;
        vcdrom_devtype.threads_num = num_threads;
 
-       res = init_scst_vdisk(&vdisk_devtype);
+       res = init_scst_vdisk(&vdisk_file_devtype);
        if (res != 0)
                goto out_free_slab;
 
@@ -3437,18 +3459,25 @@ static int __init init_scst_vdisk_driver(void)
        if (res != 0)
                goto out_free_vdisk;
 
+       res = init_scst_vdisk(&vdisk_null_devtype);
+       if (res != 0)
+               goto out_free_blk;
+
        res = init_scst_vdisk(&vcdrom_devtype);
        if (res != 0)
-               goto out_err;
+               goto out_free_null;
 
 out:
        return res;
 
-out_err:
+out_free_null:
+       exit_scst_vdisk(&vdisk_null_devtype, &vdisk_dev_list);
+
+out_free_blk:
        exit_scst_vdisk(&vdisk_blk_devtype, &vdisk_dev_list);
 
 out_free_vdisk:
-       exit_scst_vdisk(&vdisk_devtype, &vdisk_dev_list);
+       exit_scst_vdisk(&vdisk_file_devtype, &vdisk_dev_list);
 
 out_free_slab:
        kmem_cache_destroy(vdisk_thr_cachep);
@@ -3457,8 +3486,9 @@ out_free_slab:
 
 static void __exit exit_scst_vdisk_driver(void)
 {
+       exit_scst_vdisk(&vdisk_null_devtype, &vdisk_dev_list);
        exit_scst_vdisk(&vdisk_blk_devtype, &vdisk_dev_list);
-       exit_scst_vdisk(&vdisk_devtype, &vdisk_dev_list);
+       exit_scst_vdisk(&vdisk_file_devtype, &vdisk_dev_list);
        exit_scst_vdisk(&vcdrom_devtype, &vcdrom_dev_list);
        kmem_cache_destroy(vdisk_thr_cachep);
 }
index 3b30a13..3fb7690 100644 (file)
@@ -541,17 +541,15 @@ static struct scst_tgt_dev *scst_alloc_add_tgt_dev(struct scst_session *sess,
 
        if (dev->handler->parse_atomic &&
            (sess->tgt->tgtt->preprocessing_done == NULL)) {
-               if (sess->tgt->tgtt->rdy_to_xfer_atomic ||
-                   (sess->tgt->tgtt->rdy_to_xfer == NULL))
+               if (sess->tgt->tgtt->rdy_to_xfer_atomic)
                        __set_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
                                &tgt_dev->tgt_dev_flags);
-               if (dev->handler->exec_atomic || (dev->handler->exec == NULL))
+               if (dev->handler->exec_atomic)
                        __set_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
                                &tgt_dev->tgt_dev_flags);
        }
-       if (dev->handler->exec_atomic || (dev->handler->exec == NULL)) {
-               if (sess->tgt->tgtt->rdy_to_xfer_atomic ||
-                   (sess->tgt->tgtt->rdy_to_xfer == NULL))
+       if (dev->handler->exec_atomic) {
+               if (sess->tgt->tgtt->rdy_to_xfer_atomic)
                        __set_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
                                &tgt_dev->tgt_dev_flags);
                __set_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
@@ -559,8 +557,7 @@ static struct scst_tgt_dev *scst_alloc_add_tgt_dev(struct scst_session *sess,
                __set_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC,
                        &tgt_dev->tgt_dev_flags);
        }
-       if ((dev->handler->dev_done_atomic ||
-            (dev->handler->dev_done == NULL)) &&
+       if (dev->handler->dev_done_atomic &&
            sess->tgt->tgtt->xmit_response_atomic) {
                __set_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC,
                        &tgt_dev->tgt_dev_flags);
@@ -957,7 +954,6 @@ struct scst_cmd *scst_create_prepare_internal_cmd(
 
        res->cmd_lists = orig_cmd->cmd_lists;
        res->sess = orig_cmd->sess;
-       res->state = SCST_CMD_STATE_PRE_PARSE;
        res->atomic = scst_cmd_atomic(orig_cmd);
        res->internal = 1;
        res->tgtt = orig_cmd->tgtt;
@@ -968,9 +964,10 @@ struct scst_cmd *scst_create_prepare_internal_cmd(
        res->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
        res->data_direction = SCST_DATA_UNKNOWN;
        res->orig_cmd = orig_cmd;
-
        res->bufflen = bufsize;
 
+       res->state = SCST_CMD_STATE_PRE_PARSE;
+
 out:
        TRACE_EXIT_HRES((unsigned long)res);
        return res;
@@ -988,7 +985,7 @@ void scst_free_internal_cmd(struct scst_cmd *cmd)
 
 int scst_prepare_request_sense(struct scst_cmd *orig_cmd)
 {
-       int res = SCST_CMD_STATE_RES_CONT_NEXT;
+       int res = 0;
 #define sbuf_size 252
        static const uint8_t request_sense[6] =
            { REQUEST_SENSE, 0, 0, 0, sbuf_size, 0 };
@@ -1003,12 +1000,16 @@ int scst_prepare_request_sense(struct scst_cmd *orig_cmd)
        memcpy(rs_cmd->cdb, request_sense, sizeof(request_sense));
        rs_cmd->cdb_len = sizeof(request_sense);
        rs_cmd->data_direction = SCST_DATA_READ;
+       rs_cmd->expected_data_direction = rs_cmd->data_direction;
+       rs_cmd->expected_transfer_len = sbuf_size;
+       rs_cmd->expected_values_set = 1;
 
        TRACE(TRACE_MGMT_MINOR, "Adding REQUEST SENSE cmd %p to head of active "
                "cmd list ", rs_cmd);
        spin_lock_irq(&rs_cmd->cmd_lists->cmd_list_lock);
        list_add(&rs_cmd->cmd_list_entry, &rs_cmd->cmd_lists->active_cmd_list);
        spin_unlock_irq(&rs_cmd->cmd_lists->cmd_list_lock);
+       wake_up(&rs_cmd->cmd_lists->cmd_list_waitQ);
 
 out:
        TRACE_EXIT_RES(res);
@@ -1028,16 +1029,7 @@ struct scst_cmd *scst_complete_request_sense(struct scst_cmd *req_cmd)
 
        TRACE_ENTRY();
 
-       if (req_cmd->dev->handler->dev_done != NULL) {
-               int rc;
-               TRACE_DBG("Calling dev handler %s dev_done(%p)",
-                     req_cmd->dev->handler->name, req_cmd);
-               rc = req_cmd->dev->handler->dev_done(req_cmd);
-               TRACE_DBG("Dev handler %s dev_done() returned %d",
-                     req_cmd->dev->handler->name, rc);
-       }
-
-       sBUG_ON(orig_cmd);
+       sBUG_ON(orig_cmd == NULL);
 
        len = scst_get_buf_first(req_cmd, &buf);
 
@@ -1372,6 +1364,7 @@ struct scst_cmd *scst_alloc_cmd(gfp_t gfp_mask)
 #endif
 
        cmd->state = SCST_CMD_STATE_INIT_WAIT;
+       cmd->start_time = jiffies;
        atomic_set(&cmd->cmd_ref, 1);
        cmd->cmd_lists = &scst_main_cmd_lists;
        INIT_LIST_HEAD(&cmd->mgmt_cmd_list);
@@ -1466,7 +1459,7 @@ void scst_free_cmd(struct scst_cmd *cmd)
 
        if (likely(cmd->tgt_dev != NULL)) {
 #ifdef CONFIG_SCST_EXTRACHECKS
-               if (unlikely(!cmd->sent_to_midlev)) {
+               if (unlikely(!cmd->sent_for_exec)) {
                        PRINT_ERROR("Finishing not executed cmd %p (opcode "
                             "%d, target %s, lun %lld, sn %ld, expected_sn %ld)",
                             cmd, cmd->cdb[0], cmd->tgtt->name,
@@ -3033,6 +3026,14 @@ int scst_inc_on_dev_cmd(struct scst_cmd *cmd)
        cmd->dec_on_dev_needed = 1;
        TRACE_DBG("New on_dev_count %d", atomic_read(&dev->on_dev_count));
 
+       if (unlikely(cmd->internal) && (cmd->cdb[0] == REQUEST_SENSE)) {
+               /*
+                * The original command can already block the device, so
+                * REQUEST SENSE command should always pass.
+                */
+               goto out;
+       }
+
 #ifdef CONFIG_SCST_STRICT_SERIALIZING
        spin_lock_bh(&dev->dev_lock);
        if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
@@ -3119,8 +3120,10 @@ void scst_unblock_cmds(struct scst_device *dev)
                 * can't change behind us, if the corresponding cmd is in
                 * blocked_cmd_list, but we could be called before
                 * scst_inc_expected_sn().
+                *
+                * For HQ commands SN is not set.
                 */
-               if (likely(!cmd->internal && !cmd->retry)) {
+               if (likely(!cmd->internal && cmd->sn_set)) {
                        typeof(cmd->tgt_dev->expected_sn) expected_sn;
                        if (cmd->tgt_dev == NULL)
                                sBUG();
@@ -3176,7 +3179,7 @@ static void __scst_unblock_deferred(struct scst_tgt_dev *tgt_dev,
 
        if (out_of_sn_cmd->sn == tgt_dev->expected_sn) {
                scst_inc_expected_sn(tgt_dev, out_of_sn_cmd->sn_slot);
-               scst_make_deferred_commands_active(tgt_dev, out_of_sn_cmd);
+               scst_make_deferred_commands_active(tgt_dev);
        } else {
                out_of_sn_cmd->out_of_sn = 1;
                spin_lock_irq(&tgt_dev->sn_lock);
@@ -3230,7 +3233,7 @@ void scst_on_hq_cmd_response(struct scst_cmd *cmd)
         * unneeded run of the deferred commands.
         */
        if (tgt_dev->hq_cmd_count == 0)
-               scst_make_deferred_commands_active(tgt_dev, cmd);
+               scst_make_deferred_commands_active(tgt_dev);
 
 out:
        TRACE_EXIT();
index c1791d0..7da5b06 100644 (file)
@@ -202,6 +202,9 @@ int __scst_register_target_template(struct scst_tgt_template *vtt,
                        goto out_err;
        }
 
+       if (vtt->rdy_to_xfer == NULL)
+               vtt->rdy_to_xfer_atomic = 1;
+
        if (mutex_lock_interruptible(&m) != 0)
                goto out_err;
 
@@ -1484,7 +1487,9 @@ static int scst_add(struct device *cdev, struct class_interface *intf)
 #else
        scsidp = to_scsi_device(cdev->parent);
 #endif
-       res = scst_register_device(scsidp);
+
+       if (strcmp(scsidp->host->hostt->name, SCST_LOCAL_NAME) != 0)
+               res = scst_register_device(scsidp);
 
        TRACE_EXIT();
        return res;
@@ -1505,7 +1510,9 @@ static void scst_remove(struct device *cdev, struct class_interface *intf)
 #else
        scsidp = to_scsi_device(cdev->parent);
 #endif
-       scst_unregister_device(scsidp);
+
+       if (strcmp(scsidp->host->hostt->name, SCST_LOCAL_NAME) != 0)
+               scst_unregister_device(scsidp);
 
        TRACE_EXIT();
        return;
index 5f5c49c..50d6562 100644 (file)
@@ -98,11 +98,12 @@ extern unsigned long scst_trace_flag;
 #define SCST_FLAG_SUSPENDED                 1
 
 /**
- ** Return codes for cmd state process functions
+ ** Return codes for cmd state process functions. Codes are the same as
+ ** for SCST_EXEC_* to avoid translation to them and, hence, have better code.
  **/
-#define SCST_CMD_STATE_RES_CONT_SAME         0
-#define SCST_CMD_STATE_RES_CONT_NEXT         1
-#define SCST_CMD_STATE_RES_NEED_THREAD       2
+#define SCST_CMD_STATE_RES_CONT_NEXT         SCST_EXEC_COMPLETED
+#define SCST_CMD_STATE_RES_CONT_SAME         SCST_EXEC_NOT_COMPLETED
+#define SCST_CMD_STATE_RES_NEED_THREAD       SCST_EXEC_NEED_THREAD
 
 /** Name of the "default" security group **/
 #define SCST_DEFAULT_ACG_NAME                "Default"
@@ -232,16 +233,13 @@ static inline struct scst_cmd *scst_check_deferred_commands(
 }
 
 static inline void scst_make_deferred_commands_active(
-       struct scst_tgt_dev *tgt_dev, struct scst_cmd *curr_cmd)
+       struct scst_tgt_dev *tgt_dev)
 {
        struct scst_cmd *c;
 
        c = __scst_check_deferred_commands(tgt_dev);
        if (c != NULL) {
                TRACE_SN("Adding cmd %p to active cmd list", c);
-
-               EXTRACHECKS_BUG_ON(c->cmd_lists != curr_cmd->cmd_lists);
-
                spin_lock_irq(&c->cmd_lists->cmd_list_lock);
                list_add_tail(&c->cmd_list_entry,
                        &c->cmd_lists->active_cmd_list);
@@ -439,35 +437,33 @@ extern void scst_block_dev_cmd(struct scst_cmd *cmd, int outstanding);
 extern void scst_unblock_dev(struct scst_device *dev);
 extern void scst_unblock_dev_cmd(struct scst_cmd *cmd);
 
-static inline void __scst_dec_on_dev_cmd(struct scst_device *dev,
-       int unblock_dev)
+/* No locks */
+static inline void scst_dec_on_dev_cmd(struct scst_cmd *cmd)
 {
+       struct scst_device *dev = cmd->dev;
+       bool unblock_dev = cmd->inc_blocking;
+
+       if (cmd->inc_blocking) {
+               TRACE_MGMT_DBG("cmd %p (tag %llu): unblocking dev %p", cmd,
+                              (long long unsigned int)cmd->tag, cmd->dev);
+               cmd->inc_blocking = 0;
+       }
+       cmd->dec_on_dev_needed = 0;
+
        if (unblock_dev)
                scst_unblock_dev(dev);
+
        atomic_dec(&dev->on_dev_count);
        smp_mb__after_atomic_dec();
+
        TRACE_DBG("New on_dev_count %d", atomic_read(&dev->on_dev_count));
+
        sBUG_ON(atomic_read(&dev->on_dev_count) < 0);
+
        if (unlikely(dev->block_count != 0))
                wake_up_all(&dev->on_dev_waitQ);
-}
 
-static inline int scst_pre_dec_on_dev_cmd(struct scst_cmd *cmd)
-{
-       int cmd_blocking = cmd->inc_blocking;
-       if (cmd_blocking) {
-               TRACE_MGMT_DBG("cmd %p (tag %llu): unblocking dev %p", cmd,
-                              (long long unsigned int)cmd->tag, cmd->dev);
-               cmd->inc_blocking = 0;
-       }
-       cmd->dec_on_dev_needed = 0;
-       return cmd_blocking;
-}
-
-static inline void scst_dec_on_dev_cmd(struct scst_cmd *cmd)
-{
-       int cmd_blocking = scst_pre_dec_on_dev_cmd(cmd);
-       __scst_dec_on_dev_cmd(cmd->dev, cmd_blocking);
+       return;
 }
 
 static inline void __scst_get(int barrier)
index 8d28b08..1483c92 100644 (file)
@@ -1566,8 +1566,16 @@ static ssize_t scst_proc_groups_devices_write(struct file *file, const char __us
 
                while (isspace(*e) && *e != '\0')
                        e++;
-               if (!strncasecmp("READ_ONLY", e, 9))
-                       read_only = 1;
+
+               if (*e != '\0') {
+                       if (!strncasecmp("READ_ONLY", e, 9))
+                               read_only = 1;
+                       else {
+                               PRINT_ERROR("Unknown option \"%s\"", e);
+                               res = -EINVAL;
+                               goto out_free_up;
+                       }
+               }
 
                list_for_each_entry(acg_dev_tmp, &acg->acg_dev_list,
                                    acg_dev_list_entry) {
@@ -1901,8 +1909,8 @@ static int scst_groups_devices_show(struct seq_file *seq, void *v)
                goto out;
        }
 
-       seq_printf(seq, "%-60s%s  %s\n", "Device (host:ch:id:lun or name)",
-                      "Virtual lun", "Options");
+       seq_printf(seq, "%-60s%-13s%s\n", "Device (host:ch:id:lun or name)",
+                      "LUN", "Options");
 
        list_for_each_entry(acg_dev, &acg->acg_dev_list, acg_dev_list_entry) {
                if (acg_dev->dev->virt_id == 0) {
@@ -1915,13 +1923,19 @@ static int scst_groups_devices_show(struct seq_file *seq, void *v)
                                        acg_dev->dev->scsi_dev->channel,
                                        acg_dev->dev->scsi_dev->id);
                        seq_printf(seq, "%s", conv);
-                       sprintf(conv, "%%-%dd%%4d%%12s\n", 60 - size);
-                       seq_printf(seq, conv,
+
+                       /*
+                        * For some reason the third string argument always
+                        * shown as NULL, so we have to split it on 2 calls.
+                        */
+                       sprintf(conv, "%%-%dd%%-13d", 60 - size);
+                       size += seq_printf(seq, conv,
                                        acg_dev->dev->scsi_dev->lun,
-                                       acg_dev->lun,
-                                       acg_dev->rd_only_flag ? "RO" : "");
+                                       acg_dev->lun);
+                       seq_printf(seq, "%s\n",
+                               acg_dev->rd_only_flag ? "RO" : "");
                } else {
-                       seq_printf(seq, "%-60s%4Ld%12s\n",
+                       seq_printf(seq, "%-60s%-13lld%s\n",
                                       acg_dev->dev->virt_name,
                                       (long long unsigned int)acg_dev->lun,
                                       acg_dev->rd_only_flag ? "RO" : "");
index 68196ac..0c0b83c 100644 (file)
@@ -77,8 +77,6 @@ struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
        cmd->tgt = sess->tgt;
        cmd->tgtt = sess->tgt->tgtt;
 
-       cmd->start_time = jiffies;
-
        /*
         * For both wrong lun and CDB defer the error reporting for
         * scst_cmd_init_done()
@@ -274,7 +272,7 @@ void scst_cmd_init_done(struct scst_cmd *cmd, int pref_context)
        }
 
        cmd->state = SCST_CMD_STATE_INIT;
-       /* cmd must be inited here to keep the order */
+       /* cmd must be inited here to preserve the order */
        pref_context = scst_init_cmd(cmd, pref_context);
        if (unlikely(pref_context < 0))
                goto out;
@@ -329,8 +327,6 @@ static int scst_pre_parse(struct scst_cmd *cmd)
                 ((dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) ||
                  (cmd->queue_type == SCST_CMD_QUEUE_ORDERED)));
 
-       sBUG_ON(cmd->internal);
-
        /*
         * Expected transfer data supplied by the SCSI transport via the
         * target driver are untrusted, so we prefer to fetch them from CDB.
@@ -469,6 +465,18 @@ static int scst_parse_cmd(struct scst_cmd *cmd)
        TRACE_ENTRY();
 
        if (likely(!scst_is_cmd_local(cmd))) {
+               if (unlikely(!dev->handler->parse_atomic &&
+                            scst_cmd_atomic(cmd))) {
+                       /*
+                        * It shouldn't be because of SCST_TGT_DEV_AFTER_*
+                        * optimization.
+                        */
+                       TRACE_DBG("Dev handler %s parse() needs thread "
+                               "context, rescheduling", dev->handler->name);
+                       res = SCST_CMD_STATE_RES_NEED_THREAD;
+                       goto out;
+               }
+
                TRACE_DBG("Calling dev handler %s parse(%p)",
                      dev->handler->name, cmd);
                TRACE_BUFF_FLAG(TRACE_SND_BOT, "Parsing: ", cmd->cdb, cmd->cdb_len);
@@ -603,7 +611,9 @@ set_res:
        case SCST_CMD_STATE_DEV_PARSE:
        case SCST_CMD_STATE_RDY_TO_XFER:
        case SCST_CMD_STATE_TGT_PRE_EXEC:
-       case SCST_CMD_STATE_SEND_TO_MIDLEV:
+       case SCST_CMD_STATE_SEND_FOR_EXEC:
+       case SCST_CMD_STATE_LOCAL_EXEC:
+       case SCST_CMD_STATE_REAL_EXEC:
        case SCST_CMD_STATE_PRE_DEV_DONE:
        case SCST_CMD_STATE_DEV_DONE:
        case SCST_CMD_STATE_PRE_XMIT_RESP:
@@ -876,12 +886,23 @@ static int scst_rdy_to_xfer(struct scst_cmd *cmd)
                goto out_dev_done;
        }
 
-       if (cmd->tgtt->rdy_to_xfer == NULL) {
+       if ((cmd->tgtt->rdy_to_xfer == NULL) || unlikely(cmd->internal)) {
                cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
                res = SCST_CMD_STATE_RES_CONT_SAME;
                goto out;
        }
 
+       if (unlikely(!cmd->tgtt->rdy_to_xfer_atomic && scst_cmd_atomic(cmd))) {
+               /*
+                * It shouldn't be because of SCST_TGT_DEV_AFTER_*
+                * optimization.
+                */
+               TRACE_DBG("Target driver %s rdy_to_xfer() needs thread "
+                             "context, rescheduling", cmd->tgtt->name);
+               res = SCST_CMD_STATE_RES_NEED_THREAD;
+               goto out;
+       }
+
        while (1) {
                int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
 
@@ -1057,9 +1078,9 @@ static int scst_tgt_pre_exec(struct scst_cmd *cmd)
 
        TRACE_ENTRY();
 
-       cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
+       cmd->state = SCST_CMD_STATE_SEND_FOR_EXEC;
 
-       if (cmd->tgtt->pre_exec == NULL)
+       if ((cmd->tgtt->pre_exec == NULL) || unlikely(cmd->internal))
                goto out;
 
        TRACE_DBG("Calling pre_exec(%p)", cmd);
@@ -1136,9 +1157,9 @@ static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
                                        rq_sense, rq_sense_len);
        }
 
-       TRACE(TRACE_SCSI, "result=%x, cmd->status=%x, resid=%d, "
+       TRACE(TRACE_SCSI, "cmd%p, result=%x, cmd->status=%x, resid=%d, "
              "cmd->msg_status=%x, cmd->host_status=%x, "
-             "cmd->driver_status=%x", result, cmd->status, resid,
+             "cmd->driver_status=%x", cmd, result, cmd->status, resid,
              cmd->msg_status, cmd->host_status, cmd->driver_status);
 
        cmd->completed = 1;
@@ -1277,8 +1298,8 @@ static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state)
        if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
            (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
            (next_state != SCST_CMD_STATE_FINISHED)) {
-               PRINT_ERROR("scst_cmd_done_local() received invalid cmd "
-                           "state %d (opcode %d)", next_state, cmd->cdb[0]);
+               PRINT_ERROR("%s() received invalid cmd state %d (opcode %d)",
+                       __func__, next_state, cmd->cdb[0]);
                scst_set_cmd_error(cmd,
                                   SCST_LOAD_SENSE(scst_sense_hardw_error));
                scst_set_cmd_abnormal_done_state(cmd);
@@ -1392,7 +1413,7 @@ out_compl:
 
 out_done:
        /* Report the result */
-       scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
+       cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
 
        TRACE_EXIT();
        return SCST_EXEC_COMPLETED;
@@ -1422,11 +1443,6 @@ static int scst_pre_select(struct scst_cmd *cmd)
                goto out;
        }
 
-       if (cmd->local_exec_done)
-               goto out;
-
-       cmd->local_exec_done = 1;
-
        scst_block_dev_cmd(cmd, 1);
 
        /* Check for local events will be done when cmd will be executed */
@@ -1449,11 +1465,6 @@ static int scst_reserve_local(struct scst_cmd *cmd)
                goto out;
        }
 
-       if (cmd->local_exec_done)
-               goto out;
-
-       cmd->local_exec_done = 1;
-
        if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
                PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
                     "(lun=%lld)", (long long unsigned int)cmd->lun);
@@ -1495,7 +1506,7 @@ out:
 
 out_done:
        /* Report the result */
-       scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
+       cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
        res = SCST_EXEC_COMPLETED;
        goto out;
 }
@@ -1513,11 +1524,6 @@ static int scst_release_local(struct scst_cmd *cmd)
                goto out;
        }
 
-       if (cmd->local_exec_done)
-               goto out;
-
-       cmd->local_exec_done = 1;
-
        dev = cmd->dev;
 
        if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
@@ -1563,7 +1569,7 @@ out:
 out_done:
        res = SCST_EXEC_COMPLETED;
        /* Report the result */
-       scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
+       cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
        goto out;
 }
 
@@ -1649,138 +1655,116 @@ out_uncomplete:
 }
 EXPORT_SYMBOL(scst_check_local_events);
 
-/*
- * The result of cmd execution, if any, should be reported
- * via scst_cmd_done_local()
- */
-static int scst_pre_exec(struct scst_cmd *cmd)
+/* No locks */
+void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
 {
-       int res = SCST_EXEC_NOT_COMPLETED;
-       struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
+       if (slot == NULL)
+               goto inc;
 
-       TRACE_ENTRY();
+       /* Optimized for lockless fast path */
 
-       /*
-        * This function can be called several times for the same cmd, so it
-        * can't change any state in a non-reentrable way or use something
-        * like local_exec_done!!
-        */
+       TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
+               atomic_read(slot));
 
-       /* Check READ_ONLY device status */
-       if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
-           (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
-            cmd->cdb[0] == WRITE_10 ||
-            cmd->cdb[0] == WRITE_12 ||
-            cmd->cdb[0] == WRITE_16 ||
-            cmd->cdb[0] == WRITE_VERIFY ||
-            cmd->cdb[0] == WRITE_VERIFY_12 ||
-            cmd->cdb[0] == WRITE_VERIFY_16 ||
-            (cmd->dev->handler->type == TYPE_TAPE &&
-             (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
-               scst_set_cmd_error(cmd,
-                          SCST_LOAD_SENSE(scst_sense_data_protect));
-               goto out_done;
+       if (!atomic_dec_and_test(slot))
+               goto out;
+
+       TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
+               tgt_dev->num_free_sn_slots);
+       if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
+               spin_lock_irq(&tgt_dev->sn_lock);
+               if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
+                       if (tgt_dev->num_free_sn_slots < 0)
+                               tgt_dev->cur_sn_slot = slot;
+                       smp_mb(); /* to be in-sync with SIMPLE case in scst_cmd_set_sn() */
+                       tgt_dev->num_free_sn_slots++;
+                       TRACE_SN("Incremented num_free_sn_slots (%d)",
+                               tgt_dev->num_free_sn_slots);
+
+               }
+               spin_unlock_irq(&tgt_dev->sn_lock);
        }
 
-out:
-       TRACE_EXIT_RES(res);
-       return res;
+inc:
+       /*
+        * No locks is needed, because only one thread at time can
+        * be here (serialized by sn). Also it is supposed that there
+        * could not be half-incremented halves.
+        */
+       tgt_dev->expected_sn++;
+       smp_mb(); /* write must be before def_cmd_count read */
+       TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
 
-out_done:
-       res = SCST_EXEC_COMPLETED;
-       /* Report the result */
-       scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
-       goto out;
+out:
+       return;
 }
 
-/*
- * The result of cmd execution, if any, should be reported
- * via scst_cmd_done_local()
- */
-static inline int scst_local_exec(struct scst_cmd *cmd)
+/* No locks */
+static struct scst_cmd *scst_post_exec_sn(struct scst_cmd *cmd,
+       bool make_active)
 {
-       int res = SCST_EXEC_NOT_COMPLETED;
+       /* For HQ commands SN is not set */
+       bool inc_expected_sn = !cmd->inc_expected_sn_on_done &&
+                              cmd->sn_set && !cmd->retry;
+       struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
+       struct scst_cmd *res;
 
        TRACE_ENTRY();
 
-       /*
-        * Adding new commands here don't forget to update
-        * scst_is_cmd_local() in scst.h, if necessary
-        */
+       if (inc_expected_sn)
+               scst_inc_expected_sn(tgt_dev, cmd->sn_slot);
 
-       switch (cmd->cdb[0]) {
-       case MODE_SELECT:
-       case MODE_SELECT_10:
-       case LOG_SELECT:
-               res = scst_pre_select(cmd);
-               break;
-       case RESERVE:
-       case RESERVE_10:
-               res = scst_reserve_local(cmd);
-               break;
-       case RELEASE:
-       case RELEASE_10:
-               res = scst_release_local(cmd);
-               break;
-       case REPORT_LUNS:
-               res = scst_report_luns_local(cmd);
-               break;
-       }
+       if (make_active) {
+               scst_make_deferred_commands_active(tgt_dev);
+               res = NULL;
+       } else
+               res = scst_check_deferred_commands(tgt_dev);
 
-       TRACE_EXIT_RES(res);
+       TRACE_EXIT_HRES(res);
        return res;
 }
 
 /* cmd must be additionally referenced to not die inside */
-static int scst_do_send_to_midlev(struct scst_cmd *cmd)
+static int scst_do_real_exec(struct scst_cmd *cmd)
 {
-       int rc = SCST_EXEC_NOT_COMPLETED;
+       int res = SCST_EXEC_NOT_COMPLETED, rc;
        struct scst_device *dev = cmd->dev;
        struct scst_dev_type *handler = dev->handler;
 
        TRACE_ENTRY();
 
-       cmd->sent_to_midlev = 1;
-       cmd->state = SCST_CMD_STATE_EXECUTING;
-       cmd->scst_cmd_done = scst_cmd_done_local;
-
-       rc = scst_pre_exec(cmd);
-       if (rc != SCST_EXEC_NOT_COMPLETED) {
-               if (rc == SCST_EXEC_COMPLETED)
-                       goto out;
-               else if (rc == SCST_EXEC_NEED_THREAD)
-                       goto out_clear;
-               else
-                       goto out_rc_error;
-       }
-
-       rc = scst_local_exec(cmd);
-       if (rc != SCST_EXEC_NOT_COMPLETED) {
-               if (rc == SCST_EXEC_COMPLETED)
-                       goto out;
-               else if (rc == SCST_EXEC_NEED_THREAD)
-                       goto out_clear;
-               else
-                       goto out_rc_error;
-       }
+       cmd->state = SCST_CMD_STATE_REAL_EXECUTING;
 
        if (!handler->exec_sync)
                cmd->context_processable = 0;
 
        if (handler->exec) {
+               if (unlikely(!dev->handler->exec_atomic &&
+                            scst_cmd_atomic(cmd))) {
+                       /*
+                        * It shouldn't be because of SCST_TGT_DEV_AFTER_*
+                        * optimization.
+                        */
+                       TRACE_DBG("Dev handler %s exec() needs thread "
+                               "context, rescheduling", dev->handler->name);
+                       res = SCST_EXEC_NEED_THREAD;
+                       goto out_restore;
+               }
+
                TRACE_DBG("Calling dev handler %s exec(%p)",
                      handler->name, cmd);
-               TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb, cmd->cdb_len);
-               cmd->scst_cmd_done = scst_cmd_done_local;
-               rc = handler->exec(cmd);
+               TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb,
+                       cmd->cdb_len);
+               res = handler->exec(cmd);
                TRACE_DBG("Dev handler %s exec() returned %d",
-                     handler->name, rc);
-               if (rc == SCST_EXEC_COMPLETED)
-                       goto out;
-               else if (rc == SCST_EXEC_NEED_THREAD)
-                       goto out_clear;
-               else if (rc != SCST_EXEC_NOT_COMPLETED)
-                       goto out_rc_error;
+                     handler->name, res);
+
+               if (res == SCST_EXEC_COMPLETED)
+                       goto out_complete;
+               else if (res == SCST_EXEC_NEED_THREAD)
+                       goto out_restore;
+
+               sBUG_ON(res != SCST_EXEC_NOT_COMPLETED);
        }
 
        TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
@@ -1792,25 +1776,25 @@ static int scst_do_send_to_midlev(struct scst_cmd *cmd)
                goto out_error;
        }
 
-       rc = scst_check_local_events(cmd);
-       if (unlikely(rc != 0))
+       res = scst_check_local_events(cmd);
+       if (unlikely(res != 0))
                goto out_done;
 
 #ifndef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
-       if (scst_cmd_atomic(cmd)) {
+       if (unlikely(scst_cmd_atomic(cmd))) {
                TRACE_DBG("Pass-through exec() can not be called in atomic "
                        "context, rescheduling to the thread (handler %s)",
                        handler->name);
-               rc = SCST_EXEC_NEED_THREAD;
-               goto out_clear;
+               res = SCST_EXEC_NEED_THREAD;
+               goto out_restore;
        }
 #endif
 
 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
        if (unlikely(scst_alloc_request(cmd) != 0)) {
                if (scst_cmd_atomic(cmd)) {
-                       rc = SCST_EXEC_NEED_THREAD;
-                       goto out_clear;
+                       res = SCST_EXEC_NEED_THREAD;
+                       goto out_restore;
                } else {
                        PRINT_INFO("%s", "Unable to allocate request, "
                                "sending BUSY status");
@@ -1829,34 +1813,30 @@ static int scst_do_send_to_midlev(struct scst_cmd *cmd)
                        scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
        if (unlikely(rc != 0)) {
                if (scst_cmd_atomic(cmd)) {
-                       rc = SCST_EXEC_NEED_THREAD;
-                       goto out_clear;
+                       res = SCST_EXEC_NEED_THREAD;
+                       goto out_restore;
                } else {
-                       PRINT_INFO("scst_exec_req() failed: %d", rc);
+                       PRINT_ERROR("scst_exec_req() failed: %d", res);
                        goto out_error;
                }
        }
 #endif
 
-       rc = SCST_EXEC_COMPLETED;
+out_complete:
+       res = SCST_EXEC_COMPLETED;
 
 out:
        TRACE_EXIT();
-       return rc;
+       return res;
 
-out_clear:
+out_restore:
        /* Restore the state */
-       cmd->sent_to_midlev = 0;
-       cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
+       cmd->state = SCST_CMD_STATE_REAL_EXEC;
        goto out;
 
-out_rc_error:
-       PRINT_ERROR("Dev handler %s exec() or scst_local_exec() returned "
-                   "invalid code %d", handler->name, rc);
-       /* go through */
-
 out_error:
        scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
+       goto out_done;
 
 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
 out_busy:
@@ -1865,101 +1845,222 @@ out_busy:
 #endif
 
 out_done:
-       rc = SCST_EXEC_COMPLETED;
+       res = SCST_EXEC_COMPLETED;
        /* Report the result */
-       scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
-       goto out;
+       cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
+       goto out_complete;
 }
 
-/* No locks */
-void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
+static inline int scst_real_exec(struct scst_cmd *cmd)
 {
-       if (slot == NULL)
-               goto inc;
+       int res;
 
-       /* Optimized for lockless fast path */
+       TRACE_ENTRY();
 
-       TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
-               atomic_read(slot));
+       BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
+       BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
+       BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
 
-       if (!atomic_dec_and_test(slot))
-               goto out;
+       __scst_cmd_get(cmd);
 
-       TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
-               tgt_dev->num_free_sn_slots);
-       if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
-               spin_lock_irq(&tgt_dev->sn_lock);
-               if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
-                       if (tgt_dev->num_free_sn_slots < 0)
-                               tgt_dev->cur_sn_slot = slot;
-                       smp_mb(); /* to be in-sync with SIMPLE case in scst_cmd_set_sn() */
-                       tgt_dev->num_free_sn_slots++;
-                       TRACE_SN("Incremented num_free_sn_slots (%d)",
-                               tgt_dev->num_free_sn_slots);
+       res = scst_do_real_exec(cmd);
 
-               }
-               spin_unlock_irq(&tgt_dev->sn_lock);
+       if (likely(res == SCST_EXEC_COMPLETED)) {
+               scst_post_exec_sn(cmd, true);
+               if (cmd->dev->scsi_dev != NULL)
+                       generic_unplug_device(cmd->dev->scsi_dev->request_queue);
+       } else
+               sBUG_ON(res != SCST_EXEC_NEED_THREAD);
+
+       __scst_cmd_put(cmd);
+
+       /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
+
+       TRACE_EXIT_RES(res);
+       return res;
+}
+
+static int scst_do_local_exec(struct scst_cmd *cmd)
+{
+       int res;
+       struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
+
+       TRACE_ENTRY();
+
+       /* Check READ_ONLY device status */
+       if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
+           (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
+            cmd->cdb[0] == WRITE_10 ||
+            cmd->cdb[0] == WRITE_12 ||
+            cmd->cdb[0] == WRITE_16 ||
+            cmd->cdb[0] == WRITE_VERIFY ||
+            cmd->cdb[0] == WRITE_VERIFY_12 ||
+            cmd->cdb[0] == WRITE_VERIFY_16 ||
+            (cmd->dev->handler->type == TYPE_TAPE &&
+             (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
+               scst_set_cmd_error(cmd,
+                          SCST_LOAD_SENSE(scst_sense_data_protect));
+               goto out_done;
        }
 
-inc:
        /*
-        * No locks is needed, because only one thread at time can
-        * be here (serialized by sn). Also it is supposed that there
-        * could not be half-incremented halves.
+        * Adding new commands here don't forget to update
+        * scst_is_cmd_local() in scst.h, if necessary
         */
-       tgt_dev->expected_sn++;
-       smp_mb(); /* write must be before def_cmd_count read */
-       TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
+
+       switch (cmd->cdb[0]) {
+       case MODE_SELECT:
+       case MODE_SELECT_10:
+       case LOG_SELECT:
+               res = scst_pre_select(cmd);
+               break;
+       case RESERVE:
+       case RESERVE_10:
+               res = scst_reserve_local(cmd);
+               break;
+       case RELEASE:
+       case RELEASE_10:
+               res = scst_release_local(cmd);
+               break;
+       case REPORT_LUNS:
+               res = scst_report_luns_local(cmd);
+               break;
+       default:
+               res = SCST_EXEC_NOT_COMPLETED;
+               break;
+       }
 
 out:
-       return;
+       TRACE_EXIT_RES(res);
+       return res;
+
+out_done:
+       /* Report the result */
+       cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
+       res = SCST_EXEC_COMPLETED;
+       goto out;
 }
 
-static int scst_process_internal_cmd(struct scst_cmd *cmd)
+static int scst_local_exec(struct scst_cmd *cmd)
 {
-       int res = SCST_CMD_STATE_RES_CONT_NEXT, rc;
+       int res;
 
        TRACE_ENTRY();
 
+       BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
+       BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
+       BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
+
        __scst_cmd_get(cmd);
 
-       rc = scst_do_send_to_midlev(cmd);
-       if (rc == SCST_EXEC_NEED_THREAD) {
-               TRACE_DBG("%s", "scst_do_send_to_midlev() requested "
-                     "thread context, rescheduling");
-               res = SCST_CMD_STATE_RES_NEED_THREAD;
-       } else {
-               struct scst_device *dev = cmd->dev;
-               sBUG_ON(rc != SCST_EXEC_COMPLETED);
-               if (dev->scsi_dev != NULL)
-                       generic_unplug_device(dev->scsi_dev->request_queue);
-       }
+       res = scst_do_local_exec(cmd);
+       if (likely(res == SCST_EXEC_NOT_COMPLETED))
+               cmd->state = SCST_CMD_STATE_REAL_EXEC;
+       else if (res == SCST_EXEC_COMPLETED)
+               scst_post_exec_sn(cmd, true);
+       else
+               sBUG_ON(res != SCST_EXEC_NEED_THREAD);
 
        __scst_cmd_put(cmd);
 
+       /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
        TRACE_EXIT_RES(res);
        return res;
 }
 
-static int scst_send_to_midlev(struct scst_cmd **active_cmd)
+static int scst_exec(struct scst_cmd **active_cmd)
 {
-       int res, rc;
        struct scst_cmd *cmd = *active_cmd;
        struct scst_cmd *ref_cmd;
-       struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
        struct scst_device *dev = cmd->dev;
-       typeof(tgt_dev->expected_sn) expected_sn;
-       int count;
+       int res = SCST_CMD_STATE_RES_CONT_NEXT, count;
 
        TRACE_ENTRY();
 
-       res = SCST_CMD_STATE_RES_CONT_NEXT;
-
-       if (unlikely(cmd->internal || cmd->retry)) {
-               res = scst_process_internal_cmd(cmd);
+       if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
                goto out;
+
+       /* To protect tgt_dev */
+       ref_cmd = cmd;
+       __scst_cmd_get(ref_cmd);
+
+       count = 0;
+       while (1) {
+               int rc;
+
+               cmd->sent_for_exec = 1;
+               cmd->scst_cmd_done = scst_cmd_done_local;
+               cmd->state = SCST_CMD_STATE_LOCAL_EXEC;
+
+               rc = scst_do_local_exec(cmd);
+               if (likely(rc == SCST_EXEC_NOT_COMPLETED))
+                       /* Nothing to do */;
+               else if (rc == SCST_EXEC_NEED_THREAD) {
+                       TRACE_DBG("%s", "scst_do_local_exec() requested "
+                               "thread context, rescheduling");
+                       scst_dec_on_dev_cmd(cmd);
+                       res = SCST_CMD_STATE_RES_NEED_THREAD;
+                       break;
+               } else {
+                       sBUG_ON(rc != SCST_EXEC_COMPLETED);
+                       goto done;
+               }
+
+               cmd->state = SCST_CMD_STATE_REAL_EXEC;
+
+               rc = scst_do_real_exec(cmd);
+               if (likely(rc == SCST_EXEC_COMPLETED))
+                       /* Nothing to do */;
+               else if (rc == SCST_EXEC_NEED_THREAD) {
+                       TRACE_DBG("scst_real_exec() requested thread "
+                               "context, rescheduling (cmd %p)", cmd);
+                       scst_dec_on_dev_cmd(cmd);
+                       res = SCST_CMD_STATE_RES_NEED_THREAD;
+                       break;
+               } else
+                       sBUG();
+
+done:
+               count++;
+
+               cmd = scst_post_exec_sn(cmd, false);
+               if (cmd == NULL)
+                       break;
+
+               if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
+                       break;
+
+               __scst_cmd_put(ref_cmd);
+               ref_cmd = cmd;
+               __scst_cmd_get(ref_cmd);
        }
 
+       *active_cmd = cmd;
+
+       if (count == 0)
+               goto out_put;
+
+       if (dev->scsi_dev != NULL)
+               generic_unplug_device(dev->scsi_dev->request_queue);
+
+out_put:
+       __scst_cmd_put(ref_cmd);
+       /* !! At this point sess, dev and tgt_dev can be already freed !! */
+
+out:
+       TRACE_EXIT_RES(res);
+       return res;
+}
+
+static int scst_send_for_exec(struct scst_cmd **active_cmd)
+{
+       int res;
+       struct scst_cmd *cmd = *active_cmd;
+       struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
+       typeof(tgt_dev->expected_sn) expected_sn;
+
+       TRACE_ENTRY();
+
 #ifdef CONFIG_SCST_MEASURE_LATENCY
        if (cmd->pre_exec_finish == 0) {
                struct timespec ts;
@@ -1971,11 +2072,8 @@ static int scst_send_to_midlev(struct scst_cmd **active_cmd)
        }
 #endif
 
-       if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
-               goto out;
-
-       ref_cmd = cmd;
-       __scst_cmd_get(ref_cmd);
+       if (unlikely(cmd->internal))
+               goto exec;
 
        if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
                goto exec;
@@ -1992,13 +2090,11 @@ static int scst_send_to_midlev(struct scst_cmd **active_cmd)
 
                expected_sn = tgt_dev->expected_sn;
                if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
-                       /* We are under IRQ lock, but dev->dev_lock is BH one */
-                       int cmd_blocking = scst_pre_dec_on_dev_cmd(cmd);
                        if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
                                /* Necessary to allow aborting out of sn cmds */
-                               TRACE_MGMT_DBG("Aborting out of sn cmd %p (tag %llu)",
-                                              cmd,
-                                              (long long unsigned)cmd->tag);
+                               TRACE_MGMT_DBG("Aborting out of sn cmd %p "
+                                       "(tag %llu, sn %lu)", cmd,
+                                       (long long unsigned)cmd->tag, cmd->sn);
                                tgt_dev->def_cmd_count--;
                                scst_set_cmd_abnormal_done_state(cmd);
                                res = SCST_CMD_STATE_RES_CONT_SAME;
@@ -2008,12 +2104,10 @@ static int scst_send_to_midlev(struct scst_cmd **active_cmd)
                                        cmd->sn_set, expected_sn);
                                list_add_tail(&cmd->sn_cmd_list_entry,
                                              &tgt_dev->deferred_cmd_list);
+                               res = SCST_CMD_STATE_RES_CONT_NEXT;
                        }
                        spin_unlock_irq(&tgt_dev->sn_lock);
-
-                       __scst_dec_on_dev_cmd(dev, cmd_blocking);
-
-                       goto out_put;
+                       goto out;
                } else {
                        TRACE_SN("Somebody incremented expected_sn %ld, "
                                "continuing", expected_sn);
@@ -2023,51 +2117,7 @@ static int scst_send_to_midlev(struct scst_cmd **active_cmd)
        }
 
 exec:
-       count = 0;
-       while (1) {
-               atomic_t *slot = cmd->sn_slot;
-               /* For HQ commands SN is not set */
-               int inc_expected_sn = !cmd->inc_expected_sn_on_done &&
-                                     cmd->sn_set;
-
-               rc = scst_do_send_to_midlev(cmd);
-               if (rc == SCST_EXEC_NEED_THREAD) {
-                       TRACE_DBG("%s", "scst_do_send_to_midlev() requested "
-                             "thread context, rescheduling");
-                       res = SCST_CMD_STATE_RES_NEED_THREAD;
-                       scst_dec_on_dev_cmd(cmd);
-                       *active_cmd = cmd;
-                       if (count != 0)
-                               goto out_unplug;
-                       else
-                               goto out_put;
-               }
-               sBUG_ON(rc != SCST_EXEC_COMPLETED);
-
-               count++;
-
-               if (inc_expected_sn)
-                       scst_inc_expected_sn(tgt_dev, slot);
-
-               cmd = scst_check_deferred_commands(tgt_dev);
-               if (cmd == NULL)
-                       break;
-
-               if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
-                       break;
-
-               __scst_cmd_put(ref_cmd);
-               ref_cmd = cmd;
-               __scst_cmd_get(ref_cmd);
-       }
-
-out_unplug:
-       if (dev->scsi_dev != NULL)
-               generic_unplug_device(dev->scsi_dev->request_queue);
-
-out_put:
-       __scst_cmd_put(ref_cmd);
-       /* !! At this point sess, dev and tgt_dev can be already freed !! */
+       res = scst_exec(active_cmd);
 
 out:
        TRACE_EXIT_HRES(res);
@@ -2127,10 +2177,10 @@ static int scst_check_sense(struct scst_cmd *cmd)
                                        cmd->resp_data_len =
                                                cmd->dbl_ua_orig_resp_data_len;
 
+                                       cmd->state = SCST_CMD_STATE_REAL_EXEC;
                                        cmd->retry = 1;
-                                       cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
                                        res = 1;
-                                       goto out_unlock;
+                                       goto out;
                                }
                        }
                        scst_dev_check_set_UA(dev, cmd, cmd->sense,
@@ -2158,10 +2208,6 @@ static int scst_check_sense(struct scst_cmd *cmd)
 out:
        TRACE_EXIT_RES(res);
        return res;
-
-out_unlock:
-       spin_unlock_bh(&dev->dev_lock);
-       goto out;
 }
 
 static int scst_check_auto_sense(struct scst_cmd *cmd)
@@ -2196,38 +2242,28 @@ static int scst_check_auto_sense(struct scst_cmd *cmd)
        return res;
 }
 
-static int scst_done_cmd_check(struct scst_cmd **pcmd, int *pres)
+static int scst_pre_dev_done(struct scst_cmd *cmd)
 {
-       int res = 0, rc;
-       struct scst_cmd *cmd = *pcmd;
+       int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
 
        TRACE_ENTRY();
 
-       if (unlikely(cmd->cdb[0] == REQUEST_SENSE)) {
-               if (cmd->internal) {
-                       cmd = scst_complete_request_sense(cmd);
-                       *pcmd = cmd;
-               }
-       } else if (unlikely(scst_check_auto_sense(cmd))) {
+       if (unlikely(scst_check_auto_sense(cmd))) {
                PRINT_INFO("Command finished with CHECK CONDITION, but "
                            "without sense data (opcode 0x%x), issuing "
                            "REQUEST SENSE", cmd->cdb[0]);
                rc = scst_prepare_request_sense(cmd);
-               if (rc > 0) {
-                       *pres = rc;
-                       res = 1;
-                       goto out;
-               } else {
+               if (rc == 0)
+                       res = SCST_CMD_STATE_RES_CONT_NEXT;
+               else {
                        PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
                                    "returning HARDWARE ERROR");
                        scst_set_cmd_error(cmd,
                                SCST_LOAD_SENSE(scst_sense_hardw_error));
                }
-       } else if (unlikely(scst_check_sense(cmd))) {
-               *pres = SCST_CMD_STATE_RES_CONT_SAME;
-               res = 1;
                goto out;
-       }
+       } else if (unlikely(scst_check_sense(cmd)))
+               goto out;
 
        if (likely(scsi_status_is_good(cmd->status))) {
                unsigned char type = cmd->dev->handler->type;
@@ -2238,6 +2274,7 @@ static int scst_done_cmd_check(struct scst_cmd **pcmd, int *pres)
                     type == TYPE_TAPE)) {
                        int32_t length;
                        uint8_t *address;
+                       bool err = false;
 
                        length = scst_get_buf_first(cmd, &address);
                        if (length < 0) {
@@ -2246,14 +2283,15 @@ static int scst_done_cmd_check(struct scst_cmd **pcmd, int *pres)
                                scst_set_cmd_error(cmd,
                                        SCST_LOAD_SENSE(
                                                scst_sense_hardw_error));
-                               goto out;
-                       } else if (length == 0) {
-                               goto out;
+                               err = true;
                        } else if (length > 2 && cmd->cdb[0] == MODE_SENSE)
                                address[2] |= 0x80;   /* Write Protect*/
                        else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
                                address[3] |= 0x80;   /* Write Protect*/
                        scst_put_buf(cmd, address);
+
+                       if (err)
+                               goto out;
                }
 
                /*
@@ -2265,6 +2303,7 @@ static int scst_done_cmd_check(struct scst_cmd **pcmd, int *pres)
                    (cmd->resp_data_len > SCST_INQ_BYTE3)) {
                        uint8_t *buffer;
                        int buflen;
+                       bool err = false;
 
                        /* ToDo: all pages ?? */
                        buflen = scst_get_buf_first(cmd, &buffer);
@@ -2284,10 +2323,13 @@ static int scst_done_cmd_check(struct scst_cmd **pcmd, int *pres)
                                    "buffer");
                                scst_set_cmd_error(cmd,
                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
+                               err = true;
                        }
-
                        if (buflen > 0)
                                scst_put_buf(cmd, buffer);
+
+                       if (err)
+                               goto out;
                }
 
                if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
@@ -2296,8 +2338,6 @@ static int scst_done_cmd_check(struct scst_cmd **pcmd, int *pres)
                        TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded (LUN %lld)",
                                (long long unsigned int)cmd->lun);
                        cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
-                       *pres = SCST_CMD_STATE_RES_CONT_SAME;
-                       res = 1;
                        goto out;
                }
        } else {
@@ -2336,31 +2376,14 @@ static int scst_done_cmd_check(struct scst_cmd **pcmd, int *pres)
                              "MODE PARAMETERS CHANGED UA (lun %lld)",
                              (long long unsigned int)cmd->lun);
                        cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
-                       *pres = SCST_CMD_STATE_RES_CONT_SAME;
-                       res = 1;
                        goto out;
                }
        }
 
-out:
-       TRACE_EXIT_RES(res);
-       return res;
-}
-
-static int scst_pre_dev_done(struct scst_cmd **pcmd)
-{
-       int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
-
-       TRACE_ENTRY();
-
-       rc = scst_done_cmd_check(pcmd, &res);
-       if (rc)
-               goto out;
-
-       (*pcmd)->state = SCST_CMD_STATE_DEV_DONE;
+       cmd->state = SCST_CMD_STATE_DEV_DONE;
 
 out:
-       TRACE_EXIT_HRES(res);
+       TRACE_EXIT_RES(res);
        return res;
 }
 
@@ -2440,25 +2463,41 @@ static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
        if (likely(cmd->sn_set))
                scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
 
-       scst_make_deferred_commands_active(cmd->tgt_dev, cmd);
+       scst_make_deferred_commands_active(cmd->tgt_dev);
 }
 
-static int scst_dev_done(struct scst_cmd *cmd)
+static int scst_dev_done(struct scst_cmd **pcmd)
 {
        int res = SCST_CMD_STATE_RES_CONT_SAME;
+       struct scst_cmd *cmd = *pcmd;
        int state;
+       struct scst_device *dev = cmd->dev;
 
        TRACE_ENTRY();
 
        state = SCST_CMD_STATE_PRE_XMIT_RESP;
+
        if (likely(!scst_is_cmd_local(cmd)) &&
-           likely(cmd->dev->handler->dev_done != NULL)) {
+           likely(dev->handler->dev_done != NULL)) {
                int rc;
+
+               if (unlikely(!dev->handler->dev_done_atomic &&
+                            scst_cmd_atomic(cmd))) {
+                       /*
+                        * It shouldn't be because of SCST_TGT_DEV_AFTER_*
+                        * optimization.
+                        */
+                       TRACE_DBG("Dev handler %s dev_done() needs thread "
+                             "context, rescheduling", dev->handler->name);
+                       res = SCST_CMD_STATE_RES_NEED_THREAD;
+                       goto out;
+               }
+
                TRACE_DBG("Calling dev handler %s dev_done(%p)",
-                     cmd->dev->handler->name, cmd);
-               rc = cmd->dev->handler->dev_done(cmd);
+                     dev->handler->name, cmd);
+               rc = dev->handler->dev_done(cmd);
                TRACE_DBG("Dev handler %s dev_done() returned %d",
-                     cmd->dev->handler->name, rc);
+                     dev->handler->name, rc);
                if (rc != SCST_CMD_STATE_DEFAULT)
                        state = rc;
        }
@@ -2470,20 +2509,21 @@ static int scst_dev_done(struct scst_cmd *cmd)
        case SCST_CMD_STATE_PREPARE_SPACE:
        case SCST_CMD_STATE_RDY_TO_XFER:
        case SCST_CMD_STATE_TGT_PRE_EXEC:
-       case SCST_CMD_STATE_SEND_TO_MIDLEV:
+       case SCST_CMD_STATE_SEND_FOR_EXEC:
+       case SCST_CMD_STATE_LOCAL_EXEC:
+       case SCST_CMD_STATE_REAL_EXEC:
        case SCST_CMD_STATE_PRE_DEV_DONE:
        case SCST_CMD_STATE_MODE_SELECT_CHECKS:
        case SCST_CMD_STATE_DEV_DONE:
        case SCST_CMD_STATE_XMIT_RESP:
        case SCST_CMD_STATE_FINISHED:
                cmd->state = state;
-               res = SCST_CMD_STATE_RES_CONT_SAME;
                break;
 
        case SCST_CMD_STATE_NEED_THREAD_CTX:
                TRACE_DBG("Dev handler %s dev_done() requested "
                      "thread context, rescheduling",
-                     cmd->dev->handler->name);
+                     dev->handler->name);
                res = SCST_CMD_STATE_RES_NEED_THREAD;
                break;
 
@@ -2491,16 +2531,15 @@ static int scst_dev_done(struct scst_cmd *cmd)
                if (state >= 0) {
                        PRINT_ERROR("Dev handler %s dev_done() returned "
                                "invalid cmd state %d",
-                               cmd->dev->handler->name, state);
+                               dev->handler->name, state);
                } else {
                        PRINT_ERROR("Dev handler %s dev_done() returned "
-                               "error %d", cmd->dev->handler->name,
+                               "error %d", dev->handler->name,
                                state);
                }
                scst_set_cmd_error(cmd,
                           SCST_LOAD_SENSE(scst_sense_hardw_error));
                scst_set_cmd_abnormal_done_state(cmd);
-               res = SCST_CMD_STATE_RES_CONT_SAME;
                break;
        }
 
@@ -2510,9 +2549,13 @@ static int scst_dev_done(struct scst_cmd *cmd)
        if (likely(cmd->dec_on_dev_needed))
                scst_dec_on_dev_cmd(cmd);
 
-       if (cmd->inc_expected_sn_on_done && cmd->sent_to_midlev)
+       if (cmd->inc_expected_sn_on_done && cmd->sent_for_exec)
                scst_inc_check_expected_sn(cmd);
 
+       if (unlikely(cmd->cdb[0] == REQUEST_SENSE) && (cmd->internal))
+               *pcmd = scst_complete_request_sense(cmd);
+
+out:
        TRACE_EXIT_HRES(res);
        return res;
 }
@@ -2546,11 +2589,11 @@ static int scst_pre_xmit_response(struct scst_cmd *cmd)
                if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
                        scst_on_hq_cmd_response(cmd);
 
-               if (unlikely(!cmd->sent_to_midlev)) {
+               if (unlikely(!cmd->sent_for_exec)) {
                        TRACE_SN("cmd %p was not sent to mid-lev (sn %ld, set %d)",
                                cmd, cmd->sn, cmd->sn_set);
                        scst_unblock_deferred(cmd->tgt_dev, cmd);
-                       cmd->sent_to_midlev = 1;
+                       cmd->sent_for_exec = 1;
                }
        }
 
@@ -2619,6 +2662,18 @@ static int scst_xmit_response(struct scst_cmd *cmd)
 
        TRACE_ENTRY();
 
+       if (unlikely(!cmd->tgtt->xmit_response_atomic &&
+                    scst_cmd_atomic(cmd))) {
+               /*
+                * It shouldn't be because of SCST_TGT_DEV_AFTER_*
+                * optimization.
+                */
+               TRACE_DBG("Target driver %s xmit_response() needs thread "
+                             "context, rescheduling", cmd->tgtt->name);
+               res = SCST_CMD_STATE_RES_NEED_THREAD;
+               goto out;
+       }
+
        while (1) {
                int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
 
@@ -3168,7 +3223,7 @@ void scst_process_active_cmd(struct scst_cmd *cmd, int context)
                        res = scst_tgt_pre_exec(cmd);
                        break;
 
-               case SCST_CMD_STATE_SEND_TO_MIDLEV:
+               case SCST_CMD_STATE_SEND_FOR_EXEC:
                        if (tm_dbg_check_cmd(cmd) != 0) {
                                res = SCST_CMD_STATE_RES_CONT_NEXT;
                                TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
@@ -3176,12 +3231,22 @@ void scst_process_active_cmd(struct scst_cmd *cmd, int context)
                                        (long long unsigned int)cmd->tag);
                                break;
                        }
-                       res = scst_send_to_midlev(&cmd);
+                       res = scst_send_for_exec(&cmd);
+                       /* !! At this point cmd, sess & tgt_dev can be already freed !! */
+                       break;
+
+               case SCST_CMD_STATE_LOCAL_EXEC:
+                       res = scst_local_exec(cmd);
+                       /* !! At this point cmd, sess & tgt_dev can be already freed !! */
+                       break;
+
+               case SCST_CMD_STATE_REAL_EXEC:
+                       res = scst_real_exec(cmd);
                        /* !! At this point cmd, sess & tgt_dev can be already freed !! */
                        break;
 
                case SCST_CMD_STATE_PRE_DEV_DONE:
-                       res = scst_pre_dev_done(&cmd);
+                       res = scst_pre_dev_done(cmd);
                        EXTRACHECKS_BUG_ON(res ==
                                SCST_CMD_STATE_RES_NEED_THREAD);
                        break;
@@ -3191,7 +3256,7 @@ void scst_process_active_cmd(struct scst_cmd *cmd, int context)
                        break;
 
                case SCST_CMD_STATE_DEV_DONE:
-                       res = scst_dev_done(cmd);
+                       res = scst_dev_done(&cmd);
                        break;
 
                case SCST_CMD_STATE_PRE_XMIT_RESP:
@@ -3229,7 +3294,9 @@ void scst_process_active_cmd(struct scst_cmd *cmd, int context)
                case SCST_CMD_STATE_PREPARE_SPACE:
                case SCST_CMD_STATE_RDY_TO_XFER:
                case SCST_CMD_STATE_TGT_PRE_EXEC:
-               case SCST_CMD_STATE_SEND_TO_MIDLEV:
+               case SCST_CMD_STATE_SEND_FOR_EXEC:
+               case SCST_CMD_STATE_LOCAL_EXEC:
+               case SCST_CMD_STATE_REAL_EXEC:
                case SCST_CMD_STATE_PRE_DEV_DONE:
                case SCST_CMD_STATE_MODE_SELECT_CHECKS:
                case SCST_CMD_STATE_DEV_DONE:
@@ -3702,10 +3769,10 @@ void scst_abort_cmd(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd,
                 * command actually gets executed *after* new commands sent
                 * after this TM command completed.
                 */
-               TRACE_MGMT_DBG("cmd %p (tag %llu) being executed/xmitted "
-                       "(state %d, op %x, proc time %ld sec., timeout %d "
-                       "sec.), deferring ABORT...", cmd,
-                       (long long unsigned int)cmd->tag, cmd->state,
+               TRACE_MGMT_DBG("cmd %p (tag %llu, sn %lu) being "
+                       "executed/xmitted (state %d, op %x, proc time %ld sec., "
+                       "timeout %d sec.), deferring ABORT...", cmd,
+                       (long long unsigned int)cmd->tag, cmd->sn, cmd->state,
                        cmd->cdb[0], (long)(jiffies - cmd->start_time) / HZ,
                        cmd->timeout / HZ);
 
@@ -3809,8 +3876,8 @@ static void scst_unblock_aborted_cmds(int scst_mutex_held)
                                        sn_cmd_list_entry) {
                                if (__scst_check_unblock_aborted_cmd(cmd,
                                                &cmd->sn_cmd_list_entry)) {
-                                       TRACE_MGMT_DBG("Unblockd aborted SN "
-                                               "cmd %p", cmd);
+                                       TRACE_MGMT_DBG("Unblocked aborted SN "
+                                               "cmd %p (sn %lu)", cmd, cmd->sn);
                                        tgt_dev->def_cmd_count--;
                                }
                        }