scst/src/scst_targ.c

   1 /*
   2  *  scst_targ.c
   3  *
   4  *  Copyright (C) 2004 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
   5  *  Copyright (C) 2004 - 2005 Leonid Stoljar
   6  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
   7  *
   8  *  This program is free software; you can redistribute it and/or
   9  *  modify it under the terms of the GNU General Public License
  10  *  as published by the Free Software Foundation, version 2
  11  *  of the License.
  12  *
  13  *  This program is distributed in the hope that it will be useful,
  14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  *  GNU General Public License for more details.
  17  */
  18
  19 #include <linux/init.h>
  20 #include <linux/kernel.h>
  21 #include <linux/errno.h>
  22 #include <linux/list.h>
  23 #include <linux/spinlock.h>
  24 #include <linux/slab.h>
  25 #include <linux/sched.h>
  26 #include <linux/smp_lock.h>
  27 #include <linux/unistd.h>
  28 #include <linux/string.h>
  29 #include <linux/kthread.h>
  30 #include <linux/delay.h>
  31
  32 #include "scst.h"
  33 #include "scst_priv.h"
  34
  35 static void scst_cmd_set_sn(struct scst_cmd *cmd);
  36 static int __scst_init_cmd(struct scst_cmd *cmd);
  37 static void scst_finish_cmd_mgmt(struct scst_cmd *cmd);
  38 static struct scst_cmd *__scst_find_cmd_by_tag(struct scst_session *sess,
  39         uint64_t tag);
  40 static void scst_proccess_redirect_cmd(struct scst_cmd *cmd,
  41         enum scst_exec_context context, int check_retries);
  42
  43 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
  44 {
  45         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
  46         unsigned long flags;
  47
  48         spin_lock_irqsave(&t->tasklet_lock, flags);
  49         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
  50                 smp_processor_id());
  51         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
  52         spin_unlock_irqrestore(&t->tasklet_lock, flags);
  53
  54         tasklet_schedule(&t->tasklet);
  55 }
  56
  57 /*
  58  * Must not be called in parallel with scst_unregister_session_ex() for the
  59  * same sess
  60  */
  61 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
  62                              const uint8_t *lun, int lun_len,
  63                              const uint8_t *cdb, int cdb_len, int atomic)
  64 {
  65         struct scst_cmd *cmd;
  66
  67         TRACE_ENTRY();
  68
  69 #ifdef CONFIG_SCST_EXTRACHECKS
  70         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
  71                 PRINT_CRIT_ERROR("%s",
  72                         "New cmd while shutting down the session");
  73                 sBUG();
  74         }
  75 #endif
  76
  77         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
  78         if (cmd == NULL)
  79                 goto out;
  80
  81         cmd->sess = sess;
  82         cmd->tgt = sess->tgt;
  83         cmd->tgtt = sess->tgt->tgtt;
  84
  85         /*
  86          * For both wrong lun and CDB defer the error reporting for
  87          * scst_cmd_init_done()
  88          */
  89
  90         cmd->lun = scst_unpack_lun(lun, lun_len);
  91
  92         if (cdb_len <= SCST_MAX_CDB_SIZE) {
  93                 memcpy(cmd->cdb, cdb, cdb_len);
  94                 cmd->cdb_len = cdb_len;
  95         }
  96
  97         TRACE_DBG("cmd %p, sess %p", cmd, sess);
  98         scst_sess_get(sess);
  99
 100 out:
 101         TRACE_EXIT();
 102         return cmd;
 103 }
 104 EXPORT_SYMBOL(scst_rx_cmd);
 105
 106 /*
 107  * No locks, but might be on IRQ. Returns 0 on success, <0 if processing of
 108  * this command should be stopped.
 109  */
 110 static int scst_init_cmd(struct scst_cmd *cmd, enum scst_exec_context *context)
 111 {
 112         int rc, res = 0;
 113
 114         TRACE_ENTRY();
 115
 116         /* See the comment in scst_do_job_init() */
 117         if (unlikely(!list_empty(&scst_init_cmd_list))) {
 118                 TRACE_MGMT_DBG("%s", "init cmd list busy");
 119                 goto out_redirect;
 120         }
 121         /*
 122          * Memory barrier isn't necessary here, because CPU appears to
 123          * be self-consistent and we don't care about the race, described
 124          * in comment in scst_do_job_init().
 125          */
 126
 127         rc = __scst_init_cmd(cmd);
 128         if (unlikely(rc > 0))
 129                 goto out_redirect;
 130         else if (unlikely(rc != 0)) {
 131                 res = 1;
 132                 goto out;
 133         }
 134
 135         /* Small context optimization */
 136         if (((*context == SCST_CONTEXT_TASKLET) ||
 137              (*context == SCST_CONTEXT_DIRECT_ATOMIC) ||
 138              ((*context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd))) &&
 139               scst_cmd_is_expected_set(cmd)) {
 140                 if (cmd->expected_data_direction & SCST_DATA_WRITE) {
 141                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
 142                                         &cmd->tgt_dev->tgt_dev_flags))
 143                                 *context = SCST_CONTEXT_THREAD;
 144                 } else {
 145                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
 146                                         &cmd->tgt_dev->tgt_dev_flags))
 147                                 *context = SCST_CONTEXT_THREAD;
 148                 }
 149         }
 150
 151 out:
 152         TRACE_EXIT_RES(res);
 153         return res;
 154
 155 out_redirect:
 156         if (cmd->preprocessing_only) {
 157                 /*
 158                  * Poor man solution for single threaded targets, where
 159                  * blocking receiver at least sometimes means blocking all.
 160                  */
 161                 sBUG_ON(*context != SCST_CONTEXT_DIRECT);
 162                 scst_set_busy(cmd);
 163                 scst_set_cmd_abnormal_done_state(cmd);
 164                 res = 1;
 165                 /* Keep initiator away from too many BUSY commands */
 166                 msleep(50);
 167         } else {
 168                 unsigned long flags;
 169                 spin_lock_irqsave(&scst_init_lock, flags);
 170                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
 171                         "%d)", cmd, atomic_read(&scst_cmd_count));
 172                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
 173                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
 174                         scst_init_poll_cnt++;
 175                 spin_unlock_irqrestore(&scst_init_lock, flags);
 176                 wake_up(&scst_init_cmd_list_waitQ);
 177                 res = -1;
 178         }
 179         goto out;
 180 }
 181
 182 #ifdef CONFIG_SCST_MEASURE_LATENCY
 183 static inline uint64_t scst_sec_to_nsec(time_t sec)
 184 {
 185         return (uint64_t)sec * 1000000000;
 186 }
 187 #endif
 188
 189 void scst_cmd_init_done(struct scst_cmd *cmd,
 190         enum scst_exec_context pref_context)
 191 {
 192         unsigned long flags;
 193         struct scst_session *sess = cmd->sess;
 194         int rc;
 195
 196         TRACE_ENTRY();
 197
 198 #ifdef CONFIG_SCST_MEASURE_LATENCY
 199         {
 200                 struct timespec ts;
 201                 getnstimeofday(&ts);
 202                 cmd->start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
 203                 TRACE_DBG("cmd %p (sess %p): start %lld (tv_sec %ld, "
 204                         "tv_nsec %ld)", cmd, sess, cmd->start, ts.tv_sec,
 205                         ts.tv_nsec);
 206         }
 207 #endif
 208
 209         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
 210         TRACE(TRACE_SCSI, "tag=%llu, lun=%lld, CDB len=%d, queue_type=%x "
 211                 "(cmd %p)", (long long unsigned int)cmd->tag,
 212                 (long long unsigned int)cmd->lun, cmd->cdb_len,
 213                 cmd->queue_type, cmd);
 214         PRINT_BUFF_FLAG(TRACE_SCSI|TRACE_RCV_BOT, "Recieving CDB",
 215                 cmd->cdb, cmd->cdb_len);
 216
 217 #ifdef CONFIG_SCST_EXTRACHECKS
 218         if (unlikely((in_irq() || irqs_disabled())) &&
 219             ((pref_context == SCST_CONTEXT_DIRECT) ||
 220              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
 221                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
 222                         "SCST_CONTEXT_THREAD instead\n", pref_context,
 223                         cmd->tgtt->name);
 224                 pref_context = SCST_CONTEXT_THREAD;
 225         }
 226 #endif
 227
 228         atomic_inc(&sess->sess_cmd_count);
 229
 230         spin_lock_irqsave(&sess->sess_list_lock, flags);
 231
 232         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
 233                 /*
 234                  * We have to always keep command in the search list from the
 235                  * very beginning, because otherwise it can be missed during
 236                  * TM processing. This check is needed because there might be
 237                  * old, i.e. deferred, commands and new, i.e. just coming, ones.
 238                  */
 239                 if (cmd->search_cmd_list_entry.next == NULL)
 240                         list_add_tail(&cmd->search_cmd_list_entry,
 241                                 &sess->search_cmd_list);
 242                 switch (sess->init_phase) {
 243                 case SCST_SESS_IPH_SUCCESS:
 244                         break;
 245                 case SCST_SESS_IPH_INITING:
 246                         TRACE_DBG("Adding cmd %p to init deferred cmd list",
 247                                   cmd);
 248                         list_add_tail(&cmd->cmd_list_entry,
 249                                 &sess->init_deferred_cmd_list);
 250                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
 251                         goto out;
 252                 case SCST_SESS_IPH_FAILED:
 253                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
 254                         scst_set_busy(cmd);
 255                         scst_set_cmd_abnormal_done_state(cmd);
 256                         goto active;
 257                 default:
 258                         sBUG();
 259                 }
 260         } else
 261                 list_add_tail(&cmd->search_cmd_list_entry,
 262                               &sess->search_cmd_list);
 263
 264         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
 265
 266         if (unlikely(cmd->lun == NO_SUCH_LUN)) {
 267                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
 268                 scst_set_cmd_error(cmd,
 269                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
 270                 scst_set_cmd_abnormal_done_state(cmd);
 271                 goto active;
 272         }
 273
 274         if (unlikely(cmd->cdb_len == 0)) {
 275                 PRINT_ERROR("%s", "Wrong CDB len, finishing cmd");
 276                 scst_set_cmd_error(cmd,
 277                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
 278                 scst_set_cmd_abnormal_done_state(cmd);
 279                 goto active;
 280         }
 281
 282         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
 283                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
 284                 scst_set_cmd_error(cmd,
 285                         SCST_LOAD_SENSE(scst_sense_invalid_message));
 286                 scst_set_cmd_abnormal_done_state(cmd);
 287                 goto active;
 288         }
 289
 290         /*
 291          * Cmd must be inited here to preserve the order. In case if cmd
 292          * already preliminary completed by target driver we need to init
 293          * cmd anyway to find out in which format we should return sense.
 294          */
 295         cmd->state = SCST_CMD_STATE_INIT;
 296         rc = scst_init_cmd(cmd, &pref_context);
 297         if (unlikely(rc < 0))
 298                 goto out;
 299         else if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION)) {
 300                 if (rc == 0) {
 301                         /* Target driver preliminary completed cmd */
 302                         scst_set_cmd_abnormal_done_state(cmd);
 303                 }
 304         }
 305
 306 active:
 307         /* Here cmd must not be in any cmd list, no locks */
 308         switch (pref_context) {
 309         case SCST_CONTEXT_TASKLET:
 310                 scst_schedule_tasklet(cmd);
 311                 break;
 312
 313         case SCST_CONTEXT_DIRECT:
 314                 scst_process_active_cmd(cmd, false);
 315                 /* For *NEED_THREAD wake_up() is already done */
 316                 break;
 317
 318         case SCST_CONTEXT_DIRECT_ATOMIC:
 319                 scst_process_active_cmd(cmd, true);
 320                 /* For *NEED_THREAD wake_up() is already done */
 321                 break;
 322
 323         default:
 324                 PRINT_ERROR("Context %x is undefined, using the thread one",
 325                         pref_context);
 326                 /* go through */
 327         case SCST_CONTEXT_THREAD:
 328                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
 329                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
 330                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
 331                         list_add(&cmd->cmd_list_entry,
 332                                 &cmd->cmd_lists->active_cmd_list);
 333                 else
 334                         list_add_tail(&cmd->cmd_list_entry,
 335                                 &cmd->cmd_lists->active_cmd_list);
 336                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
 337                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
 338                 break;
 339         }
 340
 341 out:
 342         TRACE_EXIT();
 343         return;
 344 }
 345 EXPORT_SYMBOL(scst_cmd_init_done);
 346
 347 static int scst_pre_parse(struct scst_cmd *cmd)
 348 {
 349         int res = SCST_CMD_STATE_RES_CONT_SAME;
 350         struct scst_device *dev = cmd->dev;
 351         int rc;
 352
 353         TRACE_ENTRY();
 354
 355         cmd->inc_expected_sn_on_done = dev->handler->exec_sync ||
 356              (!dev->has_own_order_mgmt &&
 357               (dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER ||
 358                cmd->queue_type == SCST_CMD_QUEUE_ORDERED));
 359
 360         /*
 361          * Expected transfer data supplied by the SCSI transport via the
 362          * target driver are untrusted, so we prefer to fetch them from CDB.
 363          * Additionally, not all transports support supplying the expected
 364          * transfer data.
 365          */
 366
 367         rc = scst_get_cdb_info(cmd);
 368         if (unlikely(rc != 0)) {
 369                 if (rc > 0) {
 370                         PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 371                         goto out_xmit;
 372                 }
 373                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
 374                         "Should you update scst_scsi_op_table?",
 375                         cmd->cdb[0], dev->handler->name);
 376                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 377 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
 378                 if (scst_cmd_is_expected_set(cmd)) {
 379                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
 380                                 "direction %d, transfer_len %d",
 381                                 cmd->expected_data_direction,
 382                                 cmd->expected_transfer_len);
 383                         cmd->data_direction = cmd->expected_data_direction;
 384
 385                         cmd->bufflen = cmd->expected_transfer_len;
 386                         /* Restore (possibly) lost CDB length */
 387                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
 388                         if (cmd->cdb_len == -1) {
 389                                 PRINT_ERROR("Unable to get CDB length for "
 390                                         "opcode 0x%02x. Returning INVALID "
 391                                         "OPCODE", cmd->cdb[0]);
 392                                 scst_set_cmd_error(cmd,
 393                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
 394                                 goto out_xmit;
 395                         }
 396                 } else {
 397                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
 398                              "target %s not supplied expected values",
 399                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
 400                         scst_set_cmd_error(cmd,
 401                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
 402                         goto out_xmit;
 403                 }
 404 #else
 405                 scst_set_cmd_error(cmd,
 406                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
 407                 goto out_xmit;
 408 #endif
 409         } else {
 410                 TRACE(TRACE_SCSI, "op_name <%s> (cmd %p), direction=%d "
 411                         "(expected %d, set %s), transfer_len=%d (expected "
 412                         "len %d), flags=%d", cmd->op_name, cmd,
 413                         cmd->data_direction, cmd->expected_data_direction,
 414                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
 415                         cmd->bufflen, cmd->expected_transfer_len,
 416                         cmd->op_flags);
 417
 418                 if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
 419                         if (scst_cmd_is_expected_set(cmd)) {
 420                                 /*
 421                                  * Command data length can't be easily
 422                                  * determined from the CDB. ToDo, all such
 423                                  * commands processing should be fixed. Until
 424                                  * it's done, get the length from the supplied
 425                                  * expected value, but limit it to some
 426                                  * reasonable value (15MB).
 427                                  */
 428                                 cmd->bufflen = min(cmd->expected_transfer_len,
 429                                                         15*1024*1024);
 430                                 cmd->op_flags &= ~SCST_UNKNOWN_LENGTH;
 431                         } else
 432                                 cmd->bufflen = 0;
 433                 }
 434         }
 435
 436         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
 437                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
 438                             "(opcode 0x%02x)", cmd->cdb[0]);
 439                 scst_set_cmd_error(cmd,
 440                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
 441                 goto out_xmit;
 442         }
 443
 444         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
 445                 PRINT_ERROR("Linked commands are not supported "
 446                             "(opcode 0x%02x)", cmd->cdb[0]);
 447                 scst_set_cmd_error(cmd,
 448                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
 449                 goto out_xmit;
 450         }
 451
 452         cmd->state = SCST_CMD_STATE_DEV_PARSE;
 453
 454 out:
 455         TRACE_EXIT_RES(res);
 456         return res;
 457
 458 out_xmit:
 459         scst_set_cmd_abnormal_done_state(cmd);
 460         res = SCST_CMD_STATE_RES_CONT_SAME;
 461         goto out;
 462 }
 463
 464 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
 465 static bool scst_is_allowed_to_mismatch_cmd(struct scst_cmd *cmd)
 466 {
 467         bool res = false;
 468
 469         switch (cmd->cdb[0]) {
 470         case TEST_UNIT_READY:
 471                 /* Crazy VMware people sometimes do TUR with READ direction */
 472                 res = true;
 473                 break;
 474         case VERIFY:
 475         case VERIFY_6:
 476         case VERIFY_12:
 477         case VERIFY_16:
 478                 /* VERIFY commands with BYTCHK unset shouldn't fail here */
 479                 if ((cmd->op_flags & SCST_VERIFY_BYTCHK_MISMATCH_ALLOWED) &&
 480                     (cmd->cdb[1] & BYTCHK) == 0)
 481                         res = true;
 482                 break;
 483         }
 484
 485         return res;
 486 }
 487 #endif
 488
 489 static int scst_parse_cmd(struct scst_cmd *cmd)
 490 {
 491         int res = SCST_CMD_STATE_RES_CONT_SAME;
 492         int state;
 493         struct scst_device *dev = cmd->dev;
 494         int orig_bufflen = cmd->bufflen;
 495
 496         TRACE_ENTRY();
 497
 498         if (likely(!scst_is_cmd_local(cmd))) {
 499                 if (unlikely(!dev->handler->parse_atomic &&
 500                              scst_cmd_atomic(cmd))) {
 501                         /*
 502                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
 503                          * optimization.
 504                          */
 505                         TRACE_DBG("Dev handler %s parse() needs thread "
 506                                 "context, rescheduling", dev->handler->name);
 507                         res = SCST_CMD_STATE_RES_NEED_THREAD;
 508                         goto out;
 509                 }
 510
 511                 TRACE_DBG("Calling dev handler %s parse(%p)",
 512                       dev->handler->name, cmd);
 513                 TRACE_BUFF_FLAG(TRACE_SND_BOT, "Parsing: ",
 514                                 cmd->cdb, cmd->cdb_len);
 515                 state = dev->handler->parse(cmd);
 516                 /* Caution: cmd can be already dead here */
 517                 TRACE_DBG("Dev handler %s parse() returned %d",
 518                         dev->handler->name, state);
 519
 520                 switch (state) {
 521                 case SCST_CMD_STATE_NEED_THREAD_CTX:
 522                         TRACE_DBG("Dev handler %s parse() requested thread "
 523                               "context, rescheduling", dev->handler->name);
 524                         res = SCST_CMD_STATE_RES_NEED_THREAD;
 525                         goto out;
 526
 527                 case SCST_CMD_STATE_STOP:
 528                         TRACE_DBG("Dev handler %s parse() requested stop "
 529                                 "processing", dev->handler->name);
 530                         res = SCST_CMD_STATE_RES_CONT_NEXT;
 531                         goto out;
 532                 }
 533
 534                 if (state == SCST_CMD_STATE_DEFAULT)
 535                         state = SCST_CMD_STATE_PREPARE_SPACE;
 536         } else
 537                 state = SCST_CMD_STATE_PREPARE_SPACE;
 538
 539         if (cmd->data_len == -1)
 540                 cmd->data_len = cmd->bufflen;
 541
 542         if (cmd->bufflen == 0) {
 543                 /*
 544                  * According to SPC bufflen 0 for data transfer commands isn't
 545                  * an error, so we need to fix the transfer direction.
 546                  */
 547                 cmd->data_direction = SCST_DATA_NONE;
 548         }
 549
 550         if (cmd->dh_data_buf_alloced &&
 551             unlikely((orig_bufflen > cmd->bufflen))) {
 552                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
 553                         "is less, than required (size %d)", cmd->bufflen,
 554                         orig_bufflen);
 555                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 556                 goto out_error;
 557         }
 558
 559         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
 560                 goto set_res;
 561
 562         if (unlikely((cmd->bufflen == 0) &&
 563                      (cmd->op_flags & SCST_UNKNOWN_LENGTH))) {
 564                 PRINT_ERROR("Unknown data transfer length for opcode 0x%x "
 565                         "(handler %s, target %s)", cmd->cdb[0],
 566                         dev->handler->name, cmd->tgtt->name);
 567                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 568                 goto out_error;
 569         }
 570
 571 #ifdef CONFIG_SCST_EXTRACHECKS
 572         if ((cmd->bufflen != 0) &&
 573             ((cmd->data_direction == SCST_DATA_NONE) ||
 574              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
 575                 PRINT_ERROR("Dev handler %s parse() returned "
 576                         "invalid cmd data_direction %d, bufflen %d, state %d "
 577                         "or sg %p (opcode 0x%x)", dev->handler->name,
 578                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
 579                         cmd->cdb[0]);
 580                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 581                 goto out_error;
 582         }
 583 #endif
 584
 585         if (scst_cmd_is_expected_set(cmd)) {
 586 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
 587 #       ifdef CONFIG_SCST_EXTRACHECKS
 588                 if ((cmd->data_direction != cmd->expected_data_direction) ||
 589                     (cmd->bufflen != cmd->expected_transfer_len)) {
 590                         PRINT_WARNING("Expected values don't match decoded "
 591                                 "ones: data_direction %d, "
 592                                 "expected_data_direction %d, "
 593                                 "bufflen %d, expected_transfer_len %d",
 594                                 cmd->data_direction,
 595                                 cmd->expected_data_direction,
 596                                 cmd->bufflen, cmd->expected_transfer_len);
 597                         PRINT_BUFFER("Suspicious CDB", cmd->cdb, cmd->cdb_len);
 598                 }
 599 #       endif
 600                 cmd->data_direction = cmd->expected_data_direction;
 601                 cmd->bufflen = cmd->expected_transfer_len;
 602 #else
 603                 if (unlikely(cmd->data_direction !=
 604                                 cmd->expected_data_direction)) {
 605                         if (((cmd->expected_data_direction != SCST_DATA_NONE) ||
 606                              (cmd->bufflen != 0)) &&
 607                             !scst_is_allowed_to_mismatch_cmd(cmd)) {
 608                                 PRINT_ERROR("Expected data direction %d for "
 609                                         "opcode 0x%02x (handler %s, target %s) "
 610                                         "doesn't match "
 611                                         "decoded value %d",
 612                                         cmd->expected_data_direction,
 613                                         cmd->cdb[0], dev->handler->name,
 614                                         cmd->tgtt->name, cmd->data_direction);
 615                                 PRINT_BUFFER("Failed CDB",
 616                                         cmd->cdb, cmd->cdb_len);
 617                                 scst_set_cmd_error(cmd,
 618                                    SCST_LOAD_SENSE(scst_sense_invalid_message));
 619                                 goto out_dev_done;
 620                         }
 621                 }
 622                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
 623                         TRACE(TRACE_MGMT_MINOR, "Warning: expected "
 624                                 "transfer length %d for opcode 0x%02x "
 625                                 "(handler %s, target %s) doesn't match "
 626                                 "decoded value %d. Faulty initiator "
 627                                 "(e.g. VMware is known to be such) or "
 628                                 "scst_scsi_op_table should be updated?",
 629                                 cmd->expected_transfer_len, cmd->cdb[0],
 630                                 dev->handler->name, cmd->tgtt->name,
 631                                 cmd->bufflen);
 632                         PRINT_BUFF_FLAG(TRACE_MGMT_MINOR, "Suspicious CDB",
 633                                 cmd->cdb, cmd->cdb_len);
 634                 }
 635 #endif
 636         }
 637
 638         if (unlikely(cmd->data_direction == SCST_DATA_UNKNOWN)) {
 639                 PRINT_ERROR("Unknown data direction. Opcode 0x%x, handler %s, "
 640                         "target %s", cmd->cdb[0], dev->handler->name,
 641                         cmd->tgtt->name);
 642                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 643                 goto out_error;
 644         }
 645
 646 set_res:
 647         switch (state) {
 648         case SCST_CMD_STATE_PREPARE_SPACE:
 649         case SCST_CMD_STATE_PRE_PARSE:
 650         case SCST_CMD_STATE_DEV_PARSE:
 651         case SCST_CMD_STATE_RDY_TO_XFER:
 652         case SCST_CMD_STATE_TGT_PRE_EXEC:
 653         case SCST_CMD_STATE_SEND_FOR_EXEC:
 654         case SCST_CMD_STATE_LOCAL_EXEC:
 655         case SCST_CMD_STATE_REAL_EXEC:
 656         case SCST_CMD_STATE_PRE_DEV_DONE:
 657         case SCST_CMD_STATE_DEV_DONE:
 658         case SCST_CMD_STATE_PRE_XMIT_RESP:
 659         case SCST_CMD_STATE_XMIT_RESP:
 660         case SCST_CMD_STATE_FINISHED:
 661         case SCST_CMD_STATE_FINISHED_INTERNAL:
 662                 cmd->state = state;
 663                 res = SCST_CMD_STATE_RES_CONT_SAME;
 664                 break;
 665
 666         default:
 667                 if (state >= 0) {
 668                         PRINT_ERROR("Dev handler %s parse() returned "
 669                              "invalid cmd state %d (opcode %d)",
 670                              dev->handler->name, state, cmd->cdb[0]);
 671                 } else {
 672                         PRINT_ERROR("Dev handler %s parse() returned "
 673                                 "error %d (opcode %d)", dev->handler->name,
 674                                 state, cmd->cdb[0]);
 675                 }
 676                 goto out_error;
 677         }
 678
 679         if (cmd->resp_data_len == -1) {
 680                 if (cmd->data_direction & SCST_DATA_READ)
 681                         cmd->resp_data_len = cmd->bufflen;
 682                 else
 683                          cmd->resp_data_len = 0;
 684         }
 685
 686 out:
 687         TRACE_EXIT_HRES(res);
 688         return res;
 689
 690 out_error:
 691         /* dev_done() will be called as part of the regular cmd's finish */
 692         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
 693
 694 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
 695 out_dev_done:
 696 #endif
 697         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
 698         res = SCST_CMD_STATE_RES_CONT_SAME;
 699         goto out;
 700 }
 701
 702 static int scst_prepare_space(struct scst_cmd *cmd)
 703 {
 704         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
 705
 706         TRACE_ENTRY();
 707
 708         if (cmd->data_direction == SCST_DATA_NONE)
 709                 goto prep_done;
 710
 711         if (cmd->tgt_need_alloc_data_buf) {
 712                 int orig_bufflen = cmd->bufflen;
 713
 714                 TRACE_MEM("Custom tgt data buf allocation requested (cmd %p)",
 715                         cmd);
 716
 717                 r = cmd->tgtt->alloc_data_buf(cmd);
 718                 if (r > 0)
 719                         goto alloc;
 720                 else if (r == 0) {
 721                         if (unlikely(cmd->bufflen == 0)) {
 722                                 /* See comment in scst_alloc_space() */
 723                                 if (cmd->sg == NULL)
 724                                         goto alloc;
 725                         }
 726
 727                         cmd->tgt_data_buf_alloced = 1;
 728
 729                         if (unlikely(orig_bufflen < cmd->bufflen)) {
 730                                 PRINT_ERROR("Target driver allocated data "
 731                                         "buffer (size %d), is less, than "
 732                                         "required (size %d)", orig_bufflen,
 733                                         cmd->bufflen);
 734                                 goto out_error;
 735                         }
 736                         TRACE_MEM("tgt_data_buf_alloced (cmd %p)", cmd);
 737                 } else
 738                         goto check;
 739         }
 740
 741 alloc:
 742         if (!cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
 743                 r = scst_alloc_space(cmd);
 744         } else if (cmd->dh_data_buf_alloced && !cmd->tgt_data_buf_alloced) {
 745                 TRACE_MEM("dh_data_buf_alloced set (cmd %p)", cmd);
 746                 r = 0;
 747         } else if (cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
 748                 TRACE_MEM("tgt_data_buf_alloced set (cmd %p)", cmd);
 749                 cmd->sg = cmd->tgt_sg;
 750                 cmd->sg_cnt = cmd->tgt_sg_cnt;
 751                 cmd->in_sg = cmd->tgt_in_sg;
 752                 cmd->in_sg_cnt = cmd->tgt_in_sg_cnt;
 753                 r = 0;
 754         } else {
 755                 TRACE_MEM("Both *_data_buf_alloced set (cmd %p, sg %p, "
 756                         "sg_cnt %d, tgt_sg %p, tgt_sg_cnt %d)", cmd, cmd->sg,
 757                         cmd->sg_cnt, cmd->tgt_sg, cmd->tgt_sg_cnt);
 758                 r = 0;
 759         }
 760
 761 check:
 762         if (r != 0) {
 763                 if (scst_cmd_atomic(cmd)) {
 764                         TRACE_MEM("%s", "Atomic memory allocation failed, "
 765                               "rescheduling to the thread");
 766                         res = SCST_CMD_STATE_RES_NEED_THREAD;
 767                         goto out;
 768                 } else
 769                         goto out_no_space;
 770         }
 771
 772 prep_done:
 773         if (cmd->preprocessing_only) {
 774                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
 775                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
 776                                 "cmd %p", cmd);
 777                         scst_set_cmd_abnormal_done_state(cmd);
 778                         res = SCST_CMD_STATE_RES_CONT_SAME;
 779                         goto out;
 780                 }
 781
 782                 res = SCST_CMD_STATE_RES_CONT_NEXT;
 783                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
 784
 785                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
 786                 cmd->tgtt->preprocessing_done(cmd);
 787                 TRACE_DBG("%s", "preprocessing_done() returned");
 788                 goto out;
 789
 790         }
 791
 792         if (cmd->data_direction & SCST_DATA_WRITE)
 793                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
 794         else
 795                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
 796
 797 out:
 798         TRACE_EXIT_HRES(res);
 799         return res;
 800
 801 out_no_space:
 802         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
 803                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
 804         scst_set_busy(cmd);
 805         scst_set_cmd_abnormal_done_state(cmd);
 806         res = SCST_CMD_STATE_RES_CONT_SAME;
 807         goto out;
 808
 809 out_error:
 810         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
 811         scst_set_cmd_abnormal_done_state(cmd);
 812         res = SCST_CMD_STATE_RES_CONT_SAME;
 813         goto out;
 814 }
 815
 816 void scst_restart_cmd(struct scst_cmd *cmd, int status,
 817         enum scst_exec_context pref_context)
 818 {
 819         TRACE_ENTRY();
 820
 821         TRACE_DBG("Preferred context: %d", pref_context);
 822         TRACE_DBG("tag=%llu, status=%#x",
 823                   (long long unsigned int)scst_cmd_get_tag(cmd),
 824                   status);
 825
 826 #ifdef CONFIG_SCST_EXTRACHECKS
 827         if ((in_irq() || irqs_disabled()) &&
 828             ((pref_context == SCST_CONTEXT_DIRECT) ||
 829              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
 830                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
 831                         "SCST_CONTEXT_THREAD instead\n", pref_context,
 832                         cmd->tgtt->name);
 833                 pref_context = SCST_CONTEXT_THREAD;
 834         }
 835 #endif
 836
 837         switch (status) {
 838         case SCST_PREPROCESS_STATUS_SUCCESS:
 839                 if (cmd->data_direction & SCST_DATA_WRITE)
 840                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
 841                 else
 842                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
 843                 if (cmd->set_sn_on_restart_cmd)
 844                         scst_cmd_set_sn(cmd);
 845                 /* Small context optimization */
 846                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
 847                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
 848                     ((pref_context == SCST_CONTEXT_SAME) &&
 849                      scst_cmd_atomic(cmd))) {
 850                         if (cmd->data_direction & SCST_DATA_WRITE) {
 851                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
 852                                                 &cmd->tgt_dev->tgt_dev_flags))
 853                                         pref_context = SCST_CONTEXT_THREAD;
 854                         } else {
 855                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
 856                                                 &cmd->tgt_dev->tgt_dev_flags))
 857                                         pref_context = SCST_CONTEXT_THREAD;
 858                         }
 859                 }
 860                 break;
 861
 862         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
 863                 scst_set_cmd_abnormal_done_state(cmd);
 864                 break;
 865
 866         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
 867                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
 868                 /* go through */
 869         case SCST_PREPROCESS_STATUS_ERROR:
 870                 if (cmd->sense != NULL)
 871                         scst_set_cmd_error(cmd,
 872                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
 873                 scst_set_cmd_abnormal_done_state(cmd);
 874                 break;
 875
 876         default:
 877                 PRINT_ERROR("%s() received unknown status %x", __func__,
 878                         status);
 879                 scst_set_cmd_abnormal_done_state(cmd);
 880                 break;
 881         }
 882
 883         scst_proccess_redirect_cmd(cmd, pref_context, 1);
 884
 885         TRACE_EXIT();
 886         return;
 887 }
 888 EXPORT_SYMBOL(scst_restart_cmd);
 889
 890 /* No locks */
 891 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
 892 {
 893         struct scst_tgt *tgt = cmd->sess->tgt;
 894         int res = 0;
 895         unsigned long flags;
 896
 897         TRACE_ENTRY();
 898
 899         spin_lock_irqsave(&tgt->tgt_lock, flags);
 900         tgt->retry_cmds++;
 901         /*
 902          * Memory barrier is needed here, because we need the exact order
 903          * between the read and write between retry_cmds and finished_cmds to
 904          * not miss the case when a command finished while we queuing it for
 905          * retry after the finished_cmds check.
 906          */
 907         smp_mb();
 908         TRACE_RETRY("TGT QUEUE FULL: incrementing retry_cmds %d",
 909               tgt->retry_cmds);
 910         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
 911                 /* At least one cmd finished, so try again */
 912                 tgt->retry_cmds--;
 913                 TRACE_RETRY("Some command(s) finished, direct retry "
 914                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
 915                       "retry_cmds=%d)", finished_cmds,
 916                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
 917                 res = -1;
 918                 goto out_unlock_tgt;
 919         }
 920
 921         TRACE_RETRY("Adding cmd %p to retry cmd list", cmd);
 922         list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
 923
 924         if (!tgt->retry_timer_active) {
 925                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
 926                 add_timer(&tgt->retry_timer);
 927                 tgt->retry_timer_active = 1;
 928         }
 929
 930 out_unlock_tgt:
 931         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
 932
 933         TRACE_EXIT_RES(res);
 934         return res;
 935 }
 936
 937 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
 938 {
 939         int res, rc;
 940
 941         TRACE_ENTRY();
 942
 943         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
 944                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
 945                 goto out_dev_done;
 946         }
 947
 948         if ((cmd->tgtt->rdy_to_xfer == NULL) || unlikely(cmd->internal)) {
 949                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
 950                 res = SCST_CMD_STATE_RES_CONT_SAME;
 951                 goto out;
 952         }
 953
 954         if (unlikely(!cmd->tgtt->rdy_to_xfer_atomic && scst_cmd_atomic(cmd))) {
 955                 /*
 956                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
 957                  * optimization.
 958                  */
 959                 TRACE_DBG("Target driver %s rdy_to_xfer() needs thread "
 960                               "context, rescheduling", cmd->tgtt->name);
 961                 res = SCST_CMD_STATE_RES_NEED_THREAD;
 962                 goto out;
 963         }
 964
 965         while (1) {
 966                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
 967
 968                 res = SCST_CMD_STATE_RES_CONT_NEXT;
 969                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
 970
 971                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
 972 #ifdef CONFIG_SCST_DEBUG_RETRY
 973                 if (((scst_random() % 100) == 75))
 974                         rc = SCST_TGT_RES_QUEUE_FULL;
 975                 else
 976 #endif
 977                         rc = cmd->tgtt->rdy_to_xfer(cmd);
 978                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
 979
 980                 if (likely(rc == SCST_TGT_RES_SUCCESS))
 981                         goto out;
 982
 983                 /* Restore the previous state */
 984                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
 985
 986                 switch (rc) {
 987                 case SCST_TGT_RES_QUEUE_FULL:
 988                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
 989                                 break;
 990                         else
 991                                 continue;
 992
 993                 case SCST_TGT_RES_NEED_THREAD_CTX:
 994                         TRACE_DBG("Target driver %s "
 995                               "rdy_to_xfer() requested thread "
 996                               "context, rescheduling", cmd->tgtt->name);
 997                         res = SCST_CMD_STATE_RES_NEED_THREAD;
 998                         break;
 999
1000                 default:
1001                         goto out_error_rc;
1002                 }
1003                 break;
1004         }
1005
1006 out:
1007         TRACE_EXIT_HRES(res);
1008         return res;
1009
1010 out_error_rc:
1011         if (rc == SCST_TGT_RES_FATAL_ERROR) {
1012                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
1013                      "fatal error", cmd->tgtt->name);
1014         } else {
1015                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
1016                             "value %d", cmd->tgtt->name, rc);
1017         }
1018         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1019
1020 out_dev_done:
1021         scst_set_cmd_abnormal_done_state(cmd);
1022         res = SCST_CMD_STATE_RES_CONT_SAME;
1023         goto out;
1024 }
1025
1026 /* No locks, but might be in IRQ */
1027 static void scst_proccess_redirect_cmd(struct scst_cmd *cmd,
1028         enum scst_exec_context context, int check_retries)
1029 {
1030         unsigned long flags;
1031
1032         TRACE_ENTRY();
1033
1034         TRACE_DBG("Context: %x", context);
1035
1036         if (context == SCST_CONTEXT_SAME)
1037                 context = scst_cmd_atomic(cmd) ? SCST_CONTEXT_DIRECT_ATOMIC :
1038                                                  SCST_CONTEXT_DIRECT;
1039
1040         switch (context) {
1041         case SCST_CONTEXT_DIRECT_ATOMIC:
1042                 scst_process_active_cmd(cmd, true);
1043                 break;
1044
1045         case SCST_CONTEXT_DIRECT:
1046                 if (check_retries)
1047                         scst_check_retries(cmd->tgt);
1048                 scst_process_active_cmd(cmd, false);
1049                 break;
1050
1051         default:
1052                 PRINT_ERROR("Context %x is unknown, using the thread one",
1053                             context);
1054                 /* go through */
1055         case SCST_CONTEXT_THREAD:
1056                 if (check_retries)
1057                         scst_check_retries(cmd->tgt);
1058                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
1059                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
1060                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
1061                         list_add(&cmd->cmd_list_entry,
1062                                 &cmd->cmd_lists->active_cmd_list);
1063                 else
1064                         list_add_tail(&cmd->cmd_list_entry,
1065                                 &cmd->cmd_lists->active_cmd_list);
1066                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
1067                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
1068                 break;
1069
1070         case SCST_CONTEXT_TASKLET:
1071                 if (check_retries)
1072                         scst_check_retries(cmd->tgt);
1073                 scst_schedule_tasklet(cmd);
1074                 break;
1075         }
1076
1077         TRACE_EXIT();
1078         return;
1079 }
1080
1081 void scst_rx_data(struct scst_cmd *cmd, int status,
1082         enum scst_exec_context pref_context)
1083 {
1084         TRACE_ENTRY();
1085
1086         TRACE_DBG("Preferred context: %d", pref_context);
1087         TRACE(TRACE_SCSI, "cmd %p, status %#x", cmd, status);
1088
1089 #ifdef CONFIG_SCST_EXTRACHECKS
1090         if ((in_irq() || irqs_disabled()) &&
1091             ((pref_context == SCST_CONTEXT_DIRECT) ||
1092              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
1093                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
1094                         "SCST_CONTEXT_THREAD instead\n", pref_context,
1095                         cmd->tgtt->name);
1096                 pref_context = SCST_CONTEXT_THREAD;
1097         }
1098 #endif
1099
1100         switch (status) {
1101         case SCST_RX_STATUS_SUCCESS:
1102 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
1103                 if (trace_flag & TRACE_RCV_BOT) {
1104                         int i;
1105                         struct scatterlist *sg;
1106                         if (cmd->in_sg != NULL)
1107                                 sg = cmd->in_sg;
1108                         else if (cmd->tgt_in_sg != NULL)
1109                                 sg = cmd->tgt_in_sg;
1110                         else if (cmd->tgt_sg != NULL)
1111                                 sg = cmd->tgt_sg;
1112                         else
1113                                 sg = cmd->sg;
1114                         if (sg != NULL) {
1115                                 TRACE_RECV_BOT("RX data for cmd %p "
1116                                         "(sg_cnt %d, sg %p, sg[0].page %p)",
1117                                         cmd, cmd->tgt_sg_cnt, sg,
1118                                         (void *)sg_page(&sg[0]));
1119                                 for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
1120                                         PRINT_BUFF_FLAG(TRACE_RCV_BOT, "RX sg",
1121                                                 sg_virt(&sg[i]), sg[i].length);
1122                                 }
1123                         }
1124                 }
1125 #endif
1126                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1127                 /* Small context optimization */
1128                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
1129                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
1130                     ((pref_context == SCST_CONTEXT_SAME) &&
1131                      scst_cmd_atomic(cmd))) {
1132                         if (!test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC,
1133                                         &cmd->tgt_dev->tgt_dev_flags))
1134                                 pref_context = SCST_CONTEXT_THREAD;
1135                 }
1136                 break;
1137
1138         case SCST_RX_STATUS_ERROR_SENSE_SET:
1139                 scst_set_cmd_abnormal_done_state(cmd);
1140                 break;
1141
1142         case SCST_RX_STATUS_ERROR_FATAL:
1143                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1144                 /* go through */
1145         case SCST_RX_STATUS_ERROR:
1146                 scst_set_cmd_error(cmd,
1147                            SCST_LOAD_SENSE(scst_sense_hardw_error));
1148                 scst_set_cmd_abnormal_done_state(cmd);
1149                 break;
1150
1151         default:
1152                 PRINT_ERROR("scst_rx_data() received unknown status %x",
1153                         status);
1154                 scst_set_cmd_abnormal_done_state(cmd);
1155                 break;
1156         }
1157
1158         scst_proccess_redirect_cmd(cmd, pref_context, 1);
1159
1160         TRACE_EXIT();
1161         return;
1162 }
1163 EXPORT_SYMBOL(scst_rx_data);
1164
1165 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
1166 {
1167         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
1168
1169         TRACE_ENTRY();
1170
1171         cmd->state = SCST_CMD_STATE_SEND_FOR_EXEC;
1172
1173         if ((cmd->tgtt->pre_exec == NULL) || unlikely(cmd->internal))
1174                 goto out;
1175
1176         TRACE_DBG("Calling pre_exec(%p)", cmd);
1177         rc = cmd->tgtt->pre_exec(cmd);
1178         TRACE_DBG("pre_exec() returned %d", rc);
1179
1180         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
1181                 switch (rc) {
1182                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
1183                         scst_set_cmd_abnormal_done_state(cmd);
1184                         break;
1185                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
1186                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1187                         /* go through */
1188                 case SCST_PREPROCESS_STATUS_ERROR:
1189                         scst_set_cmd_error(cmd,
1190                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1191                         scst_set_cmd_abnormal_done_state(cmd);
1192                         break;
1193                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1194                         TRACE_DBG("Target driver's %s pre_exec() requested "
1195                                 "thread context, rescheduling",
1196                                 cmd->tgtt->name);
1197                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1198                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1199                         break;
1200                 default:
1201                         sBUG();
1202                         break;
1203                 }
1204         }
1205
1206 out:
1207         TRACE_EXIT_RES(res);
1208         return res;
1209 }
1210
1211 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1212         const uint8_t *rq_sense, int rq_sense_len, int resid)
1213 {
1214         TRACE_ENTRY();
1215
1216 #ifdef CONFIG_SCST_MEASURE_LATENCY
1217         {
1218                 struct timespec ts;
1219                 getnstimeofday(&ts);
1220                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1221                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1222                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1223                         ts.tv_sec, ts.tv_nsec);
1224         }
1225 #endif
1226
1227         cmd->status = result & 0xff;
1228         cmd->msg_status = msg_byte(result);
1229         cmd->host_status = host_byte(result);
1230         cmd->driver_status = driver_byte(result);
1231         if (unlikely(resid != 0)) {
1232 #ifdef CONFIG_SCST_EXTRACHECKS
1233                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1234                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1235                                 "op %x)", resid, cmd->resp_data_len,
1236                                 cmd->cdb[0]);
1237                 } else
1238 #endif
1239                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1240         }
1241
1242         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION)) {
1243                 /* We might have double reset UA here */
1244                 cmd->dbl_ua_orig_resp_data_len = cmd->resp_data_len;
1245                 cmd->dbl_ua_orig_data_direction = cmd->data_direction;
1246
1247                 scst_alloc_set_sense(cmd, 1, rq_sense, rq_sense_len);
1248         }
1249
1250         TRACE(TRACE_SCSI, "cmd %p, result=%x, cmd->status=%x, resid=%d, "
1251               "cmd->msg_status=%x, cmd->host_status=%x, "
1252               "cmd->driver_status=%x (cmd %p)", cmd, result, cmd->status, resid,
1253               cmd->msg_status, cmd->host_status, cmd->driver_status, cmd);
1254
1255         cmd->completed = 1;
1256
1257         TRACE_EXIT();
1258         return;
1259 }
1260
1261 /* For small context optimization */
1262 static inline enum scst_exec_context scst_optimize_post_exec_context(
1263         struct scst_cmd *cmd, enum scst_exec_context context)
1264 {
1265         if (((context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd)) ||
1266             (context == SCST_CONTEXT_TASKLET) ||
1267             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1268                 if (!test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC,
1269                                 &cmd->tgt_dev->tgt_dev_flags))
1270                         context = SCST_CONTEXT_THREAD;
1271         }
1272         return context;
1273 }
1274
1275 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1276 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1277                                             struct scsi_request **req)
1278 {
1279         struct scst_cmd *cmd = NULL;
1280
1281         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1282                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1283
1284         if (cmd == NULL) {
1285                 PRINT_ERROR("%s", "Request with NULL cmd");
1286                 if (*req)
1287                         scsi_release_request(*req);
1288         }
1289
1290         return cmd;
1291 }
1292
1293 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1294 {
1295         struct scsi_request *req = NULL;
1296         struct scst_cmd *cmd;
1297
1298         TRACE_ENTRY();
1299
1300         cmd = scst_get_cmd(scsi_cmd, &req);
1301         if (cmd == NULL)
1302                 goto out;
1303
1304         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1305                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1306
1307         /* Clear out request structure */
1308         req->sr_use_sg = 0;
1309         req->sr_sglist_len = 0;
1310         req->sr_bufflen = 0;
1311         req->sr_buffer = NULL;
1312         req->sr_underflow = 0;
1313         req->sr_request->rq_disk = NULL; /* disown request blk */
1314
1315         scst_release_request(cmd);
1316
1317         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1318
1319         scst_proccess_redirect_cmd(cmd,
1320                 scst_optimize_post_exec_context(cmd, scst_estimate_context()),
1321                                                 0);
1322
1323 out:
1324         TRACE_EXIT();
1325         return;
1326 }
1327 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1328 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1329 {
1330         struct scst_cmd *cmd;
1331
1332         TRACE_ENTRY();
1333
1334         cmd = (struct scst_cmd *)data;
1335         if (cmd == NULL)
1336                 goto out;
1337
1338         scst_do_cmd_done(cmd, result, sense, SCSI_SENSE_BUFFERSIZE, resid);
1339
1340         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1341
1342         scst_proccess_redirect_cmd(cmd,
1343             scst_optimize_post_exec_context(cmd, scst_estimate_context()), 0);
1344
1345 out:
1346         TRACE_EXIT();
1347         return;
1348 }
1349 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1350
1351 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state,
1352         enum scst_exec_context pref_context)
1353 {
1354         TRACE_ENTRY();
1355
1356 #ifdef CONFIG_SCST_MEASURE_LATENCY
1357         {
1358                 struct timespec ts;
1359                 getnstimeofday(&ts);
1360                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1361                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1362                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1363                         ts.tv_sec, ts.tv_nsec);
1364         }
1365 #endif
1366
1367         if (next_state == SCST_CMD_STATE_DEFAULT)
1368                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1369
1370 #if defined(CONFIG_SCST_DEBUG)
1371         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1372                 if ((trace_flag & TRACE_RCV_TOP) && (cmd->sg != NULL)) {
1373                         int i;
1374                         struct scatterlist *sg = cmd->sg;
1375                         TRACE_RECV_TOP("Exec'd %d S/G(s) at %p sg[0].page at "
1376                                 "%p", cmd->sg_cnt, sg, (void *)sg_page(&sg[0]));
1377                         for (i = 0; i < cmd->sg_cnt; ++i) {
1378                                 TRACE_BUFF_FLAG(TRACE_RCV_TOP,
1379                                         "Exec'd sg", sg_virt(&sg[i]),
1380                                         sg[i].length);
1381                         }
1382                 }
1383         }
1384 #endif
1385
1386         cmd->state = next_state;
1387
1388 #ifdef CONFIG_SCST_EXTRACHECKS
1389         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1390             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1391             (next_state != SCST_CMD_STATE_FINISHED) &&
1392             (next_state != SCST_CMD_STATE_FINISHED_INTERNAL)) {
1393                 PRINT_ERROR("%s() received invalid cmd state %d (opcode %d)",
1394                         __func__, next_state, cmd->cdb[0]);
1395                 scst_set_cmd_error(cmd,
1396                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1397                 scst_set_cmd_abnormal_done_state(cmd);
1398         }
1399 #endif
1400         pref_context = scst_optimize_post_exec_context(cmd, pref_context);
1401         scst_proccess_redirect_cmd(cmd, pref_context, 0);
1402
1403         TRACE_EXIT();
1404         return;
1405 }
1406
1407 static int scst_report_luns_local(struct scst_cmd *cmd)
1408 {
1409         int res = SCST_EXEC_COMPLETED, rc;
1410         int dev_cnt = 0;
1411         int buffer_size;
1412         int i;
1413         struct scst_tgt_dev *tgt_dev = NULL;
1414         uint8_t *buffer;
1415         int offs, overflow = 0;
1416
1417         TRACE_ENTRY();
1418
1419         if (scst_cmd_atomic(cmd)) {
1420                 res = SCST_EXEC_NEED_THREAD;
1421                 goto out;
1422         }
1423
1424         rc = scst_check_local_events(cmd);
1425         if (unlikely(rc != 0))
1426                 goto out_done;
1427
1428         cmd->status = 0;
1429         cmd->msg_status = 0;
1430         cmd->host_status = DID_OK;
1431         cmd->driver_status = 0;
1432
1433         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1434                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1435                         "LUNS command", cmd->cdb[2]);
1436                 goto out_err;
1437         }
1438
1439         buffer_size = scst_get_buf_first(cmd, &buffer);
1440         if (unlikely(buffer_size == 0))
1441                 goto out_compl;
1442         else if (unlikely(buffer_size < 0))
1443                 goto out_err;
1444
1445         if (buffer_size < 16)
1446                 goto out_put_err;
1447
1448         memset(buffer, 0, buffer_size);
1449         offs = 8;
1450
1451         /* sess->sess_tgt_dev_list_hash is protected by suspended activity */
1452         for (i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1453                 struct list_head *sess_tgt_dev_list_head =
1454                         &cmd->sess->sess_tgt_dev_list_hash[i];
1455                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1456                                 sess_tgt_dev_list_entry) {
1457                         if (!overflow) {
1458                                 if (offs >= buffer_size) {
1459                                         scst_put_buf(cmd, buffer);
1460                                         buffer_size = scst_get_buf_next(cmd,
1461                                                                        &buffer);
1462                                         if (buffer_size > 0) {
1463                                                 memset(buffer, 0, buffer_size);
1464                                                 offs = 0;
1465                                         } else {
1466                                                 overflow = 1;
1467                                                 goto inc_dev_cnt;
1468                                         }
1469                                 }
1470                                 if ((buffer_size - offs) < 8) {
1471                                         PRINT_ERROR("Buffer allocated for "
1472                                                 "REPORT LUNS command doesn't "
1473                                                 "allow to fit 8 byte entry "
1474                                                 "(buffer_size=%d)",
1475                                                 buffer_size);
1476                                         goto out_put_hw_err;
1477                                 }
1478                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1479                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1480                                 offs += 8;
1481                         }
1482 inc_dev_cnt:
1483                         dev_cnt++;
1484                 }
1485         }
1486         if (!overflow)
1487                 scst_put_buf(cmd, buffer);
1488
1489         /* Set the response header */
1490         buffer_size = scst_get_buf_first(cmd, &buffer);
1491         if (unlikely(buffer_size == 0))
1492                 goto out_compl;
1493         else if (unlikely(buffer_size < 0))
1494                 goto out_err;
1495
1496         dev_cnt *= 8;
1497         buffer[0] = (dev_cnt >> 24) & 0xff;
1498         buffer[1] = (dev_cnt >> 16) & 0xff;
1499         buffer[2] = (dev_cnt >> 8) & 0xff;
1500         buffer[3] = dev_cnt & 0xff;
1501
1502         scst_put_buf(cmd, buffer);
1503
1504         dev_cnt += 8;
1505         if (dev_cnt < cmd->resp_data_len)
1506                 scst_set_resp_data_len(cmd, dev_cnt);
1507
1508 out_compl:
1509         cmd->completed = 1;
1510
1511         /* Clear left sense_reported_luns_data_changed UA, if any. */
1512
1513         mutex_lock(&scst_mutex); /* protect sess_tgt_dev_list_hash */
1514         for (i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1515                 struct list_head *sess_tgt_dev_list_head =
1516                         &cmd->sess->sess_tgt_dev_list_hash[i];
1517
1518                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1519                                 sess_tgt_dev_list_entry) {
1520                         struct scst_tgt_dev_UA *ua;
1521
1522                         spin_lock_bh(&tgt_dev->tgt_dev_lock);
1523                         list_for_each_entry(ua, &tgt_dev->UA_list,
1524                                                 UA_list_entry) {
1525                                 if (scst_analyze_sense(ua->UA_sense_buffer,
1526                                                 sizeof(ua->UA_sense_buffer),
1527                                                 SCST_SENSE_ALL_VALID,
1528                                                 SCST_LOAD_SENSE(scst_sense_reported_luns_data_changed))) {
1529                                         TRACE_MGMT_DBG("Freeing not needed "
1530                                                 "REPORTED LUNS DATA CHANGED UA "
1531                                                 "%p", ua);
1532                                         list_del(&ua->UA_list_entry);
1533                                         mempool_free(ua, scst_ua_mempool);
1534                                         break;
1535                                 }
1536                         }
1537                         spin_unlock_bh(&tgt_dev->tgt_dev_lock);
1538                 }
1539         }
1540         mutex_unlock(&scst_mutex);
1541
1542 out_done:
1543         /* Report the result */
1544         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1545
1546 out:
1547         TRACE_EXIT_RES(res);
1548         return res;
1549
1550 out_put_err:
1551         scst_put_buf(cmd, buffer);
1552
1553 out_err:
1554         scst_set_cmd_error(cmd,
1555                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1556         goto out_compl;
1557
1558 out_put_hw_err:
1559         scst_put_buf(cmd, buffer);
1560         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1561         goto out_compl;
1562 }
1563
1564 static int scst_pre_select(struct scst_cmd *cmd)
1565 {
1566         int res = SCST_EXEC_NOT_COMPLETED;
1567
1568         TRACE_ENTRY();
1569
1570         if (scst_cmd_atomic(cmd)) {
1571                 res = SCST_EXEC_NEED_THREAD;
1572                 goto out;
1573         }
1574
1575         scst_block_dev_cmd(cmd, 1);
1576
1577         /* Check for local events will be done when cmd will be executed */
1578
1579 out:
1580         TRACE_EXIT_RES(res);
1581         return res;
1582 }
1583
1584 static int scst_reserve_local(struct scst_cmd *cmd)
1585 {
1586         int res = SCST_EXEC_NOT_COMPLETED, rc;
1587         struct scst_device *dev;
1588         struct scst_tgt_dev *tgt_dev_tmp;
1589
1590         TRACE_ENTRY();
1591
1592         if (scst_cmd_atomic(cmd)) {
1593                 res = SCST_EXEC_NEED_THREAD;
1594                 goto out;
1595         }
1596
1597         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1598                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1599                      "(lun=%lld)", (long long unsigned int)cmd->lun);
1600                 scst_set_cmd_error(cmd,
1601                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1602                 goto out_done;
1603         }
1604
1605         dev = cmd->dev;
1606
1607         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1608                 scst_block_dev_cmd(cmd, 1);
1609
1610         rc = scst_check_local_events(cmd);
1611         if (unlikely(rc != 0))
1612                 goto out_done;
1613
1614         spin_lock_bh(&dev->dev_lock);
1615
1616         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1617                 spin_unlock_bh(&dev->dev_lock);
1618                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1619                 goto out_done;
1620         }
1621
1622         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1623                             dev_tgt_dev_list_entry) {
1624                 if (cmd->tgt_dev != tgt_dev_tmp)
1625                         set_bit(SCST_TGT_DEV_RESERVED,
1626                                 &tgt_dev_tmp->tgt_dev_flags);
1627         }
1628         dev->dev_reserved = 1;
1629
1630         spin_unlock_bh(&dev->dev_lock);
1631
1632 out:
1633         TRACE_EXIT_RES(res);
1634         return res;
1635
1636 out_done:
1637         /* Report the result */
1638         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1639         res = SCST_EXEC_COMPLETED;
1640         goto out;
1641 }
1642
1643 static int scst_release_local(struct scst_cmd *cmd)
1644 {
1645         int res = SCST_EXEC_NOT_COMPLETED, rc;
1646         struct scst_tgt_dev *tgt_dev_tmp;
1647         struct scst_device *dev;
1648
1649         TRACE_ENTRY();
1650
1651         if (scst_cmd_atomic(cmd)) {
1652                 res = SCST_EXEC_NEED_THREAD;
1653                 goto out;
1654         }
1655
1656         dev = cmd->dev;
1657
1658         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1659                 scst_block_dev_cmd(cmd, 1);
1660
1661         rc = scst_check_local_events(cmd);
1662         if (unlikely(rc != 0))
1663                 goto out_done;
1664
1665         spin_lock_bh(&dev->dev_lock);
1666
1667         /*
1668          * The device could be RELEASED behind us, if RESERVING session
1669          * is closed (see scst_free_tgt_dev()), but this actually doesn't
1670          * matter, so use lock and no retest for DEV_RESERVED bits again
1671          */
1672         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1673                 res = SCST_EXEC_COMPLETED;
1674                 cmd->status = 0;
1675                 cmd->msg_status = 0;
1676                 cmd->host_status = DID_OK;
1677                 cmd->driver_status = 0;
1678                 cmd->completed = 1;
1679         } else {
1680                 list_for_each_entry(tgt_dev_tmp,
1681                                     &dev->dev_tgt_dev_list,
1682                                     dev_tgt_dev_list_entry) {
1683                         clear_bit(SCST_TGT_DEV_RESERVED,
1684                                 &tgt_dev_tmp->tgt_dev_flags);
1685                 }
1686                 dev->dev_reserved = 0;
1687         }
1688
1689         spin_unlock_bh(&dev->dev_lock);
1690
1691         if (res == SCST_EXEC_COMPLETED)
1692                 goto out_done;
1693
1694 out:
1695         TRACE_EXIT_RES(res);
1696         return res;
1697
1698 out_done:
1699         res = SCST_EXEC_COMPLETED;
1700         /* Report the result */
1701         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1702         goto out;
1703 }
1704
1705 /* No locks, no IRQ or IRQ-safe context allowed */
1706 int scst_check_local_events(struct scst_cmd *cmd)
1707 {
1708         int res, rc;
1709         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1710         struct scst_device *dev = cmd->dev;
1711
1712         TRACE_ENTRY();
1713
1714         /*
1715          * There's no race here, because we need to trace commands sent
1716          * *after* dev_double_ua_possible flag was set.
1717          */
1718         if (unlikely(dev->dev_double_ua_possible))
1719                 cmd->double_ua_possible = 1;
1720
1721         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1722                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1723                 goto out_uncomplete;
1724         }
1725
1726         /* Reserve check before Unit Attention */
1727         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED,
1728                               &tgt_dev->tgt_dev_flags))) {
1729                 if (cmd->cdb[0] != INQUIRY &&
1730                     cmd->cdb[0] != REPORT_LUNS &&
1731                     cmd->cdb[0] != RELEASE &&
1732                     cmd->cdb[0] != RELEASE_10 &&
1733                     cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER &&
1734                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL ||
1735                      (cmd->cdb[4] & 3)) &&
1736                     cmd->cdb[0] != LOG_SENSE &&
1737                     cmd->cdb[0] != REQUEST_SENSE) {
1738                         scst_set_cmd_error_status(cmd,
1739                                 SAM_STAT_RESERVATION_CONFLICT);
1740                         goto out_complete;
1741                 }
1742         }
1743
1744         /* If we had internal bus reset, set the command error unit attention */
1745         if ((dev->scsi_dev != NULL) &&
1746             unlikely(dev->scsi_dev->was_reset)) {
1747                 if (scst_is_ua_command(cmd)) {
1748                         int done = 0;
1749                         /*
1750                          * Prevent more than 1 cmd to be triggered by
1751                          * was_reset.
1752                          */
1753                         spin_lock_bh(&dev->dev_lock);
1754                         if (dev->scsi_dev->was_reset) {
1755                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1756                                 scst_set_cmd_error(cmd,
1757                                           SCST_LOAD_SENSE(scst_sense_reset_UA));
1758                                 /*
1759                                  * It looks like it is safe to clear was_reset
1760                                  * here.
1761                                  */
1762                                 dev->scsi_dev->was_reset = 0;
1763                                 done = 1;
1764                         }
1765                         spin_unlock_bh(&dev->dev_lock);
1766
1767                         if (done)
1768                                 goto out_complete;
1769                 }
1770         }
1771
1772         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING,
1773                         &cmd->tgt_dev->tgt_dev_flags))) {
1774                 if (scst_is_ua_command(cmd)) {
1775                         rc = scst_set_pending_UA(cmd);
1776                         if (rc == 0)
1777                                 goto out_complete;
1778                 }
1779         }
1780
1781         res = 0;
1782
1783 out:
1784         TRACE_EXIT_RES(res);
1785         return res;
1786
1787 out_complete:
1788         res = 1;
1789         sBUG_ON(!cmd->completed);
1790         goto out;
1791
1792 out_uncomplete:
1793         res = -1;
1794         goto out;
1795 }
1796 EXPORT_SYMBOL(scst_check_local_events);
1797
1798 /* No locks */
1799 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1800 {
1801         if (slot == NULL)
1802                 goto inc;
1803
1804         /* Optimized for lockless fast path */
1805
1806         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1807                 atomic_read(slot));
1808
1809         if (!atomic_dec_and_test(slot))
1810                 goto out;
1811
1812         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1813                 tgt_dev->num_free_sn_slots);
1814         if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
1815                 spin_lock_irq(&tgt_dev->sn_lock);
1816                 if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
1817                         if (tgt_dev->num_free_sn_slots < 0)
1818                                 tgt_dev->cur_sn_slot = slot;
1819                         /*
1820                          * To be in-sync with SIMPLE case in scst_cmd_set_sn()
1821                          */
1822                         smp_mb();
1823                         tgt_dev->num_free_sn_slots++;
1824                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1825                                 tgt_dev->num_free_sn_slots);
1826
1827                 }
1828                 spin_unlock_irq(&tgt_dev->sn_lock);
1829         }
1830
1831 inc:
1832         /*
1833          * No protection of expected_sn is needed, because only one thread
1834          * at time can be here (serialized by sn). Also it is supposed that
1835          * there could not be half-incremented halves.
1836          */
1837         tgt_dev->expected_sn++;
1838         /*
1839          * Write must be before def_cmd_count read to be in sync. with
1840          * scst_post_exec_sn(). See comment in scst_send_for_exec().
1841          */
1842         smp_mb();
1843         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1844
1845 out:
1846         return;
1847 }
1848
1849 /* No locks */
1850 static struct scst_cmd *scst_post_exec_sn(struct scst_cmd *cmd,
1851         bool make_active)
1852 {
1853         /* For HQ commands SN is not set */
1854         bool inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1855                                cmd->sn_set && !cmd->retry;
1856         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1857         struct scst_cmd *res;
1858
1859         TRACE_ENTRY();
1860
1861         if (inc_expected_sn)
1862                 scst_inc_expected_sn(tgt_dev, cmd->sn_slot);
1863
1864         if (make_active) {
1865                 scst_make_deferred_commands_active(tgt_dev);
1866                 res = NULL;
1867         } else
1868                 res = scst_check_deferred_commands(tgt_dev);
1869
1870         TRACE_EXIT_HRES(res);
1871         return res;
1872 }
1873
1874 /* cmd must be additionally referenced to not die inside */
1875 static int scst_do_real_exec(struct scst_cmd *cmd)
1876 {
1877         int res = SCST_EXEC_NOT_COMPLETED;
1878 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
1879         int rc;
1880 #endif
1881         bool atomic = scst_cmd_atomic(cmd);
1882         struct scst_device *dev = cmd->dev;
1883         struct scst_dev_type *handler = dev->handler;
1884         struct io_context *old_ctx = NULL;
1885         bool ctx_changed = false;
1886
1887         TRACE_ENTRY();
1888
1889         if (!atomic)
1890                 ctx_changed = scst_set_io_context(cmd, &old_ctx);
1891
1892         cmd->state = SCST_CMD_STATE_REAL_EXECUTING;
1893
1894         if (handler->exec) {
1895                 if (unlikely(!dev->handler->exec_atomic && atomic)) {
1896                         /*
1897                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
1898                          * optimization.
1899                          */
1900                         TRACE_DBG("Dev handler %s exec() needs thread "
1901                                 "context, rescheduling", dev->handler->name);
1902                         res = SCST_EXEC_NEED_THREAD;
1903                         goto out_restore;
1904                 }
1905
1906                 TRACE_DBG("Calling dev handler %s exec(%p)",
1907                       handler->name, cmd);
1908                 TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb,
1909                         cmd->cdb_len);
1910                 res = handler->exec(cmd);
1911                 TRACE_DBG("Dev handler %s exec() returned %d",
1912                       handler->name, res);
1913
1914                 if (res == SCST_EXEC_COMPLETED)
1915                         goto out_complete;
1916                 else if (res == SCST_EXEC_NEED_THREAD)
1917                         goto out_restore;
1918
1919                 sBUG_ON(res != SCST_EXEC_NOT_COMPLETED);
1920         }
1921
1922         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1923
1924         if (unlikely(dev->scsi_dev == NULL)) {
1925                 PRINT_ERROR("Command for virtual device must be "
1926                         "processed by device handler (lun %lld)!",
1927                         (long long unsigned int)cmd->lun);
1928                 goto out_error;
1929         }
1930
1931         res = scst_check_local_events(cmd);
1932         if (unlikely(res != 0))
1933                 goto out_done;
1934
1935 #ifndef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1936         if (unlikely(atomic)) {
1937                 TRACE_DBG("Pass-through exec() can not be called in atomic "
1938                         "context, rescheduling to the thread (handler %s)",
1939                         handler->name);
1940                 res = SCST_EXEC_NEED_THREAD;
1941                 goto out_restore;
1942         }
1943 #endif
1944
1945 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1946         if (unlikely(scst_alloc_request(cmd) != 0)) {
1947                 if (atomic) {
1948                         res = SCST_EXEC_NEED_THREAD;
1949                         goto out_restore;
1950                 } else {
1951                         PRINT_INFO("%s", "Unable to allocate request, "
1952                                 "sending BUSY status");
1953                         goto out_busy;
1954                 }
1955         }
1956
1957         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1958                     (void *)cmd->scsi_req->sr_buffer,
1959                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1960                     cmd->retries);
1961 #else
1962         rc = scst_exec_req(dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1963                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1964                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1965                         atomic ? GFP_ATOMIC : GFP_KERNEL);
1966         if (unlikely(rc != 0)) {
1967                 if (atomic) {
1968                         res = SCST_EXEC_NEED_THREAD;
1969                         goto out_restore;
1970                 } else {
1971                         PRINT_ERROR("scst_exec_req() failed: %d", res);
1972                         goto out_error;
1973                 }
1974         }
1975 #endif
1976
1977 out_complete:
1978         res = SCST_EXEC_COMPLETED;
1979
1980 out_reset_ctx:
1981         if (ctx_changed)
1982                 scst_reset_io_context(cmd->tgt_dev, old_ctx);
1983
1984         TRACE_EXIT();
1985         return res;
1986
1987 out_restore:
1988         /* Restore the state */
1989         cmd->state = SCST_CMD_STATE_REAL_EXEC;
1990         goto out_reset_ctx;
1991
1992 out_error:
1993         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1994         goto out_done;
1995
1996 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1997 out_busy:
1998         scst_set_busy(cmd);
1999         /* go through */
2000 #endif
2001
2002 out_done:
2003         res = SCST_EXEC_COMPLETED;
2004         /* Report the result */
2005         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
2006         goto out_complete;
2007 }
2008
2009 static inline int scst_real_exec(struct scst_cmd *cmd)
2010 {
2011         int res;
2012
2013         TRACE_ENTRY();
2014
2015         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
2016         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
2017         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
2018
2019         __scst_cmd_get(cmd);
2020
2021         res = scst_do_real_exec(cmd);
2022
2023         if (likely(res == SCST_EXEC_COMPLETED)) {
2024                 scst_post_exec_sn(cmd, true);
2025                 if (cmd->dev->scsi_dev != NULL)
2026                         generic_unplug_device(
2027                                 cmd->dev->scsi_dev->request_queue);
2028         } else
2029                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
2030
2031         __scst_cmd_put(cmd);
2032
2033         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
2034
2035         TRACE_EXIT_RES(res);
2036         return res;
2037 }
2038
2039 static int scst_do_local_exec(struct scst_cmd *cmd)
2040 {
2041         int res;
2042         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2043
2044         TRACE_ENTRY();
2045
2046         /* Check READ_ONLY device status */
2047         if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
2048             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
2049              cmd->cdb[0] == WRITE_10 ||
2050              cmd->cdb[0] == WRITE_12 ||
2051              cmd->cdb[0] == WRITE_16 ||
2052              cmd->cdb[0] == WRITE_VERIFY ||
2053              cmd->cdb[0] == WRITE_VERIFY_12 ||
2054              cmd->cdb[0] == WRITE_VERIFY_16 ||
2055              (cmd->dev->handler->type == TYPE_TAPE &&
2056               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
2057                 PRINT_WARNING("Attempt of write access to read-only device: "
2058                         "initiator %s, LUN %lld, op %x",
2059                         cmd->sess->initiator_name, cmd->lun, cmd->cdb[0]);
2060                 scst_set_cmd_error(cmd,
2061                            SCST_LOAD_SENSE(scst_sense_data_protect));
2062                 goto out_done;
2063         }
2064
2065         /*
2066          * Adding new commands here don't forget to update
2067          * scst_is_cmd_local() in scst.h, if necessary
2068          */
2069
2070         switch (cmd->cdb[0]) {
2071         case MODE_SELECT:
2072         case MODE_SELECT_10:
2073         case LOG_SELECT:
2074                 res = scst_pre_select(cmd);
2075                 break;
2076         case RESERVE:
2077         case RESERVE_10:
2078                 res = scst_reserve_local(cmd);
2079                 break;
2080         case RELEASE:
2081         case RELEASE_10:
2082                 res = scst_release_local(cmd);
2083                 break;
2084         case REPORT_LUNS:
2085                 res = scst_report_luns_local(cmd);
2086                 break;
2087         default:
2088                 res = SCST_EXEC_NOT_COMPLETED;
2089                 break;
2090         }
2091
2092 out:
2093         TRACE_EXIT_RES(res);
2094         return res;
2095
2096 out_done:
2097         /* Report the result */
2098         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
2099         res = SCST_EXEC_COMPLETED;
2100         goto out;
2101 }
2102
2103 static int scst_local_exec(struct scst_cmd *cmd)
2104 {
2105         int res;
2106
2107         TRACE_ENTRY();
2108
2109         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
2110         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
2111         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
2112
2113         __scst_cmd_get(cmd);
2114
2115         res = scst_do_local_exec(cmd);
2116         if (likely(res == SCST_EXEC_NOT_COMPLETED))
2117                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2118         else if (res == SCST_EXEC_COMPLETED)
2119                 scst_post_exec_sn(cmd, true);
2120         else
2121                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
2122
2123         __scst_cmd_put(cmd);
2124
2125         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
2126         TRACE_EXIT_RES(res);
2127         return res;
2128 }
2129
2130 static int scst_exec(struct scst_cmd **active_cmd)
2131 {
2132         struct scst_cmd *cmd = *active_cmd;
2133         struct scst_cmd *ref_cmd;
2134         struct scst_device *dev = cmd->dev;
2135         int res = SCST_CMD_STATE_RES_CONT_NEXT, count;
2136
2137         TRACE_ENTRY();
2138
2139         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2140                 goto out;
2141
2142         /* To protect tgt_dev */
2143         ref_cmd = cmd;
2144         __scst_cmd_get(ref_cmd);
2145
2146         count = 0;
2147         while (1) {
2148                 int rc;
2149
2150                 cmd->sent_for_exec = 1;
2151                 /*
2152                  * To sync with scst_abort_cmd(). The above assignment must
2153                  * be before SCST_CMD_ABORTED test, done later in
2154                  * scst_check_local_events(). It's far from here, so the order
2155                  * is virtually guaranteed, but let's have it just in case.
2156                  */
2157                 smp_mb();
2158
2159                 cmd->scst_cmd_done = scst_cmd_done_local;
2160                 cmd->state = SCST_CMD_STATE_LOCAL_EXEC;
2161
2162                 if (cmd->tgt_data_buf_alloced && cmd->dh_data_buf_alloced &&
2163                     (cmd->data_direction & SCST_DATA_WRITE))
2164                         scst_copy_sg(cmd, SCST_SG_COPY_FROM_TARGET);
2165
2166                 rc = scst_do_local_exec(cmd);
2167                 if (likely(rc == SCST_EXEC_NOT_COMPLETED))
2168                         /* Nothing to do */;
2169                 else if (rc == SCST_EXEC_NEED_THREAD) {
2170                         TRACE_DBG("%s", "scst_do_local_exec() requested "
2171                                 "thread context, rescheduling");
2172                         scst_dec_on_dev_cmd(cmd);
2173                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2174                         break;
2175                 } else {
2176                         sBUG_ON(rc != SCST_EXEC_COMPLETED);
2177                         goto done;
2178                 }
2179
2180                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2181
2182                 rc = scst_do_real_exec(cmd);
2183                 if (likely(rc == SCST_EXEC_COMPLETED))
2184                         /* Nothing to do */;
2185                 else if (rc == SCST_EXEC_NEED_THREAD) {
2186                         TRACE_DBG("scst_real_exec() requested thread "
2187                                 "context, rescheduling (cmd %p)", cmd);
2188                         scst_dec_on_dev_cmd(cmd);
2189                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2190                         break;
2191                 } else
2192                         sBUG();
2193
2194 done:
2195                 count++;
2196
2197                 cmd = scst_post_exec_sn(cmd, false);
2198                 if (cmd == NULL)
2199                         break;
2200
2201                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2202                         break;
2203
2204                 __scst_cmd_put(ref_cmd);
2205                 ref_cmd = cmd;
2206                 __scst_cmd_get(ref_cmd);
2207         }
2208
2209         *active_cmd = cmd;
2210
2211         if (count == 0)
2212                 goto out_put;
2213
2214         if (dev->scsi_dev != NULL)
2215                 generic_unplug_device(dev->scsi_dev->request_queue);
2216
2217 out_put:
2218         __scst_cmd_put(ref_cmd);
2219         /* !! At this point sess, dev and tgt_dev can be already freed !! */
2220
2221 out:
2222         TRACE_EXIT_RES(res);
2223         return res;
2224 }
2225
2226 static int scst_send_for_exec(struct scst_cmd **active_cmd)
2227 {
2228         int res;
2229         struct scst_cmd *cmd = *active_cmd;
2230         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2231         typeof(tgt_dev->expected_sn) expected_sn;
2232
2233         TRACE_ENTRY();
2234
2235 #ifdef CONFIG_SCST_MEASURE_LATENCY
2236         if (cmd->pre_exec_finish == 0) {
2237                 struct timespec ts;
2238                 getnstimeofday(&ts);
2239                 cmd->pre_exec_finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2240                 TRACE_DBG("cmd %p (sess %p): pre_exec_finish %lld (tv_sec %ld, "
2241                         "tv_nsec %ld)", cmd, cmd->sess, cmd->pre_exec_finish,
2242                         ts.tv_sec, ts.tv_nsec);
2243         }
2244 #endif
2245
2246         if (unlikely(cmd->internal))
2247                 goto exec;
2248
2249         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2250                 goto exec;
2251
2252         sBUG_ON(!cmd->sn_set);
2253
2254         expected_sn = tgt_dev->expected_sn;
2255         /* Optimized for lockless fast path */
2256         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2257                 spin_lock_irq(&tgt_dev->sn_lock);
2258
2259                 tgt_dev->def_cmd_count++;
2260                 /*
2261                  * Memory barrier is needed here to implement lockless fast
2262                  * path. We need the exact order of read and write between
2263                  * def_cmd_count and expected_sn. Otherwise, we can miss case,
2264                  * when expected_sn was changed to be equal to cmd->sn while
2265                  * we are queuing cmd the deferred list after the expected_sn
2266                  * below. It will lead to a forever stuck command. But with
2267                  * the barrier in such case __scst_check_deferred_commands()
2268                  * will be called and it will take sn_lock, so we will be
2269                  * synchronized.
2270                  */
2271                 smp_mb();
2272
2273                 expected_sn = tgt_dev->expected_sn;
2274                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2275                         if (unlikely(test_bit(SCST_CMD_ABORTED,
2276                                               &cmd->cmd_flags))) {
2277                                 /* Necessary to allow aborting out of sn cmds */
2278                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p "
2279                                         "(tag %llu, sn %lu)", cmd,
2280                                         (long long unsigned)cmd->tag, cmd->sn);
2281                                 tgt_dev->def_cmd_count--;
2282                                 scst_set_cmd_abnormal_done_state(cmd);
2283                                 res = SCST_CMD_STATE_RES_CONT_SAME;
2284                         } else {
2285                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
2286                                         "expected_sn=%ld)", cmd, cmd->sn,
2287                                         cmd->sn_set, expected_sn);
2288                                 list_add_tail(&cmd->sn_cmd_list_entry,
2289                                               &tgt_dev->deferred_cmd_list);
2290                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2291                         }
2292                         spin_unlock_irq(&tgt_dev->sn_lock);
2293                         goto out;
2294                 } else {
2295                         TRACE_SN("Somebody incremented expected_sn %ld, "
2296                                 "continuing", expected_sn);
2297                         tgt_dev->def_cmd_count--;
2298                         spin_unlock_irq(&tgt_dev->sn_lock);
2299                 }
2300         }
2301
2302 exec:
2303         res = scst_exec(active_cmd);
2304
2305 out:
2306         TRACE_EXIT_HRES(res);
2307         return res;
2308 }
2309
2310 /* No locks supposed to be held */
2311 static int scst_check_sense(struct scst_cmd *cmd)
2312 {
2313         int res = 0;
2314         struct scst_device *dev = cmd->dev;
2315
2316         TRACE_ENTRY();
2317
2318         if (unlikely(cmd->ua_ignore))
2319                 goto out;
2320
2321         /* If we had internal bus reset behind us, set the command error UA */
2322         if ((dev->scsi_dev != NULL) &&
2323             unlikely(cmd->host_status == DID_RESET) &&
2324             scst_is_ua_command(cmd)) {
2325                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
2326                       dev->scsi_dev->was_reset, cmd->host_status);
2327                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_reset_UA));
2328                 /* It looks like it is safe to clear was_reset here */
2329                 dev->scsi_dev->was_reset = 0;
2330         }
2331
2332         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2333             SCST_SENSE_VALID(cmd->sense)) {
2334                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2335                         cmd->sense_bufflen);
2336
2337                 /* Check Unit Attention Sense Key */
2338                 if (scst_is_ua_sense(cmd->sense)) {
2339                         if (scst_analyze_sense(cmd->sense, cmd->sense_bufflen,
2340                                         SCST_SENSE_ASC_VALID,
2341                                         0, SCST_SENSE_ASC_UA_RESET, 0)) {
2342                                 if (cmd->double_ua_possible) {
2343                                         TRACE(TRACE_MGMT_MINOR, "Double UA "
2344                                                 "detected for device %p", dev);
2345                                         TRACE(TRACE_MGMT_MINOR, "Retrying cmd"
2346                                                 " %p (tag %llu)", cmd,
2347                                                 (long long unsigned)cmd->tag);
2348
2349                                         cmd->status = 0;
2350                                         cmd->msg_status = 0;
2351                                         cmd->host_status = DID_OK;
2352                                         cmd->driver_status = 0;
2353
2354                                         mempool_free(cmd->sense,
2355                                                      scst_sense_mempool);
2356                                         cmd->sense = NULL;
2357
2358                                         scst_check_restore_sg_buff(cmd);
2359
2360                                         sBUG_ON(cmd->dbl_ua_orig_resp_data_len < 0);
2361                                         cmd->data_direction =
2362                                                 cmd->dbl_ua_orig_data_direction;
2363                                         cmd->resp_data_len =
2364                                                 cmd->dbl_ua_orig_resp_data_len;
2365
2366                                         cmd->state = SCST_CMD_STATE_REAL_EXEC;
2367                                         cmd->retry = 1;
2368                                         res = 1;
2369                                         goto out;
2370                                 }
2371                         }
2372                         scst_dev_check_set_UA(dev, cmd, cmd->sense,
2373                                 cmd->sense_bufflen);
2374                 }
2375         }
2376
2377         if (unlikely(cmd->double_ua_possible)) {
2378                 if (scst_is_ua_command(cmd)) {
2379                         TRACE_MGMT_DBG("Clearing dbl_ua_possible flag (dev %p, "
2380                                 "cmd %p)", dev, cmd);
2381                         /*
2382                          * Lock used to protect other flags in the bitfield
2383                          * (just in case, actually). Those flags can't be
2384                          * changed in parallel, because the device is
2385                          * serialized.
2386                          */
2387                         spin_lock_bh(&dev->dev_lock);
2388                         dev->dev_double_ua_possible = 0;
2389                         spin_unlock_bh(&dev->dev_lock);
2390                 }
2391         }
2392
2393 out:
2394         TRACE_EXIT_RES(res);
2395         return res;
2396 }
2397
2398 static int scst_check_auto_sense(struct scst_cmd *cmd)
2399 {
2400         int res = 0;
2401
2402         TRACE_ENTRY();
2403
2404         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2405             (!SCST_SENSE_VALID(cmd->sense) ||
2406              SCST_NO_SENSE(cmd->sense))) {
2407                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2408                       "cmd->status=%x, cmd->msg_status=%x, "
2409                       "cmd->host_status=%x, cmd->driver_status=%x (cmd %p)",
2410                       cmd->status, cmd->msg_status, cmd->host_status,
2411                       cmd->driver_status, cmd);
2412                 res = 1;
2413         } else if (unlikely(cmd->host_status)) {
2414                 if ((cmd->host_status == DID_REQUEUE) ||
2415                     (cmd->host_status == DID_IMM_RETRY) ||
2416                     (cmd->host_status == DID_SOFT_ERROR) ||
2417                     (cmd->host_status == DID_ABORT)) {
2418                         scst_set_busy(cmd);
2419                 } else {
2420                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2421                                 "received, returning HARDWARE ERROR instead "
2422                                 "(cmd %p)", cmd->host_status, cmd);
2423                         scst_set_cmd_error(cmd,
2424                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2425                 }
2426         }
2427
2428         TRACE_EXIT_RES(res);
2429         return res;
2430 }
2431
2432 static int scst_pre_dev_done(struct scst_cmd *cmd)
2433 {
2434         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2435
2436         TRACE_ENTRY();
2437
2438         if (unlikely(scst_check_auto_sense(cmd))) {
2439                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2440                             "without sense data (opcode 0x%x), issuing "
2441                             "REQUEST SENSE", cmd->cdb[0]);
2442                 rc = scst_prepare_request_sense(cmd);
2443                 if (rc == 0)
2444                         res = SCST_CMD_STATE_RES_CONT_NEXT;
2445                 else {
2446                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2447                                     "returning HARDWARE ERROR");
2448                         scst_set_cmd_error(cmd,
2449                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2450                 }
2451                 goto out;
2452         } else if (unlikely(scst_check_sense(cmd)))
2453                 goto out;
2454
2455         if (likely(scsi_status_is_good(cmd->status))) {
2456                 unsigned char type = cmd->dev->handler->type;
2457                 if (unlikely((cmd->cdb[0] == MODE_SENSE ||
2458                               cmd->cdb[0] == MODE_SENSE_10)) &&
2459                     cmd->tgt_dev->acg_dev->rd_only_flag &&
2460                     (type == TYPE_DISK ||
2461                      type == TYPE_WORM ||
2462                      type == TYPE_MOD ||
2463                      type == TYPE_TAPE)) {
2464                         int32_t length;
2465                         uint8_t *address;
2466                         bool err = false;
2467
2468                         length = scst_get_buf_first(cmd, &address);
2469                         if (length < 0) {
2470                                 PRINT_ERROR("%s", "Unable to get "
2471                                         "MODE_SENSE buffer");
2472                                 scst_set_cmd_error(cmd,
2473                                         SCST_LOAD_SENSE(
2474                                                 scst_sense_hardw_error));
2475                                 err = true;
2476                         } else if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2477                                 address[2] |= 0x80;   /* Write Protect*/
2478                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2479                                 address[3] |= 0x80;   /* Write Protect*/
2480                         scst_put_buf(cmd, address);
2481
2482                         if (err)
2483                                 goto out;
2484                 }
2485
2486                 /*
2487                  * Check and clear NormACA option for the device, if necessary,
2488                  * since we don't support ACA
2489                  */
2490                 if (unlikely((cmd->cdb[0] == INQUIRY)) &&
2491                     /* Std INQUIRY data (no EVPD) */
2492                     !(cmd->cdb[1] & SCST_INQ_EVPD) &&
2493                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2494                         uint8_t *buffer;
2495                         int buflen;
2496                         bool err = false;
2497
2498                         /* ToDo: all pages ?? */
2499                         buflen = scst_get_buf_first(cmd, &buffer);
2500                         if (buflen > SCST_INQ_BYTE3) {
2501 #ifdef CONFIG_SCST_EXTRACHECKS
2502                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2503                                         PRINT_INFO("NormACA set for device: "
2504                                             "lun=%lld, type 0x%02x. Clear it, "
2505                                             "since it's unsupported.",
2506                                             (long long unsigned int)cmd->lun,
2507                                             buffer[0]);
2508                                 }
2509 #endif
2510                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2511                         } else if (buflen != 0) {
2512                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2513                                     "buffer");
2514                                 scst_set_cmd_error(cmd,
2515                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2516                                 err = true;
2517                         }
2518                         if (buflen > 0)
2519                                 scst_put_buf(cmd, buffer);
2520
2521                         if (err)
2522                                 goto out;
2523                 }
2524
2525                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2526                     (cmd->cdb[0] == MODE_SELECT_10) ||
2527                     (cmd->cdb[0] == LOG_SELECT))) {
2528                         TRACE(TRACE_SCSI,
2529                                 "MODE/LOG SELECT succeeded (LUN %lld)",
2530                                 (long long unsigned int)cmd->lun);
2531                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2532                         goto out;
2533                 }
2534         } else {
2535                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2536                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2537                                         &cmd->tgt_dev->tgt_dev_flags)) {
2538                                 struct scst_tgt_dev *tgt_dev_tmp;
2539                                 struct scst_device *dev = cmd->dev;
2540
2541                                 TRACE(TRACE_SCSI,
2542                                         "Real RESERVE failed lun=%lld, "
2543                                         "status=%x",
2544                                         (long long unsigned int)cmd->lun,
2545                                         cmd->status);
2546                                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2547                                         cmd->sense_bufflen);
2548
2549                                 /* Clearing the reservation */
2550                                 spin_lock_bh(&dev->dev_lock);
2551                                 list_for_each_entry(tgt_dev_tmp,
2552                                                     &dev->dev_tgt_dev_list,
2553                                                     dev_tgt_dev_list_entry) {
2554                                         clear_bit(SCST_TGT_DEV_RESERVED,
2555                                                 &tgt_dev_tmp->tgt_dev_flags);
2556                                 }
2557                                 dev->dev_reserved = 0;
2558                                 spin_unlock_bh(&dev->dev_lock);
2559                         }
2560                 }
2561
2562                 /* Check for MODE PARAMETERS CHANGED UA */
2563                 if ((cmd->dev->scsi_dev != NULL) &&
2564                     (cmd->status == SAM_STAT_CHECK_CONDITION) &&
2565                     SCST_SENSE_VALID(cmd->sense) &&
2566                     scst_is_ua_sense(cmd->sense) &&
2567                     scst_analyze_sense(cmd->sense, cmd->sense_bufflen,
2568                                         SCST_SENSE_ASCx_VALID,
2569                                         0, 0x2a, 0x01)) {
2570                         TRACE(TRACE_SCSI, "MODE PARAMETERS CHANGED UA (lun "
2571                                 "%lld)", (long long unsigned int)cmd->lun);
2572                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2573                         goto out;
2574                 }
2575         }
2576
2577         cmd->state = SCST_CMD_STATE_DEV_DONE;
2578
2579 out:
2580         TRACE_EXIT_RES(res);
2581         return res;
2582 }
2583
2584 static int scst_mode_select_checks(struct scst_cmd *cmd)
2585 {
2586         int res = SCST_CMD_STATE_RES_CONT_SAME;
2587         int atomic = scst_cmd_atomic(cmd);
2588
2589         TRACE_ENTRY();
2590
2591         if (likely(scsi_status_is_good(cmd->status))) {
2592                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2593                     (cmd->cdb[0] == MODE_SELECT_10) ||
2594                     (cmd->cdb[0] == LOG_SELECT))) {
2595                         struct scst_device *dev = cmd->dev;
2596                         uint8_t sense_buffer[SCST_STANDARD_SENSE_LEN];
2597
2598                         if (atomic && (dev->scsi_dev != NULL)) {
2599                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2600                                         "context required");
2601                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2602                                 goto out;
2603                         }
2604
2605                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2606                                 "setting the SELECT UA (lun=%lld)",
2607                                 (long long unsigned int)cmd->lun);
2608
2609                         spin_lock_bh(&dev->dev_lock);
2610                         if (cmd->cdb[0] == LOG_SELECT) {
2611                                 scst_set_sense(sense_buffer,
2612                                         sizeof(sense_buffer),
2613                                         dev->d_sense,
2614                                         UNIT_ATTENTION, 0x2a, 0x02);
2615                         } else {
2616                                 scst_set_sense(sense_buffer,
2617                                         sizeof(sense_buffer),
2618                                         dev->d_sense,
2619                                         UNIT_ATTENTION, 0x2a, 0x01);
2620                         }
2621                         scst_dev_check_set_local_UA(dev, cmd, sense_buffer,
2622                                 sizeof(sense_buffer));
2623                         spin_unlock_bh(&dev->dev_lock);
2624
2625                         if (dev->scsi_dev != NULL)
2626                                 scst_obtain_device_parameters(dev);
2627                 }
2628         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) &&
2629                     SCST_SENSE_VALID(cmd->sense) &&
2630                     scst_is_ua_sense(cmd->sense) &&
2631                      /* mode parameters changed */
2632                     (scst_analyze_sense(cmd->sense, cmd->sense_bufflen,
2633                                         SCST_SENSE_ASCx_VALID,
2634                                         0, 0x2a, 0x01) ||
2635                      scst_analyze_sense(cmd->sense, cmd->sense_bufflen,
2636                                         SCST_SENSE_ASC_VALID,
2637                                         0, 0x29, 0) /* reset */ ||
2638                      scst_analyze_sense(cmd->sense, cmd->sense_bufflen,
2639                                         SCST_SENSE_ASC_VALID,
2640                                         0, 0x28, 0) /* medium changed */ ||
2641                      /* cleared by another ini (just in case) */
2642                      scst_analyze_sense(cmd->sense, cmd->sense_bufflen,
2643                                         SCST_SENSE_ASC_VALID,
2644                                         0, 0x2F, 0))) {
2645                 if (atomic) {
2646                         TRACE_DBG("Possible parameters changed UA %x: "
2647                                 "thread context required", cmd->sense[12]);
2648                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2649                         goto out;
2650                 }
2651
2652                 TRACE(TRACE_SCSI, "Possible parameters changed UA %x "
2653                         "(lun %lld): getting new parameters", cmd->sense[12],
2654                         (long long unsigned int)cmd->lun);
2655
2656                 scst_obtain_device_parameters(cmd->dev);
2657         } else
2658                 sBUG();
2659
2660         cmd->state = SCST_CMD_STATE_DEV_DONE;
2661
2662 out:
2663         TRACE_EXIT_HRES(res);
2664         return res;
2665 }
2666
2667 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2668 {
2669         if (likely(cmd->sn_set))
2670                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2671
2672         scst_make_deferred_commands_active(cmd->tgt_dev);
2673 }
2674
2675 static int scst_dev_done(struct scst_cmd *cmd)
2676 {
2677         int res = SCST_CMD_STATE_RES_CONT_SAME;
2678         int state;
2679         struct scst_device *dev = cmd->dev;
2680
2681         TRACE_ENTRY();
2682
2683         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2684
2685         if (likely(!scst_is_cmd_local(cmd)) &&
2686             likely(dev->handler->dev_done != NULL)) {
2687                 int rc;
2688
2689                 if (unlikely(!dev->handler->dev_done_atomic &&
2690                              scst_cmd_atomic(cmd))) {
2691                         /*
2692                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2693                          * optimization.
2694                          */
2695                         TRACE_DBG("Dev handler %s dev_done() needs thread "
2696                               "context, rescheduling", dev->handler->name);
2697                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2698                         goto out;
2699                 }
2700
2701                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2702                       dev->handler->name, cmd);
2703                 rc = dev->handler->dev_done(cmd);
2704                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2705                       dev->handler->name, rc);
2706                 if (rc != SCST_CMD_STATE_DEFAULT)
2707                         state = rc;
2708         }
2709
2710         switch (state) {
2711         case SCST_CMD_STATE_PRE_XMIT_RESP:
2712         case SCST_CMD_STATE_DEV_PARSE:
2713         case SCST_CMD_STATE_PRE_PARSE:
2714         case SCST_CMD_STATE_PREPARE_SPACE:
2715         case SCST_CMD_STATE_RDY_TO_XFER:
2716         case SCST_CMD_STATE_TGT_PRE_EXEC:
2717         case SCST_CMD_STATE_SEND_FOR_EXEC:
2718         case SCST_CMD_STATE_LOCAL_EXEC:
2719         case SCST_CMD_STATE_REAL_EXEC:
2720         case SCST_CMD_STATE_PRE_DEV_DONE:
2721         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2722         case SCST_CMD_STATE_DEV_DONE:
2723         case SCST_CMD_STATE_XMIT_RESP:
2724         case SCST_CMD_STATE_FINISHED:
2725         case SCST_CMD_STATE_FINISHED_INTERNAL:
2726                 cmd->state = state;
2727                 break;
2728
2729         case SCST_CMD_STATE_NEED_THREAD_CTX:
2730                 TRACE_DBG("Dev handler %s dev_done() requested "
2731                       "thread context, rescheduling",
2732                       dev->handler->name);
2733                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2734                 break;
2735
2736         default:
2737                 if (state >= 0) {
2738                         PRINT_ERROR("Dev handler %s dev_done() returned "
2739                                 "invalid cmd state %d",
2740                                 dev->handler->name, state);
2741                 } else {
2742                         PRINT_ERROR("Dev handler %s dev_done() returned "
2743                                 "error %d", dev->handler->name,
2744                                 state);
2745                 }
2746                 scst_set_cmd_error(cmd,
2747                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2748                 scst_set_cmd_abnormal_done_state(cmd);
2749                 break;
2750         }
2751
2752         if (cmd->needs_unblocking)
2753                 scst_unblock_dev_cmd(cmd);
2754
2755         if (likely(cmd->dec_on_dev_needed))
2756                 scst_dec_on_dev_cmd(cmd);
2757
2758         if (cmd->inc_expected_sn_on_done && cmd->sent_for_exec)
2759                 scst_inc_check_expected_sn(cmd);
2760
2761         if (unlikely(cmd->internal))
2762                 cmd->state = SCST_CMD_STATE_FINISHED_INTERNAL;
2763
2764 out:
2765         TRACE_EXIT_HRES(res);
2766         return res;
2767 }
2768
2769 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2770 {
2771         int res;
2772
2773         TRACE_ENTRY();
2774
2775         EXTRACHECKS_BUG_ON(cmd->internal);
2776
2777 #ifdef CONFIG_SCST_DEBUG_TM
2778         if (cmd->tm_dbg_delayed &&
2779                         !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2780                 if (scst_cmd_atomic(cmd)) {
2781                         TRACE_MGMT_DBG("%s",
2782                                 "DEBUG_TM delayed cmd needs a thread");
2783                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2784                         return res;
2785                 }
2786                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2787                         cmd, cmd->tag);
2788                 schedule_timeout_uninterruptible(HZ);
2789         }
2790 #endif
2791
2792         if (likely(cmd->tgt_dev != NULL)) {
2793                 atomic_dec(&cmd->tgt_dev->tgt_dev_cmd_count);
2794                 atomic_dec(&cmd->dev->dev_cmd_count);
2795                 /* If expected values not set, expected direction is UNKNOWN */
2796                 if (cmd->expected_data_direction & SCST_DATA_WRITE)
2797                         atomic_dec(&cmd->dev->write_cmd_count);
2798
2799                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2800                         scst_on_hq_cmd_response(cmd);
2801
2802                 if (unlikely(!cmd->sent_for_exec)) {
2803                         TRACE_SN("cmd %p was not sent to mid-lev"
2804                                 " (sn %ld, set %d)",
2805                                 cmd, cmd->sn, cmd->sn_set);
2806                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2807                         cmd->sent_for_exec = 1;
2808                 }
2809         }
2810
2811         /*
2812          * If we don't remove cmd from the search list here, before
2813          * submitting it for transmittion, we will have a race, when for
2814          * some reason cmd's release is delayed after transmittion and
2815          * initiator sends cmd with the same tag => it is possible that
2816          * a wrong cmd will be found by find() functions.
2817          */
2818         spin_lock_irq(&cmd->sess->sess_list_lock);
2819         list_del(&cmd->search_cmd_list_entry);
2820         spin_unlock_irq(&cmd->sess->sess_list_lock);
2821
2822         cmd->done = 1;
2823         smp_mb(); /* to sync with scst_abort_cmd() */
2824
2825         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2826                 scst_xmit_process_aborted_cmd(cmd);
2827
2828         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2829                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu),"
2830                                 " skipping",
2831                                 cmd, (long long unsigned int)cmd->tag);
2832                 cmd->state = SCST_CMD_STATE_FINISHED;
2833                 res = SCST_CMD_STATE_RES_CONT_SAME;
2834                 goto out;
2835         }
2836
2837         if (cmd->tgt_data_buf_alloced && cmd->dh_data_buf_alloced &&
2838             (cmd->data_direction & SCST_DATA_READ))
2839                 scst_copy_sg(cmd, SCST_SG_COPY_TO_TARGET);
2840
2841         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2842         res = SCST_CMD_STATE_RES_CONT_SAME;
2843
2844 out:
2845 #ifdef CONFIG_SCST_MEASURE_LATENCY
2846         {
2847                 struct timespec ts;
2848                 uint64_t finish, scst_time, proc_time;
2849                 struct scst_session *sess = cmd->sess;
2850
2851                 getnstimeofday(&ts);
2852                 finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2853
2854                 spin_lock_bh(&sess->meas_lock);
2855
2856                 scst_time = cmd->pre_exec_finish - cmd->start;
2857                 scst_time += finish - cmd->post_exec_start;
2858                 proc_time = finish - cmd->start;
2859
2860                 sess->scst_time += scst_time;
2861                 sess->processing_time += proc_time;
2862                 sess->processed_cmds++;
2863
2864                 spin_unlock_bh(&sess->meas_lock);
2865
2866                 TRACE_DBG("cmd %p (sess %p): finish %lld (tv_sec %ld, "
2867                         "tv_nsec %ld), scst_time %lld, proc_time %lld",
2868                         cmd, sess, finish, ts.tv_sec, ts.tv_nsec, scst_time,
2869                         proc_time);
2870         }
2871 #endif
2872         TRACE_EXIT_HRES(res);
2873         return res;
2874 }
2875
2876 static int scst_xmit_response(struct scst_cmd *cmd)
2877 {
2878         int res, rc;
2879
2880         TRACE_ENTRY();
2881
2882         EXTRACHECKS_BUG_ON(cmd->internal);
2883
2884         if (unlikely(!cmd->tgtt->xmit_response_atomic &&
2885                      scst_cmd_atomic(cmd))) {
2886                 /*
2887                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2888                  * optimization.
2889                  */
2890                 TRACE_DBG("Target driver %s xmit_response() needs thread "
2891                               "context, rescheduling", cmd->tgtt->name);
2892                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2893                 goto out;
2894         }
2895
2896         while (1) {
2897                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2898
2899                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2900                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2901
2902                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2903
2904 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2905                 if (trace_flag & TRACE_SND_BOT) {
2906                         int i;
2907                         struct scatterlist *sg;
2908                         if (cmd->tgt_sg != NULL)
2909                                 sg = cmd->tgt_sg;
2910                         else
2911                                 sg = cmd->sg;
2912                         if (sg != NULL) {
2913                                 TRACE(TRACE_SND_BOT, "Xmitting data for cmd %p "
2914                                         "(sg_cnt %d, sg %p, sg[0].page %p)",
2915                                         cmd, cmd->tgt_sg_cnt, sg,
2916                                         (void *)sg_page(&sg[0]));
2917                                 for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
2918                                         PRINT_BUFF_FLAG(TRACE_SND_BOT,
2919                                                 "Xmitting sg", sg_virt(&sg[i]),
2920                                                 sg[i].length);
2921                                 }
2922                         }
2923                 }
2924 #endif
2925
2926 #ifdef CONFIG_SCST_DEBUG_RETRY
2927                 if (((scst_random() % 100) == 77))
2928                         rc = SCST_TGT_RES_QUEUE_FULL;
2929                 else
2930 #endif
2931                         rc = cmd->tgtt->xmit_response(cmd);
2932                 TRACE_DBG("xmit_response() returned %d", rc);
2933
2934                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2935                         goto out;
2936
2937                 /* Restore the previous state */
2938                 cmd->state = SCST_CMD_STATE_XMIT_RESP;
2939
2940                 switch (rc) {
2941                 case SCST_TGT_RES_QUEUE_FULL:
2942                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2943                                 break;
2944                         else
2945                                 continue;
2946
2947                 case SCST_TGT_RES_NEED_THREAD_CTX:
2948                         TRACE_DBG("Target driver %s xmit_response() "
2949                               "requested thread context, rescheduling",
2950                               cmd->tgtt->name);
2951                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2952                         break;
2953
2954                 default:
2955                         goto out_error;
2956                 }
2957                 break;
2958         }
2959
2960 out:
2961         /* Caution: cmd can be already dead here */
2962         TRACE_EXIT_HRES(res);
2963         return res;
2964
2965 out_error:
2966         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2967                 PRINT_ERROR("Target driver %s xmit_response() returned "
2968                         "fatal error", cmd->tgtt->name);
2969         } else {
2970                 PRINT_ERROR("Target driver %s xmit_response() returned "
2971                         "invalid value %d", cmd->tgtt->name, rc);
2972         }
2973         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2974         cmd->state = SCST_CMD_STATE_FINISHED;
2975         res = SCST_CMD_STATE_RES_CONT_SAME;
2976         goto out;
2977 }
2978
2979 void scst_tgt_cmd_done(struct scst_cmd *cmd,
2980         enum scst_exec_context pref_context)
2981 {
2982         TRACE_ENTRY();
2983
2984         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2985
2986         cmd->state = SCST_CMD_STATE_FINISHED;
2987         scst_proccess_redirect_cmd(cmd, pref_context, 1);
2988
2989         TRACE_EXIT();
2990         return;
2991 }
2992 EXPORT_SYMBOL(scst_tgt_cmd_done);
2993
2994 static int scst_finish_cmd(struct scst_cmd *cmd)
2995 {
2996         int res;
2997
2998         TRACE_ENTRY();
2999
3000         atomic_dec(&cmd->sess->sess_cmd_count);
3001
3002         cmd->finished = 1;
3003         smp_mb(); /* to sync with scst_abort_cmd() */
3004
3005         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
3006                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
3007                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
3008                         atomic_read(&scst_cmd_count));
3009
3010                 scst_finish_cmd_mgmt(cmd);
3011         }
3012
3013         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
3014                 if ((cmd->tgt_dev != NULL) &&
3015                     scst_is_ua_sense(cmd->sense)) {
3016                         /* This UA delivery failed, so requeue it */
3017                         TRACE_MGMT_DBG("Requeuing UA for delivery failed cmd "
3018                                 "%p", cmd);
3019                         scst_check_set_UA(cmd->tgt_dev, cmd->sense,
3020                                 cmd->sense_bufflen, SCST_SET_UA_FLAG_AT_HEAD);
3021                 }
3022         }
3023
3024         __scst_cmd_put(cmd);
3025
3026         res = SCST_CMD_STATE_RES_CONT_NEXT;
3027
3028         TRACE_EXIT_HRES(res);
3029         return res;
3030 }
3031
3032 /*
3033  * No locks, but it must be externally serialized (see comment for
3034  * scst_cmd_init_done() in scst.h)
3035  */
3036 static void scst_cmd_set_sn(struct scst_cmd *cmd)
3037 {
3038         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
3039         unsigned long flags;
3040
3041         TRACE_ENTRY();
3042
3043         if (scst_is_implicit_hq(cmd)) {
3044                 TRACE_SN("Implicit HQ cmd %p", cmd);
3045                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
3046         }
3047
3048         EXTRACHECKS_BUG_ON(cmd->sn_set || cmd->hq_cmd_inced);
3049
3050         /* Optimized for lockless fast path */
3051
3052         scst_check_debug_sn(cmd);
3053
3054         if (cmd->dev->queue_alg ==
3055                         SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) {
3056                 /*
3057                  * Not the best way, but well enough until there will be a
3058                  * possibility to specify queue type during pass-through
3059                  * commands submission.
3060                  */
3061                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
3062         }
3063
3064         switch (cmd->queue_type) {
3065         case SCST_CMD_QUEUE_SIMPLE:
3066         case SCST_CMD_QUEUE_UNTAGGED:
3067 #if 0 /* left for future performance investigations */
3068                 if (scst_cmd_is_expected_set(cmd)) {
3069                         if ((cmd->expected_data_direction == SCST_DATA_READ) &&
3070                             (atomic_read(&cmd->dev->write_cmd_count) == 0))
3071                                 goto ordered;
3072                 } else
3073                         goto ordered;
3074 #endif
3075                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
3076                         /*
3077                          * atomic_inc_return() implies memory barrier to sync
3078                          * with scst_inc_expected_sn()
3079                          */
3080                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
3081                                 tgt_dev->curr_sn++;
3082                                 TRACE_SN("Incremented curr_sn %ld",
3083                                         tgt_dev->curr_sn);
3084                         }
3085                         cmd->sn_slot = tgt_dev->cur_sn_slot;
3086                         cmd->sn = tgt_dev->curr_sn;
3087
3088                         tgt_dev->prev_cmd_ordered = 0;
3089                 } else {
3090                         TRACE(TRACE_MINOR, "***WARNING*** Not enough SN slots "
3091                                 "%zd", ARRAY_SIZE(tgt_dev->sn_slots));
3092                         goto ordered;
3093                 }
3094                 break;
3095
3096         case SCST_CMD_QUEUE_ORDERED:
3097                 TRACE_SN("ORDERED cmd %p (op %x)", cmd, cmd->cdb[0]);
3098 ordered:
3099                 if (!tgt_dev->prev_cmd_ordered) {
3100                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
3101                         if (tgt_dev->num_free_sn_slots >= 0) {
3102                                 tgt_dev->num_free_sn_slots--;
3103                                 if (tgt_dev->num_free_sn_slots >= 0) {
3104                                         int i = 0;
3105                                         /* Commands can finish in any order, so
3106                                          * we don't know which slot is empty.
3107                                          */
3108                                         while (1) {
3109                                                 tgt_dev->cur_sn_slot++;
3110                                                 if (tgt_dev->cur_sn_slot ==
3111                                                       tgt_dev->sn_slots + ARRAY_SIZE(tgt_dev->sn_slots))
3112                                                         tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
3113
3114                                                 if (atomic_read(tgt_dev->cur_sn_slot) == 0)
3115                                                         break;
3116
3117                                                 i++;
3118                                                 sBUG_ON(i == ARRAY_SIZE(tgt_dev->sn_slots));
3119                                         }
3120                                         TRACE_SN("New cur SN slot %zd",
3121                                                 tgt_dev->cur_sn_slot -
3122                                                 tgt_dev->sn_slots);
3123                                 }
3124                         }
3125                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
3126                 }
3127                 tgt_dev->prev_cmd_ordered = 1;
3128                 tgt_dev->curr_sn++;
3129                 cmd->sn = tgt_dev->curr_sn;
3130                 break;
3131
3132         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
3133                 TRACE_SN("HQ cmd %p (op %x)", cmd, cmd->cdb[0]);
3134                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
3135                 tgt_dev->hq_cmd_count++;
3136                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
3137                 cmd->hq_cmd_inced = 1;
3138                 goto out;
3139
3140         default:
3141                 sBUG();
3142         }
3143
3144         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
3145                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
3146                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
3147                 atomic_read(tgt_dev->cur_sn_slot),
3148                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
3149                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
3150
3151         cmd->sn_set = 1;
3152
3153 out:
3154         TRACE_EXIT();
3155         return;
3156 }
3157
3158 /*
3159  * Returns 0 on success, > 0 when we need to wait for unblock,
3160  * < 0 if there is no device (lun) or device type handler.
3161  *
3162  * No locks, but might be on IRQ, protection is done by the
3163  * suspended activity.
3164  */
3165 static int scst_translate_lun(struct scst_cmd *cmd)
3166 {
3167         struct scst_tgt_dev *tgt_dev = NULL;
3168         int res;
3169
3170         TRACE_ENTRY();
3171
3172         /* See comment about smp_mb() in scst_suspend_activity() */
3173         __scst_get(1);
3174
3175         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
3176                 struct list_head *sess_tgt_dev_list_head =
3177                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
3178                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %lld)", cmd,
3179                         (long long unsigned int)cmd->lun);
3180                 res = -1;
3181                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3182                                 sess_tgt_dev_list_entry) {
3183                         if (tgt_dev->lun == cmd->lun) {
3184                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
3185
3186                                 if (unlikely(tgt_dev->dev->handler ==
3187                                                 &scst_null_devtype)) {
3188                                         PRINT_INFO("Dev handler for device "
3189                                           "%lld is NULL, the device will not "
3190                                           "be visible remotely",
3191                                            (long long unsigned int)cmd->lun);
3192                                         break;
3193                                 }
3194
3195                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
3196                                 cmd->tgt_dev = tgt_dev;
3197                                 cmd->dev = tgt_dev->dev;
3198
3199                                 res = 0;
3200                                 break;
3201                         }
3202                 }
3203                 if (res != 0) {
3204                         TRACE(TRACE_MINOR,
3205                                 "tgt_dev for lun %lld not found, command to "
3206                                 "unexisting LU?",
3207                                 (long long unsigned int)cmd->lun);
3208                         __scst_put();
3209                 }
3210         } else {
3211                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3212                 __scst_put();
3213                 res = 1;
3214         }
3215
3216         TRACE_EXIT_RES(res);
3217         return res;
3218 }
3219
3220 /*
3221  * No locks, but might be on IRQ
3222  *
3223  * Returns 0 on success, > 0 when we need to wait for unblock,
3224  * < 0 if there is no device (lun) or device type handler.
3225  */
3226 static int __scst_init_cmd(struct scst_cmd *cmd)
3227 {
3228         int res = 0;
3229
3230         TRACE_ENTRY();
3231
3232         res = scst_translate_lun(cmd);
3233         if (likely(res == 0)) {
3234                 int cnt;
3235                 bool failure = false;
3236
3237                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
3238
3239                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
3240                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
3241                         TRACE(TRACE_MGMT_MINOR,
3242                                 "Too many pending commands (%d) in "
3243                                 "session, returning BUSY to initiator \"%s\"",
3244                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
3245                                   "Anonymous" : cmd->sess->initiator_name);
3246                         failure = true;
3247                 }
3248
3249                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
3250                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
3251                         if (!failure) {
3252                                 TRACE(TRACE_MGMT_MINOR,
3253                                         "Too many pending device "
3254                                         "commands (%d), returning BUSY to "
3255                                         "initiator \"%s\"", cnt,
3256                                         (cmd->sess->initiator_name[0] == '\0') ?
3257                                                 "Anonymous" :
3258                                                 cmd->sess->initiator_name);
3259                                 failure = true;
3260                         }
3261                 }
3262
3263                 /* If expected values not set, expected direction is UNKNOWN */
3264                 if (cmd->expected_data_direction & SCST_DATA_WRITE)
3265                         atomic_inc(&cmd->dev->write_cmd_count);
3266
3267                 if (unlikely(failure))
3268                         goto out_busy;
3269
3270                 if (!cmd->set_sn_on_restart_cmd)
3271                         scst_cmd_set_sn(cmd);
3272         } else if (res < 0) {
3273                 TRACE_DBG("Finishing cmd %p", cmd);
3274                 scst_set_cmd_error(cmd,
3275                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
3276                 scst_set_cmd_abnormal_done_state(cmd);
3277         } else
3278                 goto out;
3279
3280 out:
3281         TRACE_EXIT_RES(res);
3282         return res;
3283
3284 out_busy:
3285         scst_set_busy(cmd);
3286         scst_set_cmd_abnormal_done_state(cmd);
3287         goto out;
3288 }
3289
3290 /* Called under scst_init_lock and IRQs disabled */
3291 static void scst_do_job_init(void)
3292         __releases(&scst_init_lock)
3293         __acquires(&scst_init_lock)
3294 {
3295         struct scst_cmd *cmd;
3296         int susp;
3297
3298         TRACE_ENTRY();
3299
3300 restart:
3301         /*
3302          * There is no need for read barrier here, because we don't care where
3303          * this check will be done.
3304          */
3305         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
3306         if (scst_init_poll_cnt > 0)
3307                 scst_init_poll_cnt--;
3308
3309         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
3310                 int rc;
3311                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
3312                         continue;
3313                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
3314                         spin_unlock_irq(&scst_init_lock);
3315                         rc = __scst_init_cmd(cmd);
3316                         spin_lock_irq(&scst_init_lock);
3317                         if (rc > 0) {
3318                                 TRACE_MGMT_DBG("%s",
3319                                         "FLAG SUSPENDED set, restarting");
3320                                 goto restart;
3321                         }
3322                 } else {
3323                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
3324                                        cmd, (long long unsigned int)cmd->tag);
3325                         scst_set_cmd_abnormal_done_state(cmd);
3326                 }
3327
3328                 /*
3329                  * Deleting cmd from init cmd list after __scst_init_cmd()
3330                  * is necessary to keep the check in scst_init_cmd() correct
3331                  * to preserve the commands order.
3332                  *
3333                  * We don't care about the race, when init cmd list is empty
3334                  * and one command detected that it just was not empty, so
3335                  * it's inserting to it, but another command at the same time
3336                  * seeing init cmd list empty and goes directly, because it
3337                  * could affect only commands from the same initiator to the
3338                  * same tgt_dev, but scst_cmd_init_done*() doesn't guarantee
3339                  * the order in case of simultaneous such calls anyway.
3340                  */
3341                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
3342                 smp_wmb(); /* enforce the required order */
3343                 list_del(&cmd->cmd_list_entry);
3344                 spin_unlock(&scst_init_lock);
3345
3346                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
3347                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
3348                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
3349                         list_add(&cmd->cmd_list_entry,
3350                                 &cmd->cmd_lists->active_cmd_list);
3351                 else
3352                         list_add_tail(&cmd->cmd_list_entry,
3353                                 &cmd->cmd_lists->active_cmd_list);
3354                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3355                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
3356
3357                 spin_lock(&scst_init_lock);
3358                 goto restart;
3359         }
3360
3361         /* It isn't really needed, but let's keep it */
3362         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
3363                 goto restart;
3364
3365         TRACE_EXIT();
3366         return;
3367 }
3368
3369 static inline int test_init_cmd_list(void)
3370 {
3371         int res = (!list_empty(&scst_init_cmd_list) &&
3372                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
3373                   unlikely(kthread_should_stop()) ||
3374                   (scst_init_poll_cnt > 0);
3375         return res;
3376 }
3377
3378 int scst_init_thread(void *arg)
3379 {
3380         TRACE_ENTRY();
3381
3382         PRINT_INFO("Init thread started, PID %d", current->pid);
3383
3384         current->flags |= PF_NOFREEZE;
3385
3386         set_user_nice(current, -10);
3387
3388         spin_lock_irq(&scst_init_lock);
3389         while (!kthread_should_stop()) {
3390                 wait_queue_t wait;
3391                 init_waitqueue_entry(&wait, current);
3392
3393                 if (!test_init_cmd_list()) {
3394                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
3395                                                  &wait);
3396                         for (;;) {
3397                                 set_current_state(TASK_INTERRUPTIBLE);
3398                                 if (test_init_cmd_list())
3399                                         break;
3400                                 spin_unlock_irq(&scst_init_lock);
3401                                 schedule();
3402                                 spin_lock_irq(&scst_init_lock);
3403                         }
3404                         set_current_state(TASK_RUNNING);
3405                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
3406                 }
3407                 scst_do_job_init();
3408         }
3409         spin_unlock_irq(&scst_init_lock);
3410
3411         /*
3412          * If kthread_should_stop() is true, we are guaranteed to be
3413          * on the module unload, so scst_init_cmd_list must be empty.
3414          */
3415         sBUG_ON(!list_empty(&scst_init_cmd_list));
3416
3417         PRINT_INFO("Init thread PID %d finished", current->pid);
3418
3419         TRACE_EXIT();
3420         return 0;
3421 }
3422
3423 /* Called with no locks held */
3424 void scst_process_active_cmd(struct scst_cmd *cmd, bool atomic)
3425 {
3426         int res;
3427
3428         TRACE_ENTRY();
3429
3430         EXTRACHECKS_BUG_ON(in_irq() || irqs_disabled());
3431
3432         cmd->atomic = atomic;
3433
3434         TRACE_DBG("cmd %p, atomic %d", cmd, atomic);
3435
3436         do {
3437                 switch (cmd->state) {
3438                 case SCST_CMD_STATE_PRE_PARSE:
3439                         res = scst_pre_parse(cmd);
3440                         EXTRACHECKS_BUG_ON(res ==
3441                                 SCST_CMD_STATE_RES_NEED_THREAD);
3442                         break;
3443
3444                 case SCST_CMD_STATE_DEV_PARSE:
3445                         res = scst_parse_cmd(cmd);
3446                         break;
3447
3448                 case SCST_CMD_STATE_PREPARE_SPACE:
3449                         res = scst_prepare_space(cmd);
3450                         break;
3451
3452                 case SCST_CMD_STATE_RDY_TO_XFER:
3453                         res = scst_rdy_to_xfer(cmd);
3454                         break;
3455
3456                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3457                         res = scst_tgt_pre_exec(cmd);
3458                         break;
3459
3460                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3461                         if (tm_dbg_check_cmd(cmd) != 0) {
3462                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
3463                                 TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
3464                                         "because of TM DBG delay", cmd,
3465                                         (long long unsigned int)cmd->tag);
3466                                 break;
3467                         }
3468                         res = scst_send_for_exec(&cmd);
3469                         /*
3470                          * !! At this point cmd, sess & tgt_dev can already be
3471                          * freed !!
3472                          */
3473                         break;
3474
3475                 case SCST_CMD_STATE_LOCAL_EXEC:
3476                         res = scst_local_exec(cmd);
3477                         /*
3478                          * !! At this point cmd, sess & tgt_dev can already be
3479                          * freed !!
3480                          */
3481                         break;
3482
3483                 case SCST_CMD_STATE_REAL_EXEC:
3484