scst/src/scst_targ.c

   1 /*
   2  *  scst_targ.c
   3  *
   4  *  Copyright (C) 2004 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
   5  *  Copyright (C) 2004 - 2005 Leonid Stoljar
   6  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
   7  *
   8  *  This program is free software; you can redistribute it and/or
   9  *  modify it under the terms of the GNU General Public License
  10  *  as published by the Free Software Foundation, version 2
  11  *  of the License.
  12  *
  13  *  This program is distributed in the hope that it will be useful,
  14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  *  GNU General Public License for more details.
  17  */
  18
  19 #include <linux/init.h>
  20 #include <linux/kernel.h>
  21 #include <linux/errno.h>
  22 #include <linux/list.h>
  23 #include <linux/spinlock.h>
  24 #include <linux/slab.h>
  25 #include <linux/sched.h>
  26 #include <linux/smp_lock.h>
  27 #include <linux/unistd.h>
  28 #include <linux/string.h>
  29 #include <linux/kthread.h>
  30 #include <linux/delay.h>
  31
  32 #include "scst.h"
  33 #include "scst_priv.h"
  34
  35 static void scst_cmd_set_sn(struct scst_cmd *cmd);
  36 static int __scst_init_cmd(struct scst_cmd *cmd);
  37 static void scst_finish_cmd_mgmt(struct scst_cmd *cmd);
  38 static struct scst_cmd *__scst_find_cmd_by_tag(struct scst_session *sess,
  39         uint64_t tag);
  40 static void scst_proccess_redirect_cmd(struct scst_cmd *cmd,
  41         enum scst_exec_context context, int check_retries);
  42
  43 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
  44 {
  45         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
  46         unsigned long flags;
  47
  48         spin_lock_irqsave(&t->tasklet_lock, flags);
  49         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
  50                 smp_processor_id());
  51         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
  52         spin_unlock_irqrestore(&t->tasklet_lock, flags);
  53
  54         tasklet_schedule(&t->tasklet);
  55 }
  56
  57 /*
  58  * Must not be called in parallel with scst_unregister_session_ex() for the
  59  * same sess
  60  */
  61 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
  62                              const uint8_t *lun, int lun_len,
  63                              const uint8_t *cdb, int cdb_len, int atomic)
  64 {
  65         struct scst_cmd *cmd;
  66
  67         TRACE_ENTRY();
  68
  69 #ifdef CONFIG_SCST_EXTRACHECKS
  70         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
  71                 PRINT_CRIT_ERROR("%s",
  72                         "New cmd while shutting down the session");
  73                 sBUG();
  74         }
  75 #endif
  76
  77         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
  78         if (cmd == NULL)
  79                 goto out;
  80
  81         cmd->sess = sess;
  82         cmd->tgt = sess->tgt;
  83         cmd->tgtt = sess->tgt->tgtt;
  84
  85         /*
  86          * For both wrong lun and CDB defer the error reporting for
  87          * scst_cmd_init_done()
  88          */
  89
  90         cmd->lun = scst_unpack_lun(lun, lun_len);
  91
  92         if (cdb_len <= SCST_MAX_CDB_SIZE) {
  93                 memcpy(cmd->cdb, cdb, cdb_len);
  94                 cmd->cdb_len = cdb_len;
  95         }
  96
  97         TRACE_DBG("cmd %p, sess %p", cmd, sess);
  98         scst_sess_get(sess);
  99
 100 out:
 101         TRACE_EXIT();
 102         return cmd;
 103 }
 104 EXPORT_SYMBOL(scst_rx_cmd);
 105
 106 /*
 107  * No locks, but might be on IRQ. Returns 0 on success, <0 if processing of
 108  * this command should be stopped.
 109  */
 110 static int scst_init_cmd(struct scst_cmd *cmd, enum scst_exec_context *context)
 111 {
 112         int rc, res = 0;
 113
 114         TRACE_ENTRY();
 115
 116         /* See the comment in scst_do_job_init() */
 117         if (unlikely(!list_empty(&scst_init_cmd_list))) {
 118                 TRACE_MGMT_DBG("%s", "init cmd list busy");
 119                 goto out_redirect;
 120         }
 121         /*
 122          * Memory barrier isn't necessary here, because CPU appears to
 123          * be self-consistent and we don't care about the race, described
 124          * in comment in scst_do_job_init().
 125          */
 126
 127         rc = __scst_init_cmd(cmd);
 128         if (unlikely(rc > 0))
 129                 goto out_redirect;
 130         else if (unlikely(rc != 0))
 131                 goto out;
 132
 133         /* Small context optimization */
 134         if (((*context == SCST_CONTEXT_TASKLET) ||
 135              (*context == SCST_CONTEXT_DIRECT_ATOMIC) ||
 136              ((*context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd))) &&
 137               scst_cmd_is_expected_set(cmd)) {
 138                 if (cmd->expected_data_direction == SCST_DATA_WRITE) {
 139                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
 140                                         &cmd->tgt_dev->tgt_dev_flags))
 141                                 *context = SCST_CONTEXT_THREAD;
 142                 } else {
 143                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
 144                                         &cmd->tgt_dev->tgt_dev_flags))
 145                                 *context = SCST_CONTEXT_THREAD;
 146                 }
 147         }
 148
 149 out:
 150         TRACE_EXIT_RES(res);
 151         return res;
 152
 153 out_redirect:
 154         if (cmd->preprocessing_only) {
 155                 /*
 156                  * Poor man solution for single threaded targets, where
 157                  * blocking receiver at least sometimes means blocking all.
 158                  */
 159                 sBUG_ON(*context != SCST_CONTEXT_DIRECT);
 160                 scst_set_busy(cmd);
 161                 scst_set_cmd_abnormal_done_state(cmd);
 162                 /* Keep initiator away from too many BUSY commands */
 163                 msleep(50);
 164         } else {
 165                 unsigned long flags;
 166                 spin_lock_irqsave(&scst_init_lock, flags);
 167                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
 168                         "%d)", cmd, atomic_read(&scst_cmd_count));
 169                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
 170                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
 171                         scst_init_poll_cnt++;
 172                 spin_unlock_irqrestore(&scst_init_lock, flags);
 173                 wake_up(&scst_init_cmd_list_waitQ);
 174                 res = -1;
 175         }
 176         goto out;
 177 }
 178
 179 #ifdef CONFIG_SCST_MEASURE_LATENCY
 180 static inline uint64_t scst_sec_to_nsec(time_t sec)
 181 {
 182         return (uint64_t)sec * 1000000000;
 183 }
 184 #endif
 185
 186 void scst_cmd_init_done(struct scst_cmd *cmd,
 187         enum scst_exec_context pref_context)
 188 {
 189         unsigned long flags;
 190         struct scst_session *sess = cmd->sess;
 191         int rc;
 192
 193         TRACE_ENTRY();
 194
 195 #ifdef CONFIG_SCST_MEASURE_LATENCY
 196         {
 197                 struct timespec ts;
 198                 getnstimeofday(&ts);
 199                 cmd->start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
 200                 TRACE_DBG("cmd %p (sess %p): start %lld (tv_sec %ld, "
 201                         "tv_nsec %ld)", cmd, sess, cmd->start, ts.tv_sec,
 202                         ts.tv_nsec);
 203         }
 204 #endif
 205
 206         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
 207         TRACE(TRACE_SCSI, "tag=%llu, lun=%lld, CDB len=%d, queue_type=%x "
 208                 "(cmd %p)", (long long unsigned int)cmd->tag,
 209                 (long long unsigned int)cmd->lun, cmd->cdb_len,
 210                 cmd->queue_type, cmd);
 211         PRINT_BUFF_FLAG(TRACE_SCSI|TRACE_RCV_BOT, "Recieving CDB",
 212                 cmd->cdb, cmd->cdb_len);
 213
 214 #ifdef CONFIG_SCST_EXTRACHECKS
 215         if (unlikely((in_irq() || irqs_disabled())) &&
 216             ((pref_context == SCST_CONTEXT_DIRECT) ||
 217              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
 218                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
 219                         "SCST_CONTEXT_THREAD instead\n", pref_context,
 220                         cmd->tgtt->name);
 221                 pref_context = SCST_CONTEXT_THREAD;
 222         }
 223 #endif
 224
 225         atomic_inc(&sess->sess_cmd_count);
 226
 227         spin_lock_irqsave(&sess->sess_list_lock, flags);
 228
 229         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
 230                 /*
 231                  * We have to always keep command in the search list from the
 232                  * very beginning, because otherwise it can be missed during
 233                  * TM processing. This check is needed because there might be
 234                  * old, i.e. deferred, commands and new, i.e. just coming, ones.
 235                  */
 236                 if (cmd->search_cmd_list_entry.next == NULL)
 237                         list_add_tail(&cmd->search_cmd_list_entry,
 238                                 &sess->search_cmd_list);
 239                 switch (sess->init_phase) {
 240                 case SCST_SESS_IPH_SUCCESS:
 241                         break;
 242                 case SCST_SESS_IPH_INITING:
 243                         TRACE_DBG("Adding cmd %p to init deferred cmd list",
 244                                   cmd);
 245                         list_add_tail(&cmd->cmd_list_entry,
 246                                 &sess->init_deferred_cmd_list);
 247                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
 248                         goto out;
 249                 case SCST_SESS_IPH_FAILED:
 250                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
 251                         scst_set_busy(cmd);
 252                         scst_set_cmd_abnormal_done_state(cmd);
 253                         goto active;
 254                 default:
 255                         sBUG();
 256                 }
 257         } else
 258                 list_add_tail(&cmd->search_cmd_list_entry,
 259                               &sess->search_cmd_list);
 260
 261         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
 262
 263         if (unlikely(cmd->lun == NO_SUCH_LUN)) {
 264                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
 265                 scst_set_cmd_error(cmd,
 266                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
 267                 scst_set_cmd_abnormal_done_state(cmd);
 268                 goto active;
 269         }
 270
 271         if (unlikely(cmd->cdb_len == 0)) {
 272                 PRINT_ERROR("Wrong CDB len %d, finishing cmd", 0);
 273                 scst_set_cmd_error(cmd,
 274                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
 275                 scst_set_cmd_abnormal_done_state(cmd);
 276                 goto active;
 277         }
 278
 279         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
 280                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
 281                 scst_set_cmd_error(cmd,
 282                         SCST_LOAD_SENSE(scst_sense_invalid_message));
 283                 scst_set_cmd_abnormal_done_state(cmd);
 284                 goto active;
 285         }
 286
 287         cmd->state = SCST_CMD_STATE_INIT;
 288         /* cmd must be inited here to preserve the order */
 289         rc = scst_init_cmd(cmd, &pref_context);
 290         if (unlikely(rc < 0))
 291                 goto out;
 292
 293 active:
 294         /* Here cmd must not be in any cmd list, no locks */
 295         switch (pref_context) {
 296         case SCST_CONTEXT_TASKLET:
 297                 scst_schedule_tasklet(cmd);
 298                 break;
 299
 300         case SCST_CONTEXT_DIRECT:
 301                 scst_process_active_cmd(cmd, false);
 302                 /* For *NEED_THREAD wake_up() is already done */
 303                 break;
 304
 305         case SCST_CONTEXT_DIRECT_ATOMIC:
 306                 scst_process_active_cmd(cmd, true);
 307                 /* For *NEED_THREAD wake_up() is already done */
 308                 break;
 309
 310         default:
 311                 PRINT_ERROR("Context %x is undefined, using the thread one",
 312                         pref_context);
 313                 /* go through */
 314         case SCST_CONTEXT_THREAD:
 315                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
 316                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
 317                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
 318                         list_add(&cmd->cmd_list_entry,
 319                                 &cmd->cmd_lists->active_cmd_list);
 320                 else
 321                         list_add_tail(&cmd->cmd_list_entry,
 322                                 &cmd->cmd_lists->active_cmd_list);
 323                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
 324                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
 325                 break;
 326         }
 327
 328 out:
 329         TRACE_EXIT();
 330         return;
 331 }
 332 EXPORT_SYMBOL(scst_cmd_init_done);
 333
 334 static int scst_pre_parse(struct scst_cmd *cmd)
 335 {
 336         int res = SCST_CMD_STATE_RES_CONT_SAME;
 337         struct scst_device *dev = cmd->dev;
 338         int rc;
 339
 340         TRACE_ENTRY();
 341
 342         cmd->inc_expected_sn_on_done = dev->handler->exec_sync ||
 343              (!dev->has_own_order_mgmt &&
 344               (dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER ||
 345                cmd->queue_type == SCST_CMD_QUEUE_ORDERED));
 346
 347         /*
 348          * Expected transfer data supplied by the SCSI transport via the
 349          * target driver are untrusted, so we prefer to fetch them from CDB.
 350          * Additionally, not all transports support supplying the expected
 351          * transfer data.
 352          */
 353
 354         rc = scst_get_cdb_info(cmd);
 355         if (unlikely(rc != 0)) {
 356                 if (rc > 0) {
 357                         PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 358                         goto out_xmit;
 359                 }
 360                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
 361                         "Should you update scst_scsi_op_table?",
 362                         cmd->cdb[0], dev->handler->name);
 363                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 364 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
 365                 if (scst_cmd_is_expected_set(cmd)) {
 366                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
 367                                 "direction %d, transfer_len %d",
 368                                 cmd->expected_data_direction,
 369                                 cmd->expected_transfer_len);
 370                         cmd->data_direction = cmd->expected_data_direction;
 371
 372                         cmd->bufflen = cmd->expected_transfer_len;
 373                         /* Restore (likely) lost CDB length */
 374                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
 375                         if (cmd->cdb_len == -1) {
 376                                 PRINT_ERROR("Unable to get CDB length for "
 377                                         "opcode 0x%02x. Returning INVALID "
 378                                         "OPCODE", cmd->cdb[0]);
 379                                 scst_set_cmd_error(cmd,
 380                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
 381                                 goto out_xmit;
 382                         }
 383                 } else {
 384                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
 385                              "target %s not supplied expected values",
 386                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
 387                         scst_set_cmd_error(cmd,
 388                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
 389                         goto out_xmit;
 390                 }
 391 #else
 392                 scst_set_cmd_error(cmd,
 393                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
 394                 goto out_xmit;
 395 #endif
 396         } else {
 397                 TRACE(TRACE_SCSI, "op_name <%s> (cmd %p), direction=%d "
 398                         "(expected %d, set %s), transfer_len=%d (expected "
 399                         "len %d), flags=%d", cmd->op_name, cmd,
 400                         cmd->data_direction, cmd->expected_data_direction,
 401                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
 402                         cmd->bufflen, cmd->expected_transfer_len,
 403                         cmd->op_flags);
 404
 405                 if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
 406                         if (scst_cmd_is_expected_set(cmd)) {
 407                                 /*
 408                                  * Command data length can't be easily
 409                                  * determined from the CDB. ToDo, all such
 410                                  * commands processing should be fixed. Until
 411                                  * it's done, get the length from the supplied
 412                                  * expected value, but limit it to some
 413                                  * reasonable value (15MB).
 414                                  */
 415                                 cmd->bufflen = min(cmd->expected_transfer_len,
 416                                                         15*1024*1024);
 417                                 cmd->op_flags &= ~SCST_UNKNOWN_LENGTH;
 418                         } else
 419                                 cmd->bufflen = 0;
 420                 }
 421         }
 422
 423         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
 424                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
 425                             "(opcode 0x%02x)", cmd->cdb[0]);
 426                 scst_set_cmd_error(cmd,
 427                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
 428                 goto out_xmit;
 429         }
 430
 431         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
 432                 PRINT_ERROR("Linked commands are not supported "
 433                             "(opcode 0x%02x)", cmd->cdb[0]);
 434                 scst_set_cmd_error(cmd,
 435                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
 436                 goto out_xmit;
 437         }
 438
 439         cmd->state = SCST_CMD_STATE_DEV_PARSE;
 440
 441 out:
 442         TRACE_EXIT_RES(res);
 443         return res;
 444
 445 out_xmit:
 446         scst_set_cmd_abnormal_done_state(cmd);
 447         res = SCST_CMD_STATE_RES_CONT_SAME;
 448         goto out;
 449 }
 450
 451 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
 452 static bool scst_is_allowed_to_mismatch_cmd(struct scst_cmd *cmd)
 453 {
 454         bool res = false;
 455
 456         switch (cmd->cdb[0]) {
 457         case TEST_UNIT_READY:
 458                 /* Crazy VMware people sometimes do TUR with READ direction */
 459                 res = true;
 460                 break;
 461         case VERIFY:
 462         case VERIFY_6:
 463         case VERIFY_12:
 464         case VERIFY_16:
 465                 /* VERIFY commands with BYTCHK unset shouldn't fail here */
 466                 if ((cmd->op_flags & SCST_VERIFY_BYTCHK_MISMATCH_ALLOWED) &&
 467                     (cmd->cdb[1] & BYTCHK) == 0)
 468                         res = true;
 469                 break;
 470         }
 471
 472         return res;
 473 }
 474 #endif
 475
 476 static int scst_parse_cmd(struct scst_cmd *cmd)
 477 {
 478         int res = SCST_CMD_STATE_RES_CONT_SAME;
 479         int state;
 480         struct scst_device *dev = cmd->dev;
 481         int orig_bufflen = cmd->bufflen;
 482
 483         TRACE_ENTRY();
 484
 485         if (likely(!scst_is_cmd_local(cmd))) {
 486                 if (unlikely(!dev->handler->parse_atomic &&
 487                              scst_cmd_atomic(cmd))) {
 488                         /*
 489                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
 490                          * optimization.
 491                          */
 492                         TRACE_DBG("Dev handler %s parse() needs thread "
 493                                 "context, rescheduling", dev->handler->name);
 494                         res = SCST_CMD_STATE_RES_NEED_THREAD;
 495                         goto out;
 496                 }
 497
 498                 TRACE_DBG("Calling dev handler %s parse(%p)",
 499                       dev->handler->name, cmd);
 500                 TRACE_BUFF_FLAG(TRACE_SND_BOT, "Parsing: ",
 501                                 cmd->cdb, cmd->cdb_len);
 502                 state = dev->handler->parse(cmd);
 503                 /* Caution: cmd can be already dead here */
 504                 TRACE_DBG("Dev handler %s parse() returned %d",
 505                         dev->handler->name, state);
 506
 507                 switch (state) {
 508                 case SCST_CMD_STATE_NEED_THREAD_CTX:
 509                         TRACE_DBG("Dev handler %s parse() requested thread "
 510                               "context, rescheduling", dev->handler->name);
 511                         res = SCST_CMD_STATE_RES_NEED_THREAD;
 512                         goto out;
 513
 514                 case SCST_CMD_STATE_STOP:
 515                         TRACE_DBG("Dev handler %s parse() requested stop "
 516                                 "processing", dev->handler->name);
 517                         res = SCST_CMD_STATE_RES_CONT_NEXT;
 518                         goto out;
 519                 }
 520
 521                 if (state == SCST_CMD_STATE_DEFAULT)
 522                         state = SCST_CMD_STATE_PREPARE_SPACE;
 523         } else
 524                 state = SCST_CMD_STATE_PREPARE_SPACE;
 525
 526         if (cmd->data_len == -1)
 527                 cmd->data_len = cmd->bufflen;
 528
 529         if (cmd->bufflen == 0) {
 530                 /*
 531                  * According to SPC bufflen 0 for data transfer commands isn't
 532                  * an error, so we need to fix the transfer direction.
 533                  */
 534                 cmd->data_direction = SCST_DATA_NONE;
 535         }
 536
 537         if (cmd->dh_data_buf_alloced &&
 538             unlikely((orig_bufflen > cmd->bufflen))) {
 539                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
 540                         "is less, than required (size %d)", cmd->bufflen,
 541                         orig_bufflen);
 542                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 543                 goto out_error;
 544         }
 545
 546         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
 547                 goto set_res;
 548
 549         if (unlikely((cmd->bufflen == 0) &&
 550                      (cmd->op_flags & SCST_UNKNOWN_LENGTH))) {
 551                 PRINT_ERROR("Unknown data transfer length for opcode 0x%x "
 552                         "(handler %s, target %s)", cmd->cdb[0],
 553                         dev->handler->name, cmd->tgtt->name);
 554                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 555                 goto out_error;
 556         }
 557
 558 #ifdef CONFIG_SCST_EXTRACHECKS
 559         if ((cmd->bufflen != 0) &&
 560             ((cmd->data_direction == SCST_DATA_NONE) ||
 561              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
 562                 PRINT_ERROR("Dev handler %s parse() returned "
 563                         "invalid cmd data_direction %d, bufflen %d, state %d "
 564                         "or sg %p (opcode 0x%x)", dev->handler->name,
 565                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
 566                         cmd->cdb[0]);
 567                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 568                 goto out_error;
 569         }
 570 #endif
 571
 572         if (scst_cmd_is_expected_set(cmd)) {
 573 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
 574 #       ifdef CONFIG_SCST_EXTRACHECKS
 575                 if ((cmd->data_direction != cmd->expected_data_direction) ||
 576                     (cmd->bufflen != cmd->expected_transfer_len)) {
 577                         PRINT_WARNING("Expected values don't match decoded "
 578                                 "ones: data_direction %d, "
 579                                 "expected_data_direction %d, "
 580                                 "bufflen %d, expected_transfer_len %d",
 581                                 cmd->data_direction,
 582                                 cmd->expected_data_direction,
 583                                 cmd->bufflen, cmd->expected_transfer_len);
 584                         PRINT_BUFFER("Suspicious CDB", cmd->cdb, cmd->cdb_len);
 585                 }
 586 #       endif
 587                 cmd->data_direction = cmd->expected_data_direction;
 588                 cmd->bufflen = cmd->expected_transfer_len;
 589 #else
 590                 if (unlikely(cmd->data_direction !=
 591                                 cmd->expected_data_direction)) {
 592                         if (((cmd->expected_data_direction != SCST_DATA_NONE) ||
 593                              (cmd->bufflen != 0)) &&
 594                             !scst_is_allowed_to_mismatch_cmd(cmd)) {
 595                                 PRINT_ERROR("Expected data direction %d for "
 596                                         "opcode 0x%02x (handler %s, target %s) "
 597                                         "doesn't match "
 598                                         "decoded value %d",
 599                                         cmd->expected_data_direction,
 600                                         cmd->cdb[0], dev->handler->name,
 601                                         cmd->tgtt->name, cmd->data_direction);
 602                                 PRINT_BUFFER("Failed CDB",
 603                                         cmd->cdb, cmd->cdb_len);
 604                                 scst_set_cmd_error(cmd,
 605                                    SCST_LOAD_SENSE(scst_sense_invalid_message));
 606                                 goto out_dev_done;
 607                         }
 608                 }
 609                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
 610                         TRACE(TRACE_MGMT_MINOR, "Warning: expected "
 611                                 "transfer length %d for opcode 0x%02x "
 612                                 "(handler %s, target %s) doesn't match "
 613                                 "decoded value %d. Faulty initiator "
 614                                 "(e.g. VMware is known to be such) or "
 615                                 "scst_scsi_op_table should be updated?",
 616                                 cmd->expected_transfer_len, cmd->cdb[0],
 617                                 dev->handler->name, cmd->tgtt->name,
 618                                 cmd->bufflen);
 619                         PRINT_BUFF_FLAG(TRACE_MGMT_MINOR, "Suspicious CDB",
 620                                 cmd->cdb, cmd->cdb_len);
 621                 }
 622 #endif
 623         }
 624
 625         if (unlikely(cmd->data_direction == SCST_DATA_UNKNOWN)) {
 626                 PRINT_ERROR("Unknown data direction. Opcode 0x%x, handler %s, "
 627                         "target %s", cmd->cdb[0], dev->handler->name,
 628                         cmd->tgtt->name);
 629                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 630                 goto out_error;
 631         }
 632
 633 set_res:
 634         switch (state) {
 635         case SCST_CMD_STATE_PREPARE_SPACE:
 636         case SCST_CMD_STATE_PRE_PARSE:
 637         case SCST_CMD_STATE_DEV_PARSE:
 638         case SCST_CMD_STATE_RDY_TO_XFER:
 639         case SCST_CMD_STATE_TGT_PRE_EXEC:
 640         case SCST_CMD_STATE_SEND_FOR_EXEC:
 641         case SCST_CMD_STATE_LOCAL_EXEC:
 642         case SCST_CMD_STATE_REAL_EXEC:
 643         case SCST_CMD_STATE_PRE_DEV_DONE:
 644         case SCST_CMD_STATE_DEV_DONE:
 645         case SCST_CMD_STATE_PRE_XMIT_RESP:
 646         case SCST_CMD_STATE_XMIT_RESP:
 647         case SCST_CMD_STATE_FINISHED:
 648                 cmd->state = state;
 649                 res = SCST_CMD_STATE_RES_CONT_SAME;
 650                 break;
 651
 652         default:
 653                 if (state >= 0) {
 654                         PRINT_ERROR("Dev handler %s parse() returned "
 655                              "invalid cmd state %d (opcode %d)",
 656                              dev->handler->name, state, cmd->cdb[0]);
 657                 } else {
 658                         PRINT_ERROR("Dev handler %s parse() returned "
 659                                 "error %d (opcode %d)", dev->handler->name,
 660                                 state, cmd->cdb[0]);
 661                 }
 662                 goto out_error;
 663         }
 664
 665         if (cmd->resp_data_len == -1) {
 666                 if (cmd->data_direction == SCST_DATA_READ)
 667                         cmd->resp_data_len = cmd->bufflen;
 668                 else
 669                          cmd->resp_data_len = 0;
 670         }
 671
 672 out:
 673         TRACE_EXIT_HRES(res);
 674         return res;
 675
 676 out_error:
 677         /* dev_done() will be called as part of the regular cmd's finish */
 678         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
 679
 680 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
 681 out_dev_done:
 682 #endif
 683         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
 684         res = SCST_CMD_STATE_RES_CONT_SAME;
 685         goto out;
 686 }
 687
 688 static int scst_prepare_space(struct scst_cmd *cmd)
 689 {
 690         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
 691
 692         TRACE_ENTRY();
 693
 694         if (cmd->data_direction == SCST_DATA_NONE)
 695                 goto prep_done;
 696
 697         if (cmd->tgt_need_alloc_data_buf) {
 698                 int orig_bufflen = cmd->bufflen;
 699
 700                 TRACE_MEM("Custom tgt data buf allocation requested (cmd %p)",
 701                         cmd);
 702
 703                 r = cmd->tgtt->alloc_data_buf(cmd);
 704                 if (r > 0)
 705                         goto alloc;
 706                 else if (r == 0) {
 707                         if (unlikely(cmd->bufflen == 0)) {
 708                                 /* See comment in scst_alloc_space() */
 709                                 if (cmd->sg == NULL)
 710                                         goto alloc;
 711                         }
 712
 713                         cmd->tgt_data_buf_alloced = 1;
 714
 715                         if (unlikely(orig_bufflen < cmd->bufflen)) {
 716                                 PRINT_ERROR("Target driver allocated data "
 717                                         "buffer (size %d), is less, than "
 718                                         "required (size %d)", orig_bufflen,
 719                                         cmd->bufflen);
 720                                 goto out_error;
 721                         }
 722                         TRACE_MEM("tgt_data_buf_alloced (cmd %p)", cmd);
 723                 } else
 724                         goto check;
 725         }
 726
 727 alloc:
 728         if (!cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
 729                 r = scst_alloc_space(cmd);
 730                 cmd->tgt_sg = cmd->sg;
 731                 cmd->tgt_sg_cnt = cmd->sg_cnt;
 732         } else if (cmd->dh_data_buf_alloced && !cmd->tgt_data_buf_alloced) {
 733                 TRACE_MEM("dh_data_buf_alloced set (cmd %p)", cmd);
 734                 cmd->tgt_sg = cmd->sg;
 735                 cmd->tgt_sg_cnt = cmd->sg_cnt;
 736                 r = 0;
 737         } else if (cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
 738                 TRACE_MEM("tgt_data_buf_alloced set (cmd %p)", cmd);
 739                 cmd->sg = cmd->tgt_sg;
 740                 cmd->sg_cnt = cmd->tgt_sg_cnt;
 741                 r = 0;
 742         } else {
 743                 TRACE_MEM("Both *_data_buf_alloced set (cmd %p, sg %p, "
 744                         "sg_cnt %d, tgt_sg %p, tgt_sg_cnt %d)", cmd, cmd->sg,
 745                         cmd->sg_cnt, cmd->tgt_sg, cmd->tgt_sg_cnt);
 746                 r = 0;
 747         }
 748
 749 check:
 750         if (r != 0) {
 751                 if (scst_cmd_atomic(cmd)) {
 752                         TRACE_MEM("%s", "Atomic memory allocation failed, "
 753                               "rescheduling to the thread");
 754                         res = SCST_CMD_STATE_RES_NEED_THREAD;
 755                         goto out;
 756                 } else
 757                         goto out_no_space;
 758         }
 759
 760 prep_done:
 761         if (cmd->preprocessing_only) {
 762                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
 763                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
 764                                 "cmd %p", cmd);
 765                         scst_set_cmd_abnormal_done_state(cmd);
 766                         res = SCST_CMD_STATE_RES_CONT_SAME;
 767                         goto out;
 768                 }
 769
 770                 res = SCST_CMD_STATE_RES_CONT_NEXT;
 771                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
 772
 773                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
 774                 cmd->tgtt->preprocessing_done(cmd);
 775                 TRACE_DBG("%s", "preprocessing_done() returned");
 776                 goto out;
 777
 778         }
 779
 780         switch (cmd->data_direction) {
 781         case SCST_DATA_WRITE:
 782                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
 783                 break;
 784
 785         default:
 786                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
 787                 break;
 788         }
 789
 790 out:
 791         TRACE_EXIT_HRES(res);
 792         return res;
 793
 794 out_no_space:
 795         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
 796                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
 797         scst_set_busy(cmd);
 798         scst_set_cmd_abnormal_done_state(cmd);
 799         res = SCST_CMD_STATE_RES_CONT_SAME;
 800         goto out;
 801
 802 out_error:
 803         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
 804         scst_set_cmd_abnormal_done_state(cmd);
 805         res = SCST_CMD_STATE_RES_CONT_SAME;
 806         goto out;
 807 }
 808
 809 void scst_restart_cmd(struct scst_cmd *cmd, int status,
 810         enum scst_exec_context pref_context)
 811 {
 812         TRACE_ENTRY();
 813
 814         TRACE_DBG("Preferred context: %d", pref_context);
 815         TRACE_DBG("tag=%llu, status=%#x",
 816                   (long long unsigned int)scst_cmd_get_tag(cmd),
 817                   status);
 818
 819 #ifdef CONFIG_SCST_EXTRACHECKS
 820         if ((in_irq() || irqs_disabled()) &&
 821             ((pref_context == SCST_CONTEXT_DIRECT) ||
 822              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
 823                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
 824                         "SCST_CONTEXT_THREAD instead\n", pref_context,
 825                         cmd->tgtt->name);
 826                 pref_context = SCST_CONTEXT_THREAD;
 827         }
 828 #endif
 829
 830         switch (status) {
 831         case SCST_PREPROCESS_STATUS_SUCCESS:
 832                 switch (cmd->data_direction) {
 833                 case SCST_DATA_WRITE:
 834                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
 835                         break;
 836                 default:
 837                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
 838                         break;
 839                 }
 840                 if (cmd->set_sn_on_restart_cmd)
 841                         scst_cmd_set_sn(cmd);
 842                 /* Small context optimization */
 843                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
 844                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
 845                     ((pref_context == SCST_CONTEXT_SAME) &&
 846                      scst_cmd_atomic(cmd))) {
 847                         if (cmd->data_direction == SCST_DATA_WRITE) {
 848                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
 849                                                 &cmd->tgt_dev->tgt_dev_flags))
 850                                         pref_context = SCST_CONTEXT_THREAD;
 851                         } else {
 852                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
 853                                                 &cmd->tgt_dev->tgt_dev_flags))
 854                                         pref_context = SCST_CONTEXT_THREAD;
 855                         }
 856                 }
 857                 break;
 858
 859         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
 860                 scst_set_cmd_abnormal_done_state(cmd);
 861                 break;
 862
 863         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
 864                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
 865                 /* go through */
 866         case SCST_PREPROCESS_STATUS_ERROR:
 867                 scst_set_cmd_error(cmd,
 868                            SCST_LOAD_SENSE(scst_sense_hardw_error));
 869                 scst_set_cmd_abnormal_done_state(cmd);
 870                 break;
 871
 872         default:
 873                 PRINT_ERROR("%s() received unknown status %x", __func__,
 874                         status);
 875                 scst_set_cmd_abnormal_done_state(cmd);
 876                 break;
 877         }
 878
 879         scst_proccess_redirect_cmd(cmd, pref_context, 1);
 880
 881         TRACE_EXIT();
 882         return;
 883 }
 884 EXPORT_SYMBOL(scst_restart_cmd);
 885
 886 /* No locks */
 887 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
 888 {
 889         struct scst_tgt *tgt = cmd->sess->tgt;
 890         int res = 0;
 891         unsigned long flags;
 892
 893         TRACE_ENTRY();
 894
 895         spin_lock_irqsave(&tgt->tgt_lock, flags);
 896         tgt->retry_cmds++;
 897         /*
 898          * Memory barrier is needed here, because we need the exact order
 899          * between the read and write between retry_cmds and finished_cmds to
 900          * not miss the case when a command finished while we queuing it for
 901          * retry after the finished_cmds check.
 902          */
 903         smp_mb();
 904         TRACE_RETRY("TGT QUEUE FULL: incrementing retry_cmds %d",
 905               tgt->retry_cmds);
 906         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
 907                 /* At least one cmd finished, so try again */
 908                 tgt->retry_cmds--;
 909                 TRACE_RETRY("Some command(s) finished, direct retry "
 910                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
 911                       "retry_cmds=%d)", finished_cmds,
 912                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
 913                 res = -1;
 914                 goto out_unlock_tgt;
 915         }
 916
 917         TRACE_RETRY("Adding cmd %p to retry cmd list", cmd);
 918         list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
 919
 920         if (!tgt->retry_timer_active) {
 921                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
 922                 add_timer(&tgt->retry_timer);
 923                 tgt->retry_timer_active = 1;
 924         }
 925
 926 out_unlock_tgt:
 927         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
 928
 929         TRACE_EXIT_RES(res);
 930         return res;
 931 }
 932
 933 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
 934 {
 935         int res, rc;
 936
 937         TRACE_ENTRY();
 938
 939         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
 940                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
 941                 goto out_dev_done;
 942         }
 943
 944         if ((cmd->tgtt->rdy_to_xfer == NULL) || unlikely(cmd->internal)) {
 945                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
 946                 res = SCST_CMD_STATE_RES_CONT_SAME;
 947                 goto out;
 948         }
 949
 950         if (unlikely(!cmd->tgtt->rdy_to_xfer_atomic && scst_cmd_atomic(cmd))) {
 951                 /*
 952                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
 953                  * optimization.
 954                  */
 955                 TRACE_DBG("Target driver %s rdy_to_xfer() needs thread "
 956                               "context, rescheduling", cmd->tgtt->name);
 957                 res = SCST_CMD_STATE_RES_NEED_THREAD;
 958                 goto out;
 959         }
 960
 961         while (1) {
 962                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
 963
 964                 res = SCST_CMD_STATE_RES_CONT_NEXT;
 965                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
 966
 967                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
 968 #ifdef CONFIG_SCST_DEBUG_RETRY
 969                 if (((scst_random() % 100) == 75))
 970                         rc = SCST_TGT_RES_QUEUE_FULL;
 971                 else
 972 #endif
 973                         rc = cmd->tgtt->rdy_to_xfer(cmd);
 974                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
 975
 976                 if (likely(rc == SCST_TGT_RES_SUCCESS))
 977                         goto out;
 978
 979                 /* Restore the previous state */
 980                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
 981
 982                 switch (rc) {
 983                 case SCST_TGT_RES_QUEUE_FULL:
 984                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
 985                                 break;
 986                         else
 987                                 continue;
 988
 989                 case SCST_TGT_RES_NEED_THREAD_CTX:
 990                         TRACE_DBG("Target driver %s "
 991                               "rdy_to_xfer() requested thread "
 992                               "context, rescheduling", cmd->tgtt->name);
 993                         res = SCST_CMD_STATE_RES_NEED_THREAD;
 994                         break;
 995
 996                 default:
 997                         goto out_error_rc;
 998                 }
 999                 break;
1000         }
1001
1002 out:
1003         TRACE_EXIT_HRES(res);
1004         return res;
1005
1006 out_error_rc:
1007         if (rc == SCST_TGT_RES_FATAL_ERROR) {
1008                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
1009                      "fatal error", cmd->tgtt->name);
1010         } else {
1011                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
1012                             "value %d", cmd->tgtt->name, rc);
1013         }
1014         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1015
1016 out_dev_done:
1017         scst_set_cmd_abnormal_done_state(cmd);
1018         res = SCST_CMD_STATE_RES_CONT_SAME;
1019         goto out;
1020 }
1021
1022 /* No locks, but might be in IRQ */
1023 static void scst_proccess_redirect_cmd(struct scst_cmd *cmd,
1024         enum scst_exec_context context, int check_retries)
1025 {
1026         unsigned long flags;
1027
1028         TRACE_ENTRY();
1029
1030         TRACE_DBG("Context: %x", context);
1031
1032         if (context == SCST_CONTEXT_SAME)
1033                 context = scst_cmd_atomic(cmd) ? SCST_CONTEXT_DIRECT_ATOMIC :
1034                                                  SCST_CONTEXT_DIRECT;
1035
1036         switch (context) {
1037         case SCST_CONTEXT_DIRECT_ATOMIC:
1038                 scst_process_active_cmd(cmd, true);
1039                 break;
1040
1041         case SCST_CONTEXT_DIRECT:
1042                 if (check_retries)
1043                         scst_check_retries(cmd->tgt);
1044                 scst_process_active_cmd(cmd, false);
1045                 break;
1046
1047         default:
1048                 PRINT_ERROR("Context %x is unknown, using the thread one",
1049                             context);
1050                 /* go through */
1051         case SCST_CONTEXT_THREAD:
1052                 if (check_retries)
1053                         scst_check_retries(cmd->tgt);
1054                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
1055                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
1056                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
1057                         list_add(&cmd->cmd_list_entry,
1058                                 &cmd->cmd_lists->active_cmd_list);
1059                 else
1060                         list_add_tail(&cmd->cmd_list_entry,
1061                                 &cmd->cmd_lists->active_cmd_list);
1062                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
1063                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
1064                 break;
1065
1066         case SCST_CONTEXT_TASKLET:
1067                 if (check_retries)
1068                         scst_check_retries(cmd->tgt);
1069                 scst_schedule_tasklet(cmd);
1070                 break;
1071         }
1072
1073         TRACE_EXIT();
1074         return;
1075 }
1076
1077 void scst_rx_data(struct scst_cmd *cmd, int status,
1078         enum scst_exec_context pref_context)
1079 {
1080         TRACE_ENTRY();
1081
1082         TRACE_DBG("Preferred context: %d", pref_context);
1083         TRACE(TRACE_SCSI, "cmd %p, status %#x", cmd, status);
1084
1085 #ifdef CONFIG_SCST_EXTRACHECKS
1086         if ((in_irq() || irqs_disabled()) &&
1087             ((pref_context == SCST_CONTEXT_DIRECT) ||
1088              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
1089                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
1090                         "SCST_CONTEXT_THREAD instead\n", pref_context,
1091                         cmd->tgtt->name);
1092                 pref_context = SCST_CONTEXT_THREAD;
1093         }
1094 #endif
1095
1096         switch (status) {
1097         case SCST_RX_STATUS_SUCCESS:
1098 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
1099                 if (cmd->tgt_sg) {
1100                         int i;
1101                         struct scatterlist *sg = cmd->tgt_sg;
1102                         TRACE_RECV_BOT("RX data for cmd %p "
1103                                 "(sg_cnt %d, sg %p, sg[0].page %p)", cmd,
1104                                 cmd->tgt_sg_cnt, sg, (void *)sg_page(&sg[0]));
1105                         for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
1106                                 PRINT_BUFF_FLAG(TRACE_RCV_BOT, "RX sg",
1107                                         sg_virt(&sg[i]), sg[i].length);
1108                         }
1109                 }
1110 #endif
1111                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1112                 /* Small context optimization */
1113                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
1114                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
1115                     ((pref_context == SCST_CONTEXT_SAME) &&
1116                      scst_cmd_atomic(cmd))) {
1117                         if (!test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC,
1118                                         &cmd->tgt_dev->tgt_dev_flags))
1119                                 pref_context = SCST_CONTEXT_THREAD;
1120                 }
1121                 break;
1122
1123         case SCST_RX_STATUS_ERROR_SENSE_SET:
1124                 scst_set_cmd_abnormal_done_state(cmd);
1125                 break;
1126
1127         case SCST_RX_STATUS_ERROR_FATAL:
1128                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1129                 /* go through */
1130         case SCST_RX_STATUS_ERROR:
1131                 scst_set_cmd_error(cmd,
1132                            SCST_LOAD_SENSE(scst_sense_hardw_error));
1133                 scst_set_cmd_abnormal_done_state(cmd);
1134                 break;
1135
1136         default:
1137                 PRINT_ERROR("scst_rx_data() received unknown status %x",
1138                         status);
1139                 scst_set_cmd_abnormal_done_state(cmd);
1140                 break;
1141         }
1142
1143         scst_proccess_redirect_cmd(cmd, pref_context, 1);
1144
1145         TRACE_EXIT();
1146         return;
1147 }
1148 EXPORT_SYMBOL(scst_rx_data);
1149
1150 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
1151 {
1152         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
1153
1154         TRACE_ENTRY();
1155
1156         cmd->state = SCST_CMD_STATE_SEND_FOR_EXEC;
1157
1158         if ((cmd->tgtt->pre_exec == NULL) || unlikely(cmd->internal))
1159                 goto out;
1160
1161         TRACE_DBG("Calling pre_exec(%p)", cmd);
1162         rc = cmd->tgtt->pre_exec(cmd);
1163         TRACE_DBG("pre_exec() returned %d", rc);
1164
1165         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
1166                 switch (rc) {
1167                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
1168                         scst_set_cmd_abnormal_done_state(cmd);
1169                         break;
1170                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
1171                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1172                         /* go through */
1173                 case SCST_PREPROCESS_STATUS_ERROR:
1174                         scst_set_cmd_error(cmd,
1175                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1176                         scst_set_cmd_abnormal_done_state(cmd);
1177                         break;
1178                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1179                         TRACE_DBG("Target driver's %s pre_exec() requested "
1180                                 "thread context, rescheduling",
1181                                 cmd->tgtt->name);
1182                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1183                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1184                         break;
1185                 default:
1186                         sBUG();
1187                         break;
1188                 }
1189         }
1190
1191 out:
1192         TRACE_EXIT_RES(res);
1193         return res;
1194 }
1195
1196 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1197         const uint8_t *rq_sense, int rq_sense_len, int resid)
1198 {
1199         TRACE_ENTRY();
1200
1201 #ifdef CONFIG_SCST_MEASURE_LATENCY
1202         {
1203                 struct timespec ts;
1204                 getnstimeofday(&ts);
1205                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1206                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1207                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1208                         ts.tv_sec, ts.tv_nsec);
1209         }
1210 #endif
1211
1212         cmd->status = result & 0xff;
1213         cmd->msg_status = msg_byte(result);
1214         cmd->host_status = host_byte(result);
1215         cmd->driver_status = driver_byte(result);
1216         if (unlikely(resid != 0)) {
1217 #ifdef CONFIG_SCST_EXTRACHECKS
1218                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1219                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1220                                 "op %x)", resid, cmd->resp_data_len,
1221                                 cmd->cdb[0]);
1222                 } else
1223 #endif
1224                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1225         }
1226
1227         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION)) {
1228                 /* We might have double reset UA here */
1229                 cmd->dbl_ua_orig_resp_data_len = cmd->resp_data_len;
1230                 cmd->dbl_ua_orig_data_direction = cmd->data_direction;
1231
1232                 scst_alloc_set_sense(cmd, 1, rq_sense, rq_sense_len);
1233         }
1234
1235         TRACE(TRACE_SCSI, "cmd %p, result=%x, cmd->status=%x, resid=%d, "
1236               "cmd->msg_status=%x, cmd->host_status=%x, "
1237               "cmd->driver_status=%x (cmd %p)", cmd, result, cmd->status, resid,
1238               cmd->msg_status, cmd->host_status, cmd->driver_status, cmd);
1239
1240         cmd->completed = 1;
1241
1242         TRACE_EXIT();
1243         return;
1244 }
1245
1246 /* For small context optimization */
1247 static inline enum scst_exec_context scst_optimize_post_exec_context(
1248         struct scst_cmd *cmd, enum scst_exec_context context)
1249 {
1250         if (((context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd)) ||
1251             (context == SCST_CONTEXT_TASKLET) ||
1252             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1253                 if (!test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC,
1254                                 &cmd->tgt_dev->tgt_dev_flags))
1255                         context = SCST_CONTEXT_THREAD;
1256         }
1257         return context;
1258 }
1259
1260 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1261 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1262                                             struct scsi_request **req)
1263 {
1264         struct scst_cmd *cmd = NULL;
1265
1266         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1267                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1268
1269         if (cmd == NULL) {
1270                 PRINT_ERROR("%s", "Request with NULL cmd");
1271                 if (*req)
1272                         scsi_release_request(*req);
1273         }
1274
1275         return cmd;
1276 }
1277
1278 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1279 {
1280         struct scsi_request *req = NULL;
1281         struct scst_cmd *cmd;
1282
1283         TRACE_ENTRY();
1284
1285         cmd = scst_get_cmd(scsi_cmd, &req);
1286         if (cmd == NULL)
1287                 goto out;
1288
1289         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1290                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1291
1292         /* Clear out request structure */
1293         req->sr_use_sg = 0;
1294         req->sr_sglist_len = 0;
1295         req->sr_bufflen = 0;
1296         req->sr_buffer = NULL;
1297         req->sr_underflow = 0;
1298         req->sr_request->rq_disk = NULL; /* disown request blk */
1299
1300         scst_release_request(cmd);
1301
1302         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1303
1304         scst_proccess_redirect_cmd(cmd,
1305                 scst_optimize_post_exec_context(cmd, scst_estimate_context()),
1306                                                 0);
1307
1308 out:
1309         TRACE_EXIT();
1310         return;
1311 }
1312 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1313 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1314 {
1315         struct scst_cmd *cmd;
1316
1317         TRACE_ENTRY();
1318
1319         cmd = (struct scst_cmd *)data;
1320         if (cmd == NULL)
1321                 goto out;
1322
1323         scst_do_cmd_done(cmd, result, sense, SCST_SENSE_BUFFERSIZE, resid);
1324
1325         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1326
1327         scst_proccess_redirect_cmd(cmd,
1328                 scst_optimize_post_exec_context(cmd, scst_estimate_context()),
1329                                                 0);
1330
1331 out:
1332         TRACE_EXIT();
1333         return;
1334 }
1335 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1336
1337 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state,
1338         enum scst_exec_context pref_context)
1339 {
1340         TRACE_ENTRY();
1341
1342 #ifdef CONFIG_SCST_MEASURE_LATENCY
1343         {
1344                 struct timespec ts;
1345                 getnstimeofday(&ts);
1346                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1347                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1348                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1349                         ts.tv_sec, ts.tv_nsec);
1350         }
1351 #endif
1352
1353         if (next_state == SCST_CMD_STATE_DEFAULT)
1354                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1355
1356 #if defined(CONFIG_SCST_DEBUG)
1357         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1358                 if (cmd->sg) {
1359                         int i;
1360                         struct scatterlist *sg = cmd->sg;
1361                         TRACE_RECV_TOP("Exec'd %d S/G(s) at %p sg[0].page at "
1362                                 "%p", cmd->sg_cnt, sg, (void *)sg_page(&sg[0]));
1363                         for (i = 0; i < cmd->sg_cnt; ++i) {
1364                                 TRACE_BUFF_FLAG(TRACE_RCV_TOP,
1365                                         "Exec'd sg", sg_virt(&sg[i]),
1366                                         sg[i].length);
1367                         }
1368                 }
1369         }
1370 #endif
1371
1372         cmd->state = next_state;
1373
1374 #ifdef CONFIG_SCST_EXTRACHECKS
1375         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1376             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1377             (next_state != SCST_CMD_STATE_FINISHED)) {
1378                 PRINT_ERROR("%s() received invalid cmd state %d (opcode %d)",
1379                         __func__, next_state, cmd->cdb[0]);
1380                 scst_set_cmd_error(cmd,
1381                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1382                 scst_set_cmd_abnormal_done_state(cmd);
1383         }
1384 #endif
1385         pref_context = scst_optimize_post_exec_context(cmd, pref_context);
1386         scst_proccess_redirect_cmd(cmd, pref_context, 0);
1387
1388         TRACE_EXIT();
1389         return;
1390 }
1391
1392 static int scst_report_luns_local(struct scst_cmd *cmd)
1393 {
1394         int rc;
1395         int dev_cnt = 0;
1396         int buffer_size;
1397         int i;
1398         struct scst_tgt_dev *tgt_dev = NULL;
1399         uint8_t *buffer;
1400         int offs, overflow = 0;
1401
1402         TRACE_ENTRY();
1403
1404         rc = scst_check_local_events(cmd);
1405         if (unlikely(rc != 0))
1406                 goto out_done;
1407
1408         cmd->status = 0;
1409         cmd->msg_status = 0;
1410         cmd->host_status = DID_OK;
1411         cmd->driver_status = 0;
1412
1413         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1414                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1415                         "LUNS command", cmd->cdb[2]);
1416                 goto out_err;
1417         }
1418
1419         buffer_size = scst_get_buf_first(cmd, &buffer);
1420         if (unlikely(buffer_size == 0))
1421                 goto out_compl;
1422         else if (unlikely(buffer_size < 0))
1423                 goto out_err;
1424
1425         if (buffer_size < 16)
1426                 goto out_put_err;
1427
1428         memset(buffer, 0, buffer_size);
1429         offs = 8;
1430
1431         /* sess->sess_tgt_dev_list_hash is protected by suspended activity */
1432         for (i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1433                 struct list_head *sess_tgt_dev_list_head =
1434                         &cmd->sess->sess_tgt_dev_list_hash[i];
1435                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1436                                 sess_tgt_dev_list_entry) {
1437                         if (!overflow) {
1438                                 if (offs >= buffer_size) {
1439                                         scst_put_buf(cmd, buffer);
1440                                         buffer_size = scst_get_buf_next(cmd,
1441                                                                        &buffer);
1442                                         if (buffer_size > 0) {
1443                                                 memset(buffer, 0, buffer_size);
1444                                                 offs = 0;
1445                                         } else {
1446                                                 overflow = 1;
1447                                                 goto inc_dev_cnt;
1448                                         }
1449                                 }
1450                                 if ((buffer_size - offs) < 8) {
1451                                         PRINT_ERROR("Buffer allocated for "
1452                                                 "REPORT LUNS command doesn't "
1453                                                 "allow to fit 8 byte entry "
1454                                                 "(buffer_size=%d)",
1455                                                 buffer_size);
1456                                         goto out_put_hw_err;
1457                                 }
1458                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1459                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1460                                 offs += 8;
1461                         }
1462 inc_dev_cnt:
1463                         dev_cnt++;
1464                 }
1465         }
1466         if (!overflow)
1467                 scst_put_buf(cmd, buffer);
1468
1469         /* Set the response header */
1470         buffer_size = scst_get_buf_first(cmd, &buffer);
1471         if (unlikely(buffer_size == 0))
1472                 goto out_compl;
1473         else if (unlikely(buffer_size < 0))
1474                 goto out_err;
1475
1476         dev_cnt *= 8;
1477         buffer[0] = (dev_cnt >> 24) & 0xff;
1478         buffer[1] = (dev_cnt >> 16) & 0xff;
1479         buffer[2] = (dev_cnt >> 8) & 0xff;
1480         buffer[3] = dev_cnt & 0xff;
1481
1482         scst_put_buf(cmd, buffer);
1483
1484         dev_cnt += 8;
1485         if (dev_cnt < cmd->resp_data_len)
1486                 scst_set_resp_data_len(cmd, dev_cnt);
1487
1488 out_compl:
1489         cmd->completed = 1;
1490
1491 out_done:
1492         /* Report the result */
1493         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1494
1495         TRACE_EXIT();
1496         return SCST_EXEC_COMPLETED;
1497
1498 out_put_err:
1499         scst_put_buf(cmd, buffer);
1500
1501 out_err:
1502         scst_set_cmd_error(cmd,
1503                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1504         goto out_compl;
1505
1506 out_put_hw_err:
1507         scst_put_buf(cmd, buffer);
1508         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1509         goto out_compl;
1510 }
1511
1512 static int scst_pre_select(struct scst_cmd *cmd)
1513 {
1514         int res = SCST_EXEC_NOT_COMPLETED;
1515
1516         TRACE_ENTRY();
1517
1518         if (scst_cmd_atomic(cmd)) {
1519                 res = SCST_EXEC_NEED_THREAD;
1520                 goto out;
1521         }
1522
1523         scst_block_dev_cmd(cmd, 1);
1524
1525         /* Check for local events will be done when cmd will be executed */
1526
1527 out:
1528         TRACE_EXIT_RES(res);
1529         return res;
1530 }
1531
1532 static int scst_reserve_local(struct scst_cmd *cmd)
1533 {
1534         int res = SCST_EXEC_NOT_COMPLETED, rc;
1535         struct scst_device *dev;
1536         struct scst_tgt_dev *tgt_dev_tmp;
1537
1538         TRACE_ENTRY();
1539
1540         if (scst_cmd_atomic(cmd)) {
1541                 res = SCST_EXEC_NEED_THREAD;
1542                 goto out;
1543         }
1544
1545         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1546                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1547                      "(lun=%lld)", (long long unsigned int)cmd->lun);
1548                 scst_set_cmd_error(cmd,
1549                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1550                 goto out_done;
1551         }
1552
1553         dev = cmd->dev;
1554
1555         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1556                 scst_block_dev_cmd(cmd, 1);
1557
1558         rc = scst_check_local_events(cmd);
1559         if (unlikely(rc != 0))
1560                 goto out_done;
1561
1562         spin_lock_bh(&dev->dev_lock);
1563
1564         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1565                 spin_unlock_bh(&dev->dev_lock);
1566                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1567                 goto out_done;
1568         }
1569
1570         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1571                             dev_tgt_dev_list_entry) {
1572                 if (cmd->tgt_dev != tgt_dev_tmp)
1573                         set_bit(SCST_TGT_DEV_RESERVED,
1574                                 &tgt_dev_tmp->tgt_dev_flags);
1575         }
1576         dev->dev_reserved = 1;
1577
1578         spin_unlock_bh(&dev->dev_lock);
1579
1580 out:
1581         TRACE_EXIT_RES(res);
1582         return res;
1583
1584 out_done:
1585         /* Report the result */
1586         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1587         res = SCST_EXEC_COMPLETED;
1588         goto out;
1589 }
1590
1591 static int scst_release_local(struct scst_cmd *cmd)
1592 {
1593         int res = SCST_EXEC_NOT_COMPLETED, rc;
1594         struct scst_tgt_dev *tgt_dev_tmp;
1595         struct scst_device *dev;
1596
1597         TRACE_ENTRY();
1598
1599         if (scst_cmd_atomic(cmd)) {
1600                 res = SCST_EXEC_NEED_THREAD;
1601                 goto out;
1602         }
1603
1604         dev = cmd->dev;
1605
1606         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1607                 scst_block_dev_cmd(cmd, 1);
1608
1609         rc = scst_check_local_events(cmd);
1610         if (unlikely(rc != 0))
1611                 goto out_done;
1612
1613         spin_lock_bh(&dev->dev_lock);
1614
1615         /*
1616          * The device could be RELEASED behind us, if RESERVING session
1617          * is closed (see scst_free_tgt_dev()), but this actually doesn't
1618          * matter, so use lock and no retest for DEV_RESERVED bits again
1619          */
1620         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1621                 res = SCST_EXEC_COMPLETED;
1622                 cmd->status = 0;
1623                 cmd->msg_status = 0;
1624                 cmd->host_status = DID_OK;
1625                 cmd->driver_status = 0;
1626                 cmd->completed = 1;
1627         } else {
1628                 list_for_each_entry(tgt_dev_tmp,
1629                                     &dev->dev_tgt_dev_list,
1630                                     dev_tgt_dev_list_entry) {
1631                         clear_bit(SCST_TGT_DEV_RESERVED,
1632                                 &tgt_dev_tmp->tgt_dev_flags);
1633                 }
1634                 dev->dev_reserved = 0;
1635         }
1636
1637         spin_unlock_bh(&dev->dev_lock);
1638
1639         if (res == SCST_EXEC_COMPLETED)
1640                 goto out_done;
1641
1642 out:
1643         TRACE_EXIT_RES(res);
1644         return res;
1645
1646 out_done:
1647         res = SCST_EXEC_COMPLETED;
1648         /* Report the result */
1649         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1650         goto out;
1651 }
1652
1653 /* No locks, no IRQ or IRQ-safe context allowed */
1654 int scst_check_local_events(struct scst_cmd *cmd)
1655 {
1656         int res, rc;
1657         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1658         struct scst_device *dev = cmd->dev;
1659
1660         TRACE_ENTRY();
1661
1662         /*
1663          * There's no race here, because we need to trace commands sent
1664          * *after* dev_double_ua_possible flag was set.
1665          */
1666         if (unlikely(dev->dev_double_ua_possible))
1667                 cmd->double_ua_possible = 1;
1668
1669         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1670                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1671                 goto out_uncomplete;
1672         }
1673
1674         /* Reserve check before Unit Attention */
1675         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED,
1676                               &tgt_dev->tgt_dev_flags))) {
1677                 if (cmd->cdb[0] != INQUIRY &&
1678                     cmd->cdb[0] != REPORT_LUNS &&
1679                     cmd->cdb[0] != RELEASE &&
1680                     cmd->cdb[0] != RELEASE_10 &&
1681                     cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER &&
1682                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL ||
1683                      (cmd->cdb[4] & 3)) &&
1684                     cmd->cdb[0] != LOG_SENSE &&
1685                     cmd->cdb[0] != REQUEST_SENSE) {
1686                         scst_set_cmd_error_status(cmd,
1687                                 SAM_STAT_RESERVATION_CONFLICT);
1688                         goto out_complete;
1689                 }
1690         }
1691
1692         /* If we had internal bus reset, set the command error unit attention */
1693         if ((dev->scsi_dev != NULL) &&
1694             unlikely(dev->scsi_dev->was_reset)) {
1695                 if (scst_is_ua_command(cmd)) {
1696                         int done = 0;
1697                         /*
1698                          * Prevent more than 1 cmd to be triggered by
1699                          * was_reset.
1700                          */
1701                         spin_lock_bh(&dev->dev_lock);
1702                         if (dev->scsi_dev->was_reset) {
1703                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1704                                 scst_set_cmd_error(cmd,
1705                                           SCST_LOAD_SENSE(scst_sense_reset_UA));
1706                                 /*
1707                                  * It looks like it is safe to clear was_reset
1708                                  * here.
1709                                  */
1710                                 dev->scsi_dev->was_reset = 0;
1711                                 done = 1;
1712                         }
1713                         spin_unlock_bh(&dev->dev_lock);
1714
1715                         if (done)
1716                                 goto out_complete;
1717                 }
1718         }
1719
1720         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING,
1721                         &cmd->tgt_dev->tgt_dev_flags))) {
1722                 if (scst_is_ua_command(cmd)) {
1723                         rc = scst_set_pending_UA(cmd);
1724                         if (rc == 0)
1725                                 goto out_complete;
1726                 }
1727         }
1728
1729         res = 0;
1730
1731 out:
1732         TRACE_EXIT_RES(res);
1733         return res;
1734
1735 out_complete:
1736         res = 1;
1737         sBUG_ON(!cmd->completed);
1738         goto out;
1739
1740 out_uncomplete:
1741         res = -1;
1742         goto out;
1743 }
1744 EXPORT_SYMBOL(scst_check_local_events);
1745
1746 /* No locks */
1747 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1748 {
1749         if (slot == NULL)
1750                 goto inc;
1751
1752         /* Optimized for lockless fast path */
1753
1754         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1755                 atomic_read(slot));
1756
1757         if (!atomic_dec_and_test(slot))
1758                 goto out;
1759
1760         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1761                 tgt_dev->num_free_sn_slots);
1762         if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
1763                 spin_lock_irq(&tgt_dev->sn_lock);
1764                 if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
1765                         if (tgt_dev->num_free_sn_slots < 0)
1766                                 tgt_dev->cur_sn_slot = slot;
1767                         /*
1768                          * To be in-sync with SIMPLE case in scst_cmd_set_sn()
1769                          */
1770                         smp_mb();
1771                         tgt_dev->num_free_sn_slots++;
1772                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1773                                 tgt_dev->num_free_sn_slots);
1774
1775                 }
1776                 spin_unlock_irq(&tgt_dev->sn_lock);
1777         }
1778
1779 inc:
1780         /*
1781          * No protection of expected_sn is needed, because only one thread
1782          * at time can be here (serialized by sn). Also it is supposed that
1783          * there could not be half-incremented halves.
1784          */
1785         tgt_dev->expected_sn++;
1786         /*
1787          * Write must be before def_cmd_count read to be in sync. with
1788          * scst_post_exec_sn(). See comment in scst_send_for_exec().
1789          */
1790         smp_mb();
1791         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1792
1793 out:
1794         return;
1795 }
1796
1797 /* No locks */
1798 static struct scst_cmd *scst_post_exec_sn(struct scst_cmd *cmd,
1799         bool make_active)
1800 {
1801         /* For HQ commands SN is not set */
1802         bool inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1803                                cmd->sn_set && !cmd->retry;
1804         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1805         struct scst_cmd *res;
1806
1807         TRACE_ENTRY();
1808
1809         if (inc_expected_sn)
1810                 scst_inc_expected_sn(tgt_dev, cmd->sn_slot);
1811
1812         if (make_active) {
1813                 scst_make_deferred_commands_active(tgt_dev);
1814                 res = NULL;
1815         } else
1816                 res = scst_check_deferred_commands(tgt_dev);
1817
1818         TRACE_EXIT_HRES(res);
1819         return res;
1820 }
1821
1822 /* cmd must be additionally referenced to not die inside */
1823 static int scst_do_real_exec(struct scst_cmd *cmd)
1824 {
1825         int res = SCST_EXEC_NOT_COMPLETED;
1826 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
1827         int rc;
1828 #endif
1829         struct scst_device *dev = cmd->dev;
1830         struct scst_dev_type *handler = dev->handler;
1831
1832         TRACE_ENTRY();
1833
1834         cmd->state = SCST_CMD_STATE_REAL_EXECUTING;
1835
1836         if (handler->exec) {
1837                 if (unlikely(!dev->handler->exec_atomic &&
1838                              scst_cmd_atomic(cmd))) {
1839                         /*
1840                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
1841                          * optimization.
1842                          */
1843                         TRACE_DBG("Dev handler %s exec() needs thread "
1844                                 "context, rescheduling", dev->handler->name);
1845                         res = SCST_EXEC_NEED_THREAD;
1846                         goto out_restore;
1847                 }
1848
1849                 TRACE_DBG("Calling dev handler %s exec(%p)",
1850                       handler->name, cmd);
1851                 TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb,
1852                         cmd->cdb_len);
1853                 res = handler->exec(cmd);
1854                 TRACE_DBG("Dev handler %s exec() returned %d",
1855                       handler->name, res);
1856
1857                 if (res == SCST_EXEC_COMPLETED)
1858                         goto out_complete;
1859                 else if (res == SCST_EXEC_NEED_THREAD)
1860                         goto out_restore;
1861
1862                 sBUG_ON(res != SCST_EXEC_NOT_COMPLETED);
1863         }
1864
1865         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1866
1867         if (unlikely(dev->scsi_dev == NULL)) {
1868                 PRINT_ERROR("Command for virtual device must be "
1869                         "processed by device handler (lun %lld)!",
1870                         (long long unsigned int)cmd->lun);
1871                 goto out_error;
1872         }
1873
1874         res = scst_check_local_events(cmd);
1875         if (unlikely(res != 0))
1876                 goto out_done;
1877
1878 #ifndef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1879         if (unlikely(scst_cmd_atomic(cmd))) {
1880                 TRACE_DBG("Pass-through exec() can not be called in atomic "
1881                         "context, rescheduling to the thread (handler %s)",
1882                         handler->name);
1883                 res = SCST_EXEC_NEED_THREAD;
1884                 goto out_restore;
1885         }
1886 #endif
1887
1888 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1889         if (unlikely(scst_alloc_request(cmd) != 0)) {
1890                 if (scst_cmd_atomic(cmd)) {
1891                         res = SCST_EXEC_NEED_THREAD;
1892                         goto out_restore;
1893                 } else {
1894                         PRINT_INFO("%s", "Unable to allocate request, "
1895                                 "sending BUSY status");
1896                         goto out_busy;
1897                 }
1898         }
1899
1900         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1901                     (void *)cmd->scsi_req->sr_buffer,
1902                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1903                     cmd->retries);
1904 #else
1905         rc = scst_exec_req(dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1906                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1907                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1908                         scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
1909         if (unlikely(rc != 0)) {
1910                 if (scst_cmd_atomic(cmd)) {
1911                         res = SCST_EXEC_NEED_THREAD;
1912                         goto out_restore;
1913                 } else {
1914                         PRINT_ERROR("scst_exec_req() failed: %d", res);
1915                         goto out_error;
1916                 }
1917         }
1918 #endif
1919
1920 out_complete:
1921         res = SCST_EXEC_COMPLETED;
1922
1923 out:
1924         TRACE_EXIT();
1925         return res;
1926
1927 out_restore:
1928         /* Restore the state */
1929         cmd->state = SCST_CMD_STATE_REAL_EXEC;
1930         goto out;
1931
1932 out_error:
1933         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1934         goto out_done;
1935
1936 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1937 out_busy:
1938         scst_set_busy(cmd);
1939         /* go through */
1940 #endif
1941
1942 out_done:
1943         res = SCST_EXEC_COMPLETED;
1944         /* Report the result */
1945         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1946         goto out_complete;
1947 }
1948
1949 static inline int scst_real_exec(struct scst_cmd *cmd)
1950 {
1951         int res;
1952
1953         TRACE_ENTRY();
1954
1955         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
1956         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
1957         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
1958
1959         __scst_cmd_get(cmd);
1960
1961         res = scst_do_real_exec(cmd);
1962
1963         if (likely(res == SCST_EXEC_COMPLETED)) {
1964                 scst_post_exec_sn(cmd, true);
1965                 if (cmd->dev->scsi_dev != NULL)
1966                         generic_unplug_device(
1967                                 cmd->dev->scsi_dev->request_queue);
1968         } else
1969                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
1970
1971         __scst_cmd_put(cmd);
1972
1973         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
1974
1975         TRACE_EXIT_RES(res);
1976         return res;
1977 }
1978
1979 static int scst_do_local_exec(struct scst_cmd *cmd)
1980 {
1981         int res;
1982         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1983
1984         TRACE_ENTRY();
1985
1986         /* Check READ_ONLY device status */
1987         if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
1988             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
1989              cmd->cdb[0] == WRITE_10 ||
1990              cmd->cdb[0] == WRITE_12 ||
1991              cmd->cdb[0] == WRITE_16 ||
1992              cmd->cdb[0] == WRITE_VERIFY ||
1993              cmd->cdb[0] == WRITE_VERIFY_12 ||
1994              cmd->cdb[0] == WRITE_VERIFY_16 ||
1995              (cmd->dev->handler->type == TYPE_TAPE &&
1996               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
1997                 scst_set_cmd_error(cmd,
1998                            SCST_LOAD_SENSE(scst_sense_data_protect));
1999                 goto out_done;
2000         }
2001
2002         /*
2003          * Adding new commands here don't forget to update
2004          * scst_is_cmd_local() in scst.h, if necessary
2005          */
2006
2007         switch (cmd->cdb[0]) {
2008         case MODE_SELECT:
2009         case MODE_SELECT_10:
2010         case LOG_SELECT:
2011                 res = scst_pre_select(cmd);
2012                 break;
2013         case RESERVE:
2014         case RESERVE_10:
2015                 res = scst_reserve_local(cmd);
2016                 break;
2017         case RELEASE:
2018         case RELEASE_10:
2019                 res = scst_release_local(cmd);
2020                 break;
2021         case REPORT_LUNS:
2022                 res = scst_report_luns_local(cmd);
2023                 break;
2024         default:
2025                 res = SCST_EXEC_NOT_COMPLETED;
2026                 break;
2027         }
2028
2029 out:
2030         TRACE_EXIT_RES(res);
2031         return res;
2032
2033 out_done:
2034         /* Report the result */
2035         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
2036         res = SCST_EXEC_COMPLETED;
2037         goto out;
2038 }
2039
2040 static int scst_local_exec(struct scst_cmd *cmd)
2041 {
2042         int res;
2043
2044         TRACE_ENTRY();
2045
2046         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
2047         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
2048         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
2049
2050         __scst_cmd_get(cmd);
2051
2052         res = scst_do_local_exec(cmd);
2053         if (likely(res == SCST_EXEC_NOT_COMPLETED))
2054                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2055         else if (res == SCST_EXEC_COMPLETED)
2056                 scst_post_exec_sn(cmd, true);
2057         else
2058                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
2059
2060         __scst_cmd_put(cmd);
2061
2062         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
2063         TRACE_EXIT_RES(res);
2064         return res;
2065 }
2066
2067 static int scst_exec(struct scst_cmd **active_cmd)
2068 {
2069         struct scst_cmd *cmd = *active_cmd;
2070         struct scst_cmd *ref_cmd;
2071         struct scst_device *dev = cmd->dev;
2072         int res = SCST_CMD_STATE_RES_CONT_NEXT, count;
2073
2074         TRACE_ENTRY();
2075
2076         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2077                 goto out;
2078
2079         /* To protect tgt_dev */
2080         ref_cmd = cmd;
2081         __scst_cmd_get(ref_cmd);
2082
2083         count = 0;
2084         while (1) {
2085                 int rc;
2086
2087                 cmd->sent_for_exec = 1;
2088                 cmd->scst_cmd_done = scst_cmd_done_local;
2089                 cmd->state = SCST_CMD_STATE_LOCAL_EXEC;
2090
2091                 if (cmd->tgt_data_buf_alloced && cmd->dh_data_buf_alloced &&
2092                     (cmd->data_direction == SCST_DATA_WRITE))
2093                         scst_copy_sg(cmd, SCST_SG_COPY_FROM_TARGET);
2094
2095                 rc = scst_do_local_exec(cmd);
2096                 if (likely(rc == SCST_EXEC_NOT_COMPLETED))
2097                         /* Nothing to do */;
2098                 else if (rc == SCST_EXEC_NEED_THREAD) {
2099                         TRACE_DBG("%s", "scst_do_local_exec() requested "
2100                                 "thread context, rescheduling");
2101                         scst_dec_on_dev_cmd(cmd);
2102                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2103                         break;
2104                 } else {
2105                         sBUG_ON(rc != SCST_EXEC_COMPLETED);
2106                         goto done;
2107                 }
2108
2109                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2110
2111                 rc = scst_do_real_exec(cmd);
2112                 if (likely(rc == SCST_EXEC_COMPLETED))
2113                         /* Nothing to do */;
2114                 else if (rc == SCST_EXEC_NEED_THREAD) {
2115                         TRACE_DBG("scst_real_exec() requested thread "
2116                                 "context, rescheduling (cmd %p)", cmd);
2117                         scst_dec_on_dev_cmd(cmd);
2118                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2119                         break;
2120                 } else
2121                         sBUG();
2122
2123 done:
2124                 count++;
2125
2126                 cmd = scst_post_exec_sn(cmd, false);
2127                 if (cmd == NULL)
2128                         break;
2129
2130                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2131                         break;
2132
2133                 __scst_cmd_put(ref_cmd);
2134                 ref_cmd = cmd;
2135                 __scst_cmd_get(ref_cmd);
2136         }
2137
2138         *active_cmd = cmd;
2139
2140         if (count == 0)
2141                 goto out_put;
2142
2143         if (dev->scsi_dev != NULL)
2144                 generic_unplug_device(dev->scsi_dev->request_queue);
2145
2146 out_put:
2147         __scst_cmd_put(ref_cmd);
2148         /* !! At this point sess, dev and tgt_dev can be already freed !! */
2149
2150 out:
2151         TRACE_EXIT_RES(res);
2152         return res;
2153 }
2154
2155 static int scst_send_for_exec(struct scst_cmd **active_cmd)
2156 {
2157         int res;
2158         struct scst_cmd *cmd = *active_cmd;
2159         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2160         typeof(tgt_dev->expected_sn) expected_sn;
2161
2162         TRACE_ENTRY();
2163
2164 #ifdef CONFIG_SCST_MEASURE_LATENCY
2165         if (cmd->pre_exec_finish == 0) {
2166                 struct timespec ts;
2167                 getnstimeofday(&ts);
2168                 cmd->pre_exec_finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2169                 TRACE_DBG("cmd %p (sess %p): pre_exec_finish %lld (tv_sec %ld, "
2170                         "tv_nsec %ld)", cmd, cmd->sess, cmd->pre_exec_finish,
2171                         ts.tv_sec, ts.tv_nsec);
2172         }
2173 #endif
2174
2175         if (unlikely(cmd->internal))
2176                 goto exec;
2177
2178         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2179                 goto exec;
2180
2181         sBUG_ON(!cmd->sn_set);
2182
2183         expected_sn = tgt_dev->expected_sn;
2184         /* Optimized for lockless fast path */
2185         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2186                 spin_lock_irq(&tgt_dev->sn_lock);
2187
2188                 tgt_dev->def_cmd_count++;
2189                 /*
2190                  * Memory barrier is needed here to implement lockless fast
2191                  * path. We need the exact order of read and write between
2192                  * def_cmd_count and expected_sn. Otherwise, we can miss case,
2193                  * when expected_sn was changed to be equal to cmd->sn while
2194                  * we are queuing cmd the deferred list after the expected_sn
2195                  * below. It will lead to a forever stuck command. But with
2196                  * the barrier in such case __scst_check_deferred_commands()
2197                  * will be called and it will take sn_lock, so we will be
2198                  * synchronized.
2199                  */
2200                 smp_mb();
2201
2202                 expected_sn = tgt_dev->expected_sn;
2203                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2204                         if (unlikely(test_bit(SCST_CMD_ABORTED,
2205                                               &cmd->cmd_flags))) {
2206                                 /* Necessary to allow aborting out of sn cmds */
2207                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p "
2208                                         "(tag %llu, sn %lu)", cmd,
2209                                         (long long unsigned)cmd->tag, cmd->sn);
2210                                 tgt_dev->def_cmd_count--;
2211                                 scst_set_cmd_abnormal_done_state(cmd);
2212                                 res = SCST_CMD_STATE_RES_CONT_SAME;
2213                         } else {
2214                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
2215                                         "expected_sn=%ld)", cmd, cmd->sn,
2216                                         cmd->sn_set, expected_sn);
2217                                 list_add_tail(&cmd->sn_cmd_list_entry,
2218                                               &tgt_dev->deferred_cmd_list);
2219                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2220                         }
2221                         spin_unlock_irq(&tgt_dev->sn_lock);
2222                         goto out;
2223                 } else {
2224                         TRACE_SN("Somebody incremented expected_sn %ld, "
2225                                 "continuing", expected_sn);
2226                         tgt_dev->def_cmd_count--;
2227                         spin_unlock_irq(&tgt_dev->sn_lock);
2228                 }
2229         }
2230
2231 exec:
2232         res = scst_exec(active_cmd);
2233
2234 out:
2235         TRACE_EXIT_HRES(res);
2236         return res;
2237 }
2238
2239 /* No locks supposed to be held */
2240 static int scst_check_sense(struct scst_cmd *cmd)
2241 {
2242         int res = 0;
2243         struct scst_device *dev = cmd->dev;
2244
2245         TRACE_ENTRY();
2246
2247         if (unlikely(cmd->ua_ignore))
2248                 goto out;
2249
2250         /* If we had internal bus reset behind us, set the command error UA */
2251         if ((dev->scsi_dev != NULL) &&
2252             unlikely(cmd->host_status == DID_RESET) &&
2253             scst_is_ua_command(cmd)) {
2254                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
2255                       dev->scsi_dev->was_reset, cmd->host_status);
2256                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_reset_UA));
2257                 /* It looks like it is safe to clear was_reset here */
2258                 dev->scsi_dev->was_reset = 0;
2259         }
2260
2261         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2262             SCST_SENSE_VALID(cmd->sense)) {
2263                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2264                         SCST_SENSE_BUFFERSIZE);
2265
2266                 /* Check Unit Attention Sense Key */
2267                 if (scst_is_ua_sense(cmd->sense)) {
2268                         if (cmd->sense[12] == SCST_SENSE_ASC_UA_RESET) {
2269                                 if (cmd->double_ua_possible) {
2270                                         TRACE(TRACE_MGMT_MINOR, "Double UA "
2271                                                 "detected for device %p", dev);
2272                                         TRACE(TRACE_MGMT_MINOR, "Retrying cmd"
2273                                                 " %p (tag %llu)", cmd,
2274                                                 (long long unsigned)cmd->tag);
2275
2276                                         cmd->status = 0;
2277                                         cmd->msg_status = 0;
2278                                         cmd->host_status = DID_OK;
2279                                         cmd->driver_status = 0;
2280
2281                                         mempool_free(cmd->sense,
2282                                                      scst_sense_mempool);
2283                                         cmd->sense = NULL;
2284
2285                                         scst_check_restore_sg_buff(cmd);
2286
2287                                         sBUG_ON(cmd->dbl_ua_orig_resp_data_len < 0);
2288                                         cmd->data_direction =
2289                                                 cmd->dbl_ua_orig_data_direction;
2290                                         cmd->resp_data_len =
2291                                                 cmd->dbl_ua_orig_resp_data_len;
2292
2293                                         cmd->state = SCST_CMD_STATE_REAL_EXEC;
2294                                         cmd->retry = 1;
2295                                         res = 1;
2296                                         goto out;
2297                                 }
2298                         }
2299                         scst_dev_check_set_UA(dev, cmd, cmd->sense,
2300                                 SCST_SENSE_BUFFERSIZE);
2301                 }
2302         }
2303
2304         if (unlikely(cmd->double_ua_possible)) {
2305                 if (scst_is_ua_command(cmd)) {
2306                         TRACE_MGMT_DBG("Clearing dbl_ua_possible flag (dev %p, "
2307                                 "cmd %p)", dev, cmd);
2308                         /*
2309                          * Lock used to protect other flags in the bitfield
2310                          * (just in case, actually). Those flags can't be
2311                          * changed in parallel, because the device is
2312                          * serialized.
2313                          */
2314                         spin_lock_bh(&dev->dev_lock);
2315                         dev->dev_double_ua_possible = 0;
2316                         spin_unlock_bh(&dev->dev_lock);
2317                 }
2318         }
2319
2320 out:
2321         TRACE_EXIT_RES(res);
2322         return res;
2323 }
2324
2325 static int scst_check_auto_sense(struct scst_cmd *cmd)
2326 {
2327         int res = 0;
2328
2329         TRACE_ENTRY();
2330
2331         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2332             (!SCST_SENSE_VALID(cmd->sense) ||
2333              SCST_NO_SENSE(cmd->sense))) {
2334                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2335                       "cmd->status=%x, cmd->msg_status=%x, "
2336                       "cmd->host_status=%x, cmd->driver_status=%x (cmd %p)",
2337                       cmd->status, cmd->msg_status, cmd->host_status,
2338                       cmd->driver_status, cmd);
2339                 res = 1;
2340         } else if (unlikely(cmd->host_status)) {
2341                 if ((cmd->host_status == DID_REQUEUE) ||
2342                     (cmd->host_status == DID_IMM_RETRY) ||
2343                     (cmd->host_status == DID_SOFT_ERROR) ||
2344                     (cmd->host_status == DID_ABORT)) {
2345                         scst_set_busy(cmd);
2346                 } else {
2347                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2348                                 "received, returning HARDWARE ERROR instead "
2349                                 "(cmd %p)", cmd->host_status, cmd);
2350                         scst_set_cmd_error(cmd,
2351                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2352                 }
2353         }
2354
2355         TRACE_EXIT_RES(res);
2356         return res;
2357 }
2358
2359 static int scst_pre_dev_done(struct scst_cmd *cmd)
2360 {
2361         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2362
2363         TRACE_ENTRY();
2364
2365         if (unlikely(scst_check_auto_sense(cmd))) {
2366                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2367                             "without sense data (opcode 0x%x), issuing "
2368                             "REQUEST SENSE", cmd->cdb[0]);
2369                 rc = scst_prepare_request_sense(cmd);
2370                 if (rc == 0)
2371                         res = SCST_CMD_STATE_RES_CONT_NEXT;
2372                 else {
2373                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2374                                     "returning HARDWARE ERROR");
2375                         scst_set_cmd_error(cmd,
2376                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2377                 }
2378                 goto out;
2379         } else if (unlikely(scst_check_sense(cmd)))
2380                 goto out;
2381
2382         if (likely(scsi_status_is_good(cmd->status))) {
2383                 unsigned char type = cmd->dev->handler->type;
2384                 if (unlikely((cmd->cdb[0] == MODE_SENSE ||
2385                               cmd->cdb[0] == MODE_SENSE_10)) &&
2386                     cmd->tgt_dev->acg_dev->rd_only_flag &&
2387                     (type == TYPE_DISK ||
2388                      type == TYPE_WORM ||
2389                      type == TYPE_MOD ||
2390                      type == TYPE_TAPE)) {
2391                         int32_t length;
2392                         uint8_t *address;
2393                         bool err = false;
2394
2395                         length = scst_get_buf_first(cmd, &address);
2396                         if (length < 0) {
2397                                 PRINT_ERROR("%s", "Unable to get "
2398                                         "MODE_SENSE buffer");
2399                                 scst_set_cmd_error(cmd,
2400                                         SCST_LOAD_SENSE(
2401                                                 scst_sense_hardw_error));
2402                                 err = true;
2403                         } else if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2404                                 address[2] |= 0x80;   /* Write Protect*/
2405                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2406                                 address[3] |= 0x80;   /* Write Protect*/
2407                         scst_put_buf(cmd, address);
2408
2409                         if (err)
2410                                 goto out;
2411                 }
2412
2413                 /*
2414                  * Check and clear NormACA option for the device, if necessary,
2415                  * since we don't support ACA
2416                  */
2417                 if (unlikely((cmd->cdb[0] == INQUIRY)) &&
2418                     /* Std INQUIRY data (no EVPD) */
2419                     !(cmd->cdb[1] & SCST_INQ_EVPD) &&
2420                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2421                         uint8_t *buffer;
2422                         int buflen;
2423                         bool err = false;
2424
2425                         /* ToDo: all pages ?? */
2426                         buflen = scst_get_buf_first(cmd, &buffer);
2427                         if (buflen > SCST_INQ_BYTE3) {
2428 #ifdef CONFIG_SCST_EXTRACHECKS
2429                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2430                                         PRINT_INFO("NormACA set for device: "
2431                                             "lun=%lld, type 0x%02x. Clear it, "
2432                                             "since it's unsupported.",
2433                                             (long long unsigned int)cmd->lun,
2434                                             buffer[0]);
2435                                 }
2436 #endif
2437                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2438                         } else if (buflen != 0) {
2439                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2440                                     "buffer");
2441                                 scst_set_cmd_error(cmd,
2442                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2443                                 err = true;
2444                         }
2445                         if (buflen > 0)
2446                                 scst_put_buf(cmd, buffer);
2447
2448                         if (err)
2449                                 goto out;
2450                 }
2451
2452                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2453                     (cmd->cdb[0] == MODE_SELECT_10) ||
2454                     (cmd->cdb[0] == LOG_SELECT))) {
2455                         TRACE(TRACE_SCSI,
2456                                 "MODE/LOG SELECT succeeded (LUN %lld)",
2457                                 (long long unsigned int)cmd->lun);
2458                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2459                         goto out;
2460                 }
2461         } else {
2462                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2463                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2464                                         &cmd->tgt_dev->tgt_dev_flags)) {
2465                                 struct scst_tgt_dev *tgt_dev_tmp;
2466                                 struct scst_device *dev = cmd->dev;
2467
2468                                 TRACE(TRACE_SCSI,
2469                                         "Real RESERVE failed lun=%lld, "
2470                                         "status=%x",
2471                                         (long long unsigned int)cmd->lun,
2472                                         cmd->status);
2473                                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2474                                         SCST_SENSE_BUFFERSIZE);
2475
2476                                 /* Clearing the reservation */
2477                                 spin_lock_bh(&dev->dev_lock);
2478                                 list_for_each_entry(tgt_dev_tmp,
2479                                                     &dev->dev_tgt_dev_list,
2480                                                     dev_tgt_dev_list_entry) {
2481                                         clear_bit(SCST_TGT_DEV_RESERVED,
2482                                                 &tgt_dev_tmp->tgt_dev_flags);
2483                                 }
2484                                 dev->dev_reserved = 0;
2485                                 spin_unlock_bh(&dev->dev_lock);
2486                         }
2487                 }
2488
2489                 /* Check for MODE PARAMETERS CHANGED UA */
2490                 if ((cmd->dev->scsi_dev != NULL) &&
2491                     (cmd->status == SAM_STAT_CHECK_CONDITION) &&
2492                     SCST_SENSE_VALID(cmd->sense) &&
2493                     scst_is_ua_sense(cmd->sense) &&
2494                     (cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) {
2495                         TRACE(TRACE_SCSI,
2496                               "MODE PARAMETERS CHANGED UA (lun %lld)",
2497                               (long long unsigned int)cmd->lun);
2498                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2499                         goto out;
2500                 }
2501         }
2502
2503         cmd->state = SCST_CMD_STATE_DEV_DONE;
2504
2505 out:
2506         TRACE_EXIT_RES(res);
2507         return res;
2508 }
2509
2510 static int scst_mode_select_checks(struct scst_cmd *cmd)
2511 {
2512         int res = SCST_CMD_STATE_RES_CONT_SAME;
2513         int atomic = scst_cmd_atomic(cmd);
2514
2515         TRACE_ENTRY();
2516
2517         if (likely(scsi_status_is_good(cmd->status))) {
2518                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2519                     (cmd->cdb[0] == MODE_SELECT_10) ||
2520                     (cmd->cdb[0] == LOG_SELECT))) {
2521                         struct scst_device *dev = cmd->dev;
2522                         if (atomic && (dev->scsi_dev != NULL)) {
2523                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2524                                         "context required");
2525                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2526                                 goto out;
2527                         }
2528
2529                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2530                                 "setting the SELECT UA (lun=%lld)",
2531                                 (long long unsigned int)cmd->lun);
2532
2533                         spin_lock_bh(&dev->dev_lock);
2534                         spin_lock(&scst_temp_UA_lock);
2535                         if (cmd->cdb[0] == LOG_SELECT) {
2536                                 scst_set_sense(scst_temp_UA,
2537                                         sizeof(scst_temp_UA),
2538                                         UNIT_ATTENTION, 0x2a, 0x02);
2539                         } else {
2540                                 scst_set_sense(scst_temp_UA,
2541                                         sizeof(scst_temp_UA),
2542                                         UNIT_ATTENTION, 0x2a, 0x01);
2543                         }
2544                         scst_dev_check_set_local_UA(dev, cmd, scst_temp_UA,
2545                                 sizeof(scst_temp_UA));
2546                         spin_unlock(&scst_temp_UA_lock);
2547                         spin_unlock_bh(&dev->dev_lock);
2548
2549                         if (dev->scsi_dev != NULL)
2550                                 scst_obtain_device_parameters(dev);
2551                 }
2552         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) &&
2553                     SCST_SENSE_VALID(cmd->sense) &&
2554                     scst_is_ua_sense(cmd->sense) &&
2555                     (((cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) ||
2556                      (cmd->sense[12] == 0x29) /* reset */ ||
2557                      (cmd->sense[12] == 0x28) /* medium changed */ ||
2558                      /* cleared by another ini (just in case) */
2559                      (cmd->sense[12] == 0x2F))) {
2560                 if (atomic) {
2561                         TRACE_DBG("Possible parameters changed UA %x: "
2562                                 "thread context required", cmd->sense[12]);
2563                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2564                         goto out;
2565                 }
2566
2567                 TRACE(TRACE_SCSI, "Possible parameters changed UA %x "
2568                         "(lun %lld): getting new parameters", cmd->sense[12],
2569                         (long long unsigned int)cmd->lun);
2570
2571                 scst_obtain_device_parameters(cmd->dev);
2572         } else
2573                 sBUG();
2574
2575         cmd->state = SCST_CMD_STATE_DEV_DONE;
2576
2577 out:
2578         TRACE_EXIT_HRES(res);
2579         return res;
2580 }
2581
2582 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2583 {
2584         if (likely(cmd->sn_set))
2585                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2586
2587         scst_make_deferred_commands_active(cmd->tgt_dev);
2588 }
2589
2590 static int scst_dev_done(struct scst_cmd **pcmd)
2591 {
2592         int res = SCST_CMD_STATE_RES_CONT_SAME;
2593         struct scst_cmd *cmd = *pcmd;
2594         int state;
2595         struct scst_device *dev = cmd->dev;
2596
2597         TRACE_ENTRY();
2598
2599         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2600
2601         if (likely(!scst_is_cmd_local(cmd)) &&
2602             likely(dev->handler->dev_done != NULL)) {
2603                 int rc;
2604
2605                 if (unlikely(!dev->handler->dev_done_atomic &&
2606                              scst_cmd_atomic(cmd))) {
2607                         /*
2608                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2609                          * optimization.
2610                          */
2611                         TRACE_DBG("Dev handler %s dev_done() needs thread "
2612                               "context, rescheduling", dev->handler->name);
2613                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2614                         goto out;
2615                 }
2616
2617                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2618                       dev->handler->name, cmd);
2619                 rc = dev->handler->dev_done(cmd);
2620                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2621                       dev->handler->name, rc);
2622                 if (rc != SCST_CMD_STATE_DEFAULT)
2623                         state = rc;
2624         }
2625
2626         switch (state) {
2627         case SCST_CMD_STATE_PRE_XMIT_RESP:
2628         case SCST_CMD_STATE_DEV_PARSE:
2629         case SCST_CMD_STATE_PRE_PARSE:
2630         case SCST_CMD_STATE_PREPARE_SPACE:
2631         case SCST_CMD_STATE_RDY_TO_XFER:
2632         case SCST_CMD_STATE_TGT_PRE_EXEC:
2633         case SCST_CMD_STATE_SEND_FOR_EXEC:
2634         case SCST_CMD_STATE_LOCAL_EXEC:
2635         case SCST_CMD_STATE_REAL_EXEC:
2636         case SCST_CMD_STATE_PRE_DEV_DONE:
2637         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2638         case SCST_CMD_STATE_DEV_DONE:
2639         case SCST_CMD_STATE_XMIT_RESP:
2640         case SCST_CMD_STATE_FINISHED:
2641                 cmd->state = state;
2642                 break;
2643
2644         case SCST_CMD_STATE_NEED_THREAD_CTX:
2645                 TRACE_DBG("Dev handler %s dev_done() requested "
2646                       "thread context, rescheduling",
2647                       dev->handler->name);
2648                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2649                 break;
2650
2651         default:
2652                 if (state >= 0) {
2653                         PRINT_ERROR("Dev handler %s dev_done() returned "
2654                                 "invalid cmd state %d",
2655                                 dev->handler->name, state);
2656                 } else {
2657                         PRINT_ERROR("Dev handler %s dev_done() returned "
2658                                 "error %d", dev->handler->name,
2659                                 state);
2660                 }
2661                 scst_set_cmd_error(cmd,
2662                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2663                 scst_set_cmd_abnormal_done_state(cmd);
2664                 break;
2665         }
2666
2667         if (cmd->needs_unblocking)
2668                 scst_unblock_dev_cmd(cmd);
2669
2670         if (likely(cmd->dec_on_dev_needed))
2671                 scst_dec_on_dev_cmd(cmd);
2672
2673         if (cmd->inc_expected_sn_on_done && cmd->sent_for_exec)
2674                 scst_inc_check_expected_sn(cmd);
2675
2676         if (unlikely(cmd->cdb[0] == REQUEST_SENSE) && (cmd->internal))
2677                 *pcmd = scst_complete_request_sense(cmd);
2678
2679 out:
2680         TRACE_EXIT_HRES(res);
2681         return res;
2682 }
2683
2684 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2685 {
2686         int res;
2687
2688         TRACE_ENTRY();
2689
2690 #ifdef CONFIG_SCST_DEBUG_TM
2691         if (cmd->tm_dbg_delayed &&
2692                         !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2693                 if (scst_cmd_atomic(cmd)) {
2694                         TRACE_MGMT_DBG("%s",
2695                                 "DEBUG_TM delayed cmd needs a thread");
2696                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2697                         return res;
2698                 }
2699                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2700                         cmd, cmd->tag);
2701                 schedule_timeout_uninterruptible(HZ);
2702         }
2703 #endif
2704
2705         if (likely(cmd->tgt_dev != NULL)) {
2706                 atomic_dec(&cmd->tgt_dev->tgt_dev_cmd_count);
2707                 atomic_dec(&cmd->dev->dev_cmd_count);
2708                 /* If expected values not set, expected direction is UNKNOWN */
2709                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
2710                         atomic_dec(&cmd->dev->write_cmd_count);
2711
2712                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2713                         scst_on_hq_cmd_response(cmd);
2714
2715                 if (unlikely(!cmd->sent_for_exec)) {
2716                         TRACE_SN("cmd %p was not sent to mid-lev"
2717                                 " (sn %ld, set %d)",
2718                                 cmd, cmd->sn, cmd->sn_set);
2719                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2720                         cmd->sent_for_exec = 1;
2721                 }
2722         }
2723
2724         /*
2725          * If we don't remove cmd from the search list here, before
2726          * submitting it for transmittion, we will have a race, when for
2727          * some reason cmd's release is delayed after transmittion and
2728          * initiator sends cmd with the same tag => it is possible that
2729          * a wrong cmd will be found by find() functions.
2730          */
2731         spin_lock_irq(&cmd->sess->sess_list_lock);
2732         list_del(&cmd->search_cmd_list_entry);
2733         spin_unlock_irq(&cmd->sess->sess_list_lock);
2734
2735         cmd->done = 1;
2736         smp_mb(); /* to sync with scst_abort_cmd() */
2737
2738         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2739                 scst_xmit_process_aborted_cmd(cmd);
2740
2741         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2742                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu),"
2743                                 " skipping",
2744                                 cmd, (long long unsigned int)cmd->tag);
2745                 cmd->state = SCST_CMD_STATE_FINISHED;
2746                 res = SCST_CMD_STATE_RES_CONT_SAME;
2747                 goto out;
2748         }
2749
2750         if (cmd->tgt_data_buf_alloced && cmd->dh_data_buf_alloced &&
2751             (cmd->data_direction == SCST_DATA_READ))
2752                 scst_copy_sg(cmd, SCST_SG_COPY_TO_TARGET);
2753
2754         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2755         res = SCST_CMD_STATE_RES_CONT_SAME;
2756
2757 out:
2758 #ifdef CONFIG_SCST_MEASURE_LATENCY
2759         {
2760                 struct timespec ts;
2761                 uint64_t finish, scst_time, proc_time;
2762                 struct scst_session *sess = cmd->sess;
2763
2764                 getnstimeofday(&ts);
2765                 finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2766
2767                 spin_lock_bh(&sess->meas_lock);
2768
2769                 scst_time = cmd->pre_exec_finish - cmd->start;
2770                 scst_time += finish - cmd->post_exec_start;
2771                 proc_time = finish - cmd->start;
2772
2773                 sess->scst_time += scst_time;
2774                 sess->processing_time += proc_time;
2775                 sess->processed_cmds++;
2776
2777                 spin_unlock_bh(&sess->meas_lock);
2778
2779                 TRACE_DBG("cmd %p (sess %p): finish %lld (tv_sec %ld, "
2780                         "tv_nsec %ld), scst_time %lld, proc_time %lld",
2781                         cmd, sess, finish, ts.tv_sec, ts.tv_nsec, scst_time,
2782                         proc_time);
2783         }
2784 #endif
2785         TRACE_EXIT_HRES(res);
2786         return res;
2787 }
2788
2789 static int scst_xmit_response(struct scst_cmd *cmd)
2790 {
2791         int res, rc;
2792
2793         TRACE_ENTRY();
2794
2795         if (unlikely(!cmd->tgtt->xmit_response_atomic &&
2796                      scst_cmd_atomic(cmd))) {
2797                 /*
2798                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2799                  * optimization.
2800                  */
2801                 TRACE_DBG("Target driver %s xmit_response() needs thread "
2802                               "context, rescheduling", cmd->tgtt->name);
2803                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2804                 goto out;
2805         }
2806
2807         while (1) {
2808                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2809
2810                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2811                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2812
2813                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2814
2815 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2816                 if (cmd->tgt_sg) {
2817                         int i;
2818                         struct scatterlist *sg = cmd->tgt_sg;
2819                         TRACE(TRACE_SND_BOT, "Xmitting data for cmd %p "
2820                                 "(sg_cnt %d, sg %p, sg[0].page %p)", cmd,
2821                                 cmd->tgt_sg_cnt, sg, (void *)sg_page(&sg[0]));
2822                         for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
2823                                 PRINT_BUFF_FLAG(TRACE_SND_BOT, "Xmitting sg",
2824                                         sg_virt(&sg[i]), sg[i].length);
2825                         }
2826                 }
2827 #endif
2828
2829 #ifdef CONFIG_SCST_DEBUG_RETRY
2830                 if (((scst_random() % 100) == 77))
2831                         rc = SCST_TGT_RES_QUEUE_FULL;
2832                 else
2833 #endif
2834                         rc = cmd->tgtt->xmit_response(cmd);
2835                 TRACE_DBG("xmit_response() returned %d", rc);
2836
2837                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2838                         goto out;
2839
2840                 /* Restore the previous state */
2841                 cmd->state = SCST_CMD_STATE_XMIT_RESP;
2842
2843                 switch (rc) {
2844                 case SCST_TGT_RES_QUEUE_FULL:
2845                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2846                                 break;
2847                         else
2848                                 continue;
2849
2850                 case SCST_TGT_RES_NEED_THREAD_CTX:
2851                         TRACE_DBG("Target driver %s xmit_response() "
2852                               "requested thread context, rescheduling",
2853                               cmd->tgtt->name);
2854                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2855                         break;
2856
2857                 default:
2858                         goto out_error;
2859                 }
2860                 break;
2861         }
2862
2863 out:
2864         /* Caution: cmd can be already dead here */
2865         TRACE_EXIT_HRES(res);
2866         return res;
2867
2868 out_error:
2869         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2870                 PRINT_ERROR("Target driver %s xmit_response() returned "
2871                         "fatal error", cmd->tgtt->name);
2872         } else {
2873                 PRINT_ERROR("Target driver %s xmit_response() returned "
2874                         "invalid value %d", cmd->tgtt->name, rc);
2875         }
2876         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2877         cmd->state = SCST_CMD_STATE_FINISHED;
2878         res = SCST_CMD_STATE_RES_CONT_SAME;
2879         goto out;
2880 }
2881
2882 void scst_tgt_cmd_done(struct scst_cmd *cmd,
2883         enum scst_exec_context pref_context)
2884 {
2885         TRACE_ENTRY();
2886
2887         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2888
2889         cmd->state = SCST_CMD_STATE_FINISHED;
2890         scst_proccess_redirect_cmd(cmd, pref_context, 1);
2891
2892         TRACE_EXIT();
2893         return;
2894 }
2895 EXPORT_SYMBOL(scst_tgt_cmd_done);
2896
2897 static int scst_finish_cmd(struct scst_cmd *cmd)
2898 {
2899         int res;
2900
2901         TRACE_ENTRY();
2902
2903         atomic_dec(&cmd->sess->sess_cmd_count);
2904
2905         cmd->finished = 1;
2906         smp_mb(); /* to sync with scst_abort_cmd() */
2907
2908         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2909                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
2910                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
2911                         atomic_read(&scst_cmd_count));
2912
2913                 scst_finish_cmd_mgmt(cmd);
2914         }
2915
2916         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
2917                 if ((cmd->tgt_dev != NULL) &&
2918                     scst_is_ua_sense(cmd->sense)) {
2919                         /* This UA delivery failed, so requeue it */
2920                         TRACE_MGMT_DBG("Requeuing UA for delivery failed cmd "
2921                                 "%p", cmd);
2922                         scst_check_set_UA(cmd->tgt_dev, cmd->sense,
2923                                         SCST_SENSE_BUFFERSIZE, 1);
2924                 }
2925         }
2926
2927         __scst_cmd_put(cmd);
2928
2929         res = SCST_CMD_STATE_RES_CONT_NEXT;
2930
2931         TRACE_EXIT_HRES(res);
2932         return res;
2933 }
2934
2935 /*
2936  * No locks, but it must be externally serialized (see comment for
2937  * scst_cmd_init_done() in scst.h)
2938  */
2939 static void scst_cmd_set_sn(struct scst_cmd *cmd)
2940 {
2941         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2942         unsigned long flags;
2943
2944         TRACE_ENTRY();
2945
2946         if (scst_is_implicit_hq(cmd)) {
2947                 TRACE_SN("Implicit HQ cmd %p", cmd);
2948                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
2949         }
2950
2951         EXTRACHECKS_BUG_ON(cmd->sn_set || cmd->hq_cmd_inced);
2952
2953         /* Optimized for lockless fast path */
2954
2955         scst_check_debug_sn(cmd);
2956
2957         if (cmd->dev->queue_alg ==
2958                         SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) {
2959                 /*
2960                  * Not the best way, but well enough until there will be a
2961                  * possibility to specify queue type during pass-through
2962                  * commands submission.
2963                  */
2964                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
2965         }
2966
2967         switch (cmd->queue_type) {
2968         case SCST_CMD_QUEUE_SIMPLE:
2969         case SCST_CMD_QUEUE_UNTAGGED:
2970 #if 0 /* left for future performance investigations */
2971                 if (scst_cmd_is_expected_set(cmd)) {
2972                         if ((cmd->expected_data_direction == SCST_DATA_READ) &&
2973                             (atomic_read(&cmd->dev->write_cmd_count) == 0))
2974                                 goto ordered;
2975                 } else
2976                         goto ordered;
2977 #endif
2978                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
2979                         /*
2980                          * atomic_inc_return() implies memory barrier to sync
2981                          * with scst_inc_expected_sn()
2982                          */
2983                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
2984                                 tgt_dev->curr_sn++;
2985                                 TRACE_SN("Incremented curr_sn %ld",
2986                                         tgt_dev->curr_sn);
2987                         }
2988                         cmd->sn_slot = tgt_dev->cur_sn_slot;
2989                         cmd->sn = tgt_dev->curr_sn;
2990
2991                         tgt_dev->prev_cmd_ordered = 0;
2992                 } else {
2993                         TRACE(TRACE_MINOR, "***WARNING*** Not enough SN slots "
2994                                 "%zd", ARRAY_SIZE(tgt_dev->sn_slots));
2995                         goto ordered;
2996                 }
2997                 break;
2998
2999         case SCST_CMD_QUEUE_ORDERED:
3000                 TRACE_SN("ORDERED cmd %p (op %x)", cmd, cmd->cdb[0]);
3001 ordered:
3002                 if (!tgt_dev->prev_cmd_ordered) {
3003                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
3004                         if (tgt_dev->num_free_sn_slots >= 0) {
3005                                 tgt_dev->num_free_sn_slots--;
3006                                 if (tgt_dev->num_free_sn_slots >= 0) {
3007                                         int i = 0;
3008                                         /* Commands can finish in any order, so
3009                                          * we don't know which slot is empty.
3010                                          */
3011                                         while (1) {
3012                                                 tgt_dev->cur_sn_slot++;
3013                                                 if (tgt_dev->cur_sn_slot ==
3014                                                       tgt_dev->sn_slots + ARRAY_SIZE(tgt_dev->sn_slots))
3015                                                         tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
3016
3017                                                 if (atomic_read(tgt_dev->cur_sn_slot) == 0)
3018                                                         break;
3019
3020                                                 i++;
3021                                                 sBUG_ON(i == ARRAY_SIZE(tgt_dev->sn_slots));
3022                                         }
3023                                         TRACE_SN("New cur SN slot %zd",
3024                                                 tgt_dev->cur_sn_slot -
3025                                                 tgt_dev->sn_slots);
3026                                 }
3027                         }
3028                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
3029                 }
3030                 tgt_dev->prev_cmd_ordered = 1;
3031                 tgt_dev->curr_sn++;
3032                 cmd->sn = tgt_dev->curr_sn;
3033                 break;
3034
3035         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
3036                 TRACE_SN("HQ cmd %p (op %x)", cmd, cmd->cdb[0]);
3037                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
3038                 tgt_dev->hq_cmd_count++;
3039                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
3040                 cmd->hq_cmd_inced = 1;
3041                 goto out;
3042
3043         default:
3044                 sBUG();
3045         }
3046
3047         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
3048                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
3049                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
3050                 atomic_read(tgt_dev->cur_sn_slot),
3051                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
3052                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
3053
3054         cmd->sn_set = 1;
3055
3056 out:
3057         TRACE_EXIT();
3058         return;
3059 }
3060
3061 /*
3062  * Returns 0 on success, > 0 when we need to wait for unblock,
3063  * < 0 if there is no device (lun) or device type handler.
3064  *
3065  * No locks, but might be on IRQ, protection is done by the
3066  * suspended activity.
3067  */
3068 static int scst_translate_lun(struct scst_cmd *cmd)
3069 {
3070         struct scst_tgt_dev *tgt_dev = NULL;
3071         int res;
3072
3073         TRACE_ENTRY();
3074
3075         /* See comment about smp_mb() in scst_suspend_activity() */
3076         __scst_get(1);
3077
3078         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
3079                 struct list_head *sess_tgt_dev_list_head =
3080                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
3081                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %lld)", cmd,
3082                         (long long unsigned int)cmd->lun);
3083                 res = -1;
3084                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3085                                 sess_tgt_dev_list_entry) {
3086                         if (tgt_dev->lun == cmd->lun) {
3087                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
3088
3089                                 if (unlikely(tgt_dev->dev->handler ==
3090                                                 &scst_null_devtype)) {
3091                                         PRINT_INFO("Dev handler for device "
3092                                           "%lld is NULL, the device will not "
3093                                           "be visible remotely",
3094                                            (long long unsigned int)cmd->lun);
3095                                         break;
3096                                 }
3097
3098                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
3099                                 cmd->tgt_dev = tgt_dev;
3100                                 cmd->dev = tgt_dev->dev;
3101
3102                                 res = 0;
3103                                 break;
3104                         }
3105                 }
3106                 if (res != 0) {
3107                         TRACE(TRACE_MINOR,
3108                                 "tgt_dev for lun %lld not found, command to "
3109                                 "unexisting LU?",
3110                                 (long long unsigned int)cmd->lun);
3111                         __scst_put();
3112                 }
3113         } else {
3114                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3115                 __scst_put();
3116                 res = 1;
3117         }
3118
3119         TRACE_EXIT_RES(res);
3120         return res;
3121 }
3122
3123 /*
3124  * No locks, but might be on IRQ
3125  *
3126  * Returns 0 on success, > 0 when we need to wait for unblock,
3127  * < 0 if there is no device (lun) or device type handler.
3128  */
3129 static int __scst_init_cmd(struct scst_cmd *cmd)
3130 {
3131         int res = 0;
3132
3133         TRACE_ENTRY();
3134
3135         res = scst_translate_lun(cmd);
3136         if (likely(res == 0)) {
3137                 int cnt;
3138                 bool failure = false;
3139
3140                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
3141
3142                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
3143                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
3144                         TRACE(TRACE_MGMT_MINOR,
3145                                 "Too many pending commands (%d) in "
3146                                 "session, returning BUSY to initiator \"%s\"",
3147                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
3148                                   "Anonymous" : cmd->sess->initiator_name);
3149                         failure = true;
3150                 }
3151
3152                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
3153                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
3154                         if (!failure) {
3155                                 TRACE(TRACE_MGMT_MINOR,
3156                                         "Too many pending device "
3157                                         "commands (%d), returning BUSY to "
3158                                         "initiator \"%s\"", cnt,
3159                                         (cmd->sess->initiator_name[0] == '\0') ?
3160                                                 "Anonymous" :
3161                                                 cmd->sess->initiator_name);
3162                                 failure = true;
3163                         }
3164                 }
3165
3166                 /* If expected values not set, expected direction is UNKNOWN */
3167                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
3168                         atomic_inc(&cmd->dev->write_cmd_count);
3169
3170                 if (unlikely(failure))
3171                         goto out_busy;
3172
3173                 if (!cmd->set_sn_on_restart_cmd)
3174                         scst_cmd_set_sn(cmd);
3175         } else if (res < 0) {
3176                 TRACE_DBG("Finishing cmd %p", cmd);
3177                 scst_set_cmd_error(cmd,
3178                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
3179                 scst_set_cmd_abnormal_done_state(cmd);
3180         } else
3181                 goto out;
3182
3183 out:
3184         TRACE_EXIT_RES(res);
3185         return res;
3186
3187 out_busy:
3188         scst_set_busy(cmd);
3189         scst_set_cmd_abnormal_done_state(cmd);
3190         goto out;
3191 }
3192
3193 /* Called under scst_init_lock and IRQs disabled */
3194 static void scst_do_job_init(void)
3195         __releases(&scst_init_lock)
3196         __acquires(&scst_init_lock)
3197 {
3198         struct scst_cmd *cmd;
3199         int susp;
3200
3201         TRACE_ENTRY();
3202
3203 restart:
3204         /*
3205          * There is no need for read barrier here, because we don't care where
3206          * this check will be done.
3207          */
3208         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
3209         if (scst_init_poll_cnt > 0)
3210                 scst_init_poll_cnt--;
3211
3212         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
3213                 int rc;
3214                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
3215                         continue;
3216                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
3217                         spin_unlock_irq(&scst_init_lock);
3218                         rc = __scst_init_cmd(cmd);
3219                         spin_lock_irq(&scst_init_lock);
3220                         if (rc > 0) {
3221                                 TRACE_MGMT_DBG("%s",
3222                                         "FLAG SUSPENDED set, restarting");
3223                                 goto restart;
3224                         }
3225                 } else {
3226                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
3227                                        cmd, (long long unsigned int)cmd->tag);
3228                         scst_set_cmd_abnormal_done_state(cmd);
3229                 }
3230
3231                 /*
3232                  * Deleting cmd from init cmd list after __scst_init_cmd()
3233                  * is necessary to keep the check in scst_init_cmd() correct
3234                  * to preserve the commands order.
3235                  *
3236                  * We don't care about the race, when init cmd list is empty
3237                  * and one command detected that it just was not empty, so
3238                  * it's inserting to it, but another command at the same time
3239                  * seeing init cmd list empty and goes directly, because it
3240                  * could affect only commands from the same initiator to the
3241                  * same tgt_dev, but scst_cmd_init_done*() doesn't guarantee
3242                  * the order in case of simultaneous such calls anyway.
3243                  */
3244                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
3245                 smp_wmb(); /* enforce the required order */
3246                 list_del(&cmd->cmd_list_entry);
3247                 spin_unlock(&scst_init_lock);
3248
3249                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
3250                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
3251                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
3252                         list_add(&cmd->cmd_list_entry,
3253                                 &cmd->cmd_lists->active_cmd_list);
3254                 else
3255                         list_add_tail(&cmd->cmd_list_entry,
3256                                 &cmd->cmd_lists->active_cmd_list);
3257                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3258                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
3259
3260                 spin_lock(&scst_init_lock);
3261                 goto restart;
3262         }
3263
3264         /* It isn't really needed, but let's keep it */
3265         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
3266                 goto restart;
3267
3268         TRACE_EXIT();
3269         return;
3270 }
3271
3272 static inline int test_init_cmd_list(void)
3273 {
3274         int res = (!list_empty(&scst_init_cmd_list) &&
3275                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
3276                   unlikely(kthread_should_stop()) ||
3277                   (scst_init_poll_cnt > 0);
3278         return res;
3279 }
3280
3281 int scst_init_cmd_thread(void *arg)
3282 {
3283         TRACE_ENTRY();
3284
3285         PRINT_INFO("Init thread started, PID %d", current->pid);
3286
3287         current->flags |= PF_NOFREEZE;
3288
3289         set_user_nice(current, -10);
3290
3291         spin_lock_irq(&scst_init_lock);
3292         while (!kthread_should_stop()) {
3293                 wait_queue_t wait;
3294                 init_waitqueue_entry(&wait, current);
3295
3296                 if (!test_init_cmd_list()) {
3297                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
3298                                                  &wait);
3299                         for (;;) {
3300                                 set_current_state(TASK_INTERRUPTIBLE);
3301                                 if (test_init_cmd_list())
3302                                         break;
3303                                 spin_unlock_irq(&scst_init_lock);
3304                                 schedule();
3305                                 spin_lock_irq(&scst_init_lock);
3306                         }
3307                         set_current_state(TASK_RUNNING);
3308                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
3309                 }
3310                 scst_do_job_init();
3311         }
3312         spin_unlock_irq(&scst_init_lock);
3313
3314         /*
3315          * If kthread_should_stop() is true, we are guaranteed to be
3316          * on the module unload, so scst_init_cmd_list must be empty.
3317          */
3318         sBUG_ON(!list_empty(&scst_init_cmd_list));
3319
3320         PRINT_INFO("Init thread PID %d finished", current->pid);
3321
3322         TRACE_EXIT();
3323         return 0;
3324 }
3325
3326 /* Called with no locks held */
3327 void scst_process_active_cmd(struct scst_cmd *cmd, bool atomic)
3328 {
3329         int res;
3330
3331         TRACE_ENTRY();
3332
3333         EXTRACHECKS_BUG_ON(in_irq() || irqs_disabled());
3334
3335         cmd->atomic = atomic;
3336
3337         TRACE_DBG("cmd %p, atomic %d", cmd, atomic);
3338
3339         do {
3340                 switch (cmd->state) {
3341                 case SCST_CMD_STATE_PRE_PARSE:
3342                         res = scst_pre_parse(cmd);
3343                         EXTRACHECKS_BUG_ON(res ==
3344                                 SCST_CMD_STATE_RES_NEED_THREAD);
3345                         break;
3346
3347                 case SCST_CMD_STATE_DEV_PARSE:
3348                         res = scst_parse_cmd(cmd);
3349                         break;
3350
3351                 case SCST_CMD_STATE_PREPARE_SPACE:
3352                         res = scst_prepare_space(cmd);
3353                         break;
3354
3355                 case SCST_CMD_STATE_RDY_TO_XFER:
3356                         res = scst_rdy_to_xfer(cmd);
3357                         break;
3358
3359                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3360                         res = scst_tgt_pre_exec(cmd);
3361                         break;
3362
3363                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3364                         if (tm_dbg_check_cmd(cmd) != 0) {
3365                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
3366                                 TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
3367                                         "because of TM DBG delay", cmd,
3368                                         (long long unsigned int)cmd->tag);
3369                                 break;
3370                         }
3371                         res = scst_send_for_exec(&cmd);
3372                         /*
3373                          * !! At this point cmd, sess & tgt_dev can already be
3374                          * freed !!
3375                          */
3376                         break;
3377
3378                 case SCST_CMD_STATE_LOCAL_EXEC:
3379                         res = scst_local_exec(cmd);
3380                         /*
3381                          * !! At this point cmd, sess & tgt_dev can already be
3382                          * freed !!
3383                          */
3384                         break;
3385
3386                 case SCST_CMD_STATE_REAL_EXEC:
3387                         res = scst_real_exec(cmd);
3388                         /*
3389                          * !! At this point cmd, sess & tgt_dev can already be
3390                          * freed !!
3391                          */
3392                         break;
3393
3394                 case SCST_CMD_STATE_PRE_DEV_DONE:
3395                         res = scst_pre_dev_done(cmd);
3396                         EXTRACHECKS_BUG_ON(res ==
3397                                 SCST_CMD_STATE_RES_NEED_THREAD);
3398                         break;
3399
3400                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3401                         res = scst_mode_select_checks(cmd);
3402                         break;
3403
3404                 case SCST_CMD_STATE_DEV_DONE:
3405                         res = scst_dev_done(&cmd);
3406                         break;
3407
3408                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3409                         res = scst_pre_xmit_response(cmd);
3410                         EXTRACHECKS_BUG_ON(res ==
3411                                 SCST_CMD_STATE_RES_NEED_THREAD);
3412                         break;
3413
3414                 case SCST_CMD_STATE_XMIT_RESP:
3415                         res = scst_xmit_response(cmd);
3416                         break;
3417
3418                 case SCST_CMD_STATE_FINISHED:
3419                         res = scst_finish_cmd(cmd);
3420                         EXTRACHECKS_BUG_ON(res ==
3421                                 SCST_CMD_STATE_RES_NEED_THREAD);
3422                         break;
3423
3424                 default:
3425                         PRINT_CRIT_ERROR("cmd (%p) in state %d, but shouldn't "
3426                                 "be", cmd, cmd->state);
3427                         sBUG();
3428                         res = SCST_CMD_STATE_RES_CONT_NEXT;
3429                         break;
3430                 }
3431         } while (res == SCST_CMD_STATE_RES_CONT_SAME);
3432
3433         if (res == SCST_CMD_STATE_RES_CONT_NEXT) {
3434                 /* None */
3435         } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
3436                 spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3437                 switch (cmd->state) {
3438                 case SCST_CMD_STATE_PRE_PARSE:
3439                 case SCST_CMD_STATE_DEV_PARSE:
3440                 case SCST_CMD_STATE_PREPARE_SPACE:
3441                 case SCST_CMD_STATE_RDY_TO_XFER:
3442                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3443                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3444                 case SCST_CMD_STATE_LOCAL_EXEC:
3445                 case SCST_CMD_STATE_REAL_EXEC:
3446                 case SCST_CMD_STATE_PRE_DEV_DONE:
3447                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3448                 case SCST_CMD_STATE_DEV_DONE:
3449                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3450                 case SCST_CMD_STATE_XMIT_RESP:
3451                 case SCST_CMD_STATE_FINISHED:
3452                         TRACE_DBG("Adding cmd %p to head of active cmd list",
3453                                   cmd);
3454                         list_add(&cmd->cmd_list_entry,
3455                                 &cmd->cmd_lists->active_cmd_list);
3456                         break;
3457 #ifdef CONFIG_SCST_EXTRACHECKS
3458                 /* not very valid commands */
3459                 case SCST_CMD_STATE_DEFAULT:
3460                 case SCST_CMD_STATE_NEED_THREAD_CTX:
3461                         PRINT_CRIT_ERROR("cmd %p is in invalid state %d)", cmd,
3462                                 cmd->state);
3463                         spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3464                         sBUG();
3465                         spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3466                         break;
3467 #endif
3468                 default:
3469                         break;
3470                 }
3471                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3472                 spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3473         } else
3474                 sBUG();
3475
3476         TRACE_EXIT();
3477         return;
3478 }
3479 EXPORT_SYMBOL(scst_process_active_cmd);
3480
3481 /* Called under cmd_list_lock and IRQs disabled */
3482 static void scst_do_job_active(struct list_head *cmd_list,
3483         spinlock_t *cmd_list_lock, bool atomic)
3484         __releases(cmd_list_lock)
3485         __acquires(cmd_list_lock)
3486 {
3487         TRACE_ENTRY();
3488
3489         while (!list_empty(cmd_list)) {
3490                 struct scst_cmd *cmd = list_entry(cmd_list->next, typeof(*cmd),
3491                                         cmd_list_entry);
3492                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
3493                 list_del(&cmd->cmd_list_entry);
3494                 spin_unlock_irq(cmd_list_lock);
3495                 scst_process_active_cmd(cmd, atomic);
3496                 spin_lock_irq(cmd_list_lock);
3497         }
3498
3499         TRACE_EXIT();
3500 &n