scst/src/scst_targ.c

   1 /*
   2  *  scst_targ.c
   3  *
   4  *  Copyright (C) 2004 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
   5  *  Copyright (C) 2004 - 2005 Leonid Stoljar
   6  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
   7  *
   8  *  This program is free software; you can redistribute it and/or
   9  *  modify it under the terms of the GNU General Public License
  10  *  as published by the Free Software Foundation, version 2
  11  *  of the License.
  12  *
  13  *  This program is distributed in the hope that it will be useful,
  14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  *  GNU General Public License for more details.
  17  */
  18
  19 #include <linux/init.h>
  20 #include <linux/kernel.h>
  21 #include <linux/errno.h>
  22 #include <linux/list.h>
  23 #include <linux/spinlock.h>
  24 #include <linux/slab.h>
  25 #include <linux/sched.h>
  26 #include <linux/smp_lock.h>
  27 #include <linux/unistd.h>
  28 #include <linux/string.h>
  29 #include <linux/kthread.h>
  30 #include <linux/delay.h>
  31
  32 #include "scst.h"
  33 #include "scst_priv.h"
  34
  35 static void scst_cmd_set_sn(struct scst_cmd *cmd);
  36 static int __scst_init_cmd(struct scst_cmd *cmd);
  37 static void scst_finish_cmd_mgmt(struct scst_cmd *cmd);
  38 static struct scst_cmd *__scst_find_cmd_by_tag(struct scst_session *sess,
  39         uint64_t tag);
  40 static void scst_proccess_redirect_cmd(struct scst_cmd *cmd,
  41         enum scst_exec_context context, int check_retries);
  42
  43 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
  44 {
  45         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
  46         unsigned long flags;
  47
  48         spin_lock_irqsave(&t->tasklet_lock, flags);
  49         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
  50                 smp_processor_id());
  51         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
  52         spin_unlock_irqrestore(&t->tasklet_lock, flags);
  53
  54         tasklet_schedule(&t->tasklet);
  55 }
  56
  57 /*
  58  * Must not be called in parallel with scst_unregister_session_ex() for the
  59  * same sess
  60  */
  61 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
  62                              const uint8_t *lun, int lun_len,
  63                              const uint8_t *cdb, int cdb_len, int atomic)
  64 {
  65         struct scst_cmd *cmd;
  66
  67         TRACE_ENTRY();
  68
  69 #ifdef CONFIG_SCST_EXTRACHECKS
  70         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
  71                 PRINT_CRIT_ERROR("%s",
  72                         "New cmd while shutting down the session");
  73                 sBUG();
  74         }
  75 #endif
  76
  77         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
  78         if (cmd == NULL)
  79                 goto out;
  80
  81         cmd->sess = sess;
  82         cmd->tgt = sess->tgt;
  83         cmd->tgtt = sess->tgt->tgtt;
  84
  85         /*
  86          * For both wrong lun and CDB defer the error reporting for
  87          * scst_cmd_init_done()
  88          */
  89
  90         cmd->lun = scst_unpack_lun(lun, lun_len);
  91
  92         if (cdb_len <= SCST_MAX_CDB_SIZE) {
  93                 memcpy(cmd->cdb, cdb, cdb_len);
  94                 cmd->cdb_len = cdb_len;
  95         }
  96
  97         TRACE_DBG("cmd %p, sess %p", cmd, sess);
  98         scst_sess_get(sess);
  99
 100 out:
 101         TRACE_EXIT();
 102         return cmd;
 103 }
 104 EXPORT_SYMBOL(scst_rx_cmd);
 105
 106 /*
 107  * No locks, but might be on IRQ. Returns 0 on success, <0 if processing of
 108  * this command should be stopped.
 109  */
 110 static int scst_init_cmd(struct scst_cmd *cmd, enum scst_exec_context *context)
 111 {
 112         int rc, res = 0;
 113
 114         TRACE_ENTRY();
 115
 116         /* See the comment in scst_do_job_init() */
 117         if (unlikely(!list_empty(&scst_init_cmd_list))) {
 118                 TRACE_MGMT_DBG("%s", "init cmd list busy");
 119                 goto out_redirect;
 120         }
 121         /*
 122          * Memory barrier isn't necessary here, because CPU appears to
 123          * be self-consistent and we don't care about the race, described
 124          * in comment in scst_do_job_init().
 125          */
 126
 127         rc = __scst_init_cmd(cmd);
 128         if (unlikely(rc > 0))
 129                 goto out_redirect;
 130         else if (unlikely(rc != 0))
 131                 goto out;
 132
 133         /* Small context optimization */
 134         if (((*context == SCST_CONTEXT_TASKLET) ||
 135              (*context == SCST_CONTEXT_DIRECT_ATOMIC) ||
 136              ((*context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd))) &&
 137               scst_cmd_is_expected_set(cmd)) {
 138                 if (cmd->expected_data_direction & SCST_DATA_WRITE) {
 139                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
 140                                         &cmd->tgt_dev->tgt_dev_flags))
 141                                 *context = SCST_CONTEXT_THREAD;
 142                 } else {
 143                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
 144                                         &cmd->tgt_dev->tgt_dev_flags))
 145                                 *context = SCST_CONTEXT_THREAD;
 146                 }
 147         }
 148
 149 out:
 150         TRACE_EXIT_RES(res);
 151         return res;
 152
 153 out_redirect:
 154         if (cmd->preprocessing_only) {
 155                 /*
 156                  * Poor man solution for single threaded targets, where
 157                  * blocking receiver at least sometimes means blocking all.
 158                  */
 159                 sBUG_ON(*context != SCST_CONTEXT_DIRECT);
 160                 scst_set_busy(cmd);
 161                 scst_set_cmd_abnormal_done_state(cmd);
 162                 /* Keep initiator away from too many BUSY commands */
 163                 msleep(50);
 164         } else {
 165                 unsigned long flags;
 166                 spin_lock_irqsave(&scst_init_lock, flags);
 167                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
 168                         "%d)", cmd, atomic_read(&scst_cmd_count));
 169                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
 170                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
 171                         scst_init_poll_cnt++;
 172                 spin_unlock_irqrestore(&scst_init_lock, flags);
 173                 wake_up(&scst_init_cmd_list_waitQ);
 174                 res = -1;
 175         }
 176         goto out;
 177 }
 178
 179 #ifdef CONFIG_SCST_MEASURE_LATENCY
 180 static inline uint64_t scst_sec_to_nsec(time_t sec)
 181 {
 182         return (uint64_t)sec * 1000000000;
 183 }
 184 #endif
 185
 186 void scst_cmd_init_done(struct scst_cmd *cmd,
 187         enum scst_exec_context pref_context)
 188 {
 189         unsigned long flags;
 190         struct scst_session *sess = cmd->sess;
 191         int rc;
 192
 193         TRACE_ENTRY();
 194
 195 #ifdef CONFIG_SCST_MEASURE_LATENCY
 196         {
 197                 struct timespec ts;
 198                 getnstimeofday(&ts);
 199                 cmd->start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
 200                 TRACE_DBG("cmd %p (sess %p): start %lld (tv_sec %ld, "
 201                         "tv_nsec %ld)", cmd, sess, cmd->start, ts.tv_sec,
 202                         ts.tv_nsec);
 203         }
 204 #endif
 205
 206         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
 207         TRACE(TRACE_SCSI, "tag=%llu, lun=%lld, CDB len=%d, queue_type=%x "
 208                 "(cmd %p)", (long long unsigned int)cmd->tag,
 209                 (long long unsigned int)cmd->lun, cmd->cdb_len,
 210                 cmd->queue_type, cmd);
 211         PRINT_BUFF_FLAG(TRACE_SCSI|TRACE_RCV_BOT, "Recieving CDB",
 212                 cmd->cdb, cmd->cdb_len);
 213
 214 #ifdef CONFIG_SCST_EXTRACHECKS
 215         if (unlikely((in_irq() || irqs_disabled())) &&
 216             ((pref_context == SCST_CONTEXT_DIRECT) ||
 217              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
 218                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
 219                         "SCST_CONTEXT_THREAD instead\n", pref_context,
 220                         cmd->tgtt->name);
 221                 pref_context = SCST_CONTEXT_THREAD;
 222         }
 223 #endif
 224
 225         atomic_inc(&sess->sess_cmd_count);
 226
 227         spin_lock_irqsave(&sess->sess_list_lock, flags);
 228
 229         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
 230                 /*
 231                  * We have to always keep command in the search list from the
 232                  * very beginning, because otherwise it can be missed during
 233                  * TM processing. This check is needed because there might be
 234                  * old, i.e. deferred, commands and new, i.e. just coming, ones.
 235                  */
 236                 if (cmd->search_cmd_list_entry.next == NULL)
 237                         list_add_tail(&cmd->search_cmd_list_entry,
 238                                 &sess->search_cmd_list);
 239                 switch (sess->init_phase) {
 240                 case SCST_SESS_IPH_SUCCESS:
 241                         break;
 242                 case SCST_SESS_IPH_INITING:
 243                         TRACE_DBG("Adding cmd %p to init deferred cmd list",
 244                                   cmd);
 245                         list_add_tail(&cmd->cmd_list_entry,
 246                                 &sess->init_deferred_cmd_list);
 247                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
 248                         goto out;
 249                 case SCST_SESS_IPH_FAILED:
 250                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
 251                         scst_set_busy(cmd);
 252                         scst_set_cmd_abnormal_done_state(cmd);
 253                         goto active;
 254                 default:
 255                         sBUG();
 256                 }
 257         } else
 258                 list_add_tail(&cmd->search_cmd_list_entry,
 259                               &sess->search_cmd_list);
 260
 261         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
 262
 263         if (unlikely(cmd->lun == NO_SUCH_LUN)) {
 264                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
 265                 scst_set_cmd_error(cmd,
 266                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
 267                 scst_set_cmd_abnormal_done_state(cmd);
 268                 goto active;
 269         }
 270
 271         if (unlikely(cmd->cdb_len == 0)) {
 272                 PRINT_ERROR("%s", "Wrong CDB len, finishing cmd");
 273                 scst_set_cmd_error(cmd,
 274                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
 275                 scst_set_cmd_abnormal_done_state(cmd);
 276                 goto active;
 277         }
 278
 279         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
 280                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
 281                 scst_set_cmd_error(cmd,
 282                         SCST_LOAD_SENSE(scst_sense_invalid_message));
 283                 scst_set_cmd_abnormal_done_state(cmd);
 284                 goto active;
 285         }
 286
 287         cmd->state = SCST_CMD_STATE_INIT;
 288         /* cmd must be inited here to preserve the order */
 289         rc = scst_init_cmd(cmd, &pref_context);
 290         if (unlikely(rc < 0))
 291                 goto out;
 292
 293         if (unlikely(cmd->status != SAM_STAT_GOOD))
 294                 scst_set_cmd_abnormal_done_state(cmd);
 295
 296 active:
 297         /* Here cmd must not be in any cmd list, no locks */
 298         switch (pref_context) {
 299         case SCST_CONTEXT_TASKLET:
 300                 scst_schedule_tasklet(cmd);
 301                 break;
 302
 303         case SCST_CONTEXT_DIRECT:
 304                 scst_process_active_cmd(cmd, false);
 305                 /* For *NEED_THREAD wake_up() is already done */
 306                 break;
 307
 308         case SCST_CONTEXT_DIRECT_ATOMIC:
 309                 scst_process_active_cmd(cmd, true);
 310                 /* For *NEED_THREAD wake_up() is already done */
 311                 break;
 312
 313         default:
 314                 PRINT_ERROR("Context %x is undefined, using the thread one",
 315                         pref_context);
 316                 /* go through */
 317         case SCST_CONTEXT_THREAD:
 318                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
 319                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
 320                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
 321                         list_add(&cmd->cmd_list_entry,
 322                                 &cmd->cmd_lists->active_cmd_list);
 323                 else
 324                         list_add_tail(&cmd->cmd_list_entry,
 325                                 &cmd->cmd_lists->active_cmd_list);
 326                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
 327                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
 328                 break;
 329         }
 330
 331 out:
 332         TRACE_EXIT();
 333         return;
 334 }
 335 EXPORT_SYMBOL(scst_cmd_init_done);
 336
 337 static int scst_pre_parse(struct scst_cmd *cmd)
 338 {
 339         int res = SCST_CMD_STATE_RES_CONT_SAME;
 340         struct scst_device *dev = cmd->dev;
 341         int rc;
 342
 343         TRACE_ENTRY();
 344
 345         cmd->inc_expected_sn_on_done = dev->handler->exec_sync ||
 346              (!dev->has_own_order_mgmt &&
 347               (dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER ||
 348                cmd->queue_type == SCST_CMD_QUEUE_ORDERED));
 349
 350         /*
 351          * Expected transfer data supplied by the SCSI transport via the
 352          * target driver are untrusted, so we prefer to fetch them from CDB.
 353          * Additionally, not all transports support supplying the expected
 354          * transfer data.
 355          */
 356
 357         rc = scst_get_cdb_info(cmd);
 358         if (unlikely(rc != 0)) {
 359                 if (rc > 0) {
 360                         PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 361                         goto out_xmit;
 362                 }
 363                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
 364                         "Should you update scst_scsi_op_table?",
 365                         cmd->cdb[0], dev->handler->name);
 366                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 367 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
 368                 if (scst_cmd_is_expected_set(cmd)) {
 369                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
 370                                 "direction %d, transfer_len %d",
 371                                 cmd->expected_data_direction,
 372                                 cmd->expected_transfer_len);
 373                         cmd->data_direction = cmd->expected_data_direction;
 374
 375                         cmd->bufflen = cmd->expected_transfer_len;
 376                         /* Restore (possibly) lost CDB length */
 377                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
 378                         if (cmd->cdb_len == -1) {
 379                                 PRINT_ERROR("Unable to get CDB length for "
 380                                         "opcode 0x%02x. Returning INVALID "
 381                                         "OPCODE", cmd->cdb[0]);
 382                                 scst_set_cmd_error(cmd,
 383                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
 384                                 goto out_xmit;
 385                         }
 386                 } else {
 387                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
 388                              "target %s not supplied expected values",
 389                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
 390                         scst_set_cmd_error(cmd,
 391                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
 392                         goto out_xmit;
 393                 }
 394 #else
 395                 scst_set_cmd_error(cmd,
 396                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
 397                 goto out_xmit;
 398 #endif
 399         } else {
 400                 TRACE(TRACE_SCSI, "op_name <%s> (cmd %p), direction=%d "
 401                         "(expected %d, set %s), transfer_len=%d (expected "
 402                         "len %d), flags=%d", cmd->op_name, cmd,
 403                         cmd->data_direction, cmd->expected_data_direction,
 404                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
 405                         cmd->bufflen, cmd->expected_transfer_len,
 406                         cmd->op_flags);
 407
 408                 if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
 409                         if (scst_cmd_is_expected_set(cmd)) {
 410                                 /*
 411                                  * Command data length can't be easily
 412                                  * determined from the CDB. ToDo, all such
 413                                  * commands processing should be fixed. Until
 414                                  * it's done, get the length from the supplied
 415                                  * expected value, but limit it to some
 416                                  * reasonable value (15MB).
 417                                  */
 418                                 cmd->bufflen = min(cmd->expected_transfer_len,
 419                                                         15*1024*1024);
 420                                 cmd->op_flags &= ~SCST_UNKNOWN_LENGTH;
 421                         } else
 422                                 cmd->bufflen = 0;
 423                 }
 424         }
 425
 426         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
 427                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
 428                             "(opcode 0x%02x)", cmd->cdb[0]);
 429                 scst_set_cmd_error(cmd,
 430                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
 431                 goto out_xmit;
 432         }
 433
 434         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
 435                 PRINT_ERROR("Linked commands are not supported "
 436                             "(opcode 0x%02x)", cmd->cdb[0]);
 437                 scst_set_cmd_error(cmd,
 438                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
 439                 goto out_xmit;
 440         }
 441
 442         cmd->state = SCST_CMD_STATE_DEV_PARSE;
 443
 444 out:
 445         TRACE_EXIT_RES(res);
 446         return res;
 447
 448 out_xmit:
 449         scst_set_cmd_abnormal_done_state(cmd);
 450         res = SCST_CMD_STATE_RES_CONT_SAME;
 451         goto out;
 452 }
 453
 454 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
 455 static bool scst_is_allowed_to_mismatch_cmd(struct scst_cmd *cmd)
 456 {
 457         bool res = false;
 458
 459         switch (cmd->cdb[0]) {
 460         case TEST_UNIT_READY:
 461                 /* Crazy VMware people sometimes do TUR with READ direction */
 462                 res = true;
 463                 break;
 464         case VERIFY:
 465         case VERIFY_6:
 466         case VERIFY_12:
 467         case VERIFY_16:
 468                 /* VERIFY commands with BYTCHK unset shouldn't fail here */
 469                 if ((cmd->op_flags & SCST_VERIFY_BYTCHK_MISMATCH_ALLOWED) &&
 470                     (cmd->cdb[1] & BYTCHK) == 0)
 471                         res = true;
 472                 break;
 473         }
 474
 475         return res;
 476 }
 477 #endif
 478
 479 static int scst_parse_cmd(struct scst_cmd *cmd)
 480 {
 481         int res = SCST_CMD_STATE_RES_CONT_SAME;
 482         int state;
 483         struct scst_device *dev = cmd->dev;
 484         int orig_bufflen = cmd->bufflen;
 485
 486         TRACE_ENTRY();
 487
 488         if (likely(!scst_is_cmd_local(cmd))) {
 489                 if (unlikely(!dev->handler->parse_atomic &&
 490                              scst_cmd_atomic(cmd))) {
 491                         /*
 492                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
 493                          * optimization.
 494                          */
 495                         TRACE_DBG("Dev handler %s parse() needs thread "
 496                                 "context, rescheduling", dev->handler->name);
 497                         res = SCST_CMD_STATE_RES_NEED_THREAD;
 498                         goto out;
 499                 }
 500
 501                 TRACE_DBG("Calling dev handler %s parse(%p)",
 502                       dev->handler->name, cmd);
 503                 TRACE_BUFF_FLAG(TRACE_SND_BOT, "Parsing: ",
 504                                 cmd->cdb, cmd->cdb_len);
 505                 state = dev->handler->parse(cmd);
 506                 /* Caution: cmd can be already dead here */
 507                 TRACE_DBG("Dev handler %s parse() returned %d",
 508                         dev->handler->name, state);
 509
 510                 switch (state) {
 511                 case SCST_CMD_STATE_NEED_THREAD_CTX:
 512                         TRACE_DBG("Dev handler %s parse() requested thread "
 513                               "context, rescheduling", dev->handler->name);
 514                         res = SCST_CMD_STATE_RES_NEED_THREAD;
 515                         goto out;
 516
 517                 case SCST_CMD_STATE_STOP:
 518                         TRACE_DBG("Dev handler %s parse() requested stop "
 519                                 "processing", dev->handler->name);
 520                         res = SCST_CMD_STATE_RES_CONT_NEXT;
 521                         goto out;
 522                 }
 523
 524                 if (state == SCST_CMD_STATE_DEFAULT)
 525                         state = SCST_CMD_STATE_PREPARE_SPACE;
 526         } else
 527                 state = SCST_CMD_STATE_PREPARE_SPACE;
 528
 529         if (cmd->data_len == -1)
 530                 cmd->data_len = cmd->bufflen;
 531
 532         if (cmd->bufflen == 0) {
 533                 /*
 534                  * According to SPC bufflen 0 for data transfer commands isn't
 535                  * an error, so we need to fix the transfer direction.
 536                  */
 537                 cmd->data_direction = SCST_DATA_NONE;
 538         }
 539
 540         if (cmd->dh_data_buf_alloced &&
 541             unlikely((orig_bufflen > cmd->bufflen))) {
 542                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
 543                         "is less, than required (size %d)", cmd->bufflen,
 544                         orig_bufflen);
 545                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 546                 goto out_error;
 547         }
 548
 549         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
 550                 goto set_res;
 551
 552         if (unlikely((cmd->bufflen == 0) &&
 553                      (cmd->op_flags & SCST_UNKNOWN_LENGTH))) {
 554                 PRINT_ERROR("Unknown data transfer length for opcode 0x%x "
 555                         "(handler %s, target %s)", cmd->cdb[0],
 556                         dev->handler->name, cmd->tgtt->name);
 557                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 558                 goto out_error;
 559         }
 560
 561 #ifdef CONFIG_SCST_EXTRACHECKS
 562         if ((cmd->bufflen != 0) &&
 563             ((cmd->data_direction == SCST_DATA_NONE) ||
 564              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
 565                 PRINT_ERROR("Dev handler %s parse() returned "
 566                         "invalid cmd data_direction %d, bufflen %d, state %d "
 567                         "or sg %p (opcode 0x%x)", dev->handler->name,
 568                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
 569                         cmd->cdb[0]);
 570                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 571                 goto out_error;
 572         }
 573 #endif
 574
 575         if (scst_cmd_is_expected_set(cmd)) {
 576 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
 577 #       ifdef CONFIG_SCST_EXTRACHECKS
 578                 if ((cmd->data_direction != cmd->expected_data_direction) ||
 579                     (cmd->bufflen != cmd->expected_transfer_len)) {
 580                         PRINT_WARNING("Expected values don't match decoded "
 581                                 "ones: data_direction %d, "
 582                                 "expected_data_direction %d, "
 583                                 "bufflen %d, expected_transfer_len %d",
 584                                 cmd->data_direction,
 585                                 cmd->expected_data_direction,
 586                                 cmd->bufflen, cmd->expected_transfer_len);
 587                         PRINT_BUFFER("Suspicious CDB", cmd->cdb, cmd->cdb_len);
 588                 }
 589 #       endif
 590                 cmd->data_direction = cmd->expected_data_direction;
 591                 cmd->bufflen = cmd->expected_transfer_len;
 592 #else
 593                 if (unlikely(cmd->data_direction !=
 594                                 cmd->expected_data_direction)) {
 595                         if (((cmd->expected_data_direction != SCST_DATA_NONE) ||
 596                              (cmd->bufflen != 0)) &&
 597                             !scst_is_allowed_to_mismatch_cmd(cmd)) {
 598                                 PRINT_ERROR("Expected data direction %d for "
 599                                         "opcode 0x%02x (handler %s, target %s) "
 600                                         "doesn't match "
 601                                         "decoded value %d",
 602                                         cmd->expected_data_direction,
 603                                         cmd->cdb[0], dev->handler->name,
 604                                         cmd->tgtt->name, cmd->data_direction);
 605                                 PRINT_BUFFER("Failed CDB",
 606                                         cmd->cdb, cmd->cdb_len);
 607                                 scst_set_cmd_error(cmd,
 608                                    SCST_LOAD_SENSE(scst_sense_invalid_message));
 609                                 goto out_dev_done;
 610                         }
 611                 }
 612                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
 613                         TRACE(TRACE_MGMT_MINOR, "Warning: expected "
 614                                 "transfer length %d for opcode 0x%02x "
 615                                 "(handler %s, target %s) doesn't match "
 616                                 "decoded value %d. Faulty initiator "
 617                                 "(e.g. VMware is known to be such) or "
 618                                 "scst_scsi_op_table should be updated?",
 619                                 cmd->expected_transfer_len, cmd->cdb[0],
 620                                 dev->handler->name, cmd->tgtt->name,
 621                                 cmd->bufflen);
 622                         PRINT_BUFF_FLAG(TRACE_MGMT_MINOR, "Suspicious CDB",
 623                                 cmd->cdb, cmd->cdb_len);
 624                 }
 625 #endif
 626         }
 627
 628         if (unlikely(cmd->data_direction == SCST_DATA_UNKNOWN)) {
 629                 PRINT_ERROR("Unknown data direction. Opcode 0x%x, handler %s, "
 630                         "target %s", cmd->cdb[0], dev->handler->name,
 631                         cmd->tgtt->name);
 632                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 633                 goto out_error;
 634         }
 635
 636 set_res:
 637         switch (state) {
 638         case SCST_CMD_STATE_PREPARE_SPACE:
 639         case SCST_CMD_STATE_PRE_PARSE:
 640         case SCST_CMD_STATE_DEV_PARSE:
 641         case SCST_CMD_STATE_RDY_TO_XFER:
 642         case SCST_CMD_STATE_TGT_PRE_EXEC:
 643         case SCST_CMD_STATE_SEND_FOR_EXEC:
 644         case SCST_CMD_STATE_LOCAL_EXEC:
 645         case SCST_CMD_STATE_REAL_EXEC:
 646         case SCST_CMD_STATE_PRE_DEV_DONE:
 647         case SCST_CMD_STATE_DEV_DONE:
 648         case SCST_CMD_STATE_PRE_XMIT_RESP:
 649         case SCST_CMD_STATE_XMIT_RESP:
 650         case SCST_CMD_STATE_FINISHED:
 651         case SCST_CMD_STATE_FINISHED_INTERNAL:
 652                 cmd->state = state;
 653                 res = SCST_CMD_STATE_RES_CONT_SAME;
 654                 break;
 655
 656         default:
 657                 if (state >= 0) {
 658                         PRINT_ERROR("Dev handler %s parse() returned "
 659                              "invalid cmd state %d (opcode %d)",
 660                              dev->handler->name, state, cmd->cdb[0]);
 661                 } else {
 662                         PRINT_ERROR("Dev handler %s parse() returned "
 663                                 "error %d (opcode %d)", dev->handler->name,
 664                                 state, cmd->cdb[0]);
 665                 }
 666                 goto out_error;
 667         }
 668
 669         if (cmd->resp_data_len == -1) {
 670                 if (cmd->data_direction & SCST_DATA_READ)
 671                         cmd->resp_data_len = cmd->bufflen;
 672                 else
 673                          cmd->resp_data_len = 0;
 674         }
 675
 676 out:
 677         TRACE_EXIT_HRES(res);
 678         return res;
 679
 680 out_error:
 681         /* dev_done() will be called as part of the regular cmd's finish */
 682         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
 683
 684 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
 685 out_dev_done:
 686 #endif
 687         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
 688         res = SCST_CMD_STATE_RES_CONT_SAME;
 689         goto out;
 690 }
 691
 692 static int scst_prepare_space(struct scst_cmd *cmd)
 693 {
 694         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
 695
 696         TRACE_ENTRY();
 697
 698         if (cmd->data_direction == SCST_DATA_NONE)
 699                 goto prep_done;
 700
 701         if (cmd->tgt_need_alloc_data_buf) {
 702                 int orig_bufflen = cmd->bufflen;
 703
 704                 TRACE_MEM("Custom tgt data buf allocation requested (cmd %p)",
 705                         cmd);
 706
 707                 r = cmd->tgtt->alloc_data_buf(cmd);
 708                 if (r > 0)
 709                         goto alloc;
 710                 else if (r == 0) {
 711                         if (unlikely(cmd->bufflen == 0)) {
 712                                 /* See comment in scst_alloc_space() */
 713                                 if (cmd->sg == NULL)
 714                                         goto alloc;
 715                         }
 716
 717                         cmd->tgt_data_buf_alloced = 1;
 718
 719                         if (unlikely(orig_bufflen < cmd->bufflen)) {
 720                                 PRINT_ERROR("Target driver allocated data "
 721                                         "buffer (size %d), is less, than "
 722                                         "required (size %d)", orig_bufflen,
 723                                         cmd->bufflen);
 724                                 goto out_error;
 725                         }
 726                         TRACE_MEM("tgt_data_buf_alloced (cmd %p)", cmd);
 727                 } else
 728                         goto check;
 729         }
 730
 731 alloc:
 732         if (!cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
 733                 r = scst_alloc_space(cmd);
 734         } else if (cmd->dh_data_buf_alloced && !cmd->tgt_data_buf_alloced) {
 735                 TRACE_MEM("dh_data_buf_alloced set (cmd %p)", cmd);
 736                 r = 0;
 737         } else if (cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
 738                 TRACE_MEM("tgt_data_buf_alloced set (cmd %p)", cmd);
 739                 cmd->sg = cmd->tgt_sg;
 740                 cmd->sg_cnt = cmd->tgt_sg_cnt;
 741                 cmd->in_sg = cmd->tgt_in_sg;
 742                 cmd->in_sg_cnt = cmd->tgt_in_sg_cnt;
 743                 r = 0;
 744         } else {
 745                 TRACE_MEM("Both *_data_buf_alloced set (cmd %p, sg %p, "
 746                         "sg_cnt %d, tgt_sg %p, tgt_sg_cnt %d)", cmd, cmd->sg,
 747                         cmd->sg_cnt, cmd->tgt_sg, cmd->tgt_sg_cnt);
 748                 r = 0;
 749         }
 750
 751 check:
 752         if (r != 0) {
 753                 if (scst_cmd_atomic(cmd)) {
 754                         TRACE_MEM("%s", "Atomic memory allocation failed, "
 755                               "rescheduling to the thread");
 756                         res = SCST_CMD_STATE_RES_NEED_THREAD;
 757                         goto out;
 758                 } else
 759                         goto out_no_space;
 760         }
 761
 762 prep_done:
 763         if (cmd->preprocessing_only) {
 764                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
 765                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
 766                                 "cmd %p", cmd);
 767                         scst_set_cmd_abnormal_done_state(cmd);
 768                         res = SCST_CMD_STATE_RES_CONT_SAME;
 769                         goto out;
 770                 }
 771
 772                 res = SCST_CMD_STATE_RES_CONT_NEXT;
 773                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
 774
 775                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
 776                 cmd->tgtt->preprocessing_done(cmd);
 777                 TRACE_DBG("%s", "preprocessing_done() returned");
 778                 goto out;
 779
 780         }
 781
 782         if (cmd->data_direction & SCST_DATA_WRITE)
 783                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
 784         else
 785                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
 786
 787 out:
 788         TRACE_EXIT_HRES(res);
 789         return res;
 790
 791 out_no_space:
 792         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
 793                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
 794         scst_set_busy(cmd);
 795         scst_set_cmd_abnormal_done_state(cmd);
 796         res = SCST_CMD_STATE_RES_CONT_SAME;
 797         goto out;
 798
 799 out_error:
 800         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
 801         scst_set_cmd_abnormal_done_state(cmd);
 802         res = SCST_CMD_STATE_RES_CONT_SAME;
 803         goto out;
 804 }
 805
 806 void scst_restart_cmd(struct scst_cmd *cmd, int status,
 807         enum scst_exec_context pref_context)
 808 {
 809         TRACE_ENTRY();
 810
 811         TRACE_DBG("Preferred context: %d", pref_context);
 812         TRACE_DBG("tag=%llu, status=%#x",
 813                   (long long unsigned int)scst_cmd_get_tag(cmd),
 814                   status);
 815
 816 #ifdef CONFIG_SCST_EXTRACHECKS
 817         if ((in_irq() || irqs_disabled()) &&
 818             ((pref_context == SCST_CONTEXT_DIRECT) ||
 819              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
 820                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
 821                         "SCST_CONTEXT_THREAD instead\n", pref_context,
 822                         cmd->tgtt->name);
 823                 pref_context = SCST_CONTEXT_THREAD;
 824         }
 825 #endif
 826
 827         switch (status) {
 828         case SCST_PREPROCESS_STATUS_SUCCESS:
 829                 if (cmd->data_direction & SCST_DATA_WRITE)
 830                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
 831                 else
 832                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
 833                 if (cmd->set_sn_on_restart_cmd)
 834                         scst_cmd_set_sn(cmd);
 835                 /* Small context optimization */
 836                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
 837                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
 838                     ((pref_context == SCST_CONTEXT_SAME) &&
 839                      scst_cmd_atomic(cmd))) {
 840                         if (cmd->data_direction & SCST_DATA_WRITE) {
 841                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
 842                                                 &cmd->tgt_dev->tgt_dev_flags))
 843                                         pref_context = SCST_CONTEXT_THREAD;
 844                         } else {
 845                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
 846                                                 &cmd->tgt_dev->tgt_dev_flags))
 847                                         pref_context = SCST_CONTEXT_THREAD;
 848                         }
 849                 }
 850                 break;
 851
 852         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
 853                 scst_set_cmd_abnormal_done_state(cmd);
 854                 break;
 855
 856         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
 857                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
 858                 /* go through */
 859         case SCST_PREPROCESS_STATUS_ERROR:
 860                 if (cmd->sense != NULL)
 861                         scst_set_cmd_error(cmd,
 862                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
 863                 scst_set_cmd_abnormal_done_state(cmd);
 864                 break;
 865
 866         default:
 867                 PRINT_ERROR("%s() received unknown status %x", __func__,
 868                         status);
 869                 scst_set_cmd_abnormal_done_state(cmd);
 870                 break;
 871         }
 872
 873         scst_proccess_redirect_cmd(cmd, pref_context, 1);
 874
 875         TRACE_EXIT();
 876         return;
 877 }
 878 EXPORT_SYMBOL(scst_restart_cmd);
 879
 880 /* No locks */
 881 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
 882 {
 883         struct scst_tgt *tgt = cmd->sess->tgt;
 884         int res = 0;
 885         unsigned long flags;
 886
 887         TRACE_ENTRY();
 888
 889         spin_lock_irqsave(&tgt->tgt_lock, flags);
 890         tgt->retry_cmds++;
 891         /*
 892          * Memory barrier is needed here, because we need the exact order
 893          * between the read and write between retry_cmds and finished_cmds to
 894          * not miss the case when a command finished while we queuing it for
 895          * retry after the finished_cmds check.
 896          */
 897         smp_mb();
 898         TRACE_RETRY("TGT QUEUE FULL: incrementing retry_cmds %d",
 899               tgt->retry_cmds);
 900         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
 901                 /* At least one cmd finished, so try again */
 902                 tgt->retry_cmds--;
 903                 TRACE_RETRY("Some command(s) finished, direct retry "
 904                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
 905                       "retry_cmds=%d)", finished_cmds,
 906                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
 907                 res = -1;
 908                 goto out_unlock_tgt;
 909         }
 910
 911         TRACE_RETRY("Adding cmd %p to retry cmd list", cmd);
 912         list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
 913
 914         if (!tgt->retry_timer_active) {
 915                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
 916                 add_timer(&tgt->retry_timer);
 917                 tgt->retry_timer_active = 1;
 918         }
 919
 920 out_unlock_tgt:
 921         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
 922
 923         TRACE_EXIT_RES(res);
 924         return res;
 925 }
 926
 927 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
 928 {
 929         int res, rc;
 930
 931         TRACE_ENTRY();
 932
 933         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
 934                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
 935                 goto out_dev_done;
 936         }
 937
 938         if ((cmd->tgtt->rdy_to_xfer == NULL) || unlikely(cmd->internal)) {
 939                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
 940                 res = SCST_CMD_STATE_RES_CONT_SAME;
 941                 goto out;
 942         }
 943
 944         if (unlikely(!cmd->tgtt->rdy_to_xfer_atomic && scst_cmd_atomic(cmd))) {
 945                 /*
 946                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
 947                  * optimization.
 948                  */
 949                 TRACE_DBG("Target driver %s rdy_to_xfer() needs thread "
 950                               "context, rescheduling", cmd->tgtt->name);
 951                 res = SCST_CMD_STATE_RES_NEED_THREAD;
 952                 goto out;
 953         }
 954
 955         while (1) {
 956                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
 957
 958                 res = SCST_CMD_STATE_RES_CONT_NEXT;
 959                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
 960
 961                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
 962 #ifdef CONFIG_SCST_DEBUG_RETRY
 963                 if (((scst_random() % 100) == 75))
 964                         rc = SCST_TGT_RES_QUEUE_FULL;
 965                 else
 966 #endif
 967                         rc = cmd->tgtt->rdy_to_xfer(cmd);
 968                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
 969
 970                 if (likely(rc == SCST_TGT_RES_SUCCESS))
 971                         goto out;
 972
 973                 /* Restore the previous state */
 974                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
 975
 976                 switch (rc) {
 977                 case SCST_TGT_RES_QUEUE_FULL:
 978                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
 979                                 break;
 980                         else
 981                                 continue;
 982
 983                 case SCST_TGT_RES_NEED_THREAD_CTX:
 984                         TRACE_DBG("Target driver %s "
 985                               "rdy_to_xfer() requested thread "
 986                               "context, rescheduling", cmd->tgtt->name);
 987                         res = SCST_CMD_STATE_RES_NEED_THREAD;
 988                         break;
 989
 990                 default:
 991                         goto out_error_rc;
 992                 }
 993                 break;
 994         }
 995
 996 out:
 997         TRACE_EXIT_HRES(res);
 998         return res;
 999
1000 out_error_rc:
1001         if (rc == SCST_TGT_RES_FATAL_ERROR) {
1002                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
1003                      "fatal error", cmd->tgtt->name);
1004         } else {
1005                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
1006                             "value %d", cmd->tgtt->name, rc);
1007         }
1008         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1009
1010 out_dev_done:
1011         scst_set_cmd_abnormal_done_state(cmd);
1012         res = SCST_CMD_STATE_RES_CONT_SAME;
1013         goto out;
1014 }
1015
1016 /* No locks, but might be in IRQ */
1017 static void scst_proccess_redirect_cmd(struct scst_cmd *cmd,
1018         enum scst_exec_context context, int check_retries)
1019 {
1020         unsigned long flags;
1021
1022         TRACE_ENTRY();
1023
1024         TRACE_DBG("Context: %x", context);
1025
1026         if (context == SCST_CONTEXT_SAME)
1027                 context = scst_cmd_atomic(cmd) ? SCST_CONTEXT_DIRECT_ATOMIC :
1028                                                  SCST_CONTEXT_DIRECT;
1029
1030         switch (context) {
1031         case SCST_CONTEXT_DIRECT_ATOMIC:
1032                 scst_process_active_cmd(cmd, true);
1033                 break;
1034
1035         case SCST_CONTEXT_DIRECT:
1036                 if (check_retries)
1037                         scst_check_retries(cmd->tgt);
1038                 scst_process_active_cmd(cmd, false);
1039                 break;
1040
1041         default:
1042                 PRINT_ERROR("Context %x is unknown, using the thread one",
1043                             context);
1044                 /* go through */
1045         case SCST_CONTEXT_THREAD:
1046                 if (check_retries)
1047                         scst_check_retries(cmd->tgt);
1048                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
1049                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
1050                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
1051                         list_add(&cmd->cmd_list_entry,
1052                                 &cmd->cmd_lists->active_cmd_list);
1053                 else
1054                         list_add_tail(&cmd->cmd_list_entry,
1055                                 &cmd->cmd_lists->active_cmd_list);
1056                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
1057                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
1058                 break;
1059
1060         case SCST_CONTEXT_TASKLET:
1061                 if (check_retries)
1062                         scst_check_retries(cmd->tgt);
1063                 scst_schedule_tasklet(cmd);
1064                 break;
1065         }
1066
1067         TRACE_EXIT();
1068         return;
1069 }
1070
1071 void scst_rx_data(struct scst_cmd *cmd, int status,
1072         enum scst_exec_context pref_context)
1073 {
1074         TRACE_ENTRY();
1075
1076         TRACE_DBG("Preferred context: %d", pref_context);
1077         TRACE(TRACE_SCSI, "cmd %p, status %#x", cmd, status);
1078
1079 #ifdef CONFIG_SCST_EXTRACHECKS
1080         if ((in_irq() || irqs_disabled()) &&
1081             ((pref_context == SCST_CONTEXT_DIRECT) ||
1082              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
1083                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
1084                         "SCST_CONTEXT_THREAD instead\n", pref_context,
1085                         cmd->tgtt->name);
1086                 pref_context = SCST_CONTEXT_THREAD;
1087         }
1088 #endif
1089
1090         switch (status) {
1091         case SCST_RX_STATUS_SUCCESS:
1092 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
1093                 if (trace_flag & TRACE_RCV_BOT) {
1094                         int i;
1095                         struct scatterlist *sg;
1096                         if (cmd->in_sg != NULL)
1097                                 sg = cmd->in_sg;
1098                         else if (cmd->tgt_in_sg != NULL)
1099                                 sg = cmd->tgt_in_sg;
1100                         else if (cmd->tgt_sg != NULL)
1101                                 sg = cmd->tgt_sg;
1102                         else
1103                                 sg = cmd->sg;
1104                         if (sg != NULL) {
1105                                 TRACE_RECV_BOT("RX data for cmd %p "
1106                                         "(sg_cnt %d, sg %p, sg[0].page %p)",
1107                                         cmd, cmd->tgt_sg_cnt, sg,
1108                                         (void *)sg_page(&sg[0]));
1109                                 for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
1110                                         PRINT_BUFF_FLAG(TRACE_RCV_BOT, "RX sg",
1111                                                 sg_virt(&sg[i]), sg[i].length);
1112                                 }
1113                         }
1114                 }
1115 #endif
1116                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1117                 /* Small context optimization */
1118                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
1119                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
1120                     ((pref_context == SCST_CONTEXT_SAME) &&
1121                      scst_cmd_atomic(cmd))) {
1122                         if (!test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC,
1123                                         &cmd->tgt_dev->tgt_dev_flags))
1124                                 pref_context = SCST_CONTEXT_THREAD;
1125                 }
1126                 break;
1127
1128         case SCST_RX_STATUS_ERROR_SENSE_SET:
1129                 scst_set_cmd_abnormal_done_state(cmd);
1130                 break;
1131
1132         case SCST_RX_STATUS_ERROR_FATAL:
1133                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1134                 /* go through */
1135         case SCST_RX_STATUS_ERROR:
1136                 scst_set_cmd_error(cmd,
1137                            SCST_LOAD_SENSE(scst_sense_hardw_error));
1138                 scst_set_cmd_abnormal_done_state(cmd);
1139                 break;
1140
1141         default:
1142                 PRINT_ERROR("scst_rx_data() received unknown status %x",
1143                         status);
1144                 scst_set_cmd_abnormal_done_state(cmd);
1145                 break;
1146         }
1147
1148         scst_proccess_redirect_cmd(cmd, pref_context, 1);
1149
1150         TRACE_EXIT();
1151         return;
1152 }
1153 EXPORT_SYMBOL(scst_rx_data);
1154
1155 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
1156 {
1157         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
1158
1159         TRACE_ENTRY();
1160
1161         cmd->state = SCST_CMD_STATE_SEND_FOR_EXEC;
1162
1163         if ((cmd->tgtt->pre_exec == NULL) || unlikely(cmd->internal))
1164                 goto out;
1165
1166         TRACE_DBG("Calling pre_exec(%p)", cmd);
1167         rc = cmd->tgtt->pre_exec(cmd);
1168         TRACE_DBG("pre_exec() returned %d", rc);
1169
1170         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
1171                 switch (rc) {
1172                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
1173                         scst_set_cmd_abnormal_done_state(cmd);
1174                         break;
1175                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
1176                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1177                         /* go through */
1178                 case SCST_PREPROCESS_STATUS_ERROR:
1179                         scst_set_cmd_error(cmd,
1180                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1181                         scst_set_cmd_abnormal_done_state(cmd);
1182                         break;
1183                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1184                         TRACE_DBG("Target driver's %s pre_exec() requested "
1185                                 "thread context, rescheduling",
1186                                 cmd->tgtt->name);
1187                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1188                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1189                         break;
1190                 default:
1191                         sBUG();
1192                         break;
1193                 }
1194         }
1195
1196 out:
1197         TRACE_EXIT_RES(res);
1198         return res;
1199 }
1200
1201 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1202         const uint8_t *rq_sense, int rq_sense_len, int resid)
1203 {
1204         TRACE_ENTRY();
1205
1206 #ifdef CONFIG_SCST_MEASURE_LATENCY
1207         {
1208                 struct timespec ts;
1209                 getnstimeofday(&ts);
1210                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1211                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1212                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1213                         ts.tv_sec, ts.tv_nsec);
1214         }
1215 #endif
1216
1217         cmd->status = result & 0xff;
1218         cmd->msg_status = msg_byte(result);
1219         cmd->host_status = host_byte(result);
1220         cmd->driver_status = driver_byte(result);
1221         if (unlikely(resid != 0)) {
1222 #ifdef CONFIG_SCST_EXTRACHECKS
1223                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1224                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1225                                 "op %x)", resid, cmd->resp_data_len,
1226                                 cmd->cdb[0]);
1227                 } else
1228 #endif
1229                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1230         }
1231
1232         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION)) {
1233                 /* We might have double reset UA here */
1234                 cmd->dbl_ua_orig_resp_data_len = cmd->resp_data_len;
1235                 cmd->dbl_ua_orig_data_direction = cmd->data_direction;
1236
1237                 scst_alloc_set_sense(cmd, 1, rq_sense, rq_sense_len);
1238         }
1239
1240         TRACE(TRACE_SCSI, "cmd %p, result=%x, cmd->status=%x, resid=%d, "
1241               "cmd->msg_status=%x, cmd->host_status=%x, "
1242               "cmd->driver_status=%x (cmd %p)", cmd, result, cmd->status, resid,
1243               cmd->msg_status, cmd->host_status, cmd->driver_status, cmd);
1244
1245         cmd->completed = 1;
1246
1247         TRACE_EXIT();
1248         return;
1249 }
1250
1251 /* For small context optimization */
1252 static inline enum scst_exec_context scst_optimize_post_exec_context(
1253         struct scst_cmd *cmd, enum scst_exec_context context)
1254 {
1255         if (((context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd)) ||
1256             (context == SCST_CONTEXT_TASKLET) ||
1257             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1258                 if (!test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC,
1259                                 &cmd->tgt_dev->tgt_dev_flags))
1260                         context = SCST_CONTEXT_THREAD;
1261         }
1262         return context;
1263 }
1264
1265 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1266 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1267                                             struct scsi_request **req)
1268 {
1269         struct scst_cmd *cmd = NULL;
1270
1271         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1272                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1273
1274         if (cmd == NULL) {
1275                 PRINT_ERROR("%s", "Request with NULL cmd");
1276                 if (*req)
1277                         scsi_release_request(*req);
1278         }
1279
1280         return cmd;
1281 }
1282
1283 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1284 {
1285         struct scsi_request *req = NULL;
1286         struct scst_cmd *cmd;
1287
1288         TRACE_ENTRY();
1289
1290         cmd = scst_get_cmd(scsi_cmd, &req);
1291         if (cmd == NULL)
1292                 goto out;
1293
1294         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1295                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1296
1297         /* Clear out request structure */
1298         req->sr_use_sg = 0;
1299         req->sr_sglist_len = 0;
1300         req->sr_bufflen = 0;
1301         req->sr_buffer = NULL;
1302         req->sr_underflow = 0;
1303         req->sr_request->rq_disk = NULL; /* disown request blk */
1304
1305         scst_release_request(cmd);
1306
1307         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1308
1309         scst_proccess_redirect_cmd(cmd,
1310                 scst_optimize_post_exec_context(cmd, scst_estimate_context()),
1311                                                 0);
1312
1313 out:
1314         TRACE_EXIT();
1315         return;
1316 }
1317 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1318 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1319 {
1320         struct scst_cmd *cmd;
1321
1322         TRACE_ENTRY();
1323
1324         cmd = (struct scst_cmd *)data;
1325         if (cmd == NULL)
1326                 goto out;
1327
1328         scst_do_cmd_done(cmd, result, sense, SCSI_SENSE_BUFFERSIZE, resid);
1329
1330         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1331
1332         scst_proccess_redirect_cmd(cmd,
1333             scst_optimize_post_exec_context(cmd, scst_estimate_context()), 0);
1334
1335 out:
1336         TRACE_EXIT();
1337         return;
1338 }
1339 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1340
1341 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state,
1342         enum scst_exec_context pref_context)
1343 {
1344         TRACE_ENTRY();
1345
1346 #ifdef CONFIG_SCST_MEASURE_LATENCY
1347         {
1348                 struct timespec ts;
1349                 getnstimeofday(&ts);
1350                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1351                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1352                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1353                         ts.tv_sec, ts.tv_nsec);
1354         }
1355 #endif
1356
1357         if (next_state == SCST_CMD_STATE_DEFAULT)
1358                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1359
1360 #if defined(CONFIG_SCST_DEBUG)
1361         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1362                 if ((trace_flag & TRACE_RCV_TOP) && (cmd->sg != NULL)) {
1363                         int i;
1364                         struct scatterlist *sg = cmd->sg;
1365                         TRACE_RECV_TOP("Exec'd %d S/G(s) at %p sg[0].page at "
1366                                 "%p", cmd->sg_cnt, sg, (void *)sg_page(&sg[0]));
1367                         for (i = 0; i < cmd->sg_cnt; ++i) {
1368                                 TRACE_BUFF_FLAG(TRACE_RCV_TOP,
1369                                         "Exec'd sg", sg_virt(&sg[i]),
1370                                         sg[i].length);
1371                         }
1372                 }
1373         }
1374 #endif
1375
1376         cmd->state = next_state;
1377
1378 #ifdef CONFIG_SCST_EXTRACHECKS
1379         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1380             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1381             (next_state != SCST_CMD_STATE_FINISHED) &&
1382             (next_state != SCST_CMD_STATE_FINISHED_INTERNAL)) {
1383                 PRINT_ERROR("%s() received invalid cmd state %d (opcode %d)",
1384                         __func__, next_state, cmd->cdb[0]);
1385                 scst_set_cmd_error(cmd,
1386                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1387                 scst_set_cmd_abnormal_done_state(cmd);
1388         }
1389 #endif
1390         pref_context = scst_optimize_post_exec_context(cmd, pref_context);
1391         scst_proccess_redirect_cmd(cmd, pref_context, 0);
1392
1393         TRACE_EXIT();
1394         return;
1395 }
1396
1397 static int scst_report_luns_local(struct scst_cmd *cmd)
1398 {
1399         int rc;
1400         int dev_cnt = 0;
1401         int buffer_size;
1402         int i;
1403         struct scst_tgt_dev *tgt_dev = NULL;
1404         uint8_t *buffer;
1405         int offs, overflow = 0;
1406
1407         TRACE_ENTRY();
1408
1409         rc = scst_check_local_events(cmd);
1410         if (unlikely(rc != 0))
1411                 goto out_done;
1412
1413         cmd->status = 0;
1414         cmd->msg_status = 0;
1415         cmd->host_status = DID_OK;
1416         cmd->driver_status = 0;
1417
1418         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1419                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1420                         "LUNS command", cmd->cdb[2]);
1421                 goto out_err;
1422         }
1423
1424         buffer_size = scst_get_buf_first(cmd, &buffer);
1425         if (unlikely(buffer_size == 0))
1426                 goto out_compl;
1427         else if (unlikely(buffer_size < 0))
1428                 goto out_err;
1429
1430         if (buffer_size < 16)
1431                 goto out_put_err;
1432
1433         memset(buffer, 0, buffer_size);
1434         offs = 8;
1435
1436         /* sess->sess_tgt_dev_list_hash is protected by suspended activity */
1437         for (i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1438                 struct list_head *sess_tgt_dev_list_head =
1439                         &cmd->sess->sess_tgt_dev_list_hash[i];
1440                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1441                                 sess_tgt_dev_list_entry) {
1442                         if (!overflow) {
1443                                 if (offs >= buffer_size) {
1444                                         scst_put_buf(cmd, buffer);
1445                                         buffer_size = scst_get_buf_next(cmd,
1446                                                                        &buffer);
1447                                         if (buffer_size > 0) {
1448                                                 memset(buffer, 0, buffer_size);
1449                                                 offs = 0;
1450                                         } else {
1451                                                 overflow = 1;
1452                                                 goto inc_dev_cnt;
1453                                         }
1454                                 }
1455                                 if ((buffer_size - offs) < 8) {
1456                                         PRINT_ERROR("Buffer allocated for "
1457                                                 "REPORT LUNS command doesn't "
1458                                                 "allow to fit 8 byte entry "
1459                                                 "(buffer_size=%d)",
1460                                                 buffer_size);
1461                                         goto out_put_hw_err;
1462                                 }
1463                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1464                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1465                                 offs += 8;
1466                         }
1467 inc_dev_cnt:
1468                         dev_cnt++;
1469                 }
1470         }
1471         if (!overflow)
1472                 scst_put_buf(cmd, buffer);
1473
1474         /* Set the response header */
1475         buffer_size = scst_get_buf_first(cmd, &buffer);
1476         if (unlikely(buffer_size == 0))
1477                 goto out_compl;
1478         else if (unlikely(buffer_size < 0))
1479                 goto out_err;
1480
1481         dev_cnt *= 8;
1482         buffer[0] = (dev_cnt >> 24) & 0xff;
1483         buffer[1] = (dev_cnt >> 16) & 0xff;
1484         buffer[2] = (dev_cnt >> 8) & 0xff;
1485         buffer[3] = dev_cnt & 0xff;
1486
1487         scst_put_buf(cmd, buffer);
1488
1489         dev_cnt += 8;
1490         if (dev_cnt < cmd->resp_data_len)
1491                 scst_set_resp_data_len(cmd, dev_cnt);
1492
1493 out_compl:
1494         cmd->completed = 1;
1495
1496         /* Clear left sense_reported_luns_data_changed UA, if any. */
1497
1498         mutex_lock(&scst_mutex); /* protect sess_tgt_dev_list_hash */
1499         for (i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1500                 struct list_head *sess_tgt_dev_list_head =
1501                         &cmd->sess->sess_tgt_dev_list_hash[i];
1502
1503                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1504                                 sess_tgt_dev_list_entry) {
1505                         struct scst_tgt_dev_UA *ua;
1506
1507                         spin_lock_bh(&tgt_dev->tgt_dev_lock);
1508                         list_for_each_entry(ua, &tgt_dev->UA_list,
1509                                                 UA_list_entry) {
1510                                 if (scst_analyze_sense(ua->UA_sense_buffer,
1511                                                 sizeof(ua->UA_sense_buffer),
1512                                                 SCST_SENSE_ALL_VALID,
1513                                                 SCST_LOAD_SENSE(scst_sense_reported_luns_data_changed))) {
1514                                         TRACE_MGMT_DBG("Freeing not needed "
1515                                                 "REPORTED LUNS DATA CHANGED UA "
1516                                                 "%p", ua);
1517                                         list_del(&ua->UA_list_entry);
1518                                         mempool_free(ua, scst_ua_mempool);
1519                                         break;
1520                                 }
1521                         }
1522                         spin_unlock_bh(&tgt_dev->tgt_dev_lock);
1523                 }
1524         }
1525         mutex_unlock(&scst_mutex);
1526
1527 out_done:
1528         /* Report the result */
1529         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1530
1531         TRACE_EXIT();
1532         return SCST_EXEC_COMPLETED;
1533
1534 out_put_err:
1535         scst_put_buf(cmd, buffer);
1536
1537 out_err:
1538         scst_set_cmd_error(cmd,
1539                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1540         goto out_compl;
1541
1542 out_put_hw_err:
1543         scst_put_buf(cmd, buffer);
1544         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1545         goto out_compl;
1546 }
1547
1548 static int scst_pre_select(struct scst_cmd *cmd)
1549 {
1550         int res = SCST_EXEC_NOT_COMPLETED;
1551
1552         TRACE_ENTRY();
1553
1554         if (scst_cmd_atomic(cmd)) {
1555                 res = SCST_EXEC_NEED_THREAD;
1556                 goto out;
1557         }
1558
1559         scst_block_dev_cmd(cmd, 1);
1560
1561         /* Check for local events will be done when cmd will be executed */
1562
1563 out:
1564         TRACE_EXIT_RES(res);
1565         return res;
1566 }
1567
1568 static int scst_reserve_local(struct scst_cmd *cmd)
1569 {
1570         int res = SCST_EXEC_NOT_COMPLETED, rc;
1571         struct scst_device *dev;
1572         struct scst_tgt_dev *tgt_dev_tmp;
1573
1574         TRACE_ENTRY();
1575
1576         if (scst_cmd_atomic(cmd)) {
1577                 res = SCST_EXEC_NEED_THREAD;
1578                 goto out;
1579         }
1580
1581         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1582                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1583                      "(lun=%lld)", (long long unsigned int)cmd->lun);
1584                 scst_set_cmd_error(cmd,
1585                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1586                 goto out_done;
1587         }
1588
1589         dev = cmd->dev;
1590
1591         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1592                 scst_block_dev_cmd(cmd, 1);
1593
1594         rc = scst_check_local_events(cmd);
1595         if (unlikely(rc != 0))
1596                 goto out_done;
1597
1598         spin_lock_bh(&dev->dev_lock);
1599
1600         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1601                 spin_unlock_bh(&dev->dev_lock);
1602                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1603                 goto out_done;
1604         }
1605
1606         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1607                             dev_tgt_dev_list_entry) {
1608                 if (cmd->tgt_dev != tgt_dev_tmp)
1609                         set_bit(SCST_TGT_DEV_RESERVED,
1610                                 &tgt_dev_tmp->tgt_dev_flags);
1611         }
1612         dev->dev_reserved = 1;
1613
1614         spin_unlock_bh(&dev->dev_lock);
1615
1616 out:
1617         TRACE_EXIT_RES(res);
1618         return res;
1619
1620 out_done:
1621         /* Report the result */
1622         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1623         res = SCST_EXEC_COMPLETED;
1624         goto out;
1625 }
1626
1627 static int scst_release_local(struct scst_cmd *cmd)
1628 {
1629         int res = SCST_EXEC_NOT_COMPLETED, rc;
1630         struct scst_tgt_dev *tgt_dev_tmp;
1631         struct scst_device *dev;
1632
1633         TRACE_ENTRY();
1634
1635         if (scst_cmd_atomic(cmd)) {
1636                 res = SCST_EXEC_NEED_THREAD;
1637                 goto out;
1638         }
1639
1640         dev = cmd->dev;
1641
1642         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1643                 scst_block_dev_cmd(cmd, 1);
1644
1645         rc = scst_check_local_events(cmd);
1646         if (unlikely(rc != 0))
1647                 goto out_done;
1648
1649         spin_lock_bh(&dev->dev_lock);
1650
1651         /*
1652          * The device could be RELEASED behind us, if RESERVING session
1653          * is closed (see scst_free_tgt_dev()), but this actually doesn't
1654          * matter, so use lock and no retest for DEV_RESERVED bits again
1655          */
1656         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1657                 res = SCST_EXEC_COMPLETED;
1658                 cmd->status = 0;
1659                 cmd->msg_status = 0;
1660                 cmd->host_status = DID_OK;
1661                 cmd->driver_status = 0;
1662                 cmd->completed = 1;
1663         } else {
1664                 list_for_each_entry(tgt_dev_tmp,
1665                                     &dev->dev_tgt_dev_list,
1666                                     dev_tgt_dev_list_entry) {
1667                         clear_bit(SCST_TGT_DEV_RESERVED,
1668                                 &tgt_dev_tmp->tgt_dev_flags);
1669                 }
1670                 dev->dev_reserved = 0;
1671         }
1672
1673         spin_unlock_bh(&dev->dev_lock);
1674
1675         if (res == SCST_EXEC_COMPLETED)
1676                 goto out_done;
1677
1678 out:
1679         TRACE_EXIT_RES(res);
1680         return res;
1681
1682 out_done:
1683         res = SCST_EXEC_COMPLETED;
1684         /* Report the result */
1685         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1686         goto out;
1687 }
1688
1689 /* No locks, no IRQ or IRQ-safe context allowed */
1690 int scst_check_local_events(struct scst_cmd *cmd)
1691 {
1692         int res, rc;
1693         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1694         struct scst_device *dev = cmd->dev;
1695
1696         TRACE_ENTRY();
1697
1698         /*
1699          * There's no race here, because we need to trace commands sent
1700          * *after* dev_double_ua_possible flag was set.
1701          */
1702         if (unlikely(dev->dev_double_ua_possible))
1703                 cmd->double_ua_possible = 1;
1704
1705         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1706                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1707                 goto out_uncomplete;
1708         }
1709
1710         /* Reserve check before Unit Attention */
1711         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED,
1712                               &tgt_dev->tgt_dev_flags))) {
1713                 if (cmd->cdb[0] != INQUIRY &&
1714                     cmd->cdb[0] != REPORT_LUNS &&
1715                     cmd->cdb[0] != RELEASE &&
1716                     cmd->cdb[0] != RELEASE_10 &&
1717                     cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER &&
1718                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL ||
1719                      (cmd->cdb[4] & 3)) &&
1720                     cmd->cdb[0] != LOG_SENSE &&
1721                     cmd->cdb[0] != REQUEST_SENSE) {
1722                         scst_set_cmd_error_status(cmd,
1723                                 SAM_STAT_RESERVATION_CONFLICT);
1724                         goto out_complete;
1725                 }
1726         }
1727
1728         /* If we had internal bus reset, set the command error unit attention */
1729         if ((dev->scsi_dev != NULL) &&
1730             unlikely(dev->scsi_dev->was_reset)) {
1731                 if (scst_is_ua_command(cmd)) {
1732                         int done = 0;
1733                         /*
1734                          * Prevent more than 1 cmd to be triggered by
1735                          * was_reset.
1736                          */
1737                         spin_lock_bh(&dev->dev_lock);
1738                         if (dev->scsi_dev->was_reset) {
1739                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1740                                 scst_set_cmd_error(cmd,
1741                                           SCST_LOAD_SENSE(scst_sense_reset_UA));
1742                                 /*
1743                                  * It looks like it is safe to clear was_reset
1744                                  * here.
1745                                  */
1746                                 dev->scsi_dev->was_reset = 0;
1747                                 done = 1;
1748                         }
1749                         spin_unlock_bh(&dev->dev_lock);
1750
1751                         if (done)
1752                                 goto out_complete;
1753                 }
1754         }
1755
1756         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING,
1757                         &cmd->tgt_dev->tgt_dev_flags))) {
1758                 if (scst_is_ua_command(cmd)) {
1759                         rc = scst_set_pending_UA(cmd);
1760                         if (rc == 0)
1761                                 goto out_complete;
1762                 }
1763         }
1764
1765         res = 0;
1766
1767 out:
1768         TRACE_EXIT_RES(res);
1769         return res;
1770
1771 out_complete:
1772         res = 1;
1773         sBUG_ON(!cmd->completed);
1774         goto out;
1775
1776 out_uncomplete:
1777         res = -1;
1778         goto out;
1779 }
1780 EXPORT_SYMBOL(scst_check_local_events);
1781
1782 /* No locks */
1783 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1784 {
1785         if (slot == NULL)
1786                 goto inc;
1787
1788         /* Optimized for lockless fast path */
1789
1790         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1791                 atomic_read(slot));
1792
1793         if (!atomic_dec_and_test(slot))
1794                 goto out;
1795
1796         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1797                 tgt_dev->num_free_sn_slots);
1798         if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
1799                 spin_lock_irq(&tgt_dev->sn_lock);
1800                 if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
1801                         if (tgt_dev->num_free_sn_slots < 0)
1802                                 tgt_dev->cur_sn_slot = slot;
1803                         /*
1804                          * To be in-sync with SIMPLE case in scst_cmd_set_sn()
1805                          */
1806                         smp_mb();
1807                         tgt_dev->num_free_sn_slots++;
1808                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1809                                 tgt_dev->num_free_sn_slots);
1810
1811                 }
1812                 spin_unlock_irq(&tgt_dev->sn_lock);
1813         }
1814
1815 inc:
1816         /*
1817          * No protection of expected_sn is needed, because only one thread
1818          * at time can be here (serialized by sn). Also it is supposed that
1819          * there could not be half-incremented halves.
1820          */
1821         tgt_dev->expected_sn++;
1822         /*
1823          * Write must be before def_cmd_count read to be in sync. with
1824          * scst_post_exec_sn(). See comment in scst_send_for_exec().
1825          */
1826         smp_mb();
1827         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1828
1829 out:
1830         return;
1831 }
1832
1833 /* No locks */
1834 static struct scst_cmd *scst_post_exec_sn(struct scst_cmd *cmd,
1835         bool make_active)
1836 {
1837         /* For HQ commands SN is not set */
1838         bool inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1839                                cmd->sn_set && !cmd->retry;
1840         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1841         struct scst_cmd *res;
1842
1843         TRACE_ENTRY();
1844
1845         if (inc_expected_sn)
1846                 scst_inc_expected_sn(tgt_dev, cmd->sn_slot);
1847
1848         if (make_active) {
1849                 scst_make_deferred_commands_active(tgt_dev);
1850                 res = NULL;
1851         } else
1852                 res = scst_check_deferred_commands(tgt_dev);
1853
1854         TRACE_EXIT_HRES(res);
1855         return res;
1856 }
1857
1858 /* cmd must be additionally referenced to not die inside */
1859 static int scst_do_real_exec(struct scst_cmd *cmd)
1860 {
1861         int res = SCST_EXEC_NOT_COMPLETED;
1862 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
1863         int rc;
1864 #endif
1865         bool atomic = scst_cmd_atomic(cmd);
1866         struct scst_device *dev = cmd->dev;
1867         struct scst_dev_type *handler = dev->handler;
1868         struct io_context *old_ctx = NULL;
1869         bool ctx_changed = false;
1870
1871         TRACE_ENTRY();
1872
1873         if (!atomic)
1874                 ctx_changed = scst_set_io_context(cmd, &old_ctx);
1875
1876         cmd->state = SCST_CMD_STATE_REAL_EXECUTING;
1877
1878         if (handler->exec) {
1879                 if (unlikely(!dev->handler->exec_atomic && atomic)) {
1880                         /*
1881                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
1882                          * optimization.
1883                          */
1884                         TRACE_DBG("Dev handler %s exec() needs thread "
1885                                 "context, rescheduling", dev->handler->name);
1886                         res = SCST_EXEC_NEED_THREAD;
1887                         goto out_restore;
1888                 }
1889
1890                 TRACE_DBG("Calling dev handler %s exec(%p)",
1891                       handler->name, cmd);
1892                 TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb,
1893                         cmd->cdb_len);
1894                 res = handler->exec(cmd);
1895                 TRACE_DBG("Dev handler %s exec() returned %d",
1896                       handler->name, res);
1897
1898                 if (res == SCST_EXEC_COMPLETED)
1899                         goto out_complete;
1900                 else if (res == SCST_EXEC_NEED_THREAD)
1901                         goto out_restore;
1902
1903                 sBUG_ON(res != SCST_EXEC_NOT_COMPLETED);
1904         }
1905
1906         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1907
1908         if (unlikely(dev->scsi_dev == NULL)) {
1909                 PRINT_ERROR("Command for virtual device must be "
1910                         "processed by device handler (lun %lld)!",
1911                         (long long unsigned int)cmd->lun);
1912                 goto out_error;
1913         }
1914
1915         res = scst_check_local_events(cmd);
1916         if (unlikely(res != 0))
1917                 goto out_done;
1918
1919 #ifndef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1920         if (unlikely(atomic)) {
1921                 TRACE_DBG("Pass-through exec() can not be called in atomic "
1922                         "context, rescheduling to the thread (handler %s)",
1923                         handler->name);
1924                 res = SCST_EXEC_NEED_THREAD;
1925                 goto out_restore;
1926         }
1927 #endif
1928
1929 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1930         if (unlikely(scst_alloc_request(cmd) != 0)) {
1931                 if (atomic) {
1932                         res = SCST_EXEC_NEED_THREAD;
1933                         goto out_restore;
1934                 } else {
1935                         PRINT_INFO("%s", "Unable to allocate request, "
1936                                 "sending BUSY status");
1937                         goto out_busy;
1938                 }
1939         }
1940
1941         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1942                     (void *)cmd->scsi_req->sr_buffer,
1943                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1944                     cmd->retries);
1945 #else
1946         rc = scst_exec_req(dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1947                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1948                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1949                         atomic ? GFP_ATOMIC : GFP_KERNEL);
1950         if (unlikely(rc != 0)) {
1951                 if (atomic) {
1952                         res = SCST_EXEC_NEED_THREAD;
1953                         goto out_restore;
1954                 } else {
1955                         PRINT_ERROR("scst_exec_req() failed: %d", res);
1956                         goto out_error;
1957                 }
1958         }
1959 #endif
1960
1961 out_complete:
1962         res = SCST_EXEC_COMPLETED;
1963
1964 out_reset_ctx:
1965         if (ctx_changed)
1966                 scst_reset_io_context(cmd->tgt_dev, old_ctx);
1967
1968         TRACE_EXIT();
1969         return res;
1970
1971 out_restore:
1972         /* Restore the state */
1973         cmd->state = SCST_CMD_STATE_REAL_EXEC;
1974         goto out_reset_ctx;
1975
1976 out_error:
1977         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1978         goto out_done;
1979
1980 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1981 out_busy:
1982         scst_set_busy(cmd);
1983         /* go through */
1984 #endif
1985
1986 out_done:
1987         res = SCST_EXEC_COMPLETED;
1988         /* Report the result */
1989         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1990         goto out_complete;
1991 }
1992
1993 static inline int scst_real_exec(struct scst_cmd *cmd)
1994 {
1995         int res;
1996
1997         TRACE_ENTRY();
1998
1999         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
2000         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
2001         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
2002
2003         __scst_cmd_get(cmd);
2004
2005         res = scst_do_real_exec(cmd);
2006
2007         if (likely(res == SCST_EXEC_COMPLETED)) {
2008                 scst_post_exec_sn(cmd, true);
2009                 if (cmd->dev->scsi_dev != NULL)
2010                         generic_unplug_device(
2011                                 cmd->dev->scsi_dev->request_queue);
2012         } else
2013                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
2014
2015         __scst_cmd_put(cmd);
2016
2017         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
2018
2019         TRACE_EXIT_RES(res);
2020         return res;
2021 }
2022
2023 static int scst_do_local_exec(struct scst_cmd *cmd)
2024 {
2025         int res;
2026         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2027
2028         TRACE_ENTRY();
2029
2030         /* Check READ_ONLY device status */
2031         if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
2032             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
2033              cmd->cdb[0] == WRITE_10 ||
2034              cmd->cdb[0] == WRITE_12 ||
2035              cmd->cdb[0] == WRITE_16 ||
2036              cmd->cdb[0] == WRITE_VERIFY ||
2037              cmd->cdb[0] == WRITE_VERIFY_12 ||
2038              cmd->cdb[0] == WRITE_VERIFY_16 ||
2039              (cmd->dev->handler->type == TYPE_TAPE &&
2040               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
2041                 PRINT_WARNING("Attempt of write access to read-only device: "
2042                         "initiator %s, LUN %lld, op %x",
2043                         cmd->sess->initiator_name, cmd->lun, cmd->cdb[0]);
2044                 scst_set_cmd_error(cmd,
2045                            SCST_LOAD_SENSE(scst_sense_data_protect));
2046                 goto out_done;
2047         }
2048
2049         /*
2050          * Adding new commands here don't forget to update
2051          * scst_is_cmd_local() in scst.h, if necessary
2052          */
2053
2054         switch (cmd->cdb[0]) {
2055         case MODE_SELECT:
2056         case MODE_SELECT_10:
2057         case LOG_SELECT:
2058                 res = scst_pre_select(cmd);
2059                 break;
2060         case RESERVE:
2061         case RESERVE_10:
2062                 res = scst_reserve_local(cmd);
2063                 break;
2064         case RELEASE:
2065         case RELEASE_10:
2066                 res = scst_release_local(cmd);
2067                 break;
2068         case REPORT_LUNS:
2069                 res = scst_report_luns_local(cmd);
2070                 break;
2071         default:
2072                 res = SCST_EXEC_NOT_COMPLETED;
2073                 break;
2074         }
2075
2076 out:
2077         TRACE_EXIT_RES(res);
2078         return res;
2079
2080 out_done:
2081         /* Report the result */
2082         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
2083         res = SCST_EXEC_COMPLETED;
2084         goto out;
2085 }
2086
2087 static int scst_local_exec(struct scst_cmd *cmd)
2088 {
2089         int res;
2090
2091         TRACE_ENTRY();
2092
2093         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
2094         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
2095         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
2096
2097         __scst_cmd_get(cmd);
2098
2099         res = scst_do_local_exec(cmd);
2100         if (likely(res == SCST_EXEC_NOT_COMPLETED))
2101                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2102         else if (res == SCST_EXEC_COMPLETED)
2103                 scst_post_exec_sn(cmd, true);
2104         else
2105                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
2106
2107         __scst_cmd_put(cmd);
2108
2109         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
2110         TRACE_EXIT_RES(res);
2111         return res;
2112 }
2113
2114 static int scst_exec(struct scst_cmd **active_cmd)
2115 {
2116         struct scst_cmd *cmd = *active_cmd;
2117         struct scst_cmd *ref_cmd;
2118         struct scst_device *dev = cmd->dev;
2119         int res = SCST_CMD_STATE_RES_CONT_NEXT, count;
2120
2121         TRACE_ENTRY();
2122
2123         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2124                 goto out;
2125
2126         /* To protect tgt_dev */
2127         ref_cmd = cmd;
2128         __scst_cmd_get(ref_cmd);
2129
2130         count = 0;
2131         while (1) {
2132                 int rc;
2133
2134                 cmd->sent_for_exec = 1;
2135                 /*
2136                  * To sync with scst_abort_cmd(). The above assignment must
2137                  * be before SCST_CMD_ABORTED test, done later in
2138                  * scst_check_local_events(). It's far from here, so the order
2139                  * is virtually guaranteed, but let's have it just in case.
2140                  */
2141                 smp_mb();
2142
2143                 cmd->scst_cmd_done = scst_cmd_done_local;
2144                 cmd->state = SCST_CMD_STATE_LOCAL_EXEC;
2145
2146                 if (cmd->tgt_data_buf_alloced && cmd->dh_data_buf_alloced &&
2147                     (cmd->data_direction & SCST_DATA_WRITE))
2148                         scst_copy_sg(cmd, SCST_SG_COPY_FROM_TARGET);
2149
2150                 rc = scst_do_local_exec(cmd);
2151                 if (likely(rc == SCST_EXEC_NOT_COMPLETED))
2152                         /* Nothing to do */;
2153                 else if (rc == SCST_EXEC_NEED_THREAD) {
2154                         TRACE_DBG("%s", "scst_do_local_exec() requested "
2155                                 "thread context, rescheduling");
2156                         scst_dec_on_dev_cmd(cmd);
2157                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2158                         break;
2159                 } else {
2160                         sBUG_ON(rc != SCST_EXEC_COMPLETED);
2161                         goto done;
2162                 }
2163
2164                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2165
2166                 rc = scst_do_real_exec(cmd);
2167                 if (likely(rc == SCST_EXEC_COMPLETED))
2168                         /* Nothing to do */;
2169                 else if (rc == SCST_EXEC_NEED_THREAD) {
2170                         TRACE_DBG("scst_real_exec() requested thread "
2171                                 "context, rescheduling (cmd %p)", cmd);
2172                         scst_dec_on_dev_cmd(cmd);
2173                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2174                         break;
2175                 } else
2176                         sBUG();
2177
2178 done:
2179                 count++;
2180
2181                 cmd = scst_post_exec_sn(cmd, false);
2182                 if (cmd == NULL)
2183                         break;
2184
2185                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2186                         break;
2187
2188                 __scst_cmd_put(ref_cmd);
2189                 ref_cmd = cmd;
2190                 __scst_cmd_get(ref_cmd);
2191         }
2192
2193         *active_cmd = cmd;
2194
2195         if (count == 0)
2196                 goto out_put;
2197
2198         if (dev->scsi_dev != NULL)
2199                 generic_unplug_device(dev->scsi_dev->request_queue);
2200
2201 out_put:
2202         __scst_cmd_put(ref_cmd);
2203         /* !! At this point sess, dev and tgt_dev can be already freed !! */
2204
2205 out:
2206         TRACE_EXIT_RES(res);
2207         return res;
2208 }
2209
2210 static int scst_send_for_exec(struct scst_cmd **active_cmd)
2211 {
2212         int res;
2213         struct scst_cmd *cmd = *active_cmd;
2214         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2215         typeof(tgt_dev->expected_sn) expected_sn;
2216
2217         TRACE_ENTRY();
2218
2219 #ifdef CONFIG_SCST_MEASURE_LATENCY
2220         if (cmd->pre_exec_finish == 0) {
2221                 struct timespec ts;
2222                 getnstimeofday(&ts);
2223                 cmd->pre_exec_finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2224                 TRACE_DBG("cmd %p (sess %p): pre_exec_finish %lld (tv_sec %ld, "
2225                         "tv_nsec %ld)", cmd, cmd->sess, cmd->pre_exec_finish,
2226                         ts.tv_sec, ts.tv_nsec);
2227         }
2228 #endif
2229
2230         if (unlikely(cmd->internal))
2231                 goto exec;
2232
2233         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2234                 goto exec;
2235
2236         sBUG_ON(!cmd->sn_set);
2237
2238         expected_sn = tgt_dev->expected_sn;
2239         /* Optimized for lockless fast path */
2240         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2241                 spin_lock_irq(&tgt_dev->sn_lock);
2242
2243                 tgt_dev->def_cmd_count++;
2244                 /*
2245                  * Memory barrier is needed here to implement lockless fast
2246                  * path. We need the exact order of read and write between
2247                  * def_cmd_count and expected_sn. Otherwise, we can miss case,
2248                  * when expected_sn was changed to be equal to cmd->sn while
2249                  * we are queuing cmd the deferred list after the expected_sn
2250                  * below. It will lead to a forever stuck command. But with
2251                  * the barrier in such case __scst_check_deferred_commands()
2252                  * will be called and it will take sn_lock, so we will be
2253                  * synchronized.
2254                  */
2255                 smp_mb();
2256
2257                 expected_sn = tgt_dev->expected_sn;
2258                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2259                         if (unlikely(test_bit(SCST_CMD_ABORTED,
2260                                               &cmd->cmd_flags))) {
2261                                 /* Necessary to allow aborting out of sn cmds */
2262                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p "
2263                                         "(tag %llu, sn %lu)", cmd,
2264                                         (long long unsigned)cmd->tag, cmd->sn);
2265                                 tgt_dev->def_cmd_count--;
2266                                 scst_set_cmd_abnormal_done_state(cmd);
2267                                 res = SCST_CMD_STATE_RES_CONT_SAME;
2268                         } else {
2269                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
2270                                         "expected_sn=%ld)", cmd, cmd->sn,
2271                                         cmd->sn_set, expected_sn);
2272                                 list_add_tail(&cmd->sn_cmd_list_entry,
2273                                               &tgt_dev->deferred_cmd_list);
2274                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2275                         }
2276                         spin_unlock_irq(&tgt_dev->sn_lock);
2277                         goto out;
2278                 } else {
2279                         TRACE_SN("Somebody incremented expected_sn %ld, "
2280                                 "continuing", expected_sn);
2281                         tgt_dev->def_cmd_count--;
2282                         spin_unlock_irq(&tgt_dev->sn_lock);
2283                 }
2284         }
2285
2286 exec:
2287         res = scst_exec(active_cmd);
2288
2289 out:
2290         TRACE_EXIT_HRES(res);
2291         return res;
2292 }
2293
2294 /* No locks supposed to be held */
2295 static int scst_check_sense(struct scst_cmd *cmd)
2296 {
2297         int res = 0;
2298         struct scst_device *dev = cmd->dev;
2299
2300         TRACE_ENTRY();
2301
2302         if (unlikely(cmd->ua_ignore))
2303                 goto out;
2304
2305         /* If we had internal bus reset behind us, set the command error UA */
2306         if ((dev->scsi_dev != NULL) &&
2307             unlikely(cmd->host_status == DID_RESET) &&
2308             scst_is_ua_command(cmd)) {
2309                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
2310                       dev->scsi_dev->was_reset, cmd->host_status);
2311                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_reset_UA));
2312                 /* It looks like it is safe to clear was_reset here */
2313                 dev->scsi_dev->was_reset = 0;
2314         }
2315
2316         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2317             SCST_SENSE_VALID(cmd->sense)) {
2318                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2319                         SCST_SENSE_BUFFERSIZE);
2320
2321                 /* Check Unit Attention Sense Key */
2322                 if (scst_is_ua_sense(cmd->sense)) {
2323                         if (scst_analyze_sense(cmd->sense,
2324                                         SCST_SENSE_BUFFERSIZE,
2325                                         SCST_SENSE_ASC_VALID,
2326                                         0, SCST_SENSE_ASC_UA_RESET, 0)) {
2327                                 if (cmd->double_ua_possible) {
2328                                         TRACE(TRACE_MGMT_MINOR, "Double UA "
2329                                                 "detected for device %p", dev);
2330                                         TRACE(TRACE_MGMT_MINOR, "Retrying cmd"
2331                                                 " %p (tag %llu)", cmd,
2332                                                 (long long unsigned)cmd->tag);
2333
2334                                         cmd->status = 0;
2335                                         cmd->msg_status = 0;
2336                                         cmd->host_status = DID_OK;
2337                                         cmd->driver_status = 0;
2338
2339                                         mempool_free(cmd->sense,
2340                                                      scst_sense_mempool);
2341                                         cmd->sense = NULL;
2342
2343                                         scst_check_restore_sg_buff(cmd);
2344
2345                                         sBUG_ON(cmd->dbl_ua_orig_resp_data_len < 0);
2346                                         cmd->data_direction =
2347                                                 cmd->dbl_ua_orig_data_direction;
2348                                         cmd->resp_data_len =
2349                                                 cmd->dbl_ua_orig_resp_data_len;
2350
2351                                         cmd->state = SCST_CMD_STATE_REAL_EXEC;
2352                                         cmd->retry = 1;
2353                                         res = 1;
2354                                         goto out;
2355                                 }
2356                         }
2357                         scst_dev_check_set_UA(dev, cmd, cmd->sense,
2358                                 SCST_SENSE_BUFFERSIZE);
2359                 }
2360         }
2361
2362         if (unlikely(cmd->double_ua_possible)) {
2363                 if (scst_is_ua_command(cmd)) {
2364                         TRACE_MGMT_DBG("Clearing dbl_ua_possible flag (dev %p, "
2365                                 "cmd %p)", dev, cmd);
2366                         /*
2367                          * Lock used to protect other flags in the bitfield
2368                          * (just in case, actually). Those flags can't be
2369                          * changed in parallel, because the device is
2370                          * serialized.
2371                          */
2372                         spin_lock_bh(&dev->dev_lock);
2373                         dev->dev_double_ua_possible = 0;
2374                         spin_unlock_bh(&dev->dev_lock);
2375                 }
2376         }
2377
2378 out:
2379         TRACE_EXIT_RES(res);
2380         return res;
2381 }
2382
2383 static int scst_check_auto_sense(struct scst_cmd *cmd)
2384 {
2385         int res = 0;
2386
2387         TRACE_ENTRY();
2388
2389         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2390             (!SCST_SENSE_VALID(cmd->sense) ||
2391              SCST_NO_SENSE(cmd->sense))) {
2392                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2393                       "cmd->status=%x, cmd->msg_status=%x, "
2394                       "cmd->host_status=%x, cmd->driver_status=%x (cmd %p)",
2395                       cmd->status, cmd->msg_status, cmd->host_status,
2396                       cmd->driver_status, cmd);
2397                 res = 1;
2398         } else if (unlikely(cmd->host_status)) {
2399                 if ((cmd->host_status == DID_REQUEUE) ||
2400                     (cmd->host_status == DID_IMM_RETRY) ||
2401                     (cmd->host_status == DID_SOFT_ERROR) ||
2402                     (cmd->host_status == DID_ABORT)) {
2403                         scst_set_busy(cmd);
2404                 } else {
2405                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2406                                 "received, returning HARDWARE ERROR instead "
2407                                 "(cmd %p)", cmd->host_status, cmd);
2408                         scst_set_cmd_error(cmd,
2409                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2410                 }
2411         }
2412
2413         TRACE_EXIT_RES(res);
2414         return res;
2415 }
2416
2417 static int scst_pre_dev_done(struct scst_cmd *cmd)
2418 {
2419         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2420
2421         TRACE_ENTRY();
2422
2423         if (unlikely(scst_check_auto_sense(cmd))) {
2424                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2425                             "without sense data (opcode 0x%x), issuing "
2426                             "REQUEST SENSE", cmd->cdb[0]);
2427                 rc = scst_prepare_request_sense(cmd);
2428                 if (rc == 0)
2429                         res = SCST_CMD_STATE_RES_CONT_NEXT;
2430                 else {
2431                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2432                                     "returning HARDWARE ERROR");
2433                         scst_set_cmd_error(cmd,
2434                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2435                 }
2436                 goto out;
2437         } else if (unlikely(scst_check_sense(cmd)))
2438                 goto out;
2439
2440         if (likely(scsi_status_is_good(cmd->status))) {
2441                 unsigned char type = cmd->dev->handler->type;
2442                 if (unlikely((cmd->cdb[0] == MODE_SENSE ||
2443                               cmd->cdb[0] == MODE_SENSE_10)) &&
2444                     cmd->tgt_dev->acg_dev->rd_only_flag &&
2445                     (type == TYPE_DISK ||
2446                      type == TYPE_WORM ||
2447                      type == TYPE_MOD ||
2448                      type == TYPE_TAPE)) {
2449                         int32_t length;
2450                         uint8_t *address;
2451                         bool err = false;
2452
2453                         length = scst_get_buf_first(cmd, &address);
2454                         if (length < 0) {
2455                                 PRINT_ERROR("%s", "Unable to get "
2456                                         "MODE_SENSE buffer");
2457                                 scst_set_cmd_error(cmd,
2458                                         SCST_LOAD_SENSE(
2459                                                 scst_sense_hardw_error));
2460                                 err = true;
2461                         } else if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2462                                 address[2] |= 0x80;   /* Write Protect*/
2463                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2464                                 address[3] |= 0x80;   /* Write Protect*/
2465                         scst_put_buf(cmd, address);
2466
2467                         if (err)
2468                                 goto out;
2469                 }
2470
2471                 /*
2472                  * Check and clear NormACA option for the device, if necessary,
2473                  * since we don't support ACA
2474                  */
2475                 if (unlikely((cmd->cdb[0] == INQUIRY)) &&
2476                     /* Std INQUIRY data (no EVPD) */
2477                     !(cmd->cdb[1] & SCST_INQ_EVPD) &&
2478                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2479                         uint8_t *buffer;
2480                         int buflen;
2481                         bool err = false;
2482
2483                         /* ToDo: all pages ?? */
2484                         buflen = scst_get_buf_first(cmd, &buffer);
2485                         if (buflen > SCST_INQ_BYTE3) {
2486 #ifdef CONFIG_SCST_EXTRACHECKS
2487                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2488                                         PRINT_INFO("NormACA set for device: "
2489                                             "lun=%lld, type 0x%02x. Clear it, "
2490                                             "since it's unsupported.",
2491                                             (long long unsigned int)cmd->lun,
2492                                             buffer[0]);
2493                                 }
2494 #endif
2495                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2496                         } else if (buflen != 0) {
2497                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2498                                     "buffer");
2499                                 scst_set_cmd_error(cmd,
2500                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2501                                 err = true;
2502                         }
2503                         if (buflen > 0)
2504                                 scst_put_buf(cmd, buffer);
2505
2506                         if (err)
2507                                 goto out;
2508                 }
2509
2510                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2511                     (cmd->cdb[0] == MODE_SELECT_10) ||
2512                     (cmd->cdb[0] == LOG_SELECT))) {
2513                         TRACE(TRACE_SCSI,
2514                                 "MODE/LOG SELECT succeeded (LUN %lld)",
2515                                 (long long unsigned int)cmd->lun);
2516                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2517                         goto out;
2518                 }
2519         } else {
2520                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2521                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2522                                         &cmd->tgt_dev->tgt_dev_flags)) {
2523                                 struct scst_tgt_dev *tgt_dev_tmp;
2524                                 struct scst_device *dev = cmd->dev;
2525
2526                                 TRACE(TRACE_SCSI,
2527                                         "Real RESERVE failed lun=%lld, "
2528                                         "status=%x",
2529                                         (long long unsigned int)cmd->lun,
2530                                         cmd->status);
2531                                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2532                                         SCST_SENSE_BUFFERSIZE);
2533
2534                                 /* Clearing the reservation */
2535                                 spin_lock_bh(&dev->dev_lock);
2536                                 list_for_each_entry(tgt_dev_tmp,
2537                                                     &dev->dev_tgt_dev_list,
2538                                                     dev_tgt_dev_list_entry) {
2539                                         clear_bit(SCST_TGT_DEV_RESERVED,
2540                                                 &tgt_dev_tmp->tgt_dev_flags);
2541                                 }
2542                                 dev->dev_reserved = 0;
2543                                 spin_unlock_bh(&dev->dev_lock);
2544                         }
2545                 }
2546
2547                 /* Check for MODE PARAMETERS CHANGED UA */
2548                 if ((cmd->dev->scsi_dev != NULL) &&
2549                     (cmd->status == SAM_STAT_CHECK_CONDITION) &&
2550                     SCST_SENSE_VALID(cmd->sense) &&
2551                     scst_is_ua_sense(cmd->sense) &&
2552                     scst_analyze_sense(cmd->sense, SCST_SENSE_BUFFERSIZE,
2553                                         SCST_SENSE_ASCx_VALID,
2554                                         0, 0x2a, 0x01)) {
2555                         TRACE(TRACE_SCSI, "MODE PARAMETERS CHANGED UA (lun "
2556                                 "%lld)", (long long unsigned int)cmd->lun);
2557                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2558                         goto out;
2559                 }
2560         }
2561
2562         cmd->state = SCST_CMD_STATE_DEV_DONE;
2563
2564 out:
2565         TRACE_EXIT_RES(res);
2566         return res;
2567 }
2568
2569 static int scst_mode_select_checks(struct scst_cmd *cmd)
2570 {
2571         int res = SCST_CMD_STATE_RES_CONT_SAME;
2572         int atomic = scst_cmd_atomic(cmd);
2573
2574         TRACE_ENTRY();
2575
2576         if (likely(scsi_status_is_good(cmd->status))) {
2577                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2578                     (cmd->cdb[0] == MODE_SELECT_10) ||
2579                     (cmd->cdb[0] == LOG_SELECT))) {
2580                         struct scst_device *dev = cmd->dev;
2581                         uint8_t sense_buffer[SCST_STANDARD_SENSE_LEN];
2582
2583                         if (atomic && (dev->scsi_dev != NULL)) {
2584                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2585                                         "context required");
2586                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2587                                 goto out;
2588                         }
2589
2590                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2591                                 "setting the SELECT UA (lun=%lld)",
2592                                 (long long unsigned int)cmd->lun);
2593
2594                         spin_lock_bh(&dev->dev_lock);
2595                         if (cmd->cdb[0] == LOG_SELECT) {
2596                                 scst_set_sense(sense_buffer,
2597                                         sizeof(sense_buffer),
2598                                         dev->d_sense,
2599                                         UNIT_ATTENTION, 0x2a, 0x02);
2600                         } else {
2601                                 scst_set_sense(sense_buffer,
2602                                         sizeof(sense_buffer),
2603                                         dev->d_sense,
2604                                         UNIT_ATTENTION, 0x2a, 0x01);
2605                         }
2606                         scst_dev_check_set_local_UA(dev, cmd, sense_buffer,
2607                                 sizeof(sense_buffer));
2608                         spin_unlock_bh(&dev->dev_lock);
2609
2610                         if (dev->scsi_dev != NULL)
2611                                 scst_obtain_device_parameters(dev);
2612                 }
2613         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) &&
2614                     SCST_SENSE_VALID(cmd->sense) &&
2615                     scst_is_ua_sense(cmd->sense) &&
2616                      /* mode parameters changed */
2617                     (scst_analyze_sense(cmd->sense, SCST_SENSE_BUFFERSIZE,
2618                                         SCST_SENSE_ASCx_VALID,
2619                                         0, 0x2a, 0x01) ||
2620                      scst_analyze_sense(cmd->sense, SCST_SENSE_BUFFERSIZE,
2621                                         SCST_SENSE_ASC_VALID,
2622                                         0, 0x29, 0) /* reset */ ||
2623                      scst_analyze_sense(cmd->sense, SCST_SENSE_BUFFERSIZE,
2624                                         SCST_SENSE_ASC_VALID,
2625                                         0, 0x28, 0) /* medium changed */ ||
2626                      /* cleared by another ini (just in case) */
2627                      scst_analyze_sense(cmd->sense, SCST_SENSE_BUFFERSIZE,
2628                                         SCST_SENSE_ASC_VALID,
2629                                         0, 0x2F, 0))) {
2630                 if (atomic) {
2631                         TRACE_DBG("Possible parameters changed UA %x: "
2632                                 "thread context required", cmd->sense[12]);
2633                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2634                         goto out;
2635                 }
2636
2637                 TRACE(TRACE_SCSI, "Possible parameters changed UA %x "
2638                         "(lun %lld): getting new parameters", cmd->sense[12],
2639                         (long long unsigned int)cmd->lun);
2640
2641                 scst_obtain_device_parameters(cmd->dev);
2642         } else
2643                 sBUG();
2644
2645         cmd->state = SCST_CMD_STATE_DEV_DONE;
2646
2647 out:
2648         TRACE_EXIT_HRES(res);
2649         return res;
2650 }
2651
2652 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2653 {
2654         if (likely(cmd->sn_set))
2655                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2656
2657         scst_make_deferred_commands_active(cmd->tgt_dev);
2658 }
2659
2660 static int scst_dev_done(struct scst_cmd *cmd)
2661 {
2662         int res = SCST_CMD_STATE_RES_CONT_SAME;
2663         int state;
2664         struct scst_device *dev = cmd->dev;
2665
2666         TRACE_ENTRY();
2667
2668         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2669
2670         if (likely(!scst_is_cmd_local(cmd)) &&
2671             likely(dev->handler->dev_done != NULL)) {
2672                 int rc;
2673
2674                 if (unlikely(!dev->handler->dev_done_atomic &&
2675                              scst_cmd_atomic(cmd))) {
2676                         /*
2677                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2678                          * optimization.
2679                          */
2680                         TRACE_DBG("Dev handler %s dev_done() needs thread "
2681                               "context, rescheduling", dev->handler->name);
2682                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2683                         goto out;
2684                 }
2685
2686                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2687                       dev->handler->name, cmd);
2688                 rc = dev->handler->dev_done(cmd);
2689                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2690                       dev->handler->name, rc);
2691                 if (rc != SCST_CMD_STATE_DEFAULT)
2692                         state = rc;
2693         }
2694
2695         switch (state) {
2696         case SCST_CMD_STATE_PRE_XMIT_RESP:
2697         case SCST_CMD_STATE_DEV_PARSE:
2698         case SCST_CMD_STATE_PRE_PARSE:
2699         case SCST_CMD_STATE_PREPARE_SPACE:
2700         case SCST_CMD_STATE_RDY_TO_XFER:
2701         case SCST_CMD_STATE_TGT_PRE_EXEC:
2702         case SCST_CMD_STATE_SEND_FOR_EXEC:
2703         case SCST_CMD_STATE_LOCAL_EXEC:
2704         case SCST_CMD_STATE_REAL_EXEC:
2705         case SCST_CMD_STATE_PRE_DEV_DONE:
2706         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2707         case SCST_CMD_STATE_DEV_DONE:
2708         case SCST_CMD_STATE_XMIT_RESP:
2709         case SCST_CMD_STATE_FINISHED:
2710         case SCST_CMD_STATE_FINISHED_INTERNAL:
2711                 cmd->state = state;
2712                 break;
2713
2714         case SCST_CMD_STATE_NEED_THREAD_CTX:
2715                 TRACE_DBG("Dev handler %s dev_done() requested "
2716                       "thread context, rescheduling",
2717                       dev->handler->name);
2718                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2719                 break;
2720
2721         default:
2722                 if (state >= 0) {
2723                         PRINT_ERROR("Dev handler %s dev_done() returned "
2724                                 "invalid cmd state %d",
2725                                 dev->handler->name, state);
2726                 } else {
2727                         PRINT_ERROR("Dev handler %s dev_done() returned "
2728                                 "error %d", dev->handler->name,
2729                                 state);
2730                 }
2731                 scst_set_cmd_error(cmd,
2732                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2733                 scst_set_cmd_abnormal_done_state(cmd);
2734                 break;
2735         }
2736
2737         if (cmd->needs_unblocking)
2738                 scst_unblock_dev_cmd(cmd);
2739
2740         if (likely(cmd->dec_on_dev_needed))
2741                 scst_dec_on_dev_cmd(cmd);
2742
2743         if (cmd->inc_expected_sn_on_done && cmd->sent_for_exec)
2744                 scst_inc_check_expected_sn(cmd);
2745
2746         if (unlikely(cmd->internal))
2747                 cmd->state = SCST_CMD_STATE_FINISHED_INTERNAL;
2748
2749 out:
2750         TRACE_EXIT_HRES(res);
2751         return res;
2752 }
2753
2754 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2755 {
2756         int res;
2757
2758         TRACE_ENTRY();
2759
2760         EXTRACHECKS_BUG_ON(cmd->internal);
2761
2762 #ifdef CONFIG_SCST_DEBUG_TM
2763         if (cmd->tm_dbg_delayed &&
2764                         !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2765                 if (scst_cmd_atomic(cmd)) {
2766                         TRACE_MGMT_DBG("%s",
2767                                 "DEBUG_TM delayed cmd needs a thread");
2768                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2769                         return res;
2770                 }
2771                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2772                         cmd, cmd->tag);
2773                 schedule_timeout_uninterruptible(HZ);
2774         }
2775 #endif
2776
2777         if (likely(cmd->tgt_dev != NULL)) {
2778                 atomic_dec(&cmd->tgt_dev->tgt_dev_cmd_count);
2779                 atomic_dec(&cmd->dev->dev_cmd_count);
2780                 /* If expected values not set, expected direction is UNKNOWN */
2781                 if (cmd->expected_data_direction & SCST_DATA_WRITE)
2782                         atomic_dec(&cmd->dev->write_cmd_count);
2783
2784                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2785                         scst_on_hq_cmd_response(cmd);
2786
2787                 if (unlikely(!cmd->sent_for_exec)) {
2788                         TRACE_SN("cmd %p was not sent to mid-lev"
2789                                 " (sn %ld, set %d)",
2790                                 cmd, cmd->sn, cmd->sn_set);
2791                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2792                         cmd->sent_for_exec = 1;
2793                 }
2794         }
2795
2796         /*
2797          * If we don't remove cmd from the search list here, before
2798          * submitting it for transmittion, we will have a race, when for
2799          * some reason cmd's release is delayed after transmittion and
2800          * initiator sends cmd with the same tag => it is possible that
2801          * a wrong cmd will be found by find() functions.
2802          */
2803         spin_lock_irq(&cmd->sess->sess_list_lock);
2804         list_del(&cmd->search_cmd_list_entry);
2805         spin_unlock_irq(&cmd->sess->sess_list_lock);
2806
2807         cmd->done = 1;
2808         smp_mb(); /* to sync with scst_abort_cmd() */
2809
2810         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2811                 scst_xmit_process_aborted_cmd(cmd);
2812
2813         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2814                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu),"
2815                                 " skipping",
2816                                 cmd, (long long unsigned int)cmd->tag);
2817                 cmd->state = SCST_CMD_STATE_FINISHED;
2818                 res = SCST_CMD_STATE_RES_CONT_SAME;
2819                 goto out;
2820         }
2821
2822         if (cmd->tgt_data_buf_alloced && cmd->dh_data_buf_alloced &&
2823             (cmd->data_direction & SCST_DATA_READ))
2824                 scst_copy_sg(cmd, SCST_SG_COPY_TO_TARGET);
2825
2826         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2827         res = SCST_CMD_STATE_RES_CONT_SAME;
2828
2829 out:
2830 #ifdef CONFIG_SCST_MEASURE_LATENCY
2831         {
2832                 struct timespec ts;
2833                 uint64_t finish, scst_time, proc_time;
2834                 struct scst_session *sess = cmd->sess;
2835
2836                 getnstimeofday(&ts);
2837                 finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2838
2839                 spin_lock_bh(&sess->meas_lock);
2840
2841                 scst_time = cmd->pre_exec_finish - cmd->start;
2842                 scst_time += finish - cmd->post_exec_start;
2843                 proc_time = finish - cmd->start;
2844
2845                 sess->scst_time += scst_time;
2846                 sess->processing_time += proc_time;
2847                 sess->processed_cmds++;
2848
2849                 spin_unlock_bh(&sess->meas_lock);
2850
2851                 TRACE_DBG("cmd %p (sess %p): finish %lld (tv_sec %ld, "
2852                         "tv_nsec %ld), scst_time %lld, proc_time %lld",
2853                         cmd, sess, finish, ts.tv_sec, ts.tv_nsec, scst_time,
2854                         proc_time);
2855         }
2856 #endif
2857         TRACE_EXIT_HRES(res);
2858         return res;
2859 }
2860
2861 static int scst_xmit_response(struct scst_cmd *cmd)
2862 {
2863         int res, rc;
2864
2865         TRACE_ENTRY();
2866
2867         EXTRACHECKS_BUG_ON(cmd->internal);
2868
2869         if (unlikely(!cmd->tgtt->xmit_response_atomic &&
2870                      scst_cmd_atomic(cmd))) {
2871                 /*
2872                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2873                  * optimization.
2874                  */
2875                 TRACE_DBG("Target driver %s xmit_response() needs thread "
2876                               "context, rescheduling", cmd->tgtt->name);
2877                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2878                 goto out;
2879         }
2880
2881         while (1) {
2882                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2883
2884                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2885                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2886
2887                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2888
2889 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2890                 if (trace_flag & TRACE_SND_BOT) {
2891                         int i;
2892                         struct scatterlist *sg;
2893                         if (cmd->tgt_sg != NULL)
2894                                 sg = cmd->tgt_sg;
2895                         else
2896                                 sg = cmd->sg;
2897                         if (sg != NULL) {
2898                                 TRACE(TRACE_SND_BOT, "Xmitting data for cmd %p "
2899                                         "(sg_cnt %d, sg %p, sg[0].page %p)",
2900                                         cmd, cmd->tgt_sg_cnt, sg,
2901                                         (void *)sg_page(&sg[0]));
2902                                 for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
2903                                         PRINT_BUFF_FLAG(TRACE_SND_BOT,
2904                                                 "Xmitting sg", sg_virt(&sg[i]),
2905                                                 sg[i].length);
2906                                 }
2907                         }
2908                 }
2909 #endif
2910
2911 #ifdef CONFIG_SCST_DEBUG_RETRY
2912                 if (((scst_random() % 100) == 77))
2913                         rc = SCST_TGT_RES_QUEUE_FULL;
2914                 else
2915 #endif
2916                         rc = cmd->tgtt->xmit_response(cmd);
2917                 TRACE_DBG("xmit_response() returned %d", rc);
2918
2919                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2920                         goto out;
2921
2922                 /* Restore the previous state */
2923                 cmd->state = SCST_CMD_STATE_XMIT_RESP;
2924
2925                 switch (rc) {
2926                 case SCST_TGT_RES_QUEUE_FULL:
2927                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2928                                 break;
2929                         else
2930                                 continue;
2931
2932                 case SCST_TGT_RES_NEED_THREAD_CTX:
2933                         TRACE_DBG("Target driver %s xmit_response() "
2934                               "requested thread context, rescheduling",
2935                               cmd->tgtt->name);
2936                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2937                         break;
2938
2939                 default:
2940                         goto out_error;
2941                 }
2942                 break;
2943         }
2944
2945 out:
2946         /* Caution: cmd can be already dead here */
2947         TRACE_EXIT_HRES(res);
2948         return res;
2949
2950 out_error:
2951         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2952                 PRINT_ERROR("Target driver %s xmit_response() returned "
2953                         "fatal error", cmd->tgtt->name);
2954         } else {
2955                 PRINT_ERROR("Target driver %s xmit_response() returned "
2956                         "invalid value %d", cmd->tgtt->name, rc);
2957         }
2958         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2959         cmd->state = SCST_CMD_STATE_FINISHED;
2960         res = SCST_CMD_STATE_RES_CONT_SAME;
2961         goto out;
2962 }
2963
2964 void scst_tgt_cmd_done(struct scst_cmd *cmd,
2965         enum scst_exec_context pref_context)
2966 {
2967         TRACE_ENTRY();
2968
2969         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2970
2971         cmd->state = SCST_CMD_STATE_FINISHED;
2972         scst_proccess_redirect_cmd(cmd, pref_context, 1);
2973
2974         TRACE_EXIT();
2975         return;
2976 }
2977 EXPORT_SYMBOL(scst_tgt_cmd_done);
2978
2979 static int scst_finish_cmd(struct scst_cmd *cmd)
2980 {
2981         int res;
2982
2983         TRACE_ENTRY();
2984
2985         atomic_dec(&cmd->sess->sess_cmd_count);
2986
2987         cmd->finished = 1;
2988         smp_mb(); /* to sync with scst_abort_cmd() */
2989
2990         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2991                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
2992                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
2993                         atomic_read(&scst_cmd_count));
2994
2995                 scst_finish_cmd_mgmt(cmd);
2996         }
2997
2998         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
2999                 if ((cmd->tgt_dev != NULL) &&
3000                     scst_is_ua_sense(cmd->sense)) {
3001                         /* This UA delivery failed, so requeue it */
3002                         TRACE_MGMT_DBG("Requeuing UA for delivery failed cmd "
3003                                 "%p", cmd);
3004                         scst_check_set_UA(cmd->tgt_dev, cmd->sense,
3005                                         SCST_SENSE_BUFFERSIZE,
3006                                         SCST_SET_UA_FLAG_AT_HEAD);
3007                 }
3008         }
3009
3010         __scst_cmd_put(cmd);
3011
3012         res = SCST_CMD_STATE_RES_CONT_NEXT;
3013
3014         TRACE_EXIT_HRES(res);
3015         return res;
3016 }
3017
3018 /*
3019  * No locks, but it must be externally serialized (see comment for
3020  * scst_cmd_init_done() in scst.h)
3021  */
3022 static void scst_cmd_set_sn(struct scst_cmd *cmd)
3023 {
3024         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
3025         unsigned long flags;
3026
3027         TRACE_ENTRY();
3028
3029         if (scst_is_implicit_hq(cmd)) {
3030                 TRACE_SN("Implicit HQ cmd %p", cmd);
3031                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
3032         }
3033
3034         EXTRACHECKS_BUG_ON(cmd->sn_set || cmd->hq_cmd_inced);
3035
3036         /* Optimized for lockless fast path */
3037
3038         scst_check_debug_sn(cmd);
3039
3040         if (cmd->dev->queue_alg ==
3041                         SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) {
3042                 /*
3043                  * Not the best way, but well enough until there will be a
3044                  * possibility to specify queue type during pass-through
3045                  * commands submission.
3046                  */
3047                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
3048         }
3049
3050         switch (cmd->queue_type) {
3051         case SCST_CMD_QUEUE_SIMPLE:
3052         case SCST_CMD_QUEUE_UNTAGGED:
3053 #if 0 /* left for future performance investigations */
3054                 if (scst_cmd_is_expected_set(cmd)) {
3055                         if ((cmd->expected_data_direction == SCST_DATA_READ) &&
3056                             (atomic_read(&cmd->dev->write_cmd_count) == 0))
3057                                 goto ordered;
3058                 } else
3059                         goto ordered;
3060 #endif
3061                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
3062                         /*
3063                          * atomic_inc_return() implies memory barrier to sync
3064                          * with scst_inc_expected_sn()
3065                          */
3066                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
3067                                 tgt_dev->curr_sn++;
3068                                 TRACE_SN("Incremented curr_sn %ld",
3069                                         tgt_dev->curr_sn);
3070                         }
3071                         cmd->sn_slot = tgt_dev->cur_sn_slot;
3072                         cmd->sn = tgt_dev->curr_sn;
3073
3074                         tgt_dev->prev_cmd_ordered = 0;
3075                 } else {
3076                         TRACE(TRACE_MINOR, "***WARNING*** Not enough SN slots "
3077                                 "%zd", ARRAY_SIZE(tgt_dev->sn_slots));
3078                         goto ordered;
3079                 }
3080                 break;
3081
3082         case SCST_CMD_QUEUE_ORDERED:
3083                 TRACE_SN("ORDERED cmd %p (op %x)", cmd, cmd->cdb[0]);
3084 ordered:
3085                 if (!tgt_dev->prev_cmd_ordered) {
3086                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
3087                         if (tgt_dev->num_free_sn_slots >= 0) {
3088                                 tgt_dev->num_free_sn_slots--;
3089                                 if (tgt_dev->num_free_sn_slots >= 0) {
3090                                         int i = 0;
3091                                         /* Commands can finish in any order, so
3092                                          * we don't know which slot is empty.
3093                                          */
3094                                         while (1) {
3095                                                 tgt_dev->cur_sn_slot++;
3096                                                 if (tgt_dev->cur_sn_slot ==
3097                                                       tgt_dev->sn_slots + ARRAY_SIZE(tgt_dev->sn_slots))
3098                                                         tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
3099
3100                                                 if (atomic_read(tgt_dev->cur_sn_slot) == 0)
3101                                                         break;
3102
3103                                                 i++;
3104                                                 sBUG_ON(i == ARRAY_SIZE(tgt_dev->sn_slots));
3105                                         }
3106                                         TRACE_SN("New cur SN slot %zd",
3107                                                 tgt_dev->cur_sn_slot -
3108                                                 tgt_dev->sn_slots);
3109                                 }
3110                         }
3111                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
3112                 }
3113                 tgt_dev->prev_cmd_ordered = 1;
3114                 tgt_dev->curr_sn++;
3115                 cmd->sn = tgt_dev->curr_sn;
3116                 break;
3117
3118         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
3119                 TRACE_SN("HQ cmd %p (op %x)", cmd, cmd->cdb[0]);
3120                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
3121                 tgt_dev->hq_cmd_count++;
3122                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
3123                 cmd->hq_cmd_inced = 1;
3124                 goto out;
3125
3126         default:
3127                 sBUG();
3128         }
3129
3130         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
3131                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
3132                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
3133                 atomic_read(tgt_dev->cur_sn_slot),
3134                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
3135                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
3136
3137         cmd->sn_set = 1;
3138
3139 out:
3140         TRACE_EXIT();
3141         return;
3142 }
3143
3144 /*
3145  * Returns 0 on success, > 0 when we need to wait for unblock,
3146  * < 0 if there is no device (lun) or device type handler.
3147  *
3148  * No locks, but might be on IRQ, protection is done by the
3149  * suspended activity.
3150  */
3151 static int scst_translate_lun(struct scst_cmd *cmd)
3152 {
3153         struct scst_tgt_dev *tgt_dev = NULL;
3154         int res;
3155
3156         TRACE_ENTRY();
3157
3158         /* See comment about smp_mb() in scst_suspend_activity() */
3159         __scst_get(1);
3160
3161         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
3162                 struct list_head *sess_tgt_dev_list_head =
3163                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
3164                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %lld)", cmd,
3165                         (long long unsigned int)cmd->lun);
3166                 res = -1;
3167                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3168                                 sess_tgt_dev_list_entry) {
3169                         if (tgt_dev->lun == cmd->lun) {
3170                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
3171
3172                                 if (unlikely(tgt_dev->dev->handler ==
3173                                                 &scst_null_devtype)) {
3174                                         PRINT_INFO("Dev handler for device "
3175                                           "%lld is NULL, the device will not "
3176                                           "be visible remotely",
3177                                            (long long unsigned int)cmd->lun);
3178                                         break;
3179                                 }
3180
3181                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
3182                                 cmd->tgt_dev = tgt_dev;
3183                                 cmd->dev = tgt_dev->dev;
3184
3185                                 res = 0;
3186                                 break;
3187                         }
3188                 }
3189                 if (res != 0) {
3190                         TRACE(TRACE_MINOR,
3191                                 "tgt_dev for lun %lld not found, command to "
3192                                 "unexisting LU?",
3193                                 (long long unsigned int)cmd->lun);
3194                         __scst_put();
3195                 }
3196         } else {
3197                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3198                 __scst_put();
3199                 res = 1;
3200         }
3201
3202         TRACE_EXIT_RES(res);
3203         return res;
3204 }
3205
3206 /*
3207  * No locks, but might be on IRQ
3208  *
3209  * Returns 0 on success, > 0 when we need to wait for unblock,
3210  * < 0 if there is no device (lun) or device type handler.
3211  */
3212 static int __scst_init_cmd(struct scst_cmd *cmd)
3213 {
3214         int res = 0;
3215
3216         TRACE_ENTRY();
3217
3218         res = scst_translate_lun(cmd);
3219         if (likely(res == 0)) {
3220                 int cnt;
3221                 bool failure = false;
3222
3223                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
3224
3225                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
3226                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
3227                         TRACE(TRACE_MGMT_MINOR,
3228                                 "Too many pending commands (%d) in "
3229                                 "session, returning BUSY to initiator \"%s\"",
3230                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
3231                                   "Anonymous" : cmd->sess->initiator_name);
3232                         failure = true;
3233                 }
3234
3235                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
3236                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
3237                         if (!failure) {
3238                                 TRACE(TRACE_MGMT_MINOR,
3239                                         "Too many pending device "
3240                                         "commands (%d), returning BUSY to "
3241                                         "initiator \"%s\"", cnt,
3242                                         (cmd->sess->initiator_name[0] == '\0') ?
3243                                                 "Anonymous" :
3244                                                 cmd->sess->initiator_name);
3245                                 failure = true;
3246                         }
3247                 }
3248
3249                 /* If expected values not set, expected direction is UNKNOWN */
3250                 if (cmd->expected_data_direction & SCST_DATA_WRITE)
3251                         atomic_inc(&cmd->dev->write_cmd_count);
3252
3253                 if (unlikely(failure))
3254                         goto out_busy;
3255
3256                 if (!cmd->set_sn_on_restart_cmd)
3257                         scst_cmd_set_sn(cmd);
3258         } else if (res < 0) {
3259                 TRACE_DBG("Finishing cmd %p", cmd);
3260                 scst_set_cmd_error(cmd,
3261                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
3262                 scst_set_cmd_abnormal_done_state(cmd);
3263         } else
3264                 goto out;
3265
3266 out:
3267         TRACE_EXIT_RES(res);
3268         return res;
3269
3270 out_busy:
3271         scst_set_busy(cmd);
3272         scst_set_cmd_abnormal_done_state(cmd);
3273         goto out;
3274 }
3275
3276 /* Called under scst_init_lock and IRQs disabled */
3277 static void scst_do_job_init(void)
3278         __releases(&scst_init_lock)
3279         __acquires(&scst_init_lock)
3280 {
3281         struct scst_cmd *cmd;
3282         int susp;
3283
3284         TRACE_ENTRY();
3285
3286 restart:
3287         /*
3288          * There is no need for read barrier here, because we don't care where
3289          * this check will be done.
3290          */
3291         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
3292         if (scst_init_poll_cnt > 0)
3293                 scst_init_poll_cnt--;
3294
3295         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
3296                 int rc;
3297                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
3298                         continue;
3299                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
3300                         spin_unlock_irq(&scst_init_lock);
3301                         rc = __scst_init_cmd(cmd);
3302                         spin_lock_irq(&scst_init_lock);
3303                         if (rc > 0) {
3304                                 TRACE_MGMT_DBG("%s",
3305                                         "FLAG SUSPENDED set, restarting");
3306                                 goto restart;
3307                         }
3308                 } else {
3309                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
3310                                        cmd, (long long unsigned int)cmd->tag);
3311                         scst_set_cmd_abnormal_done_state(cmd);
3312                 }
3313
3314                 /*
3315                  * Deleting cmd from init cmd list after __scst_init_cmd()
3316                  * is necessary to keep the check in scst_init_cmd() correct
3317                  * to preserve the commands order.
3318                  *
3319                  * We don't care about the race, when init cmd list is empty
3320                  * and one command detected that it just was not empty, so
3321                  * it's inserting to it, but another command at the same time
3322                  * seeing init cmd list empty and goes directly, because it
3323                  * could affect only commands from the same initiator to the
3324                  * same tgt_dev, but scst_cmd_init_done*() doesn't guarantee
3325                  * the order in case of simultaneous such calls anyway.
3326                  */
3327                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
3328                 smp_wmb(); /* enforce the required order */
3329                 list_del(&cmd->cmd_list_entry);
3330                 spin_unlock(&scst_init_lock);
3331
3332                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
3333                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
3334                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
3335                         list_add(&cmd->cmd_list_entry,
3336                                 &cmd->cmd_lists->active_cmd_list);
3337                 else
3338                         list_add_tail(&cmd->cmd_list_entry,
3339                                 &cmd->cmd_lists->active_cmd_list);
3340                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3341                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
3342
3343                 spin_lock(&scst_init_lock);
3344                 goto restart;
3345         }
3346
3347         /* It isn't really needed, but let's keep it */
3348         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
3349                 goto restart;
3350
3351         TRACE_EXIT();
3352         return;
3353 }
3354
3355 static inline int test_init_cmd_list(void)
3356 {
3357         int res = (!list_empty(&scst_init_cmd_list) &&
3358                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
3359                   unlikely(kthread_should_stop()) ||
3360                   (scst_init_poll_cnt > 0);
3361         return res;
3362 }
3363
3364 int scst_init_thread(void *arg)
3365 {
3366         TRACE_ENTRY();
3367
3368         PRINT_INFO("Init thread started, PID %d", current->pid);
3369
3370         current->flags |= PF_NOFREEZE;
3371
3372         set_user_nice(current, -10);
3373
3374         spin_lock_irq(&scst_init_lock);
3375         while (!kthread_should_stop()) {
3376                 wait_queue_t wait;
3377                 init_waitqueue_entry(&wait, current);
3378
3379                 if (!test_init_cmd_list()) {
3380                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
3381                                                  &wait);
3382                         for (;;) {
3383                                 set_current_state(TASK_INTERRUPTIBLE);
3384                                 if (test_init_cmd_list())
3385                                         break;
3386                                 spin_unlock_irq(&scst_init_lock);
3387                                 schedule();
3388                                 spin_lock_irq(&scst_init_lock);
3389                         }
3390                         set_current_state(TASK_RUNNING);
3391                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
3392                 }
3393                 scst_do_job_init();
3394         }
3395         spin_unlock_irq(&scst_init_lock);
3396
3397         /*
3398          * If kthread_should_stop() is true, we are guaranteed to be
3399          * on the module unload, so scst_init_cmd_list must be empty.
3400          */
3401         sBUG_ON(!list_empty(&scst_init_cmd_list));
3402
3403         PRINT_INFO("Init thread PID %d finished", current->pid);
3404
3405         TRACE_EXIT();
3406         return 0;
3407 }
3408
3409 /* Called with no locks held */
3410 void scst_process_active_cmd(struct scst_cmd *cmd, bool atomic)
3411 {
3412         int res;
3413
3414         TRACE_ENTRY();
3415
3416         EXTRACHECKS_BUG_ON(in_irq() || irqs_disabled());
3417
3418         cmd->atomic = atomic;
3419
3420         TRACE_DBG("cmd %p, atomic %d", cmd, atomic);
3421
3422         do {
3423                 switch (cmd->state) {
3424                 case SCST_CMD_STATE_PRE_PARSE:
3425                         res = scst_pre_parse(cmd);
3426                         EXTRACHECKS_BUG_ON(res ==
3427                                 SCST_CMD_STATE_RES_NEED_THREAD);
3428                         break;
3429
3430                 case SCST_CMD_STATE_DEV_PARSE:
3431                         res = scst_parse_cmd(cmd);
3432                         break;
3433
3434                 case SCST_CMD_STATE_PREPARE_SPACE:
3435                         res = scst_prepare_space(cmd);
3436                         break;
3437
3438                 case SCST_CMD_STATE_RDY_TO_XFER:
3439                         res = scst_rdy_to_xfer(cmd);
3440                         break;
3441
3442                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3443                         res = scst_tgt_pre_exec(cmd);
3444                         break;
3445
3446                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3447                         if (tm_dbg_check_cmd(cmd) != 0) {
3448                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
3449                                 TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
3450                                         "because of TM DBG delay", cmd,
3451                                         (long long unsigned int)cmd->tag);
3452                                 break;
3453                         }
3454                         res = scst_send_for_exec(&cmd);
3455                         /*
3456                          * !! At this point cmd, sess & tgt_dev can already be
3457                          * freed !!
3458                          */
3459                         break;
3460
3461                 case SCST_CMD_STATE_LOCAL_EXEC:
3462                         res = scst_local_exec(cmd);
3463                         /*
3464                          * !! At this point cmd, sess & tgt_dev can already be
3465                          * freed !!
3466                          */
3467                         break;
3468
3469                 case SCST_CMD_STATE_REAL_EXEC:
3470                         res = scst_real_exec(cmd);
3471                         /*
3472                          * !! At this point cmd, sess & tgt_dev can already be
3473                          * freed !!
3474               &