scst/src/scst_targ.c

   1 /*
   2  *  scst_targ.c
   3  *
   4  *  Copyright (C) 2004 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
   5  *  Copyright (C) 2004 - 2005 Leonid Stoljar
   6  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
   7  *
   8  *  This program is free software; you can redistribute it and/or
   9  *  modify it under the terms of the GNU General Public License
  10  *  as published by the Free Software Foundation, version 2
  11  *  of the License.
  12  *
  13  *  This program is distributed in the hope that it will be useful,
  14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  *  GNU General Public License for more details.
  17  */
  18
  19 #include <linux/init.h>
  20 #include <linux/kernel.h>
  21 #include <linux/errno.h>
  22 #include <linux/list.h>
  23 #include <linux/spinlock.h>
  24 #include <linux/slab.h>
  25 #include <linux/sched.h>
  26 #include <linux/smp_lock.h>
  27 #include <linux/unistd.h>
  28 #include <linux/string.h>
  29 #include <linux/kthread.h>
  30 #include <linux/delay.h>
  31
  32 #include "scst.h"
  33 #include "scst_priv.h"
  34
  35 static void scst_cmd_set_sn(struct scst_cmd *cmd);
  36 static int __scst_init_cmd(struct scst_cmd *cmd);
  37 static void scst_finish_cmd_mgmt(struct scst_cmd *cmd);
  38
  39 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
  40 {
  41         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
  42         unsigned long flags;
  43
  44         spin_lock_irqsave(&t->tasklet_lock, flags);
  45         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
  46                 smp_processor_id());
  47         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
  48         spin_unlock_irqrestore(&t->tasklet_lock, flags);
  49
  50         tasklet_schedule(&t->tasklet);
  51 }
  52
  53 /*
  54  * Must not be called in parallel with scst_unregister_session_ex() for the
  55  * same sess
  56  */
  57 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
  58                              const uint8_t *lun, int lun_len,
  59                              const uint8_t *cdb, int cdb_len, int atomic)
  60 {
  61         struct scst_cmd *cmd;
  62
  63         TRACE_ENTRY();
  64
  65 #ifdef CONFIG_SCST_EXTRACHECKS
  66         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
  67                 PRINT_CRIT_ERROR("%s",
  68                         "New cmd while shutting down the session");
  69                 sBUG();
  70         }
  71 #endif
  72
  73         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
  74         if (cmd == NULL)
  75                 goto out;
  76
  77         cmd->sess = sess;
  78         cmd->tgt = sess->tgt;
  79         cmd->tgtt = sess->tgt->tgtt;
  80
  81         /*
  82          * For both wrong lun and CDB defer the error reporting for
  83          * scst_cmd_init_done()
  84          */
  85
  86         cmd->lun = scst_unpack_lun(lun, lun_len);
  87
  88         if (cdb_len <= SCST_MAX_CDB_SIZE) {
  89                 memcpy(cmd->cdb, cdb, cdb_len);
  90                 cmd->cdb_len = cdb_len;
  91         }
  92
  93         TRACE_DBG("cmd %p, sess %p", cmd, sess);
  94         scst_sess_get(sess);
  95
  96 out:
  97         TRACE_EXIT();
  98         return cmd;
  99 }
 100 EXPORT_SYMBOL(scst_rx_cmd);
 101
 102 /*
 103  * No locks, but might be on IRQ. Returns 0 on success, <0 if processing of
 104  * this command should be stopped.
 105  */
 106 static int scst_init_cmd(struct scst_cmd *cmd, enum scst_exec_context *context)
 107 {
 108         int rc, res = 0;
 109
 110         TRACE_ENTRY();
 111
 112         /* See the comment in scst_do_job_init() */
 113         if (unlikely(!list_empty(&scst_init_cmd_list))) {
 114                 TRACE_MGMT_DBG("%s", "init cmd list busy");
 115                 goto out_redirect;
 116         }
 117         /*
 118          * Memory barrier isn't necessary here, because CPU appears to
 119          * be self-consistent
 120          */
 121
 122         rc = __scst_init_cmd(cmd);
 123         if (unlikely(rc > 0))
 124                 goto out_redirect;
 125         else if (unlikely(rc != 0))
 126                 goto out;
 127
 128         /* Small context optimization */
 129         if (((*context == SCST_CONTEXT_TASKLET) ||
 130              (*context == SCST_CONTEXT_DIRECT_ATOMIC) ||
 131              ((*context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd))) &&
 132               scst_cmd_is_expected_set(cmd)) {
 133                 if (cmd->expected_data_direction == SCST_DATA_WRITE) {
 134                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
 135                                         &cmd->tgt_dev->tgt_dev_flags))
 136                                 *context = SCST_CONTEXT_THREAD;
 137                 } else {
 138                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
 139                                         &cmd->tgt_dev->tgt_dev_flags))
 140                                 *context = SCST_CONTEXT_THREAD;
 141                 }
 142         }
 143
 144 out:
 145         TRACE_EXIT_RES(res);
 146         return res;
 147
 148 out_redirect:
 149         if (cmd->preprocessing_only) {
 150                 /*
 151                  * Poor man solution for single threaded targets, where
 152                  * blocking receiver at least sometimes means blocking all.
 153                  */
 154                 sBUG_ON(*context != SCST_CONTEXT_DIRECT);
 155                 scst_set_busy(cmd);
 156                 scst_set_cmd_abnormal_done_state(cmd);
 157                 /* Keep initiator away from too many BUSY commands */
 158                 msleep(50);
 159         } else {
 160                 unsigned long flags;
 161                 spin_lock_irqsave(&scst_init_lock, flags);
 162                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
 163                         "%d)", cmd, atomic_read(&scst_cmd_count));
 164                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
 165                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
 166                         scst_init_poll_cnt++;
 167                 spin_unlock_irqrestore(&scst_init_lock, flags);
 168                 wake_up(&scst_init_cmd_list_waitQ);
 169                 res = -1;
 170         }
 171         goto out;
 172 }
 173
 174 #ifdef CONFIG_SCST_MEASURE_LATENCY
 175 static inline uint64_t scst_sec_to_nsec(time_t sec)
 176 {
 177         return (uint64_t)sec * 1000000000;
 178 }
 179 #endif
 180
 181 void scst_cmd_init_done(struct scst_cmd *cmd,
 182         enum scst_exec_context pref_context)
 183 {
 184         unsigned long flags;
 185         struct scst_session *sess = cmd->sess;
 186         int rc;
 187
 188         TRACE_ENTRY();
 189
 190 #ifdef CONFIG_SCST_MEASURE_LATENCY
 191         {
 192                 struct timespec ts;
 193                 getnstimeofday(&ts);
 194                 cmd->start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
 195                 TRACE_DBG("cmd %p (sess %p): start %lld (tv_sec %ld, "
 196                         "tv_nsec %ld)", cmd, sess, cmd->start, ts.tv_sec,
 197                         ts.tv_nsec);
 198         }
 199 #endif
 200
 201         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
 202         TRACE(TRACE_SCSI, "tag=%llu, lun=%lld, CDB len=%d, queue_type=%x "
 203                 "(cmd %p)", (long long unsigned int)cmd->tag,
 204                 (long long unsigned int)cmd->lun, cmd->cdb_len,
 205                 cmd->queue_type, cmd);
 206         PRINT_BUFF_FLAG(TRACE_SCSI|TRACE_RCV_BOT, "Recieving CDB",
 207                 cmd->cdb, cmd->cdb_len);
 208
 209 #ifdef CONFIG_SCST_EXTRACHECKS
 210         if (unlikely((in_irq() || irqs_disabled())) &&
 211             ((pref_context == SCST_CONTEXT_DIRECT) ||
 212              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
 213                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
 214                         "SCST_CONTEXT_THREAD instead\n", pref_context,
 215                         cmd->tgtt->name);
 216                 pref_context = SCST_CONTEXT_THREAD;
 217         }
 218 #endif
 219
 220         atomic_inc(&sess->sess_cmd_count);
 221
 222         spin_lock_irqsave(&sess->sess_list_lock, flags);
 223
 224         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
 225                 /*
 226                  * We have to always keep command in the search list from the
 227                  * very beginning, because otherwise it can be missed during
 228                  * TM processing. This check is needed because there might be
 229                  * old, i.e. deferred, commands and new, i.e. just coming, ones.
 230                  */
 231                 if (cmd->search_cmd_list_entry.next == NULL)
 232                         list_add_tail(&cmd->search_cmd_list_entry,
 233                                 &sess->search_cmd_list);
 234                 switch (sess->init_phase) {
 235                 case SCST_SESS_IPH_SUCCESS:
 236                         break;
 237                 case SCST_SESS_IPH_INITING:
 238                         TRACE_DBG("Adding cmd %p to init deferred cmd list",
 239                                   cmd);
 240                         list_add_tail(&cmd->cmd_list_entry,
 241                                 &sess->init_deferred_cmd_list);
 242                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
 243                         goto out;
 244                 case SCST_SESS_IPH_FAILED:
 245                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
 246                         scst_set_busy(cmd);
 247                         scst_set_cmd_abnormal_done_state(cmd);
 248                         goto active;
 249                 default:
 250                         sBUG();
 251                 }
 252         } else
 253                 list_add_tail(&cmd->search_cmd_list_entry,
 254                               &sess->search_cmd_list);
 255
 256         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
 257
 258         if (unlikely(cmd->lun == NO_SUCH_LUN)) {
 259                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
 260                 scst_set_cmd_error(cmd,
 261                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
 262                 scst_set_cmd_abnormal_done_state(cmd);
 263                 goto active;
 264         }
 265
 266         if (unlikely(cmd->cdb_len == 0)) {
 267                 PRINT_ERROR("Wrong CDB len %d, finishing cmd", 0);
 268                 scst_set_cmd_error(cmd,
 269                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
 270                 scst_set_cmd_abnormal_done_state(cmd);
 271                 goto active;
 272         }
 273
 274         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
 275                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
 276                 scst_set_cmd_error(cmd,
 277                         SCST_LOAD_SENSE(scst_sense_invalid_message));
 278                 scst_set_cmd_abnormal_done_state(cmd);
 279                 goto active;
 280         }
 281
 282         cmd->state = SCST_CMD_STATE_INIT;
 283         /* cmd must be inited here to preserve the order */
 284         rc = scst_init_cmd(cmd, &pref_context);
 285         if (unlikely(rc < 0))
 286                 goto out;
 287
 288 active:
 289         /* Here cmd must not be in any cmd list, no locks */
 290         switch (pref_context) {
 291         case SCST_CONTEXT_TASKLET:
 292                 scst_schedule_tasklet(cmd);
 293                 break;
 294
 295         case SCST_CONTEXT_DIRECT:
 296                 scst_process_active_cmd(cmd, false);
 297                 /* For *NEED_THREAD wake_up() is already done */
 298                 break;
 299
 300         case SCST_CONTEXT_DIRECT_ATOMIC:
 301                 scst_process_active_cmd(cmd, true);
 302                 /* For *NEED_THREAD wake_up() is already done */
 303                 break;
 304
 305         default:
 306                 PRINT_ERROR("Context %x is undefined, using the thread one",
 307                         pref_context);
 308                 /* go through */
 309         case SCST_CONTEXT_THREAD:
 310                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
 311                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
 312                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
 313                         list_add(&cmd->cmd_list_entry,
 314                                 &cmd->cmd_lists->active_cmd_list);
 315                 else
 316                         list_add_tail(&cmd->cmd_list_entry,
 317                                 &cmd->cmd_lists->active_cmd_list);
 318                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
 319                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
 320                 break;
 321         }
 322
 323 out:
 324         TRACE_EXIT();
 325         return;
 326 }
 327 EXPORT_SYMBOL(scst_cmd_init_done);
 328
 329 static int scst_pre_parse(struct scst_cmd *cmd)
 330 {
 331         int res = SCST_CMD_STATE_RES_CONT_SAME;
 332         struct scst_device *dev = cmd->dev;
 333         int rc;
 334
 335         TRACE_ENTRY();
 336
 337         cmd->inc_expected_sn_on_done = dev->handler->exec_sync ||
 338              (!dev->has_own_order_mgmt &&
 339               (dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER ||
 340                cmd->queue_type == SCST_CMD_QUEUE_ORDERED));
 341
 342         /*
 343          * Expected transfer data supplied by the SCSI transport via the
 344          * target driver are untrusted, so we prefer to fetch them from CDB.
 345          * Additionally, not all transports support supplying the expected
 346          * transfer data.
 347          */
 348
 349         rc = scst_get_cdb_info(cmd);
 350         if (unlikely(rc != 0)) {
 351                 if (rc > 0) {
 352                         PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 353                         goto out_xmit;
 354                 }
 355                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
 356                         "Should you update scst_scsi_op_table?",
 357                         cmd->cdb[0], dev->handler->name);
 358                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 359 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
 360                 if (scst_cmd_is_expected_set(cmd)) {
 361                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
 362                                 "direction %d, transfer_len %d",
 363                                 cmd->expected_data_direction,
 364                                 cmd->expected_transfer_len);
 365                         cmd->data_direction = cmd->expected_data_direction;
 366
 367                         cmd->bufflen = cmd->expected_transfer_len;
 368                         /* Restore (likely) lost CDB length */
 369                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
 370                         if (cmd->cdb_len == -1) {
 371                                 PRINT_ERROR("Unable to get CDB length for "
 372                                         "opcode 0x%02x. Returning INVALID "
 373                                         "OPCODE", cmd->cdb[0]);
 374                                 scst_set_cmd_error(cmd,
 375                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
 376                                 goto out_xmit;
 377                         }
 378                 } else {
 379                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
 380                              "target %s not supplied expected values",
 381                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
 382                         scst_set_cmd_error(cmd,
 383                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
 384                         goto out_xmit;
 385                 }
 386 #else
 387                 scst_set_cmd_error(cmd,
 388                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
 389                 goto out_xmit;
 390 #endif
 391         } else {
 392                 TRACE(TRACE_SCSI, "op_name <%s> (cmd %p), direction=%d "
 393                         "(expected %d, set %s), transfer_len=%d (expected "
 394                         "len %d), flags=%d", cmd->op_name, cmd,
 395                         cmd->data_direction, cmd->expected_data_direction,
 396                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
 397                         cmd->bufflen, cmd->expected_transfer_len,
 398                         cmd->op_flags);
 399
 400                 if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
 401                         if (scst_cmd_is_expected_set(cmd)) {
 402                                 /*
 403                                  * Command data length can't be easily
 404                                  * determined from the CDB. ToDo, all such
 405                                  * commands processing should be fixed. Until
 406                                  * it's done, get the length from the supplied
 407                                  * expected value, but limit it to some
 408                                  * reasonable value (15MB).
 409                                  */
 410                                 cmd->bufflen = min(cmd->expected_transfer_len,
 411                                                         15*1024*1024);
 412                                 cmd->op_flags &= ~SCST_UNKNOWN_LENGTH;
 413                         } else
 414                                 cmd->bufflen = 0;
 415                 }
 416         }
 417
 418         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
 419                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
 420                             "(opcode 0x%02x)", cmd->cdb[0]);
 421                 scst_set_cmd_error(cmd,
 422                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
 423                 goto out_xmit;
 424         }
 425
 426         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
 427                 PRINT_ERROR("Linked commands are not supported "
 428                             "(opcode 0x%02x)", cmd->cdb[0]);
 429                 scst_set_cmd_error(cmd,
 430                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
 431                 goto out_xmit;
 432         }
 433
 434         cmd->state = SCST_CMD_STATE_DEV_PARSE;
 435
 436 out:
 437         TRACE_EXIT_RES(res);
 438         return res;
 439
 440 out_xmit:
 441         scst_set_cmd_abnormal_done_state(cmd);
 442         res = SCST_CMD_STATE_RES_CONT_SAME;
 443         goto out;
 444 }
 445
 446 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
 447 static bool scst_is_allowed_to_mismatch_cmd(struct scst_cmd *cmd)
 448 {
 449         bool res = false;
 450
 451         switch (cmd->cdb[0]) {
 452         case TEST_UNIT_READY:
 453                 /* Crazy VMware people sometimes do TUR with READ direction */
 454                 res = true;
 455                 break;
 456         case VERIFY:
 457         case VERIFY_6:
 458         case VERIFY_12:
 459         case VERIFY_16:
 460                 /* VERIFY commands with BYTCHK unset shouldn't fail here */
 461                 if ((cmd->op_flags & SCST_VERIFY_BYTCHK_MISMATCH_ALLOWED) &&
 462                     (cmd->cdb[1] & BYTCHK) == 0)
 463                         res = true;
 464                 break;
 465         }
 466
 467         return res;
 468 }
 469 #endif
 470
 471 static int scst_parse_cmd(struct scst_cmd *cmd)
 472 {
 473         int res = SCST_CMD_STATE_RES_CONT_SAME;
 474         int state;
 475         struct scst_device *dev = cmd->dev;
 476         int orig_bufflen = cmd->bufflen;
 477
 478         TRACE_ENTRY();
 479
 480         if (likely(!scst_is_cmd_local(cmd))) {
 481                 if (unlikely(!dev->handler->parse_atomic &&
 482                              scst_cmd_atomic(cmd))) {
 483                         /*
 484                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
 485                          * optimization.
 486                          */
 487                         TRACE_DBG("Dev handler %s parse() needs thread "
 488                                 "context, rescheduling", dev->handler->name);
 489                         res = SCST_CMD_STATE_RES_NEED_THREAD;
 490                         goto out;
 491                 }
 492
 493                 TRACE_DBG("Calling dev handler %s parse(%p)",
 494                       dev->handler->name, cmd);
 495                 TRACE_BUFF_FLAG(TRACE_SND_BOT, "Parsing: ",
 496                                 cmd->cdb, cmd->cdb_len);
 497                 state = dev->handler->parse(cmd);
 498                 /* Caution: cmd can be already dead here */
 499                 TRACE_DBG("Dev handler %s parse() returned %d",
 500                         dev->handler->name, state);
 501
 502                 switch (state) {
 503                 case SCST_CMD_STATE_NEED_THREAD_CTX:
 504                         TRACE_DBG("Dev handler %s parse() requested thread "
 505                               "context, rescheduling", dev->handler->name);
 506                         res = SCST_CMD_STATE_RES_NEED_THREAD;
 507                         goto out;
 508
 509                 case SCST_CMD_STATE_STOP:
 510                         TRACE_DBG("Dev handler %s parse() requested stop "
 511                                 "processing", dev->handler->name);
 512                         res = SCST_CMD_STATE_RES_CONT_NEXT;
 513                         goto out;
 514                 }
 515
 516                 if (state == SCST_CMD_STATE_DEFAULT)
 517                         state = SCST_CMD_STATE_PREPARE_SPACE;
 518         } else
 519                 state = SCST_CMD_STATE_PREPARE_SPACE;
 520
 521         if (cmd->data_len == -1)
 522                 cmd->data_len = cmd->bufflen;
 523
 524         if (cmd->dh_data_buf_alloced &&
 525             unlikely((orig_bufflen > cmd->bufflen))) {
 526                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
 527                         "is less, than required (size %d)", cmd->bufflen,
 528                         orig_bufflen);
 529                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 530                 goto out_error;
 531         }
 532
 533         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
 534                 goto set_res;
 535
 536         if (unlikely((cmd->bufflen == 0) &&
 537                      (cmd->op_flags & SCST_UNKNOWN_LENGTH))) {
 538                 PRINT_ERROR("Unknown data transfer length for opcode 0x%x "
 539                         "(handler %s, target %s)", cmd->cdb[0],
 540                         dev->handler->name, cmd->tgtt->name);
 541                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 542                 goto out_error;
 543         }
 544
 545 #ifdef CONFIG_SCST_EXTRACHECKS
 546         if ((cmd->bufflen != 0) &&
 547             ((cmd->data_direction == SCST_DATA_NONE) ||
 548              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
 549                 PRINT_ERROR("Dev handler %s parse() returned "
 550                         "invalid cmd data_direction %d, bufflen %d, state %d "
 551                         "or sg %p (opcode 0x%x)", dev->handler->name,
 552                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
 553                         cmd->cdb[0]);
 554                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 555                 goto out_error;
 556         }
 557 #endif
 558
 559         if (scst_cmd_is_expected_set(cmd)) {
 560 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
 561 #       ifdef CONFIG_SCST_EXTRACHECKS
 562                 if ((cmd->data_direction != cmd->expected_data_direction) ||
 563                     (cmd->bufflen != cmd->expected_transfer_len)) {
 564                         PRINT_WARNING("Expected values don't match decoded "
 565                                 "ones: data_direction %d, "
 566                                 "expected_data_direction %d, "
 567                                 "bufflen %d, expected_transfer_len %d",
 568                                 cmd->data_direction,
 569                                 cmd->expected_data_direction,
 570                                 cmd->bufflen, cmd->expected_transfer_len);
 571                         PRINT_BUFFER("Suspicious CDB", cmd->cdb, cmd->cdb_len);
 572                 }
 573 #       endif
 574                 cmd->data_direction = cmd->expected_data_direction;
 575                 cmd->bufflen = cmd->expected_transfer_len;
 576 #else
 577                 if (unlikely(cmd->data_direction !=
 578                                 cmd->expected_data_direction)) {
 579                         if (((cmd->expected_data_direction != SCST_DATA_NONE) ||
 580                              (cmd->bufflen != 0)) &&
 581                             !scst_is_allowed_to_mismatch_cmd(cmd)) {
 582                                 PRINT_ERROR("Expected data direction %d for "
 583                                         "opcode 0x%02x (handler %s, target %s) "
 584                                         "doesn't match "
 585                                         "decoded value %d",
 586                                         cmd->expected_data_direction,
 587                                         cmd->cdb[0], dev->handler->name,
 588                                         cmd->tgtt->name, cmd->data_direction);
 589                                 PRINT_BUFFER("Failed CDB",
 590                                         cmd->cdb, cmd->cdb_len);
 591                                 scst_set_cmd_error(cmd,
 592                                    SCST_LOAD_SENSE(scst_sense_invalid_message));
 593                                 goto out_dev_done;
 594                         }
 595                 }
 596                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
 597                         TRACE(TRACE_MGMT_MINOR, "Warning: expected "
 598                                 "transfer length %d for opcode 0x%02x "
 599                                 "(handler %s, target %s) doesn't match "
 600                                 "decoded value %d. Faulty initiator "
 601                                 "(e.g. VMware is known to be such) or "
 602                                 "scst_scsi_op_table should be updated?",
 603                                 cmd->expected_transfer_len, cmd->cdb[0],
 604                                 dev->handler->name, cmd->tgtt->name,
 605                                 cmd->bufflen);
 606                         PRINT_BUFF_FLAG(TRACE_MGMT_MINOR, "Suspicious CDB",
 607                                 cmd->cdb, cmd->cdb_len);
 608                 }
 609 #endif
 610         }
 611
 612         if (unlikely(cmd->data_direction == SCST_DATA_UNKNOWN)) {
 613                 PRINT_ERROR("Unknown data direction. Opcode 0x%x, handler %s, "
 614                         "target %s", cmd->cdb[0], dev->handler->name,
 615                         cmd->tgtt->name);
 616                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
 617                 goto out_error;
 618         }
 619
 620 set_res:
 621         switch (state) {
 622         case SCST_CMD_STATE_PREPARE_SPACE:
 623         case SCST_CMD_STATE_PRE_PARSE:
 624         case SCST_CMD_STATE_DEV_PARSE:
 625         case SCST_CMD_STATE_RDY_TO_XFER:
 626         case SCST_CMD_STATE_TGT_PRE_EXEC:
 627         case SCST_CMD_STATE_SEND_FOR_EXEC:
 628         case SCST_CMD_STATE_LOCAL_EXEC:
 629         case SCST_CMD_STATE_REAL_EXEC:
 630         case SCST_CMD_STATE_PRE_DEV_DONE:
 631         case SCST_CMD_STATE_DEV_DONE:
 632         case SCST_CMD_STATE_PRE_XMIT_RESP:
 633         case SCST_CMD_STATE_XMIT_RESP:
 634         case SCST_CMD_STATE_FINISHED:
 635                 cmd->state = state;
 636                 res = SCST_CMD_STATE_RES_CONT_SAME;
 637                 break;
 638
 639         default:
 640                 if (state >= 0) {
 641                         PRINT_ERROR("Dev handler %s parse() returned "
 642                              "invalid cmd state %d (opcode %d)",
 643                              dev->handler->name, state, cmd->cdb[0]);
 644                 } else {
 645                         PRINT_ERROR("Dev handler %s parse() returned "
 646                                 "error %d (opcode %d)", dev->handler->name,
 647                                 state, cmd->cdb[0]);
 648                 }
 649                 goto out_error;
 650         }
 651
 652         if (cmd->resp_data_len == -1) {
 653                 if (cmd->data_direction == SCST_DATA_READ)
 654                         cmd->resp_data_len = cmd->bufflen;
 655                 else
 656                          cmd->resp_data_len = 0;
 657         }
 658
 659 out:
 660         TRACE_EXIT_HRES(res);
 661         return res;
 662
 663 out_error:
 664         /* dev_done() will be called as part of the regular cmd's finish */
 665         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
 666
 667 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
 668 out_dev_done:
 669 #endif
 670         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
 671         res = SCST_CMD_STATE_RES_CONT_SAME;
 672         goto out;
 673 }
 674
 675 static int scst_prepare_space(struct scst_cmd *cmd)
 676 {
 677         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
 678
 679         TRACE_ENTRY();
 680
 681         if (cmd->data_direction == SCST_DATA_NONE)
 682                 goto prep_done;
 683
 684         if (cmd->tgt_need_alloc_data_buf) {
 685                 int orig_bufflen = cmd->bufflen;
 686
 687                 TRACE_MEM("Custom tgt data buf allocation requested (cmd %p)",
 688                         cmd);
 689
 690                 r = cmd->tgtt->alloc_data_buf(cmd);
 691                 if (r > 0)
 692                         goto alloc;
 693                 else if (r == 0) {
 694                         if (unlikely(cmd->bufflen == 0)) {
 695                                 /* See comment in scst_alloc_space() */
 696                                 if (cmd->sg == NULL)
 697                                         goto alloc;
 698                         }
 699
 700                         cmd->tgt_data_buf_alloced = 1;
 701
 702                         if (unlikely(orig_bufflen < cmd->bufflen)) {
 703                                 PRINT_ERROR("Target driver allocated data "
 704                                         "buffer (size %d), is less, than "
 705                                         "required (size %d)", orig_bufflen,
 706                                         cmd->bufflen);
 707                                 goto out_error;
 708                         }
 709                         TRACE_MEM("tgt_data_buf_alloced (cmd %p)", cmd);
 710                 } else
 711                         goto check;
 712         }
 713
 714 alloc:
 715         if (!cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
 716                 r = scst_alloc_space(cmd);
 717                 cmd->tgt_sg = cmd->sg;
 718                 cmd->tgt_sg_cnt = cmd->sg_cnt;
 719         } else if (cmd->dh_data_buf_alloced && !cmd->tgt_data_buf_alloced) {
 720                 TRACE_MEM("dh_data_buf_alloced set (cmd %p)", cmd);
 721                 cmd->tgt_sg = cmd->sg;
 722                 cmd->tgt_sg_cnt = cmd->sg_cnt;
 723                 r = 0;
 724         } else if (cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
 725                 TRACE_MEM("tgt_data_buf_alloced set (cmd %p)", cmd);
 726                 cmd->sg = cmd->tgt_sg;
 727                 cmd->sg_cnt = cmd->tgt_sg_cnt;
 728                 r = 0;
 729         } else {
 730                 TRACE_MEM("Both *_data_buf_alloced set (cmd %p, sg %p, "
 731                         "sg_cnt %d, tgt_sg %p, tgt_sg_cnt %d)", cmd, cmd->sg,
 732                         cmd->sg_cnt, cmd->tgt_sg, cmd->tgt_sg_cnt);
 733                 r = 0;
 734         }
 735
 736 check:
 737         if (r != 0) {
 738                 if (scst_cmd_atomic(cmd)) {
 739                         TRACE_MEM("%s", "Atomic memory allocation failed, "
 740                               "rescheduling to the thread");
 741                         res = SCST_CMD_STATE_RES_NEED_THREAD;
 742                         goto out;
 743                 } else
 744                         goto out_no_space;
 745         }
 746
 747 prep_done:
 748         if (cmd->preprocessing_only) {
 749                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
 750                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
 751                                 "cmd %p", cmd);
 752                         scst_set_cmd_abnormal_done_state(cmd);
 753                         res = SCST_CMD_STATE_RES_CONT_SAME;
 754                         goto out;
 755                 }
 756
 757                 res = SCST_CMD_STATE_RES_CONT_NEXT;
 758                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
 759
 760                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
 761                 cmd->tgtt->preprocessing_done(cmd);
 762                 TRACE_DBG("%s", "preprocessing_done() returned");
 763                 goto out;
 764
 765         }
 766
 767         switch (cmd->data_direction) {
 768         case SCST_DATA_WRITE:
 769                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
 770                 break;
 771
 772         default:
 773                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
 774                 break;
 775         }
 776
 777 out:
 778         TRACE_EXIT_HRES(res);
 779         return res;
 780
 781 out_no_space:
 782         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
 783                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
 784         scst_set_busy(cmd);
 785         scst_set_cmd_abnormal_done_state(cmd);
 786         res = SCST_CMD_STATE_RES_CONT_SAME;
 787         goto out;
 788
 789 out_error:
 790         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
 791         scst_set_cmd_abnormal_done_state(cmd);
 792         res = SCST_CMD_STATE_RES_CONT_SAME;
 793         goto out;
 794 }
 795
 796 void scst_restart_cmd(struct scst_cmd *cmd, int status,
 797         enum scst_exec_context pref_context)
 798 {
 799         TRACE_ENTRY();
 800
 801         TRACE_DBG("Preferred context: %d", pref_context);
 802         TRACE_DBG("tag=%llu, status=%#x",
 803                   (long long unsigned int)scst_cmd_get_tag(cmd),
 804                   status);
 805
 806 #ifdef CONFIG_SCST_EXTRACHECKS
 807         if ((in_irq() || irqs_disabled()) &&
 808             ((pref_context == SCST_CONTEXT_DIRECT) ||
 809              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
 810                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
 811                         "SCST_CONTEXT_THREAD instead\n", pref_context,
 812                         cmd->tgtt->name);
 813                 pref_context = SCST_CONTEXT_THREAD;
 814         }
 815 #endif
 816
 817         switch (status) {
 818         case SCST_PREPROCESS_STATUS_SUCCESS:
 819                 switch (cmd->data_direction) {
 820                 case SCST_DATA_WRITE:
 821                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
 822                         break;
 823                 default:
 824                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
 825                         break;
 826                 }
 827                 if (cmd->set_sn_on_restart_cmd)
 828                         scst_cmd_set_sn(cmd);
 829                 /* Small context optimization */
 830                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
 831                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
 832                     ((pref_context == SCST_CONTEXT_SAME) &&
 833                      scst_cmd_atomic(cmd))) {
 834                         if (cmd->data_direction == SCST_DATA_WRITE) {
 835                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
 836                                                 &cmd->tgt_dev->tgt_dev_flags))
 837                                         pref_context = SCST_CONTEXT_THREAD;
 838                         } else {
 839                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
 840                                                 &cmd->tgt_dev->tgt_dev_flags))
 841                                         pref_context = SCST_CONTEXT_THREAD;
 842                         }
 843                 }
 844                 break;
 845
 846         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
 847                 scst_set_cmd_abnormal_done_state(cmd);
 848                 break;
 849
 850         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
 851                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
 852                 /* go through */
 853         case SCST_PREPROCESS_STATUS_ERROR:
 854                 scst_set_cmd_error(cmd,
 855                            SCST_LOAD_SENSE(scst_sense_hardw_error));
 856                 scst_set_cmd_abnormal_done_state(cmd);
 857                 break;
 858
 859         default:
 860                 PRINT_ERROR("%s() received unknown status %x", __func__,
 861                         status);
 862                 scst_set_cmd_abnormal_done_state(cmd);
 863                 break;
 864         }
 865
 866         scst_proccess_redirect_cmd(cmd, pref_context, 1);
 867
 868         TRACE_EXIT();
 869         return;
 870 }
 871 EXPORT_SYMBOL(scst_restart_cmd);
 872
 873 /* No locks */
 874 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
 875 {
 876         struct scst_tgt *tgt = cmd->sess->tgt;
 877         int res = 0;
 878         unsigned long flags;
 879
 880         TRACE_ENTRY();
 881
 882         spin_lock_irqsave(&tgt->tgt_lock, flags);
 883         tgt->retry_cmds++;
 884         smp_mb();
 885         TRACE_RETRY("TGT QUEUE FULL: incrementing retry_cmds %d",
 886               tgt->retry_cmds);
 887         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
 888                 /* At least one cmd finished, so try again */
 889                 tgt->retry_cmds--;
 890                 TRACE_RETRY("Some command(s) finished, direct retry "
 891                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
 892                       "retry_cmds=%d)", finished_cmds,
 893                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
 894                 res = -1;
 895                 goto out_unlock_tgt;
 896         }
 897
 898         TRACE_RETRY("Adding cmd %p to retry cmd list", cmd);
 899         list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
 900
 901         if (!tgt->retry_timer_active) {
 902                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
 903                 add_timer(&tgt->retry_timer);
 904                 tgt->retry_timer_active = 1;
 905         }
 906
 907 out_unlock_tgt:
 908         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
 909
 910         TRACE_EXIT_RES(res);
 911         return res;
 912 }
 913
 914 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
 915 {
 916         int res, rc;
 917
 918         TRACE_ENTRY();
 919
 920         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
 921                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
 922                 goto out_dev_done;
 923         }
 924
 925         if ((cmd->tgtt->rdy_to_xfer == NULL) || unlikely(cmd->internal)) {
 926                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
 927                 res = SCST_CMD_STATE_RES_CONT_SAME;
 928                 goto out;
 929         }
 930
 931         if (unlikely(!cmd->tgtt->rdy_to_xfer_atomic && scst_cmd_atomic(cmd))) {
 932                 /*
 933                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
 934                  * optimization.
 935                  */
 936                 TRACE_DBG("Target driver %s rdy_to_xfer() needs thread "
 937                               "context, rescheduling", cmd->tgtt->name);
 938                 res = SCST_CMD_STATE_RES_NEED_THREAD;
 939                 goto out;
 940         }
 941
 942         while (1) {
 943                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
 944
 945                 res = SCST_CMD_STATE_RES_CONT_NEXT;
 946                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
 947
 948                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
 949 #ifdef CONFIG_SCST_DEBUG_RETRY
 950                 if (((scst_random() % 100) == 75))
 951                         rc = SCST_TGT_RES_QUEUE_FULL;
 952                 else
 953 #endif
 954                         rc = cmd->tgtt->rdy_to_xfer(cmd);
 955                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
 956
 957                 if (likely(rc == SCST_TGT_RES_SUCCESS))
 958                         goto out;
 959
 960                 /* Restore the previous state */
 961                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
 962
 963                 switch (rc) {
 964                 case SCST_TGT_RES_QUEUE_FULL:
 965                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
 966                                 break;
 967                         else
 968                                 continue;
 969
 970                 case SCST_TGT_RES_NEED_THREAD_CTX:
 971                         TRACE_DBG("Target driver %s "
 972                               "rdy_to_xfer() requested thread "
 973                               "context, rescheduling", cmd->tgtt->name);
 974                         res = SCST_CMD_STATE_RES_NEED_THREAD;
 975                         break;
 976
 977                 default:
 978                         goto out_error_rc;
 979                 }
 980                 break;
 981         }
 982
 983 out:
 984         TRACE_EXIT_HRES(res);
 985         return res;
 986
 987 out_error_rc:
 988         if (rc == SCST_TGT_RES_FATAL_ERROR) {
 989                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
 990                      "fatal error", cmd->tgtt->name);
 991         } else {
 992                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
 993                             "value %d", cmd->tgtt->name, rc);
 994         }
 995         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
 996
 997 out_dev_done:
 998         scst_set_cmd_abnormal_done_state(cmd);
 999         res = SCST_CMD_STATE_RES_CONT_SAME;
1000         goto out;
1001 }
1002
1003 /* No locks, but might be in IRQ */
1004 void scst_proccess_redirect_cmd(struct scst_cmd *cmd,
1005         enum scst_exec_context context, int check_retries)
1006 {
1007         unsigned long flags;
1008
1009         TRACE_ENTRY();
1010
1011         TRACE_DBG("Context: %x", context);
1012
1013         if (context == SCST_CONTEXT_SAME)
1014                 context = scst_cmd_atomic(cmd) ? SCST_CONTEXT_DIRECT_ATOMIC :
1015                                                  SCST_CONTEXT_DIRECT;
1016
1017         switch (context) {
1018         case SCST_CONTEXT_DIRECT_ATOMIC:
1019                 scst_process_active_cmd(cmd, true);
1020                 break;
1021
1022         case SCST_CONTEXT_DIRECT:
1023                 if (check_retries)
1024                         scst_check_retries(cmd->tgt);
1025                 scst_process_active_cmd(cmd, false);
1026                 break;
1027
1028         default:
1029                 PRINT_ERROR("Context %x is unknown, using the thread one",
1030                             context);
1031                 /* go through */
1032         case SCST_CONTEXT_THREAD:
1033                 if (check_retries)
1034                         scst_check_retries(cmd->tgt);
1035                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
1036                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
1037                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
1038                         list_add(&cmd->cmd_list_entry,
1039                                 &cmd->cmd_lists->active_cmd_list);
1040                 else
1041                         list_add_tail(&cmd->cmd_list_entry,
1042                                 &cmd->cmd_lists->active_cmd_list);
1043                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
1044                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
1045                 break;
1046
1047         case SCST_CONTEXT_TASKLET:
1048                 if (check_retries)
1049                         scst_check_retries(cmd->tgt);
1050                 scst_schedule_tasklet(cmd);
1051                 break;
1052         }
1053
1054         TRACE_EXIT();
1055         return;
1056 }
1057
1058 void scst_rx_data(struct scst_cmd *cmd, int status,
1059         enum scst_exec_context pref_context)
1060 {
1061         TRACE_ENTRY();
1062
1063         TRACE_DBG("Preferred context: %d", pref_context);
1064         TRACE(TRACE_SCSI, "cmd %p, status %#x", cmd, status);
1065
1066 #ifdef CONFIG_SCST_EXTRACHECKS
1067         if ((in_irq() || irqs_disabled()) &&
1068             ((pref_context == SCST_CONTEXT_DIRECT) ||
1069              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
1070                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
1071                         "SCST_CONTEXT_THREAD instead\n", pref_context,
1072                         cmd->tgtt->name);
1073                 pref_context = SCST_CONTEXT_THREAD;
1074         }
1075 #endif
1076
1077         switch (status) {
1078         case SCST_RX_STATUS_SUCCESS:
1079 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
1080                 if (cmd->tgt_sg) {
1081                         int i;
1082                         struct scatterlist *sg = cmd->tgt_sg;
1083                         TRACE_RECV_BOT("RX data for cmd %p "
1084                                 "(sg_cnt %d, sg %p, sg[0].page %p)", cmd,
1085                                 cmd->tgt_sg_cnt, sg, (void *)sg_page(&sg[0]));
1086                         for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
1087                                 PRINT_BUFF_FLAG(TRACE_RCV_BOT, "RX sg",
1088                                         sg_virt(&sg[i]), sg[i].length);
1089                         }
1090                 }
1091 #endif
1092                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1093                 /* Small context optimization */
1094                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
1095                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
1096                     ((pref_context == SCST_CONTEXT_SAME) &&
1097                      scst_cmd_atomic(cmd))) {
1098                         if (!test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC,
1099                                         &cmd->tgt_dev->tgt_dev_flags))
1100                                 pref_context = SCST_CONTEXT_THREAD;
1101                 }
1102                 break;
1103
1104         case SCST_RX_STATUS_ERROR_SENSE_SET:
1105                 scst_set_cmd_abnormal_done_state(cmd);
1106                 break;
1107
1108         case SCST_RX_STATUS_ERROR_FATAL:
1109                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1110                 /* go through */
1111         case SCST_RX_STATUS_ERROR:
1112                 scst_set_cmd_error(cmd,
1113                            SCST_LOAD_SENSE(scst_sense_hardw_error));
1114                 scst_set_cmd_abnormal_done_state(cmd);
1115                 break;
1116
1117         default:
1118                 PRINT_ERROR("scst_rx_data() received unknown status %x",
1119                         status);
1120                 scst_set_cmd_abnormal_done_state(cmd);
1121                 break;
1122         }
1123
1124         scst_proccess_redirect_cmd(cmd, pref_context, 1);
1125
1126         TRACE_EXIT();
1127         return;
1128 }
1129 EXPORT_SYMBOL(scst_rx_data);
1130
1131 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
1132 {
1133         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
1134
1135         TRACE_ENTRY();
1136
1137         cmd->state = SCST_CMD_STATE_SEND_FOR_EXEC;
1138
1139         if ((cmd->tgtt->pre_exec == NULL) || unlikely(cmd->internal))
1140                 goto out;
1141
1142         TRACE_DBG("Calling pre_exec(%p)", cmd);
1143         rc = cmd->tgtt->pre_exec(cmd);
1144         TRACE_DBG("pre_exec() returned %d", rc);
1145
1146         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
1147                 switch (rc) {
1148                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
1149                         scst_set_cmd_abnormal_done_state(cmd);
1150                         break;
1151                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
1152                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1153                         /* go through */
1154                 case SCST_PREPROCESS_STATUS_ERROR:
1155                         scst_set_cmd_error(cmd,
1156                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1157                         scst_set_cmd_abnormal_done_state(cmd);
1158                         break;
1159                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1160                         TRACE_DBG("Target driver's %s pre_exec() requested "
1161                                 "thread context, rescheduling",
1162                                 cmd->tgtt->name);
1163                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1164                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1165                         break;
1166                 default:
1167                         sBUG();
1168                         break;
1169                 }
1170         }
1171
1172 out:
1173         TRACE_EXIT_RES(res);
1174         return res;
1175 }
1176
1177 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1178         const uint8_t *rq_sense, int rq_sense_len, int resid)
1179 {
1180         TRACE_ENTRY();
1181
1182 #ifdef CONFIG_SCST_MEASURE_LATENCY
1183         {
1184                 struct timespec ts;
1185                 getnstimeofday(&ts);
1186                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1187                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1188                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1189                         ts.tv_sec, ts.tv_nsec);
1190         }
1191 #endif
1192
1193         cmd->status = result & 0xff;
1194         cmd->msg_status = msg_byte(result);
1195         cmd->host_status = host_byte(result);
1196         cmd->driver_status = driver_byte(result);
1197         if (unlikely(resid != 0)) {
1198 #ifdef CONFIG_SCST_EXTRACHECKS
1199                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1200                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1201                                 "op %x)", resid, cmd->resp_data_len,
1202                                 cmd->cdb[0]);
1203                 } else
1204 #endif
1205                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1206         }
1207
1208         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION)) {
1209                 /* We might have double reset UA here */
1210                 cmd->dbl_ua_orig_resp_data_len = cmd->resp_data_len;
1211                 cmd->dbl_ua_orig_data_direction = cmd->data_direction;
1212
1213                 scst_alloc_set_sense(cmd, 1, rq_sense, rq_sense_len);
1214         }
1215
1216         TRACE(TRACE_SCSI, "cmd %p, result=%x, cmd->status=%x, resid=%d, "
1217               "cmd->msg_status=%x, cmd->host_status=%x, "
1218               "cmd->driver_status=%x (cmd %p)", cmd, result, cmd->status, resid,
1219               cmd->msg_status, cmd->host_status, cmd->driver_status, cmd);
1220
1221         cmd->completed = 1;
1222
1223         TRACE_EXIT();
1224         return;
1225 }
1226
1227 /* For small context optimization */
1228 static inline enum scst_exec_context scst_optimize_post_exec_context(
1229         struct scst_cmd *cmd, enum scst_exec_context context)
1230 {
1231         if (((context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd)) ||
1232             (context == SCST_CONTEXT_TASKLET) ||
1233             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1234                 if (!test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC,
1235                                 &cmd->tgt_dev->tgt_dev_flags))
1236                         context = SCST_CONTEXT_THREAD;
1237         }
1238         return context;
1239 }
1240
1241 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1242 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1243                                             struct scsi_request **req)
1244 {
1245         struct scst_cmd *cmd = NULL;
1246
1247         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1248                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1249
1250         if (cmd == NULL) {
1251                 PRINT_ERROR("%s", "Request with NULL cmd");
1252                 if (*req)
1253                         scsi_release_request(*req);
1254         }
1255
1256         return cmd;
1257 }
1258
1259 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1260 {
1261         struct scsi_request *req = NULL;
1262         struct scst_cmd *cmd;
1263
1264         TRACE_ENTRY();
1265
1266         cmd = scst_get_cmd(scsi_cmd, &req);
1267         if (cmd == NULL)
1268                 goto out;
1269
1270         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1271                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1272
1273         /* Clear out request structure */
1274         req->sr_use_sg = 0;
1275         req->sr_sglist_len = 0;
1276         req->sr_bufflen = 0;
1277         req->sr_buffer = NULL;
1278         req->sr_underflow = 0;
1279         req->sr_request->rq_disk = NULL; /* disown request blk */
1280
1281         scst_release_request(cmd);
1282
1283         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1284
1285         scst_proccess_redirect_cmd(cmd,
1286                 scst_optimize_post_exec_context(cmd, scst_estimate_context()),
1287                                                 0);
1288
1289 out:
1290         TRACE_EXIT();
1291         return;
1292 }
1293 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1294 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1295 {
1296         struct scst_cmd *cmd;
1297
1298         TRACE_ENTRY();
1299
1300         cmd = (struct scst_cmd *)data;
1301         if (cmd == NULL)
1302                 goto out;
1303
1304         scst_do_cmd_done(cmd, result, sense, SCST_SENSE_BUFFERSIZE, resid);
1305
1306         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1307
1308         scst_proccess_redirect_cmd(cmd,
1309                 scst_optimize_post_exec_context(cmd, scst_estimate_context()),
1310                                                 0);
1311
1312 out:
1313         TRACE_EXIT();
1314         return;
1315 }
1316 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1317
1318 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state,
1319         enum scst_exec_context pref_context)
1320 {
1321         TRACE_ENTRY();
1322
1323 #ifdef CONFIG_SCST_MEASURE_LATENCY
1324         {
1325                 struct timespec ts;
1326                 getnstimeofday(&ts);
1327                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1328                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1329                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1330                         ts.tv_sec, ts.tv_nsec);
1331         }
1332 #endif
1333
1334         if (next_state == SCST_CMD_STATE_DEFAULT)
1335                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1336
1337 #if defined(CONFIG_SCST_DEBUG)
1338         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1339                 if (cmd->sg) {
1340                         int i;
1341                         struct scatterlist *sg = cmd->sg;
1342                         TRACE_RECV_TOP("Exec'd %d S/G(s) at %p sg[0].page at "
1343                                 "%p", cmd->sg_cnt, sg, (void *)sg_page(&sg[0]));
1344                         for (i = 0; i < cmd->sg_cnt; ++i) {
1345                                 TRACE_BUFF_FLAG(TRACE_RCV_TOP,
1346                                         "Exec'd sg", sg_virt(&sg[i]),
1347                                         sg[i].length);
1348                         }
1349                 }
1350         }
1351 #endif
1352
1353         cmd->state = next_state;
1354
1355 #ifdef CONFIG_SCST_EXTRACHECKS
1356         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1357             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1358             (next_state != SCST_CMD_STATE_FINISHED)) {
1359                 PRINT_ERROR("%s() received invalid cmd state %d (opcode %d)",
1360                         __func__, next_state, cmd->cdb[0]);
1361                 scst_set_cmd_error(cmd,
1362                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1363                 scst_set_cmd_abnormal_done_state(cmd);
1364         }
1365 #endif
1366         pref_context = scst_optimize_post_exec_context(cmd, pref_context);
1367         scst_proccess_redirect_cmd(cmd, pref_context, 0);
1368
1369         TRACE_EXIT();
1370         return;
1371 }
1372
1373 static int scst_report_luns_local(struct scst_cmd *cmd)
1374 {
1375         int rc;
1376         int dev_cnt = 0;
1377         int buffer_size;
1378         int i;
1379         struct scst_tgt_dev *tgt_dev = NULL;
1380         uint8_t *buffer;
1381         int offs, overflow = 0;
1382
1383         TRACE_ENTRY();
1384
1385         rc = scst_check_local_events(cmd);
1386         if (unlikely(rc != 0))
1387                 goto out_done;
1388
1389         cmd->status = 0;
1390         cmd->msg_status = 0;
1391         cmd->host_status = DID_OK;
1392         cmd->driver_status = 0;
1393
1394         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1395                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1396                         "LUNS command", cmd->cdb[2]);
1397                 goto out_err;
1398         }
1399
1400         buffer_size = scst_get_buf_first(cmd, &buffer);
1401         if (unlikely(buffer_size == 0))
1402                 goto out_compl;
1403         else if (unlikely(buffer_size < 0))
1404                 goto out_err;
1405
1406         if (buffer_size < 16)
1407                 goto out_put_err;
1408
1409         memset(buffer, 0, buffer_size);
1410         offs = 8;
1411
1412         /* sess->sess_tgt_dev_list_hash is protected by suspended activity */
1413         for (i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1414                 struct list_head *sess_tgt_dev_list_head =
1415                         &cmd->sess->sess_tgt_dev_list_hash[i];
1416                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1417                                 sess_tgt_dev_list_entry) {
1418                         if (!overflow) {
1419                                 if (offs >= buffer_size) {
1420                                         scst_put_buf(cmd, buffer);
1421                                         buffer_size = scst_get_buf_next(cmd,
1422                                                                        &buffer);
1423                                         if (buffer_size > 0) {
1424                                                 memset(buffer, 0, buffer_size);
1425                                                 offs = 0;
1426                                         } else {
1427                                                 overflow = 1;
1428                                                 goto inc_dev_cnt;
1429                                         }
1430                                 }
1431                                 if ((buffer_size - offs) < 8) {
1432                                         PRINT_ERROR("Buffer allocated for "
1433                                                 "REPORT LUNS command doesn't "
1434                                                 "allow to fit 8 byte entry "
1435                                                 "(buffer_size=%d)",
1436                                                 buffer_size);
1437                                         goto out_put_hw_err;
1438                                 }
1439                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1440                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1441                                 offs += 8;
1442                         }
1443 inc_dev_cnt:
1444                         dev_cnt++;
1445                 }
1446         }
1447         if (!overflow)
1448                 scst_put_buf(cmd, buffer);
1449
1450         /* Set the response header */
1451         buffer_size = scst_get_buf_first(cmd, &buffer);
1452         if (unlikely(buffer_size == 0))
1453                 goto out_compl;
1454         else if (unlikely(buffer_size < 0))
1455                 goto out_err;
1456
1457         dev_cnt *= 8;
1458         buffer[0] = (dev_cnt >> 24) & 0xff;
1459         buffer[1] = (dev_cnt >> 16) & 0xff;
1460         buffer[2] = (dev_cnt >> 8) & 0xff;
1461         buffer[3] = dev_cnt & 0xff;
1462
1463         scst_put_buf(cmd, buffer);
1464
1465         dev_cnt += 8;
1466         if (dev_cnt < cmd->resp_data_len)
1467                 scst_set_resp_data_len(cmd, dev_cnt);
1468
1469 out_compl:
1470         cmd->completed = 1;
1471
1472 out_done:
1473         /* Report the result */
1474         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1475
1476         TRACE_EXIT();
1477         return SCST_EXEC_COMPLETED;
1478
1479 out_put_err:
1480         scst_put_buf(cmd, buffer);
1481
1482 out_err:
1483         scst_set_cmd_error(cmd,
1484                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1485         goto out_compl;
1486
1487 out_put_hw_err:
1488         scst_put_buf(cmd, buffer);
1489         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1490         goto out_compl;
1491 }
1492
1493 static int scst_pre_select(struct scst_cmd *cmd)
1494 {
1495         int res = SCST_EXEC_NOT_COMPLETED;
1496
1497         TRACE_ENTRY();
1498
1499         if (scst_cmd_atomic(cmd)) {
1500                 res = SCST_EXEC_NEED_THREAD;
1501                 goto out;
1502         }
1503
1504         scst_block_dev_cmd(cmd, 1);
1505
1506         /* Check for local events will be done when cmd will be executed */
1507
1508 out:
1509         TRACE_EXIT_RES(res);
1510         return res;
1511 }
1512
1513 static int scst_reserve_local(struct scst_cmd *cmd)
1514 {
1515         int res = SCST_EXEC_NOT_COMPLETED, rc;
1516         struct scst_device *dev;
1517         struct scst_tgt_dev *tgt_dev_tmp;
1518
1519         TRACE_ENTRY();
1520
1521         if (scst_cmd_atomic(cmd)) {
1522                 res = SCST_EXEC_NEED_THREAD;
1523                 goto out;
1524         }
1525
1526         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1527                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1528                      "(lun=%lld)", (long long unsigned int)cmd->lun);
1529                 scst_set_cmd_error(cmd,
1530                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1531                 goto out_done;
1532         }
1533
1534         dev = cmd->dev;
1535
1536         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1537                 scst_block_dev_cmd(cmd, 1);
1538
1539         rc = scst_check_local_events(cmd);
1540         if (unlikely(rc != 0))
1541                 goto out_done;
1542
1543         spin_lock_bh(&dev->dev_lock);
1544
1545         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1546                 spin_unlock_bh(&dev->dev_lock);
1547                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1548                 goto out_done;
1549         }
1550
1551         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1552                             dev_tgt_dev_list_entry) {
1553                 if (cmd->tgt_dev != tgt_dev_tmp)
1554                         set_bit(SCST_TGT_DEV_RESERVED,
1555                                 &tgt_dev_tmp->tgt_dev_flags);
1556         }
1557         dev->dev_reserved = 1;
1558
1559         spin_unlock_bh(&dev->dev_lock);
1560
1561 out:
1562         TRACE_EXIT_RES(res);
1563         return res;
1564
1565 out_done:
1566         /* Report the result */
1567         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1568         res = SCST_EXEC_COMPLETED;
1569         goto out;
1570 }
1571
1572 static int scst_release_local(struct scst_cmd *cmd)
1573 {
1574         int res = SCST_EXEC_NOT_COMPLETED, rc;
1575         struct scst_tgt_dev *tgt_dev_tmp;
1576         struct scst_device *dev;
1577
1578         TRACE_ENTRY();
1579
1580         if (scst_cmd_atomic(cmd)) {
1581                 res = SCST_EXEC_NEED_THREAD;
1582                 goto out;
1583         }
1584
1585         dev = cmd->dev;
1586
1587         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1588                 scst_block_dev_cmd(cmd, 1);
1589
1590         rc = scst_check_local_events(cmd);
1591         if (unlikely(rc != 0))
1592                 goto out_done;
1593
1594         spin_lock_bh(&dev->dev_lock);
1595
1596         /*
1597          * The device could be RELEASED behind us, if RESERVING session
1598          * is closed (see scst_free_tgt_dev()), but this actually doesn't
1599          * matter, so use lock and no retest for DEV_RESERVED bits again
1600          */
1601         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1602                 res = SCST_EXEC_COMPLETED;
1603                 cmd->status = 0;
1604                 cmd->msg_status = 0;
1605                 cmd->host_status = DID_OK;
1606                 cmd->driver_status = 0;
1607                 cmd->completed = 1;
1608         } else {
1609                 list_for_each_entry(tgt_dev_tmp,
1610                                     &dev->dev_tgt_dev_list,
1611                                     dev_tgt_dev_list_entry) {
1612                         clear_bit(SCST_TGT_DEV_RESERVED,
1613                                 &tgt_dev_tmp->tgt_dev_flags);
1614                 }
1615                 dev->dev_reserved = 0;
1616         }
1617
1618         spin_unlock_bh(&dev->dev_lock);
1619
1620         if (res == SCST_EXEC_COMPLETED)
1621                 goto out_done;
1622
1623 out:
1624         TRACE_EXIT_RES(res);
1625         return res;
1626
1627 out_done:
1628         res = SCST_EXEC_COMPLETED;
1629         /* Report the result */
1630         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1631         goto out;
1632 }
1633
1634 /* No locks, no IRQ or IRQ-safe context allowed */
1635 int scst_check_local_events(struct scst_cmd *cmd)
1636 {
1637         int res, rc;
1638         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1639         struct scst_device *dev = cmd->dev;
1640
1641         TRACE_ENTRY();
1642
1643         /*
1644          * There's no race here, because we need to trace commands sent
1645          * *after* dev_double_ua_possible flag was set.
1646          */
1647         if (unlikely(dev->dev_double_ua_possible))
1648                 cmd->double_ua_possible = 1;
1649
1650         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1651                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1652                 goto out_uncomplete;
1653         }
1654
1655         /* Reserve check before Unit Attention */
1656         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED,
1657                               &tgt_dev->tgt_dev_flags))) {
1658                 if (cmd->cdb[0] != INQUIRY &&
1659                     cmd->cdb[0] != REPORT_LUNS &&
1660                     cmd->cdb[0] != RELEASE &&
1661                     cmd->cdb[0] != RELEASE_10 &&
1662                     cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER &&
1663                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL ||
1664                      (cmd->cdb[4] & 3)) &&
1665                     cmd->cdb[0] != LOG_SENSE &&
1666                     cmd->cdb[0] != REQUEST_SENSE) {
1667                         scst_set_cmd_error_status(cmd,
1668                                 SAM_STAT_RESERVATION_CONFLICT);
1669                         goto out_complete;
1670                 }
1671         }
1672
1673         /* If we had internal bus reset, set the command error unit attention */
1674         if ((dev->scsi_dev != NULL) &&
1675             unlikely(dev->scsi_dev->was_reset)) {
1676                 if (scst_is_ua_command(cmd)) {
1677                         int done = 0;
1678                         /*
1679                          * Prevent more than 1 cmd to be triggered by
1680                          * was_reset.
1681                          */
1682                         spin_lock_bh(&dev->dev_lock);
1683                         barrier(); /* to reread was_reset */
1684                         if (dev->scsi_dev->was_reset) {
1685                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1686                                 scst_set_cmd_error(cmd,
1687                                           SCST_LOAD_SENSE(scst_sense_reset_UA));
1688                                 /*
1689                                  * It looks like it is safe to clear was_reset
1690                                  * here.
1691                                  */
1692                                 dev->scsi_dev->was_reset = 0;
1693                                 done = 1;
1694                         }
1695                         spin_unlock_bh(&dev->dev_lock);
1696
1697                         if (done)
1698                                 goto out_complete;
1699                 }
1700         }
1701
1702         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING,
1703                         &cmd->tgt_dev->tgt_dev_flags))) {
1704                 if (scst_is_ua_command(cmd)) {
1705                         rc = scst_set_pending_UA(cmd);
1706                         if (rc == 0)
1707                                 goto out_complete;
1708                 }
1709         }
1710
1711         res = 0;
1712
1713 out:
1714         TRACE_EXIT_RES(res);
1715         return res;
1716
1717 out_complete:
1718         res = 1;
1719         sBUG_ON(!cmd->completed);
1720         goto out;
1721
1722 out_uncomplete:
1723         res = -1;
1724         goto out;
1725 }
1726 EXPORT_SYMBOL(scst_check_local_events);
1727
1728 /* No locks */
1729 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1730 {
1731         if (slot == NULL)
1732                 goto inc;
1733
1734         /* Optimized for lockless fast path */
1735
1736         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1737                 atomic_read(slot));
1738
1739         if (!atomic_dec_and_test(slot))
1740                 goto out;
1741
1742         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1743                 tgt_dev->num_free_sn_slots);
1744         if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
1745                 spin_lock_irq(&tgt_dev->sn_lock);
1746                 if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
1747                         if (tgt_dev->num_free_sn_slots < 0)
1748                                 tgt_dev->cur_sn_slot = slot;
1749                         /*
1750                          * To be in-sync with SIMPLE case in scst_cmd_set_sn()
1751                          */
1752                         smp_mb();
1753                         tgt_dev->num_free_sn_slots++;
1754                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1755                                 tgt_dev->num_free_sn_slots);
1756
1757                 }
1758                 spin_unlock_irq(&tgt_dev->sn_lock);
1759         }
1760
1761 inc:
1762         /*
1763          * No locks is needed, because only one thread at time can
1764          * be here (serialized by sn). Also it is supposed that there
1765          * could not be half-incremented halves.
1766          */
1767         tgt_dev->expected_sn++;
1768         smp_mb(); /* write must be before def_cmd_count read */
1769         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1770
1771 out:
1772         return;
1773 }
1774
1775 /* No locks */
1776 static struct scst_cmd *scst_post_exec_sn(struct scst_cmd *cmd,
1777         bool make_active)
1778 {
1779         /* For HQ commands SN is not set */
1780         bool inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1781                                cmd->sn_set && !cmd->retry;
1782         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1783         struct scst_cmd *res;
1784
1785         TRACE_ENTRY();
1786
1787         if (inc_expected_sn)
1788                 scst_inc_expected_sn(tgt_dev, cmd->sn_slot);
1789
1790         if (make_active) {
1791                 scst_make_deferred_commands_active(tgt_dev);
1792                 res = NULL;
1793         } else
1794                 res = scst_check_deferred_commands(tgt_dev);
1795
1796         TRACE_EXIT_HRES(res);
1797         return res;
1798 }
1799
1800 /* cmd must be additionally referenced to not die inside */
1801 static int scst_do_real_exec(struct scst_cmd *cmd)
1802 {
1803         int res = SCST_EXEC_NOT_COMPLETED;
1804 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
1805         int rc;
1806 #endif
1807         struct scst_device *dev = cmd->dev;
1808         struct scst_dev_type *handler = dev->handler;
1809
1810         TRACE_ENTRY();
1811
1812         cmd->state = SCST_CMD_STATE_REAL_EXECUTING;
1813
1814         if (handler->exec) {
1815                 if (unlikely(!dev->handler->exec_atomic &&
1816                              scst_cmd_atomic(cmd))) {
1817                         /*
1818                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
1819                          * optimization.
1820                          */
1821                         TRACE_DBG("Dev handler %s exec() needs thread "
1822                                 "context, rescheduling", dev->handler->name);
1823                         res = SCST_EXEC_NEED_THREAD;
1824                         goto out_restore;
1825                 }
1826
1827                 TRACE_DBG("Calling dev handler %s exec(%p)",
1828                       handler->name, cmd);
1829                 TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb,
1830                         cmd->cdb_len);
1831                 res = handler->exec(cmd);
1832                 TRACE_DBG("Dev handler %s exec() returned %d",
1833                       handler->name, res);
1834
1835                 if (res == SCST_EXEC_COMPLETED)
1836                         goto out_complete;
1837                 else if (res == SCST_EXEC_NEED_THREAD)
1838                         goto out_restore;
1839
1840                 sBUG_ON(res != SCST_EXEC_NOT_COMPLETED);
1841         }
1842
1843         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1844
1845         if (unlikely(dev->scsi_dev == NULL)) {
1846                 PRINT_ERROR("Command for virtual device must be "
1847                         "processed by device handler (lun %lld)!",
1848                         (long long unsigned int)cmd->lun);
1849                 goto out_error;
1850         }
1851
1852         res = scst_check_local_events(cmd);
1853         if (unlikely(res != 0))
1854                 goto out_done;
1855
1856 #ifndef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1857         if (unlikely(scst_cmd_atomic(cmd))) {
1858                 TRACE_DBG("Pass-through exec() can not be called in atomic "
1859                         "context, rescheduling to the thread (handler %s)",
1860                         handler->name);
1861                 res = SCST_EXEC_NEED_THREAD;
1862                 goto out_restore;
1863         }
1864 #endif
1865
1866 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1867         if (unlikely(scst_alloc_request(cmd) != 0)) {
1868                 if (scst_cmd_atomic(cmd)) {
1869                         res = SCST_EXEC_NEED_THREAD;
1870                         goto out_restore;
1871                 } else {
1872                         PRINT_INFO("%s", "Unable to allocate request, "
1873                                 "sending BUSY status");
1874                         goto out_busy;
1875                 }
1876         }
1877
1878         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1879                     (void *)cmd->scsi_req->sr_buffer,
1880                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1881                     cmd->retries);
1882 #else
1883         rc = scst_exec_req(dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1884                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1885                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1886                         scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
1887         if (unlikely(rc != 0)) {
1888                 if (scst_cmd_atomic(cmd)) {
1889                         res = SCST_EXEC_NEED_THREAD;
1890                         goto out_restore;
1891                 } else {
1892                         PRINT_ERROR("scst_exec_req() failed: %d", res);
1893                         goto out_error;
1894                 }
1895         }
1896 #endif
1897
1898 out_complete:
1899         res = SCST_EXEC_COMPLETED;
1900
1901 out:
1902         TRACE_EXIT();
1903         return res;
1904
1905 out_restore:
1906         /* Restore the state */
1907         cmd->state = SCST_CMD_STATE_REAL_EXEC;
1908         goto out;
1909
1910 out_error:
1911         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1912         goto out_done;
1913
1914 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1915 out_busy:
1916         scst_set_busy(cmd);
1917         /* go through */
1918 #endif
1919
1920 out_done:
1921         res = SCST_EXEC_COMPLETED;
1922         /* Report the result */
1923         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1924         goto out_complete;
1925 }
1926
1927 static inline int scst_real_exec(struct scst_cmd *cmd)
1928 {
1929         int res;
1930
1931         TRACE_ENTRY();
1932
1933         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
1934         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
1935         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
1936
1937         __scst_cmd_get(cmd);
1938
1939         res = scst_do_real_exec(cmd);
1940
1941         if (likely(res == SCST_EXEC_COMPLETED)) {
1942                 scst_post_exec_sn(cmd, true);
1943                 if (cmd->dev->scsi_dev != NULL)
1944                         generic_unplug_device(
1945                                 cmd->dev->scsi_dev->request_queue);
1946         } else
1947                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
1948
1949         __scst_cmd_put(cmd);
1950
1951         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
1952
1953         TRACE_EXIT_RES(res);
1954         return res;
1955 }
1956
1957 static int scst_do_local_exec(struct scst_cmd *cmd)
1958 {
1959         int res;
1960         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1961
1962         TRACE_ENTRY();
1963
1964         /* Check READ_ONLY device status */
1965         if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
1966             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
1967              cmd->cdb[0] == WRITE_10 ||
1968              cmd->cdb[0] == WRITE_12 ||
1969              cmd->cdb[0] == WRITE_16 ||
1970              cmd->cdb[0] == WRITE_VERIFY ||
1971              cmd->cdb[0] == WRITE_VERIFY_12 ||
1972              cmd->cdb[0] == WRITE_VERIFY_16 ||
1973              (cmd->dev->handler->type == TYPE_TAPE &&
1974               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
1975                 scst_set_cmd_error(cmd,
1976                            SCST_LOAD_SENSE(scst_sense_data_protect));
1977                 goto out_done;
1978         }
1979
1980         /*
1981          * Adding new commands here don't forget to update
1982          * scst_is_cmd_local() in scst.h, if necessary
1983          */
1984
1985         switch (cmd->cdb[0]) {
1986         case MODE_SELECT:
1987         case MODE_SELECT_10:
1988         case LOG_SELECT:
1989                 res = scst_pre_select(cmd);
1990                 break;
1991         case RESERVE:
1992         case RESERVE_10:
1993                 res = scst_reserve_local(cmd);
1994                 break;
1995         case RELEASE:
1996         case RELEASE_10:
1997                 res = scst_release_local(cmd);
1998                 break;
1999         case REPORT_LUNS:
2000                 res = scst_report_luns_local(cmd);
2001                 break;
2002         default:
2003                 res = SCST_EXEC_NOT_COMPLETED;
2004                 break;
2005         }
2006
2007 out:
2008         TRACE_EXIT_RES(res);
2009         return res;
2010
2011 out_done:
2012         /* Report the result */
2013         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
2014         res = SCST_EXEC_COMPLETED;
2015         goto out;
2016 }
2017
2018 static int scst_local_exec(struct scst_cmd *cmd)
2019 {
2020         int res;
2021
2022         TRACE_ENTRY();
2023
2024         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
2025         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
2026         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
2027
2028         __scst_cmd_get(cmd);
2029
2030         res = scst_do_local_exec(cmd);
2031         if (likely(res == SCST_EXEC_NOT_COMPLETED))
2032                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2033         else if (res == SCST_EXEC_COMPLETED)
2034                 scst_post_exec_sn(cmd, true);
2035         else
2036                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
2037
2038         __scst_cmd_put(cmd);
2039
2040         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
2041         TRACE_EXIT_RES(res);
2042         return res;
2043 }
2044
2045 static int scst_exec(struct scst_cmd **active_cmd)
2046 {
2047         struct scst_cmd *cmd = *active_cmd;
2048         struct scst_cmd *ref_cmd;
2049         struct scst_device *dev = cmd->dev;
2050         int res = SCST_CMD_STATE_RES_CONT_NEXT, count;
2051
2052         TRACE_ENTRY();
2053
2054         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2055                 goto out;
2056
2057         /* To protect tgt_dev */
2058         ref_cmd = cmd;
2059         __scst_cmd_get(ref_cmd);
2060
2061         count = 0;
2062         while (1) {
2063                 int rc;
2064
2065                 cmd->sent_for_exec = 1;
2066                 cmd->scst_cmd_done = scst_cmd_done_local;
2067                 cmd->state = SCST_CMD_STATE_LOCAL_EXEC;
2068
2069                 if (cmd->tgt_data_buf_alloced && cmd->dh_data_buf_alloced &&
2070                     (cmd->data_direction == SCST_DATA_WRITE))
2071                         scst_copy_sg(cmd, SCST_SG_COPY_FROM_TARGET);
2072
2073                 rc = scst_do_local_exec(cmd);
2074                 if (likely(rc == SCST_EXEC_NOT_COMPLETED))
2075                         /* Nothing to do */;
2076                 else if (rc == SCST_EXEC_NEED_THREAD) {
2077                         TRACE_DBG("%s", "scst_do_local_exec() requested "
2078                                 "thread context, rescheduling");
2079                         scst_dec_on_dev_cmd(cmd);
2080                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2081                         break;
2082                 } else {
2083                         sBUG_ON(rc != SCST_EXEC_COMPLETED);
2084                         goto done;
2085                 }
2086
2087                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2088
2089                 rc = scst_do_real_exec(cmd);
2090                 if (likely(rc == SCST_EXEC_COMPLETED))
2091                         /* Nothing to do */;
2092                 else if (rc == SCST_EXEC_NEED_THREAD) {
2093                         TRACE_DBG("scst_real_exec() requested thread "
2094                                 "context, rescheduling (cmd %p)", cmd);
2095                         scst_dec_on_dev_cmd(cmd);
2096                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2097                         break;
2098                 } else
2099                         sBUG();
2100
2101 done:
2102                 count++;
2103
2104                 cmd = scst_post_exec_sn(cmd, false);
2105                 if (cmd == NULL)
2106                         break;
2107
2108                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2109                         break;
2110
2111                 __scst_cmd_put(ref_cmd);
2112                 ref_cmd = cmd;
2113                 __scst_cmd_get(ref_cmd);
2114         }
2115
2116         *active_cmd = cmd;
2117
2118         if (count == 0)
2119                 goto out_put;
2120
2121         if (dev->scsi_dev != NULL)
2122                 generic_unplug_device(dev->scsi_dev->request_queue);
2123
2124 out_put:
2125         __scst_cmd_put(ref_cmd);
2126         /* !! At this point sess, dev and tgt_dev can be already freed !! */
2127
2128 out:
2129         TRACE_EXIT_RES(res);
2130         return res;
2131 }
2132
2133 static int scst_send_for_exec(struct scst_cmd **active_cmd)
2134 {
2135         int res;
2136         struct scst_cmd *cmd = *active_cmd;
2137         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2138         typeof(tgt_dev->expected_sn) expected_sn;
2139
2140         TRACE_ENTRY();
2141
2142 #ifdef CONFIG_SCST_MEASURE_LATENCY
2143         if (cmd->pre_exec_finish == 0) {
2144                 struct timespec ts;
2145                 getnstimeofday(&ts);
2146                 cmd->pre_exec_finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2147                 TRACE_DBG("cmd %p (sess %p): pre_exec_finish %lld (tv_sec %ld, "
2148                         "tv_nsec %ld)", cmd, cmd->sess, cmd->pre_exec_finish,
2149                         ts.tv_sec, ts.tv_nsec);
2150         }
2151 #endif
2152
2153         if (unlikely(cmd->internal))
2154                 goto exec;
2155
2156         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2157                 goto exec;
2158
2159         sBUG_ON(!cmd->sn_set);
2160
2161         expected_sn = tgt_dev->expected_sn;
2162         /* Optimized for lockless fast path */
2163         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2164                 spin_lock_irq(&tgt_dev->sn_lock);
2165
2166                 tgt_dev->def_cmd_count++;
2167                 smp_mb();
2168
2169                 expected_sn = tgt_dev->expected_sn;
2170                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2171                         if (unlikely(test_bit(SCST_CMD_ABORTED,
2172                                               &cmd->cmd_flags))) {
2173                                 /* Necessary to allow aborting out of sn cmds */
2174                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p "
2175                                         "(tag %llu, sn %lu)", cmd,
2176                                         (long long unsigned)cmd->tag, cmd->sn);
2177                                 tgt_dev->def_cmd_count--;
2178                                 scst_set_cmd_abnormal_done_state(cmd);
2179                                 res = SCST_CMD_STATE_RES_CONT_SAME;
2180                         } else {
2181                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
2182                                         "expected_sn=%ld)", cmd, cmd->sn,
2183                                         cmd->sn_set, expected_sn);
2184                                 list_add_tail(&cmd->sn_cmd_list_entry,
2185                                               &tgt_dev->deferred_cmd_list);
2186                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2187                         }
2188                         spin_unlock_irq(&tgt_dev->sn_lock);
2189                         goto out;
2190                 } else {
2191                         TRACE_SN("Somebody incremented expected_sn %ld, "
2192                                 "continuing", expected_sn);
2193                         tgt_dev->def_cmd_count--;
2194                         spin_unlock_irq(&tgt_dev->sn_lock);
2195                 }
2196         }
2197
2198 exec:
2199         res = scst_exec(active_cmd);
2200
2201 out:
2202         TRACE_EXIT_HRES(res);
2203         return res;
2204 }
2205
2206 /* No locks supposed to be held */
2207 static int scst_check_sense(struct scst_cmd *cmd)
2208 {
2209         int res = 0;
2210         struct scst_device *dev = cmd->dev;
2211
2212         TRACE_ENTRY();
2213
2214         if (unlikely(cmd->ua_ignore))
2215                 goto out;
2216
2217         /* If we had internal bus reset behind us, set the command error UA */
2218         if ((dev->scsi_dev != NULL) &&
2219             unlikely(cmd->host_status == DID_RESET) &&
2220             scst_is_ua_command(cmd)) {
2221                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
2222                       dev->scsi_dev->was_reset, cmd->host_status);
2223                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_reset_UA));
2224                 /* It looks like it is safe to clear was_reset here */
2225                 dev->scsi_dev->was_reset = 0;
2226         }
2227
2228         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2229             SCST_SENSE_VALID(cmd->sense)) {
2230                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2231                         SCST_SENSE_BUFFERSIZE);
2232
2233                 /* Check Unit Attention Sense Key */
2234                 if (scst_is_ua_sense(cmd->sense)) {
2235                         if (cmd->sense[12] == SCST_SENSE_ASC_UA_RESET) {
2236                                 if (cmd->double_ua_possible) {
2237                                         TRACE(TRACE_MGMT_MINOR, "Double UA "
2238                                                 "detected for device %p", dev);
2239                                         TRACE(TRACE_MGMT_MINOR, "Retrying cmd"
2240                                                 " %p (tag %llu)", cmd,
2241                                                 (long long unsigned)cmd->tag);
2242
2243                                         cmd->status = 0;
2244                                         cmd->msg_status = 0;
2245                                         cmd->host_status = DID_OK;
2246                                         cmd->driver_status = 0;
2247
2248                                         mempool_free(cmd->sense,
2249                                                      scst_sense_mempool);
2250                                         cmd->sense = NULL;
2251
2252                                         scst_check_restore_sg_buff(cmd);
2253
2254                                         sBUG_ON(cmd->dbl_ua_orig_resp_data_len < 0);
2255                                         cmd->data_direction =
2256                                                 cmd->dbl_ua_orig_data_direction;
2257                                         cmd->resp_data_len =
2258                                                 cmd->dbl_ua_orig_resp_data_len;
2259
2260                                         cmd->state = SCST_CMD_STATE_REAL_EXEC;
2261                                         cmd->retry = 1;
2262                                         res = 1;
2263                                         goto out;
2264                                 }
2265                         }
2266                         scst_dev_check_set_UA(dev, cmd, cmd->sense,
2267                                 SCST_SENSE_BUFFERSIZE);
2268                 }
2269         }
2270
2271         if (unlikely(cmd->double_ua_possible)) {
2272                 if (scst_is_ua_command(cmd)) {
2273                         TRACE_MGMT_DBG("Clearing dbl_ua_possible flag (dev %p, "
2274                                 "cmd %p)", dev, cmd);
2275                         /*
2276                          * Lock used to protect other flags in the bitfield
2277                          * (just in case, actually). Those 2 flags can't be
2278                          * changed in parallel, because the device is
2279                          * serialized.
2280                          */
2281                         spin_lock_bh(&dev->dev_lock);
2282                         dev->dev_double_ua_possible = 0;
2283                         dev->dev_serialized = 0;
2284                         spin_unlock_bh(&dev->dev_lock);
2285                 }
2286         }
2287
2288 out:
2289         TRACE_EXIT_RES(res);
2290         return res;
2291 }
2292
2293 static int scst_check_auto_sense(struct scst_cmd *cmd)
2294 {
2295         int res = 0;
2296
2297         TRACE_ENTRY();
2298
2299         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2300             (!SCST_SENSE_VALID(cmd->sense) ||
2301              SCST_NO_SENSE(cmd->sense))) {
2302                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2303                       "cmd->status=%x, cmd->msg_status=%x, "
2304                       "cmd->host_status=%x, cmd->driver_status=%x (cmd %p)",
2305                       cmd->status, cmd->msg_status, cmd->host_status,
2306                       cmd->driver_status, cmd);
2307                 res = 1;
2308         } else if (unlikely(cmd->host_status)) {
2309                 if ((cmd->host_status == DID_REQUEUE) ||
2310                     (cmd->host_status == DID_IMM_RETRY) ||
2311                     (cmd->host_status == DID_SOFT_ERROR) ||
2312                     (cmd->host_status == DID_ABORT)) {
2313                         scst_set_busy(cmd);
2314                 } else {
2315                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2316                                 "received, returning HARDWARE ERROR instead "
2317                                 "(cmd %p)", cmd->host_status, cmd);
2318                         scst_set_cmd_error(cmd,
2319                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2320                 }
2321         }
2322
2323         TRACE_EXIT_RES(res);
2324         return res;
2325 }
2326
2327 static int scst_pre_dev_done(struct scst_cmd *cmd)
2328 {
2329         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2330
2331         TRACE_ENTRY();
2332
2333         if (unlikely(scst_check_auto_sense(cmd))) {
2334                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2335                             "without sense data (opcode 0x%x), issuing "
2336                             "REQUEST SENSE", cmd->cdb[0]);
2337                 rc = scst_prepare_request_sense(cmd);
2338                 if (rc == 0)
2339                         res = SCST_CMD_STATE_RES_CONT_NEXT;
2340                 else {
2341                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2342                                     "returning HARDWARE ERROR");
2343                         scst_set_cmd_error(cmd,
2344                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2345                 }
2346                 goto out;
2347         } else if (unlikely(scst_check_sense(cmd)))
2348                 goto out;
2349
2350         if (likely(scsi_status_is_good(cmd->status))) {
2351                 unsigned char type = cmd->dev->handler->type;
2352                 if (unlikely((cmd->cdb[0] == MODE_SENSE ||
2353                               cmd->cdb[0] == MODE_SENSE_10)) &&
2354                     cmd->tgt_dev->acg_dev->rd_only_flag &&
2355                     (type == TYPE_DISK ||
2356                      type == TYPE_WORM ||
2357                      type == TYPE_MOD ||
2358                      type == TYPE_TAPE)) {
2359                         int32_t length;
2360                         uint8_t *address;
2361                         bool err = false;
2362
2363                         length = scst_get_buf_first(cmd, &address);
2364                         if (length < 0) {
2365                                 PRINT_ERROR("%s", "Unable to get "
2366                                         "MODE_SENSE buffer");
2367                                 scst_set_cmd_error(cmd,
2368                                         SCST_LOAD_SENSE(
2369                                                 scst_sense_hardw_error));
2370                                 err = true;
2371                         } else if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2372                                 address[2] |= 0x80;   /* Write Protect*/
2373                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2374                                 address[3] |= 0x80;   /* Write Protect*/
2375                         scst_put_buf(cmd, address);
2376
2377                         if (err)
2378                                 goto out;
2379                 }
2380
2381                 /*
2382                  * Check and clear NormACA option for the device, if necessary,
2383                  * since we don't support ACA
2384                  */
2385                 if (unlikely((cmd->cdb[0] == INQUIRY)) &&
2386                     /* Std INQUIRY data (no EVPD) */
2387                     !(cmd->cdb[1] & SCST_INQ_EVPD) &&
2388                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2389                         uint8_t *buffer;
2390                         int buflen;
2391                         bool err = false;
2392
2393                         /* ToDo: all pages ?? */
2394                         buflen = scst_get_buf_first(cmd, &buffer);
2395                         if (buflen > SCST_INQ_BYTE3) {
2396 #ifdef CONFIG_SCST_EXTRACHECKS
2397                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2398                                         PRINT_INFO("NormACA set for device: "
2399                                             "lun=%lld, type 0x%02x. Clear it, "
2400                                             "since it's unsupported.",
2401                                             (long long unsigned int)cmd->lun,
2402                                             buffer[0]);
2403                                 }
2404 #endif
2405                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2406                         } else if (buflen != 0) {
2407                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2408                                     "buffer");
2409                                 scst_set_cmd_error(cmd,
2410                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2411                                 err = true;
2412                         }
2413                         if (buflen > 0)
2414                                 scst_put_buf(cmd, buffer);
2415
2416                         if (err)
2417                                 goto out;
2418                 }
2419
2420                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2421                     (cmd->cdb[0] == MODE_SELECT_10) ||
2422                     (cmd->cdb[0] == LOG_SELECT))) {
2423                         TRACE(TRACE_SCSI,
2424                                 "MODE/LOG SELECT succeeded (LUN %lld)",
2425                                 (long long unsigned int)cmd->lun);
2426                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2427                         goto out;
2428                 }
2429         } else {
2430                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2431                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2432                                         &cmd->tgt_dev->tgt_dev_flags)) {
2433                                 struct scst_tgt_dev *tgt_dev_tmp;
2434                                 struct scst_device *dev = cmd->dev;
2435
2436                                 TRACE(TRACE_SCSI,
2437                                         "Real RESERVE failed lun=%lld, "
2438                                         "status=%x",
2439                                         (long long unsigned int)cmd->lun,
2440                                         cmd->status);
2441                                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2442                                         SCST_SENSE_BUFFERSIZE);
2443
2444                                 /* Clearing the reservation */
2445                                 spin_lock_bh(&dev->dev_lock);
2446                                 list_for_each_entry(tgt_dev_tmp,
2447                                                     &dev->dev_tgt_dev_list,
2448                                                     dev_tgt_dev_list_entry) {
2449                                         clear_bit(SCST_TGT_DEV_RESERVED,
2450                                                 &tgt_dev_tmp->tgt_dev_flags);
2451                                 }
2452                                 dev->dev_reserved = 0;
2453                                 spin_unlock_bh(&dev->dev_lock);
2454                         }
2455                 }
2456
2457                 /* Check for MODE PARAMETERS CHANGED UA */
2458                 if ((cmd->dev->scsi_dev != NULL) &&
2459                     (cmd->status == SAM_STAT_CHECK_CONDITION) &&
2460                     SCST_SENSE_VALID(cmd->sense) &&
2461                     scst_is_ua_sense(cmd->sense) &&
2462                     (cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) {
2463                         TRACE(TRACE_SCSI,
2464                               "MODE PARAMETERS CHANGED UA (lun %lld)",
2465                               (long long unsigned int)cmd->lun);
2466                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2467                         goto out;
2468                 }
2469         }
2470
2471         cmd->state = SCST_CMD_STATE_DEV_DONE;
2472
2473 out:
2474         TRACE_EXIT_RES(res);
2475         return res;
2476 }
2477
2478 static int scst_mode_select_checks(struct scst_cmd *cmd)
2479 {
2480         int res = SCST_CMD_STATE_RES_CONT_SAME;
2481         int atomic = scst_cmd_atomic(cmd);
2482
2483         TRACE_ENTRY();
2484
2485         if (likely(scsi_status_is_good(cmd->status))) {
2486                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2487                     (cmd->cdb[0] == MODE_SELECT_10) ||
2488                     (cmd->cdb[0] == LOG_SELECT))) {
2489                         struct scst_device *dev = cmd->dev;
2490                         if (atomic && (dev->scsi_dev != NULL)) {
2491                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2492                                         "context required");
2493                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2494                                 goto out;
2495                         }
2496
2497                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2498                                 "setting the SELECT UA (lun=%lld)",
2499                                 (long long unsigned int)cmd->lun);
2500
2501                         spin_lock_bh(&dev->dev_lock);
2502                         spin_lock(&scst_temp_UA_lock);
2503                         if (cmd->cdb[0] == LOG_SELECT) {
2504                                 scst_set_sense(scst_temp_UA,
2505                                         sizeof(scst_temp_UA),
2506                                         UNIT_ATTENTION, 0x2a, 0x02);
2507                         } else {
2508                                 scst_set_sense(scst_temp_UA,
2509                                         sizeof(scst_temp_UA),
2510                                         UNIT_ATTENTION, 0x2a, 0x01);
2511                         }
2512                         scst_dev_check_set_local_UA(dev, cmd, scst_temp_UA,
2513                                 sizeof(scst_temp_UA));
2514                         spin_unlock(&scst_temp_UA_lock);
2515                         spin_unlock_bh(&dev->dev_lock);
2516
2517                         if (dev->scsi_dev != NULL)
2518                                 scst_obtain_device_parameters(dev);
2519                 }
2520         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) &&
2521                     SCST_SENSE_VALID(cmd->sense) &&
2522                     scst_is_ua_sense(cmd->sense) &&
2523                     (((cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) ||
2524                      (cmd->sense[12] == 0x29) /* reset */ ||
2525                      (cmd->sense[12] == 0x28) /* medium changed */ ||
2526                      /* cleared by another ini (just in case) */
2527                      (cmd->sense[12] == 0x2F))) {
2528                 if (atomic) {
2529                         TRACE_DBG("Possible parameters changed UA %x: "
2530                                 "thread context required", cmd->sense[12]);
2531                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2532                         goto out;
2533                 }
2534
2535                 TRACE(TRACE_SCSI, "Possible parameters changed UA %x "
2536                         "(lun %lld): getting new parameters", cmd->sense[12],
2537                         (long long unsigned int)cmd->lun);
2538
2539                 scst_obtain_device_parameters(cmd->dev);
2540         } else
2541                 sBUG();
2542
2543         cmd->state = SCST_CMD_STATE_DEV_DONE;
2544
2545 out:
2546         TRACE_EXIT_HRES(res);
2547         return res;
2548 }
2549
2550 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2551 {
2552         if (likely(cmd->sn_set))
2553                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2554
2555         scst_make_deferred_commands_active(cmd->tgt_dev);
2556 }
2557
2558 static int scst_dev_done(struct scst_cmd **pcmd)
2559 {
2560         int res = SCST_CMD_STATE_RES_CONT_SAME;
2561         struct scst_cmd *cmd = *pcmd;
2562         int state;
2563         struct scst_device *dev = cmd->dev;
2564
2565         TRACE_ENTRY();
2566
2567         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2568
2569         if (likely(!scst_is_cmd_local(cmd)) &&
2570             likely(dev->handler->dev_done != NULL)) {
2571                 int rc;
2572
2573                 if (unlikely(!dev->handler->dev_done_atomic &&
2574                              scst_cmd_atomic(cmd))) {
2575                         /*
2576                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2577                          * optimization.
2578                          */
2579                         TRACE_DBG("Dev handler %s dev_done() needs thread "
2580                               "context, rescheduling", dev->handler->name);
2581                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2582                         goto out;
2583                 }
2584
2585                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2586                       dev->handler->name, cmd);
2587                 rc = dev->handler->dev_done(cmd);
2588                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2589                       dev->handler->name, rc);
2590                 if (rc != SCST_CMD_STATE_DEFAULT)
2591                         state = rc;
2592         }
2593
2594         switch (state) {
2595         case SCST_CMD_STATE_PRE_XMIT_RESP:
2596         case SCST_CMD_STATE_DEV_PARSE:
2597         case SCST_CMD_STATE_PRE_PARSE:
2598         case SCST_CMD_STATE_PREPARE_SPACE:
2599         case SCST_CMD_STATE_RDY_TO_XFER:
2600         case SCST_CMD_STATE_TGT_PRE_EXEC:
2601         case SCST_CMD_STATE_SEND_FOR_EXEC:
2602         case SCST_CMD_STATE_LOCAL_EXEC:
2603         case SCST_CMD_STATE_REAL_EXEC:
2604         case SCST_CMD_STATE_PRE_DEV_DONE:
2605         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2606         case SCST_CMD_STATE_DEV_DONE:
2607         case SCST_CMD_STATE_XMIT_RESP:
2608         case SCST_CMD_STATE_FINISHED:
2609                 cmd->state = state;
2610                 break;
2611
2612         case SCST_CMD_STATE_NEED_THREAD_CTX:
2613                 TRACE_DBG("Dev handler %s dev_done() requested "
2614                       "thread context, rescheduling",
2615                       dev->handler->name);
2616                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2617                 break;
2618
2619         default:
2620                 if (state >= 0) {
2621                         PRINT_ERROR("Dev handler %s dev_done() returned "
2622                                 "invalid cmd state %d",
2623                                 dev->handler->name, state);
2624                 } else {
2625                         PRINT_ERROR("Dev handler %s dev_done() returned "
2626                                 "error %d", dev->handler->name,
2627                                 state);
2628                 }
2629                 scst_set_cmd_error(cmd,
2630                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2631                 scst_set_cmd_abnormal_done_state(cmd);
2632                 break;
2633         }
2634
2635         if (cmd->needs_unblocking)
2636                 scst_unblock_dev_cmd(cmd);
2637
2638         if (likely(cmd->dec_on_dev_needed))
2639                 scst_dec_on_dev_cmd(cmd);
2640
2641         if (cmd->inc_expected_sn_on_done && cmd->sent_for_exec)
2642                 scst_inc_check_expected_sn(cmd);
2643
2644         if (unlikely(cmd->cdb[0] == REQUEST_SENSE) && (cmd->internal))
2645                 *pcmd = scst_complete_request_sense(cmd);
2646
2647 out:
2648         TRACE_EXIT_HRES(res);
2649         return res;
2650 }
2651
2652 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2653 {
2654         int res;
2655
2656         TRACE_ENTRY();
2657
2658 #ifdef CONFIG_SCST_DEBUG_TM
2659         if (cmd->tm_dbg_delayed &&
2660                         !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2661                 if (scst_cmd_atomic(cmd)) {
2662                         TRACE_MGMT_DBG("%s",
2663                                 "DEBUG_TM delayed cmd needs a thread");
2664                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2665                         return res;
2666                 }
2667                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2668                         cmd, cmd->tag);
2669                 schedule_timeout_uninterruptible(HZ);
2670         }
2671 #endif
2672
2673         if (likely(cmd->tgt_dev != NULL)) {
2674                 atomic_dec(&cmd->tgt_dev->tgt_dev_cmd_count);
2675                 atomic_dec(&cmd->dev->dev_cmd_count);
2676                 /* If expected values not set, expected direction is UNKNOWN */
2677                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
2678                         atomic_dec(&cmd->dev->write_cmd_count);
2679
2680                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2681                         scst_on_hq_cmd_response(cmd);
2682
2683                 if (unlikely(!cmd->sent_for_exec)) {
2684                         TRACE_SN("cmd %p was not sent to mid-lev"
2685                                 " (sn %ld, set %d)",
2686                                 cmd, cmd->sn, cmd->sn_set);
2687                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2688                         cmd->sent_for_exec = 1;
2689                 }
2690         }
2691
2692         /*
2693          * If we don't remove cmd from the search list here, before
2694          * submitting it for transmittion, we will have a race, when for
2695          * some reason cmd's release is delayed after transmittion and
2696          * initiator sends cmd with the same tag => it is possible that
2697          * a wrong cmd will be found by find() functions.
2698          */
2699         spin_lock_irq(&cmd->sess->sess_list_lock);
2700         list_del(&cmd->search_cmd_list_entry);
2701         spin_unlock_irq(&cmd->sess->sess_list_lock);
2702
2703         cmd->done = 1;
2704         smp_mb(); /* to sync with scst_abort_cmd() */
2705
2706         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2707                 scst_xmit_process_aborted_cmd(cmd);
2708
2709         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2710                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu),"
2711                                 " skipping",
2712                                 cmd, (long long unsigned int)cmd->tag);
2713                 cmd->state = SCST_CMD_STATE_FINISHED;
2714                 res = SCST_CMD_STATE_RES_CONT_SAME;
2715                 goto out;
2716         }
2717
2718         if (cmd->tgt_data_buf_alloced && cmd->dh_data_buf_alloced &&
2719             (cmd->data_direction == SCST_DATA_READ))
2720                 scst_copy_sg(cmd, SCST_SG_COPY_TO_TARGET);
2721
2722         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2723         res = SCST_CMD_STATE_RES_CONT_SAME;
2724
2725 out:
2726 #ifdef CONFIG_SCST_MEASURE_LATENCY
2727         {
2728                 struct timespec ts;
2729                 uint64_t finish, scst_time, proc_time;
2730                 struct scst_session *sess = cmd->sess;
2731
2732                 getnstimeofday(&ts);
2733                 finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2734
2735                 spin_lock_bh(&sess->meas_lock);
2736
2737                 scst_time = cmd->pre_exec_finish - cmd->start;
2738                 scst_time += finish - cmd->post_exec_start;
2739                 proc_time = finish - cmd->start;
2740
2741                 sess->scst_time += scst_time;
2742                 sess->processing_time += proc_time;
2743                 sess->processed_cmds++;
2744
2745                 spin_unlock_bh(&sess->meas_lock);
2746
2747                 TRACE_DBG("cmd %p (sess %p): finish %lld (tv_sec %ld, "
2748                         "tv_nsec %ld), scst_time %lld, proc_time %lld",
2749                         cmd, sess, finish, ts.tv_sec, ts.tv_nsec, scst_time,
2750                         proc_time);
2751         }
2752 #endif
2753         TRACE_EXIT_HRES(res);
2754         return res;
2755 }
2756
2757 static int scst_xmit_response(struct scst_cmd *cmd)
2758 {
2759         int res, rc;
2760
2761         TRACE_ENTRY();
2762
2763         if (unlikely(!cmd->tgtt->xmit_response_atomic &&
2764                      scst_cmd_atomic(cmd))) {
2765                 /*
2766                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2767                  * optimization.
2768                  */
2769                 TRACE_DBG("Target driver %s xmit_response() needs thread "
2770                               "context, rescheduling", cmd->tgtt->name);
2771                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2772                 goto out;
2773         }
2774
2775         while (1) {
2776                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2777
2778                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2779                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2780
2781                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2782
2783 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2784                 if (cmd->tgt_sg) {
2785                         int i;
2786                         struct scatterlist *sg = cmd->tgt_sg;
2787                         TRACE(TRACE_SND_BOT, "Xmitting data for cmd %p "
2788                                 "(sg_cnt %d, sg %p, sg[0].page %p)", cmd,
2789                                 cmd->tgt_sg_cnt, sg, (void *)sg_page(&sg[0]));
2790                         for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
2791                                 PRINT_BUFF_FLAG(TRACE_SND_BOT, "Xmitting sg",
2792                                         sg_virt(&sg[i]), sg[i].length);
2793                         }
2794                 }
2795 #endif
2796
2797 #ifdef CONFIG_SCST_DEBUG_RETRY
2798                 if (((scst_random() % 100) == 77))
2799                         rc = SCST_TGT_RES_QUEUE_FULL;
2800                 else
2801 #endif
2802                         rc = cmd->tgtt->xmit_response(cmd);
2803                 TRACE_DBG("xmit_response() returned %d", rc);
2804
2805                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2806                         goto out;
2807
2808                 /* Restore the previous state */
2809                 cmd->state = SCST_CMD_STATE_XMIT_RESP;
2810
2811                 switch (rc) {
2812                 case SCST_TGT_RES_QUEUE_FULL:
2813                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2814                                 break;
2815                         else
2816                                 continue;
2817
2818                 case SCST_TGT_RES_NEED_THREAD_CTX:
2819                         TRACE_DBG("Target driver %s xmit_response() "
2820                               "requested thread context, rescheduling",
2821                               cmd->tgtt->name);
2822                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2823                         break;
2824
2825                 default:
2826                         goto out_error;
2827                 }
2828                 break;
2829         }
2830
2831 out:
2832         /* Caution: cmd can be already dead here */
2833         TRACE_EXIT_HRES(res);
2834         return res;
2835
2836 out_error:
2837         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2838                 PRINT_ERROR("Target driver %s xmit_response() returned "
2839                         "fatal error", cmd->tgtt->name);
2840         } else {
2841                 PRINT_ERROR("Target driver %s xmit_response() returned "
2842                         "invalid value %d", cmd->tgtt->name, rc);
2843         }
2844         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2845         cmd->state = SCST_CMD_STATE_FINISHED;
2846         res = SCST_CMD_STATE_RES_CONT_SAME;
2847         goto out;
2848 }
2849
2850 void scst_tgt_cmd_done(struct scst_cmd *cmd,
2851         enum scst_exec_context pref_context)
2852 {
2853         TRACE_ENTRY();
2854
2855         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2856
2857         cmd->state = SCST_CMD_STATE_FINISHED;
2858         scst_proccess_redirect_cmd(cmd, pref_context, 1);
2859
2860         TRACE_EXIT();
2861         return;
2862 }
2863 EXPORT_SYMBOL(scst_tgt_cmd_done);
2864
2865 static int scst_finish_cmd(struct scst_cmd *cmd)
2866 {
2867         int res;
2868
2869         TRACE_ENTRY();
2870
2871         atomic_dec(&cmd->sess->sess_cmd_count);
2872
2873         cmd->finished = 1;
2874         smp_mb(); /* to sync with scst_abort_cmd() */
2875
2876         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2877                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
2878                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
2879                         atomic_read(&scst_cmd_count));
2880
2881                 scst_finish_cmd_mgmt(cmd);
2882         }
2883
2884         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
2885                 if ((cmd->tgt_dev != NULL) &&
2886                     scst_is_ua_sense(cmd->sense)) {
2887                         /* This UA delivery failed, so requeue it */
2888                         TRACE_MGMT_DBG("Requeuing UA for delivery failed cmd "
2889                                 "%p", cmd);
2890                         scst_check_set_UA(cmd->tgt_dev, cmd->sense,
2891                                         SCST_SENSE_BUFFERSIZE, 1);
2892                 }
2893         }
2894
2895         __scst_cmd_put(cmd);
2896
2897         res = SCST_CMD_STATE_RES_CONT_NEXT;
2898
2899         TRACE_EXIT_HRES(res);
2900         return res;
2901 }
2902
2903 /*
2904  * No locks, but it must be externally serialized (see comment for
2905  * scst_cmd_init_done() in scst.h)
2906  */
2907 static void scst_cmd_set_sn(struct scst_cmd *cmd)
2908 {
2909         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2910         unsigned long flags;
2911
2912         TRACE_ENTRY();
2913
2914         if (scst_is_implicit_hq(cmd)) {
2915                 TRACE_SN("Implicit HQ cmd %p", cmd);
2916                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
2917         }
2918
2919         EXTRACHECKS_BUG_ON(cmd->sn_set || cmd->hq_cmd_inced);
2920
2921         /* Optimized for lockless fast path */
2922
2923         scst_check_debug_sn(cmd);
2924
2925         if (cmd->dev->queue_alg ==
2926                         SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) {
2927                 /*
2928                  * Not the best way, but well enough until there will be a
2929                  * possibility to specify queue type during pass-through
2930                  * commands submission.
2931                  */
2932                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
2933         }
2934
2935         switch (cmd->queue_type) {
2936         case SCST_CMD_QUEUE_SIMPLE:
2937         case SCST_CMD_QUEUE_UNTAGGED:
2938 #if 0 /* left for future performance investigations */
2939                 if (scst_cmd_is_expected_set(cmd)) {
2940                         if ((cmd->expected_data_direction == SCST_DATA_READ) &&
2941                             (atomic_read(&cmd->dev->write_cmd_count) == 0))
2942                                 goto ordered;
2943                 } else
2944                         goto ordered;
2945 #endif
2946                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
2947                         /*
2948                          * atomic_inc_return() implies memory barrier to sync
2949                          * with scst_inc_expected_sn()
2950                          */
2951                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
2952                                 tgt_dev->curr_sn++;
2953                                 TRACE_SN("Incremented curr_sn %ld",
2954                                         tgt_dev->curr_sn);
2955                         }
2956                         cmd->sn_slot = tgt_dev->cur_sn_slot;
2957                         cmd->sn = tgt_dev->curr_sn;
2958
2959                         tgt_dev->prev_cmd_ordered = 0;
2960                 } else {
2961                         TRACE(TRACE_MINOR, "***WARNING*** Not enough SN slots "
2962                                 "%zd", ARRAY_SIZE(tgt_dev->sn_slots));
2963                         goto ordered;
2964                 }
2965                 break;
2966
2967         case SCST_CMD_QUEUE_ORDERED:
2968                 TRACE_SN("ORDERED cmd %p (op %x)", cmd, cmd->cdb[0]);
2969 ordered:
2970                 if (!tgt_dev->prev_cmd_ordered) {
2971                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2972                         if (tgt_dev->num_free_sn_slots >= 0) {
2973                                 tgt_dev->num_free_sn_slots--;
2974                                 if (tgt_dev->num_free_sn_slots >= 0) {
2975                                         int i = 0;
2976                                         /* Commands can finish in any order, so
2977                                          * we don't know which slot is empty.
2978                                          */
2979                                         while (1) {
2980                                                 tgt_dev->cur_sn_slot++;
2981                                                 if (tgt_dev->cur_sn_slot ==
2982                                                       tgt_dev->sn_slots + ARRAY_SIZE(tgt_dev->sn_slots))
2983                                                         tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
2984
2985                                                 if (atomic_read(tgt_dev->cur_sn_slot) == 0)
2986                                                         break;
2987
2988                                                 i++;
2989                                                 sBUG_ON(i == ARRAY_SIZE(tgt_dev->sn_slots));
2990                                         }
2991                                         TRACE_SN("New cur SN slot %zd",
2992                                                 tgt_dev->cur_sn_slot -
2993                                                 tgt_dev->sn_slots);
2994                                 }
2995                         }
2996                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2997                 }
2998                 tgt_dev->prev_cmd_ordered = 1;
2999                 tgt_dev->curr_sn++;
3000                 cmd->sn = tgt_dev->curr_sn;
3001                 break;
3002
3003         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
3004                 TRACE_SN("HQ cmd %p (op %x)", cmd, cmd->cdb[0]);
3005                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
3006                 tgt_dev->hq_cmd_count++;
3007                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
3008                 cmd->hq_cmd_inced = 1;
3009                 goto out;
3010
3011         default:
3012                 sBUG();
3013         }
3014
3015         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
3016                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
3017                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
3018                 atomic_read(tgt_dev->cur_sn_slot),
3019                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
3020                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
3021
3022         cmd->sn_set = 1;
3023
3024 out:
3025         TRACE_EXIT();
3026         return;
3027 }
3028
3029 /*
3030  * Returns 0 on success, > 0 when we need to wait for unblock,
3031  * < 0 if there is no device (lun) or device type handler.
3032  *
3033  * No locks, but might be on IRQ, protection is done by the
3034  * suspended activity.
3035  */
3036 static int scst_translate_lun(struct scst_cmd *cmd)
3037 {
3038         struct scst_tgt_dev *tgt_dev = NULL;
3039         int res;
3040
3041         TRACE_ENTRY();
3042
3043         __scst_get(1);
3044
3045         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
3046                 struct list_head *sess_tgt_dev_list_head =
3047                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
3048                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %lld)", cmd,
3049                         (long long unsigned int)cmd->lun);
3050                 res = -1;
3051                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3052                                 sess_tgt_dev_list_entry) {
3053                         if (tgt_dev->lun == cmd->lun) {
3054                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
3055
3056                                 if (unlikely(tgt_dev->dev->handler ==
3057                                                 &scst_null_devtype)) {
3058                                         PRINT_INFO("Dev handler for device "
3059                                           "%lld is NULL, the device will not "
3060                                           "be visible remotely",
3061                                            (long long unsigned int)cmd->lun);
3062                                         break;
3063                                 }
3064
3065                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
3066                                 cmd->tgt_dev = tgt_dev;
3067                                 cmd->dev = tgt_dev->dev;
3068
3069                                 res = 0;
3070                                 break;
3071                         }
3072                 }
3073                 if (res != 0) {
3074                         TRACE(TRACE_MINOR,
3075                                 "tgt_dev for lun %lld not found, command to "
3076                                 "unexisting LU?",
3077                                 (long long unsigned int)cmd->lun);
3078                         __scst_put();
3079                 }
3080         } else {
3081                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3082                 __scst_put();
3083                 res = 1;
3084         }
3085
3086         TRACE_EXIT_RES(res);
3087         return res;
3088 }
3089
3090 /*
3091  * No locks, but might be on IRQ
3092  *
3093  * Returns 0 on success, > 0 when we need to wait for unblock,
3094  * < 0 if there is no device (lun) or device type handler.
3095  */
3096 static int __scst_init_cmd(struct scst_cmd *cmd)
3097 {
3098         int res = 0;
3099
3100         TRACE_ENTRY();
3101
3102         res = scst_translate_lun(cmd);
3103         if (likely(res == 0)) {
3104                 int cnt;
3105                 bool failure = false;
3106
3107                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
3108
3109                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
3110                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
3111                         TRACE(TRACE_MGMT_MINOR,
3112                                 "Too many pending commands (%d) in "
3113                                 "session, returning BUSY to initiator \"%s\"",
3114                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
3115                                   "Anonymous" : cmd->sess->initiator_name);
3116                         failure = true;
3117                 }
3118
3119                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
3120                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
3121                         if (!failure) {
3122                                 TRACE(TRACE_MGMT_MINOR,
3123                                         "Too many pending device "
3124                                         "commands (%d), returning BUSY to "
3125                                         "initiator \"%s\"", cnt,
3126                                         (cmd->sess->initiator_name[0] == '\0') ?
3127                                                 "Anonymous" :
3128                                                 cmd->sess->initiator_name);
3129                                 failure = true;
3130                         }
3131                 }
3132
3133                 /* If expected values not set, expected direction is UNKNOWN */
3134                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
3135                         atomic_inc(&cmd->dev->write_cmd_count);
3136
3137                 if (unlikely(failure))
3138                         goto out_busy;
3139
3140                 if (!cmd->set_sn_on_restart_cmd)
3141                         scst_cmd_set_sn(cmd);
3142         } else if (res < 0) {
3143                 TRACE_DBG("Finishing cmd %p", cmd);
3144                 scst_set_cmd_error(cmd,
3145                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
3146                 scst_set_cmd_abnormal_done_state(cmd);
3147         } else
3148                 goto out;
3149
3150 out:
3151         TRACE_EXIT_RES(res);
3152         return res;
3153
3154 out_busy:
3155         scst_set_busy(cmd);
3156         scst_set_cmd_abnormal_done_state(cmd);
3157         goto out;
3158 }
3159
3160 /* Called under scst_init_lock and IRQs disabled */
3161 static void scst_do_job_init(void)
3162         __releases(&scst_init_lock)
3163         __acquires(&scst_init_lock)
3164 {
3165         struct scst_cmd *cmd;
3166         int susp;
3167
3168         TRACE_ENTRY();
3169
3170 restart:
3171         /*
3172          * There is no need for read barrier here, because we don't care where
3173          * this check will be done.
3174          */
3175         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
3176         if (scst_init_poll_cnt > 0)
3177                 scst_init_poll_cnt--;
3178
3179         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
3180                 int rc;
3181                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
3182                         continue;
3183                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
3184                         spin_unlock_irq(&scst_init_lock);
3185                         rc = __scst_init_cmd(cmd);
3186                         spin_lock_irq(&scst_init_lock);
3187                         if (rc > 0) {
3188                                 TRACE_MGMT_DBG("%s",
3189                                         "FLAG SUSPENDED set, restarting");
3190                                 goto restart;
3191                         }
3192                 } else {
3193                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
3194                                        cmd, (long long unsigned int)cmd->tag);
3195                         scst_set_cmd_abnormal_done_state(cmd);
3196                 }
3197
3198                 /*
3199                  * Deleting cmd from init cmd list after __scst_init_cmd()
3200                  * is necessary to keep the check in scst_init_cmd() correct
3201                  * to preserve the commands order.
3202                  *
3203                  * We don't care about the race, when init cmd list is empty
3204                  * and one command detected that it just was not empty, so
3205                  * it's inserting to it, but another command at the same time
3206                  * seeing init cmd list empty and goes directly, because it
3207                  * could affect only commands from the same initiator to the
3208                  * same tgt_dev, but init_cmd_done() doesn't guarantee the order
3209                  * in case of simultaneous such calls anyway.
3210                  */
3211                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
3212                 smp_wmb();
3213                 list_del(&cmd->cmd_list_entry);
3214                 spin_unlock(&scst_init_lock);
3215
3216                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
3217                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
3218                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
3219                         list_add(&cmd->cmd_list_entry,
3220                                 &cmd->cmd_lists->active_cmd_list);
3221                 else
3222                         list_add_tail(&cmd->cmd_list_entry,
3223                                 &cmd->cmd_lists->active_cmd_list);
3224                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3225                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
3226
3227                 spin_lock(&scst_init_lock);
3228                 goto restart;
3229         }
3230
3231         /* It isn't really needed, but let's keep it */
3232         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
3233                 goto restart;
3234
3235         TRACE_EXIT();
3236         return;
3237 }
3238
3239 static inline int test_init_cmd_list(void)
3240 {
3241         int res = (!list_empty(&scst_init_cmd_list) &&
3242                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
3243                   unlikely(kthread_should_stop()) ||
3244                   (scst_init_poll_cnt > 0);
3245         return res;
3246 }
3247
3248 int scst_init_cmd_thread(void *arg)
3249 {
3250         TRACE_ENTRY();
3251
3252         PRINT_INFO("Init thread started, PID %d", current->pid);
3253
3254         current->flags |= PF_NOFREEZE;
3255
3256         set_user_nice(current, -10);
3257
3258         spin_lock_irq(&scst_init_lock);
3259         while (!kthread_should_stop()) {
3260                 wait_queue_t wait;
3261                 init_waitqueue_entry(&wait, current);
3262
3263                 if (!test_init_cmd_list()) {
3264                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
3265                                                  &wait);
3266                         for (;;) {
3267                                 set_current_state(TASK_INTERRUPTIBLE);
3268                                 if (test_init_cmd_list())
3269                                         break;
3270                                 spin_unlock_irq(&scst_init_lock);
3271                                 schedule();
3272                                 spin_lock_irq(&scst_init_lock);
3273                         }
3274                         set_current_state(TASK_RUNNING);
3275                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
3276                 }
3277                 scst_do_job_init();
3278         }
3279         spin_unlock_irq(&scst_init_lock);
3280
3281         /*
3282          * If kthread_should_stop() is true, we are guaranteed to be
3283          * on the module unload, so scst_init_cmd_list must be empty.
3284          */
3285         sBUG_ON(!list_empty(&scst_init_cmd_list));
3286
3287         PRINT_INFO("Init thread PID %d finished", current->pid);
3288
3289         TRACE_EXIT();
3290         return 0;
3291 }
3292
3293 /* Called with no locks held */
3294 void scst_process_active_cmd(struct scst_cmd *cmd, bool atomic)
3295 {
3296         int res;
3297
3298         TRACE_ENTRY();
3299
3300         EXTRACHECKS_BUG_ON(in_irq() || irqs_disabled());
3301
3302         cmd->atomic = atomic;
3303
3304         TRACE_DBG("cmd %p, atomic %d", cmd, atomic);
3305
3306         do {
3307                 switch (cmd->state) {
3308                 case SCST_CMD_STATE_PRE_PARSE:
3309                         res = scst_pre_parse(cmd);
3310                         EXTRACHECKS_BUG_ON(res ==
3311                                 SCST_CMD_STATE_RES_NEED_THREAD);
3312                         break;
3313
3314                 case SCST_CMD_STATE_DEV_PARSE:
3315                         res = scst_parse_cmd(cmd);
3316                         break;
3317
3318                 case SCST_CMD_STATE_PREPARE_SPACE:
3319                         res = scst_prepare_space(cmd);
3320                         break;
3321
3322                 case SCST_CMD_STATE_RDY_TO_XFER:
3323                         res = scst_rdy_to_xfer(cmd);
3324                         break;
3325
3326                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3327                         res = scst_tgt_pre_exec(cmd);
3328                         break;
3329
3330                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3331                         if (tm_dbg_check_cmd(cmd) != 0) {
3332                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
3333                                 TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
3334                                         "because of TM DBG delay", cmd,
3335                                         (long long unsigned int)cmd->tag);
3336                                 break;
3337                         }
3338                         res = scst_send_for_exec(&cmd);
3339                         /*
3340                          * !! At this point cmd, sess & tgt_dev can already be
3341                          * freed !!
3342                          */
3343                         break;
3344
3345                 case SCST_CMD_STATE_LOCAL_EXEC:
3346                         res = scst_local_exec(cmd);
3347                         /*
3348                          * !! At this point cmd, sess & tgt_dev can already be
3349                          * freed !!
3350                          */
3351                         break;
3352
3353                 case SCST_CMD_STATE_REAL_EXEC:
3354                         res = scst_real_exec(cmd);
3355                         /*
3356                          * !! At this point cmd, sess & tgt_dev can already be
3357                          * freed !!
3358                          */
3359                         break;
3360
3361                 case SCST_CMD_STATE_PRE_DEV_DONE:
3362                         res = scst_pre_dev_done(cmd);
3363                         EXTRACHECKS_BUG_ON(res ==
3364                                 SCST_CMD_STATE_RES_NEED_THREAD);
3365                         break;
3366
3367                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3368                         res = scst_mode_select_checks(cmd);
3369                         break;
3370
3371                 case SCST_CMD_STATE_DEV_DONE:
3372                         res = scst_dev_done(&cmd);
3373                         break;
3374
3375                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3376                         res = scst_pre_xmit_response(cmd);
3377                         EXTRACHECKS_BUG_ON(res ==
3378                                 SCST_CMD_STATE_RES_NEED_THREAD);
3379                         break;
3380
3381                 case SCST_CMD_STATE_XMIT_RESP:
3382                         res = scst_xmit_response(cmd);
3383                         break;
3384
3385                 case SCST_CMD_STATE_FINISHED:
3386                         res = scst_finish_cmd(cmd);
3387                         EXTRACHECKS_BUG_ON(res ==
3388                                 SCST_CMD_STATE_RES_NEED_THREAD);
3389                         break;
3390
3391                 default:
3392                         PRINT_CRIT_ERROR("cmd (%p) in state %d, but shouldn't "
3393                                 "be", cmd, cmd->state);
3394                         sBUG();
3395                         res = SCST_CMD_STATE_RES_CONT_NEXT;
3396                         break;
3397                 }
3398         } while (res == SCST_CMD_STATE_RES_CONT_SAME);
3399
3400         if (res == SCST_CMD_STATE_RES_CONT_NEXT) {
3401                 /* None */
3402         } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
3403                 spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3404                 switch (cmd->state) {
3405                 case SCST_CMD_STATE_PRE_PARSE:
3406                 case SCST_CMD_STATE_DEV_PARSE:
3407                 case SCST_CMD_STATE_PREPARE_SPACE:
3408                 case SCST_CMD_STATE_RDY_TO_XFER:
3409                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3410                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3411                 case SCST_CMD_STATE_LOCAL_EXEC:
3412                 case SCST_CMD_STATE_REAL_EXEC:
3413                 case SCST_CMD_STATE_PRE_DEV_DONE:
3414                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3415                 case SCST_CMD_STATE_DEV_DONE:
3416                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3417                 case SCST_CMD_STATE_XMIT_RESP:
3418                 case SCST_CMD_STATE_FINISHED:
3419                         TRACE_DBG("Adding cmd %p to head of active cmd list",
3420                                   cmd);
3421                         list_add(&cmd->cmd_list_entry,
3422                                 &cmd->cmd_lists->active_cmd_list);
3423                         break;
3424 #ifdef CONFIG_SCST_EXTRACHECKS
3425                 /* not very valid commands */
3426                 case SCST_CMD_STATE_DEFAULT:
3427                 case SCST_CMD_STATE_NEED_THREAD_CTX:
3428                         PRINT_CRIT_ERROR("cmd %p is in invalid state %d)", cmd,
3429                                 cmd->state);
3430                         spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3431                         sBUG();
3432                         spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3433                         break;
3434 #endif
3435                 default:
3436                         break;
3437                 }
3438                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3439                 spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3440         } else
3441                 sBUG();
3442
3443         TRACE_EXIT();
3444         return;
3445 }
3446 EXPORT_SYMBOL(scst_process_active_cmd);
3447
3448 /* Called under cmd_list_lock and IRQs disabled */
3449 static void scst_do_job_active(struct list_head *cmd_list,
3450         spinlock_t *cmd_list_lock, bool atomic)
3451         __releases(cmd_list_lock)
3452         __acquires(cmd_list_lock)
3453 {
3454         TRACE_ENTRY();
3455
3456         while (!list_empty(cmd_list)) {
3457                 struct scst_cmd *cmd = list_entry(cmd_list->next, typeof(*cmd),
3458                                         cmd_list_entry);
3459                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
3460                 list_del(&cmd->cmd_list_entry);
3461                 spin_unlock_irq(cmd_list_lock);
3462                 scst_process_active_cmd(cmd, atomic);
3463                 spin_lock_irq(cmd_list_lock);
3464         }
3465
3466         TRACE_EXIT();
3467         return;
3468 }
3469
3470 static inline int test_cmd_lists(struct scst_cmd_lists *p_cmd_lists)
3471 {
3472         int res = !list_empty(&p_cmd_lists->active_cmd_list) ||
3473             unlikely(kthread_should_stop()) ||
3474             tm_dbg_is_release();
3475         return res;
3476 }
3477
3478 int scst_cmd_thread(void *arg)
3479 {
3480         struct scst_cmd_lists *p_cmd_lists = (struct scst_cmd_lists *)arg;
3481
3482         TRACE_ENTRY();
3483
3484         PRINT_INFO("Processing thread started, PID %d", current->pid);
3485
3486 #if 0
3487         set_user_nice(current, 10);
3488 #endif
3489         current->flags |= PF_NOFREEZE;
3490
3491         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3492         while (!kthread_should_stop()) {
3493                 wait_queue_t wait;
3494                 init_waitqueue_entry(&wait, current);
3495
3496                 if (!test_cmd_lists(p_cmd_lists)) {
3497                         add_wait_queue_exclusive(&p_cmd_lists->cmd_list_waitQ,
3498                                 &wait);
3499                         for (;;) {
3500                                 set_current_state(TASK_INTERRUPTIBLE);
3501                                 if (test_cmd_lists(p_cmd_lists))
3502                                         break;
3503                                 spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3504                                 schedule();
3505                   &n