a4f5e4f57395ee0efe1454c81fd25b2ecd19d47b
[mirror/scst/.git] / scst / src / scst_targ.c
1 /*
2  *  scst_targ.c
3  *
4  *  Copyright (C) 2004 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2004 - 2005 Leonid Stoljar
6  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
7  *
8  *  This program is free software; you can redistribute it and/or
9  *  modify it under the terms of the GNU General Public License
10  *  as published by the Free Software Foundation, version 2
11  *  of the License.
12  *
13  *  This program is distributed in the hope that it will be useful,
14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  *  GNU General Public License for more details.
17  */
18
19 #include <linux/init.h>
20 #include <linux/kernel.h>
21 #include <linux/errno.h>
22 #include <linux/list.h>
23 #include <linux/spinlock.h>
24 #include <linux/slab.h>
25 #include <linux/sched.h>
26 #include <linux/smp_lock.h>
27 #include <linux/unistd.h>
28 #include <linux/string.h>
29 #include <linux/kthread.h>
30 #include <linux/delay.h>
31
32 #include "scst.h"
33 #include "scst_priv.h"
34
35 static void scst_cmd_set_sn(struct scst_cmd *cmd);
36 static int __scst_init_cmd(struct scst_cmd *cmd);
37 static void scst_finish_cmd_mgmt(struct scst_cmd *cmd);
38
39 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
40 {
41         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
42         unsigned long flags;
43
44         spin_lock_irqsave(&t->tasklet_lock, flags);
45         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
46                 smp_processor_id());
47         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
48         spin_unlock_irqrestore(&t->tasklet_lock, flags);
49
50         tasklet_schedule(&t->tasklet);
51 }
52
53 /*
54  * Must not be called in parallel with scst_unregister_session_ex() for the
55  * same sess
56  */
57 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
58                              const uint8_t *lun, int lun_len,
59                              const uint8_t *cdb, int cdb_len, int atomic)
60 {
61         struct scst_cmd *cmd;
62
63         TRACE_ENTRY();
64
65 #ifdef CONFIG_SCST_EXTRACHECKS
66         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
67                 PRINT_CRIT_ERROR("%s", "New cmd while shutting down the session");
68                 sBUG();
69         }
70 #endif
71
72         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
73         if (cmd == NULL)
74                 goto out;
75
76         cmd->sess = sess;
77         cmd->tgt = sess->tgt;
78         cmd->tgtt = sess->tgt->tgtt;
79
80         /*
81          * For both wrong lun and CDB defer the error reporting for
82          * scst_cmd_init_done()
83          */
84
85         cmd->lun = scst_unpack_lun(lun, lun_len);
86
87         if (cdb_len <= SCST_MAX_CDB_SIZE) {
88                 memcpy(cmd->cdb, cdb, cdb_len);
89                 cmd->cdb_len = cdb_len;
90         }
91
92         TRACE_DBG("cmd %p, sess %p", cmd, sess);
93         scst_sess_get(sess);
94
95 out:
96         TRACE_EXIT();
97         return cmd;
98 }
99 EXPORT_SYMBOL(scst_rx_cmd);
100
101 static int scst_init_cmd(struct scst_cmd *cmd, int context)
102 {
103         int rc;
104
105         TRACE_ENTRY();
106
107         /* See the comment in scst_do_job_init() */
108         if (unlikely(!list_empty(&scst_init_cmd_list))) {
109                 TRACE_MGMT_DBG("%s", "init cmd list busy");
110                 goto out_redirect;
111         }
112         /*
113          * Memory barrier isn't necessary here, because CPU appears to
114          * be self-consistent
115          */
116
117         rc = __scst_init_cmd(cmd);
118         if (unlikely(rc > 0))
119                 goto out_redirect;
120         else if (unlikely(rc != 0))
121                 goto out;
122
123         /* Small context optimization */
124         if (((context == SCST_CONTEXT_TASKLET) ||
125              (context == SCST_CONTEXT_DIRECT_ATOMIC)) &&
126             scst_cmd_is_expected_set(cmd)) {
127                 if (cmd->expected_data_direction == SCST_DATA_WRITE) {
128                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
129                                         &cmd->tgt_dev->tgt_dev_flags))
130                                 context = SCST_CONTEXT_THREAD;
131                 } else {
132                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
133                                         &cmd->tgt_dev->tgt_dev_flags))
134                                 context = SCST_CONTEXT_THREAD;
135                 }
136         }
137
138 out:
139         TRACE_EXIT_RES(context);
140         return context;
141
142 out_redirect:
143         if (cmd->preprocessing_only) {
144                 /*
145                  * Poor man solution for single threaded targets, where
146                  * blocking receiver at least sometimes means blocking all.
147                  */
148                 sBUG_ON(context != SCST_CONTEXT_DIRECT);
149                 scst_set_busy(cmd);
150                 scst_set_cmd_abnormal_done_state(cmd);
151                 /* Keep initiator away from too many BUSY commands */
152                 if (!in_interrupt() && !in_atomic())
153                         msleep(50);
154                 else
155                         WARN_ON_ONCE(1);
156         } else {
157                 unsigned long flags;
158                 spin_lock_irqsave(&scst_init_lock, flags);
159                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
160                         "%d)", cmd, atomic_read(&scst_cmd_count));
161                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
162                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
163                         scst_init_poll_cnt++;
164                 spin_unlock_irqrestore(&scst_init_lock, flags);
165                 wake_up(&scst_init_cmd_list_waitQ);
166                 context = -1;
167         }
168         goto out;
169 }
170
171 #ifdef CONFIG_SCST_MEASURE_LATENCY
172 static inline uint64_t scst_sec_to_nsec(time_t sec)
173 {
174         return (uint64_t)sec * 1000000000;
175 }
176 #endif
177
178 void scst_cmd_init_done(struct scst_cmd *cmd, int pref_context)
179 {
180         unsigned long flags;
181         struct scst_session *sess = cmd->sess;
182
183         TRACE_ENTRY();
184
185 #ifdef CONFIG_SCST_MEASURE_LATENCY
186         {
187                 struct timespec ts;
188                 getnstimeofday(&ts);
189                 cmd->start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
190                 TRACE_DBG("cmd %p (sess %p): start %lld (tv_sec %ld, "
191                         "tv_nsec %ld)", cmd, sess, cmd->start, ts.tv_sec,
192                         ts.tv_nsec);
193         }
194 #endif
195
196         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
197         TRACE(TRACE_SCSI, "tag=%llu, lun=%lld, CDB len=%d",
198               (long long unsigned int)cmd->tag,
199               (long long unsigned int)cmd->lun,
200               cmd->cdb_len);
201         PRINT_BUFF_FLAG(TRACE_SCSI|TRACE_RCV_BOT, "Recieving CDB",
202                 cmd->cdb, cmd->cdb_len);
203
204 #ifdef CONFIG_SCST_EXTRACHECKS
205         if (unlikely(in_irq()) && ((pref_context == SCST_CONTEXT_DIRECT) ||
206                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
207                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
208                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
209                         cmd->tgtt->name);
210                 pref_context = SCST_CONTEXT_TASKLET;
211         }
212 #endif
213
214         atomic_inc(&sess->sess_cmd_count);
215
216         spin_lock_irqsave(&sess->sess_list_lock, flags);
217
218         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
219                 /*
220                  * We have to always keep command in the search list from the
221                  * very beginning, because otherwise it can be missed during
222                  * TM processing. This check is needed because there might be
223                  * old, i.e. deferred, commands and new, i.e. just coming, ones.
224                  */
225                 if (cmd->search_cmd_list_entry.next == NULL)
226                         list_add_tail(&cmd->search_cmd_list_entry,
227                                 &sess->search_cmd_list);
228                 switch (sess->init_phase) {
229                 case SCST_SESS_IPH_SUCCESS:
230                         break;
231                 case SCST_SESS_IPH_INITING:
232                         TRACE_DBG("Adding cmd %p to init deferred cmd list", cmd);
233                         list_add_tail(&cmd->cmd_list_entry,
234                                 &sess->init_deferred_cmd_list);
235                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
236                         goto out;
237                 case SCST_SESS_IPH_FAILED:
238                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
239                         scst_set_busy(cmd);
240                         scst_set_cmd_abnormal_done_state(cmd);
241                         goto active;
242                 default:
243                         sBUG();
244                 }
245         } else
246                 list_add_tail(&cmd->search_cmd_list_entry, &sess->search_cmd_list);
247
248         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
249
250         if (unlikely(cmd->lun == NO_SUCH_LUN)) {
251                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
252                 scst_set_cmd_error(cmd,
253                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
254                 scst_set_cmd_abnormal_done_state(cmd);
255                 goto active;
256         }
257
258         if (unlikely(cmd->cdb_len == 0)) {
259                 PRINT_ERROR("Wrong CDB len %d, finishing cmd", 0);
260                 scst_set_cmd_error(cmd,
261                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
262                 scst_set_cmd_abnormal_done_state(cmd);
263                 goto active;
264         }
265
266         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
267                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
268                 scst_set_cmd_error(cmd,
269                         SCST_LOAD_SENSE(scst_sense_invalid_message));
270                 scst_set_cmd_abnormal_done_state(cmd);
271                 goto active;
272         }
273
274         cmd->state = SCST_CMD_STATE_INIT;
275         /* cmd must be inited here to preserve the order */
276         pref_context = scst_init_cmd(cmd, pref_context);
277         if (unlikely(pref_context < 0))
278                 goto out;
279
280 active:
281         /* Here cmd must not be in any cmd list, no locks */
282         switch (pref_context) {
283         case SCST_CONTEXT_TASKLET:
284                 scst_schedule_tasklet(cmd);
285                 break;
286
287         case SCST_CONTEXT_DIRECT:
288         case SCST_CONTEXT_DIRECT_ATOMIC:
289                 scst_process_active_cmd(cmd, pref_context);
290                 /* For *NEED_THREAD wake_up() is already done */
291                 break;
292
293         default:
294                 PRINT_ERROR("Context %x is undefined, using the thread one",
295                         pref_context);
296                 /* go through */
297         case SCST_CONTEXT_THREAD:
298                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
299                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
300                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
301                         list_add(&cmd->cmd_list_entry,
302                                 &cmd->cmd_lists->active_cmd_list);
303                 else
304                         list_add_tail(&cmd->cmd_list_entry,
305                                 &cmd->cmd_lists->active_cmd_list);
306                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
307                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
308                 break;
309         }
310
311 out:
312         TRACE_EXIT();
313         return;
314 }
315 EXPORT_SYMBOL(scst_cmd_init_done);
316
317 static int scst_pre_parse(struct scst_cmd *cmd)
318 {
319         int res = SCST_CMD_STATE_RES_CONT_SAME;
320         struct scst_device *dev = cmd->dev;
321         int rc;
322
323         TRACE_ENTRY();
324
325         cmd->inc_expected_sn_on_done = dev->handler->exec_sync ||
326                 (!dev->has_own_order_mgmt &&
327                  ((dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) ||
328                   (cmd->queue_type == SCST_CMD_QUEUE_ORDERED)));
329
330         /*
331          * Expected transfer data supplied by the SCSI transport via the
332          * target driver are untrusted, so we prefer to fetch them from CDB.
333          * Additionally, not all transports support supplying the expected
334          * transfer data.
335          */
336
337         rc = scst_get_cdb_info(cmd);
338         if (unlikely(rc != 0)) {
339                 if (rc > 0) {
340                         PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
341                         goto out_xmit;
342                 }
343                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
344                         "Should you update scst_scsi_op_table?",
345                         cmd->cdb[0], dev->handler->name);
346                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
347 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
348                 if (scst_cmd_is_expected_set(cmd)) {
349                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
350                                 "direction %d, transfer_len %d",
351                                 cmd->expected_data_direction,
352                                 cmd->expected_transfer_len);
353                         cmd->data_direction = cmd->expected_data_direction;
354
355                         cmd->bufflen = cmd->expected_transfer_len;
356                         /* Restore (likely) lost CDB length */
357                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
358                         if (cmd->cdb_len == -1) {
359                                 PRINT_ERROR("Unable to get CDB length for "
360                                         "opcode 0x%02x. Returning INVALID "
361                                         "OPCODE", cmd->cdb[0]);
362                                 scst_set_cmd_error(cmd,
363                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
364                                 goto out_xmit;
365                         }
366                 } else {
367                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
368                              "target %s not supplied expected values",
369                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
370                         scst_set_cmd_error(cmd,
371                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
372                         goto out_xmit;
373                 }
374 #else
375                 scst_set_cmd_error(cmd,
376                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
377                 goto out_xmit;
378 #endif
379         } else {
380                 TRACE(TRACE_SCSI, "op_name <%s>, direction=%d (expected %d, "
381                         "set %s), transfer_len=%d (expected len %d), flags=%d",
382                         cmd->op_name, cmd->data_direction,
383                         cmd->expected_data_direction,
384                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
385                         cmd->bufflen, cmd->expected_transfer_len, cmd->op_flags);
386
387                 if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
388                         if (scst_cmd_is_expected_set(cmd)) {
389                                 /*
390                                  * Command data length can't be easily
391                                  * determined from the CDB. ToDo, all such
392                                  * commands processing should be fixed. Until
393                                  * it's done, get the length from the supplied
394                                  * expected value, but limit it to some
395                                  * reasonable value (15MB).
396                                  */
397                                 cmd->bufflen = min(cmd->expected_transfer_len,
398                                                         15*1024*1024);
399                                 cmd->op_flags &= ~SCST_UNKNOWN_LENGTH;
400                         } else
401                                 cmd->bufflen = 0;
402                 }
403         }
404
405         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
406                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
407                             "(opcode 0x%02x)", cmd->cdb[0]);
408                 scst_set_cmd_error(cmd,
409                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
410                 goto out_xmit;
411         }
412
413         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
414                 PRINT_ERROR("Linked commands are not supported "
415                             "(opcode 0x%02x)", cmd->cdb[0]);
416                 scst_set_cmd_error(cmd,
417                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
418                 goto out_xmit;
419         }
420
421         cmd->state = SCST_CMD_STATE_DEV_PARSE;
422
423 out:
424         TRACE_EXIT_RES(res);
425         return res;
426
427 out_xmit:
428         scst_set_cmd_abnormal_done_state(cmd);
429         res = SCST_CMD_STATE_RES_CONT_SAME;
430         goto out;
431 }
432
433 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
434 static bool scst_is_allowed_to_mismatch_cmd(struct scst_cmd *cmd)
435 {
436         bool res = false;
437
438         switch (cmd->cdb[0]) {
439         case TEST_UNIT_READY:
440                 /* Crazy VMware people sometimes do TUR with READ direction */
441                 res = true;
442                 break;
443         case VERIFY:
444         case VERIFY_6:
445         case VERIFY_12:
446         case VERIFY_16:
447                 /* VERIFY commands with BYTCHK unset shouldn't fail here */
448                 if (((cmd->op_flags & SCST_VERIFY_BYTCHK_MISMATCH_ALLOWED) != 0) &&
449                     (cmd->cdb[1] & BYTCHK) == 0)
450                         res = true;
451                 break;
452         }
453
454         return res;
455 }
456 #endif
457
458 static int scst_parse_cmd(struct scst_cmd *cmd)
459 {
460         int res = SCST_CMD_STATE_RES_CONT_SAME;
461         int state;
462         struct scst_device *dev = cmd->dev;
463         int orig_bufflen = cmd->bufflen;
464
465         TRACE_ENTRY();
466
467         if (likely(!scst_is_cmd_local(cmd))) {
468                 if (unlikely(!dev->handler->parse_atomic &&
469                              scst_cmd_atomic(cmd))) {
470                         /*
471                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
472                          * optimization.
473                          */
474                         TRACE_DBG("Dev handler %s parse() needs thread "
475                                 "context, rescheduling", dev->handler->name);
476                         res = SCST_CMD_STATE_RES_NEED_THREAD;
477                         goto out;
478                 }
479
480                 TRACE_DBG("Calling dev handler %s parse(%p)",
481                       dev->handler->name, cmd);
482                 TRACE_BUFF_FLAG(TRACE_SND_BOT, "Parsing: ", cmd->cdb, cmd->cdb_len);
483                 state = dev->handler->parse(cmd);
484                 /* Caution: cmd can be already dead here */
485                 TRACE_DBG("Dev handler %s parse() returned %d",
486                         dev->handler->name, state);
487
488                 switch (state) {
489                 case SCST_CMD_STATE_NEED_THREAD_CTX:
490                         TRACE_DBG("Dev handler %s parse() requested thread "
491                               "context, rescheduling", dev->handler->name);
492                         res = SCST_CMD_STATE_RES_NEED_THREAD;
493                         goto out;
494
495                 case SCST_CMD_STATE_STOP:
496                         TRACE_DBG("Dev handler %s parse() requested stop "
497                                 "processing", dev->handler->name);
498                         res = SCST_CMD_STATE_RES_CONT_NEXT;
499                         goto out;
500                 }
501
502                 if (state == SCST_CMD_STATE_DEFAULT)
503                         state = SCST_CMD_STATE_PREPARE_SPACE;
504         } else
505                 state = SCST_CMD_STATE_PREPARE_SPACE;
506
507         if (cmd->data_len == -1)
508                 cmd->data_len = cmd->bufflen;
509
510         if (cmd->data_buf_alloced && unlikely((orig_bufflen > cmd->bufflen))) {
511                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
512                         "is less, than required (size %d)", cmd->bufflen,
513                         orig_bufflen);
514                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
515                 goto out_error;
516         }
517
518         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
519                 goto set_res;
520
521         if (unlikely((cmd->bufflen == 0) &&
522                      (cmd->op_flags & SCST_UNKNOWN_LENGTH))) {
523                 PRINT_ERROR("Unknown data transfer length for opcode 0x%x "
524                         "(handler %s, target %s)", cmd->cdb[0],
525                         dev->handler->name, cmd->tgtt->name);
526                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
527                 goto out_error;
528         }
529
530 #ifdef CONFIG_SCST_EXTRACHECKS
531         if ((cmd->bufflen != 0) &&
532             ((cmd->data_direction == SCST_DATA_NONE) ||
533              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
534                 PRINT_ERROR("Dev handler %s parse() returned "
535                         "invalid cmd data_direction %d, bufflen %d, state %d "
536                         "or sg %p (opcode 0x%x)", dev->handler->name,
537                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
538                         cmd->cdb[0]);
539                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
540                 goto out_error;
541         }
542 #endif
543
544         if (scst_cmd_is_expected_set(cmd)) {
545 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
546 #       ifdef CONFIG_SCST_EXTRACHECKS
547                 if ((cmd->data_direction != cmd->expected_data_direction) ||
548                     (cmd->bufflen != cmd->expected_transfer_len)) {
549                         PRINT_WARNING("Expected values don't match decoded ones: "
550                                 "data_direction %d, expected_data_direction %d, "
551                                 "bufflen %d, expected_transfer_len %d",
552                                 cmd->data_direction, cmd->expected_data_direction,
553                                 cmd->bufflen, cmd->expected_transfer_len);
554                         PRINT_BUFFER("Suspicious CDB", cmd->cdb, cmd->cdb_len);
555                 }
556 #       endif
557                 cmd->data_direction = cmd->expected_data_direction;
558                 cmd->bufflen = cmd->expected_transfer_len;
559 #else
560                 if (unlikely(cmd->data_direction != cmd->expected_data_direction)) {
561                         if (((cmd->expected_data_direction != SCST_DATA_NONE) ||
562                              (cmd->bufflen != 0)) &&
563                             !scst_is_allowed_to_mismatch_cmd(cmd)) {
564                                 PRINT_ERROR("Expected data direction %d for opcode "
565                                         "0x%02x (handler %s, target %s) doesn't match "
566                                         "decoded value %d", cmd->expected_data_direction,
567                                         cmd->cdb[0], dev->handler->name,
568                                         cmd->tgtt->name, cmd->data_direction);
569                                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
570                                 scst_set_cmd_error(cmd,
571                                         SCST_LOAD_SENSE(scst_sense_invalid_message));
572                                 goto out_dev_done;
573                         }
574                 }
575                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
576                         static int repd;
577                         if (repd < 100) {
578                                 /*
579                                  * Intentionally unlocked. Few messages more
580                                  * or less don't matter.
581                                  */
582                                 repd++;
583                                 TRACE(TRACE_MINOR, "Warning: expected transfer "
584                                         "length %d for opcode 0x%02x (handler "
585                                         "%s, target %s) doesn't match decoded "
586                                         "value %d. Faulty initiator (e.g. "
587                                         "VMware is known to be such) or "
588                                         "scst_scsi_op_table should be updated?",
589                                         cmd->expected_transfer_len, cmd->cdb[0],
590                                         dev->handler->name, cmd->tgtt->name,
591                                         cmd->bufflen);
592                                 PRINT_BUFF_FLAG(TRACE_MINOR, "Suspicious CDB",
593                                         cmd->cdb, cmd->cdb_len);
594                         }
595                 }
596 #endif
597         }
598
599         if (unlikely(cmd->data_direction == SCST_DATA_UNKNOWN)) {
600                 PRINT_ERROR("Unknown data direction. Opcode 0x%x, handler %s, "
601                         "target %s", cmd->cdb[0], dev->handler->name,
602                         cmd->tgtt->name);
603                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
604                 goto out_error;
605         }
606
607 set_res:
608         switch (state) {
609         case SCST_CMD_STATE_PREPARE_SPACE:
610         case SCST_CMD_STATE_PRE_PARSE:
611         case SCST_CMD_STATE_DEV_PARSE:
612         case SCST_CMD_STATE_RDY_TO_XFER:
613         case SCST_CMD_STATE_TGT_PRE_EXEC:
614         case SCST_CMD_STATE_SEND_FOR_EXEC:
615         case SCST_CMD_STATE_LOCAL_EXEC:
616         case SCST_CMD_STATE_REAL_EXEC:
617         case SCST_CMD_STATE_PRE_DEV_DONE:
618         case SCST_CMD_STATE_DEV_DONE:
619         case SCST_CMD_STATE_PRE_XMIT_RESP:
620         case SCST_CMD_STATE_XMIT_RESP:
621         case SCST_CMD_STATE_FINISHED:
622                 cmd->state = state;
623                 res = SCST_CMD_STATE_RES_CONT_SAME;
624                 break;
625
626         default:
627                 if (state >= 0) {
628                         PRINT_ERROR("Dev handler %s parse() returned "
629                              "invalid cmd state %d (opcode %d)",
630                              dev->handler->name, state, cmd->cdb[0]);
631                 } else {
632                         PRINT_ERROR("Dev handler %s parse() returned "
633                                 "error %d (opcode %d)", dev->handler->name,
634                                 state, cmd->cdb[0]);
635                 }
636                 goto out_error;
637         }
638
639         if (cmd->resp_data_len == -1) {
640                 if (cmd->data_direction == SCST_DATA_READ)
641                         cmd->resp_data_len = cmd->bufflen;
642                 else
643                          cmd->resp_data_len = 0;
644         }
645
646 out:
647         TRACE_EXIT_HRES(res);
648         return res;
649
650 out_error:
651         /* dev_done() will be called as part of the regular cmd's finish */
652         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
653
654 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
655 out_dev_done:
656 #endif
657         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
658         res = SCST_CMD_STATE_RES_CONT_SAME;
659         goto out;
660 }
661
662 static int scst_prepare_space(struct scst_cmd *cmd)
663 {
664         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
665
666         TRACE_ENTRY();
667
668         if (cmd->data_direction == SCST_DATA_NONE)
669                 goto prep_done;
670
671         if (cmd->data_buf_tgt_alloc) {
672                 int orig_bufflen = cmd->bufflen;
673
674                 TRACE_MEM("%s", "Custom tgt data buf allocation requested");
675
676                 r = cmd->tgtt->alloc_data_buf(cmd);
677                 if (r > 0)
678                         goto alloc;
679                 else if (r == 0) {
680                         if (unlikely(cmd->bufflen == 0)) {
681                                 /* See comment in scst_alloc_space() */
682                                 if (cmd->sg == NULL)
683                                         goto alloc;
684                         }
685                         cmd->data_buf_alloced = 1;
686                         if (unlikely(orig_bufflen < cmd->bufflen)) {
687                                 PRINT_ERROR("Target driver allocated data "
688                                         "buffer (size %d), is less, than "
689                                         "required (size %d)", orig_bufflen,
690                                         cmd->bufflen);
691                                 goto out_error;
692                         }
693                         TRACE_MEM("%s", "data_buf_alloced, returning");
694                 }
695                 goto check;
696         }
697
698 alloc:
699         if (!cmd->data_buf_alloced)
700                 r = scst_alloc_space(cmd);
701         else
702                 TRACE_MEM("%s", "data_buf_alloced set, returning");
703
704 check:
705         if (r != 0) {
706                 if (scst_cmd_atomic(cmd)) {
707                         TRACE_MEM("%s", "Atomic memory allocation failed, "
708                               "rescheduling to the thread");
709                         res = SCST_CMD_STATE_RES_NEED_THREAD;
710                         goto out;
711                 } else
712                         goto out_no_space;
713         }
714
715 prep_done:
716         if (cmd->preprocessing_only) {
717                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
718                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
719                                 "cmd %p", cmd);
720                         scst_set_cmd_abnormal_done_state(cmd);
721                         res = SCST_CMD_STATE_RES_CONT_SAME;
722                         goto out;
723                 }
724
725                 res = SCST_CMD_STATE_RES_CONT_NEXT;
726                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
727
728                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
729                 cmd->tgtt->preprocessing_done(cmd);
730                 TRACE_DBG("%s", "preprocessing_done() returned");
731                 goto out;
732
733         }
734
735         switch (cmd->data_direction) {
736         case SCST_DATA_WRITE:
737                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
738                 break;
739
740         default:
741                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
742                 break;
743         }
744
745 out:
746         TRACE_EXIT_HRES(res);
747         return res;
748
749 out_no_space:
750         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
751                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
752         scst_set_busy(cmd);
753         scst_set_cmd_abnormal_done_state(cmd);
754         res = SCST_CMD_STATE_RES_CONT_SAME;
755         goto out;
756
757 out_error:
758         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
759         scst_set_cmd_abnormal_done_state(cmd);
760         res = SCST_CMD_STATE_RES_CONT_SAME;
761         goto out;
762 }
763
764 void scst_restart_cmd(struct scst_cmd *cmd, int status, int pref_context)
765 {
766         TRACE_ENTRY();
767
768         TRACE_DBG("Preferred context: %d", pref_context);
769         TRACE_DBG("tag=%llu, status=%#x",
770                   (long long unsigned int)scst_cmd_get_tag(cmd),
771                   status);
772
773 #ifdef CONFIG_SCST_EXTRACHECKS
774         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
775                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
776                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
777                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
778                         cmd->tgtt->name);
779                 pref_context = SCST_CONTEXT_TASKLET;
780         }
781 #endif
782
783         switch (status) {
784         case SCST_PREPROCESS_STATUS_SUCCESS:
785                 switch (cmd->data_direction) {
786                 case SCST_DATA_WRITE:
787                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
788                         break;
789                 default:
790                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
791                         break;
792                 }
793                 if (cmd->set_sn_on_restart_cmd)
794                         scst_cmd_set_sn(cmd);
795                 /* Small context optimization */
796                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
797                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) {
798                         if (cmd->data_direction == SCST_DATA_WRITE) {
799                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
800                                                 &cmd->tgt_dev->tgt_dev_flags))
801                                         pref_context = SCST_CONTEXT_THREAD;
802                         } else {
803                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
804                                                 &cmd->tgt_dev->tgt_dev_flags))
805                                         pref_context = SCST_CONTEXT_THREAD;
806                         }
807                 }
808                 break;
809
810         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
811                 scst_set_cmd_abnormal_done_state(cmd);
812                 break;
813
814         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
815                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
816                 /* go through */
817         case SCST_PREPROCESS_STATUS_ERROR:
818                 scst_set_cmd_error(cmd,
819                            SCST_LOAD_SENSE(scst_sense_hardw_error));
820                 scst_set_cmd_abnormal_done_state(cmd);
821                 break;
822
823         default:
824                 PRINT_ERROR("%s() received unknown status %x", __func__,
825                         status);
826                 scst_set_cmd_abnormal_done_state(cmd);
827                 break;
828         }
829
830         scst_proccess_redirect_cmd(cmd, pref_context, 1);
831
832         TRACE_EXIT();
833         return;
834 }
835 EXPORT_SYMBOL(scst_restart_cmd);
836
837 /* No locks */
838 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
839 {
840         struct scst_tgt *tgt = cmd->sess->tgt;
841         int res = 0;
842         unsigned long flags;
843
844         TRACE_ENTRY();
845
846         spin_lock_irqsave(&tgt->tgt_lock, flags);
847         tgt->retry_cmds++;
848         smp_mb();
849         TRACE_RETRY("TGT QUEUE FULL: incrementing retry_cmds %d",
850               tgt->retry_cmds);
851         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
852                 /* At least one cmd finished, so try again */
853                 tgt->retry_cmds--;
854                 TRACE_RETRY("Some command(s) finished, direct retry "
855                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
856                       "retry_cmds=%d)", finished_cmds,
857                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
858                 res = -1;
859                 goto out_unlock_tgt;
860         }
861
862         TRACE_RETRY("Adding cmd %p to retry cmd list", cmd);
863         list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
864
865         if (!tgt->retry_timer_active) {
866                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
867                 add_timer(&tgt->retry_timer);
868                 tgt->retry_timer_active = 1;
869         }
870
871 out_unlock_tgt:
872         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
873
874         TRACE_EXIT_RES(res);
875         return res;
876 }
877
878 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
879 {
880         int res, rc;
881
882         TRACE_ENTRY();
883
884         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
885                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
886                 goto out_dev_done;
887         }
888
889         if ((cmd->tgtt->rdy_to_xfer == NULL) || unlikely(cmd->internal)) {
890                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
891                 res = SCST_CMD_STATE_RES_CONT_SAME;
892                 goto out;
893         }
894
895         if (unlikely(!cmd->tgtt->rdy_to_xfer_atomic && scst_cmd_atomic(cmd))) {
896                 /*
897                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
898                  * optimization.
899                  */
900                 TRACE_DBG("Target driver %s rdy_to_xfer() needs thread "
901                               "context, rescheduling", cmd->tgtt->name);
902                 res = SCST_CMD_STATE_RES_NEED_THREAD;
903                 goto out;
904         }
905
906         while (1) {
907                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
908
909                 res = SCST_CMD_STATE_RES_CONT_NEXT;
910                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
911
912                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
913 #ifdef CONFIG_SCST_DEBUG_RETRY
914                 if (((scst_random() % 100) == 75))
915                         rc = SCST_TGT_RES_QUEUE_FULL;
916                 else
917 #endif
918                         rc = cmd->tgtt->rdy_to_xfer(cmd);
919                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
920
921                 if (likely(rc == SCST_TGT_RES_SUCCESS))
922                         goto out;
923
924                 /* Restore the previous state */
925                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
926
927                 switch (rc) {
928                 case SCST_TGT_RES_QUEUE_FULL:
929                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
930                                 break;
931                         else
932                                 continue;
933
934                 case SCST_TGT_RES_NEED_THREAD_CTX:
935                         TRACE_DBG("Target driver %s "
936                               "rdy_to_xfer() requested thread "
937                               "context, rescheduling", cmd->tgtt->name);
938                         res = SCST_CMD_STATE_RES_NEED_THREAD;
939                         break;
940
941                 default:
942                         goto out_error_rc;
943                 }
944                 break;
945         }
946
947 out:
948         TRACE_EXIT_HRES(res);
949         return res;
950
951 out_error_rc:
952         if (rc == SCST_TGT_RES_FATAL_ERROR) {
953                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
954                      "fatal error", cmd->tgtt->name);
955         } else {
956                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
957                             "value %d", cmd->tgtt->name, rc);
958         }
959         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
960
961 out_dev_done:
962         scst_set_cmd_abnormal_done_state(cmd);
963         res = SCST_CMD_STATE_RES_CONT_SAME;
964         goto out;
965 }
966
967 /* No locks, but might be in IRQ */
968 void scst_proccess_redirect_cmd(struct scst_cmd *cmd, int context,
969         int check_retries)
970 {
971         unsigned long flags;
972
973         TRACE_ENTRY();
974
975         TRACE_DBG("Context: %x", context);
976
977         switch (context & ~SCST_CONTEXT_PROCESSABLE) {
978         case SCST_CONTEXT_DIRECT_ATOMIC:
979                 context &= ~SCST_CONTEXT_PROCESSABLE;
980                 /* go through */
981         case SCST_CONTEXT_DIRECT:
982                 if (check_retries)
983                         scst_check_retries(cmd->tgt);
984                 scst_process_active_cmd(cmd, context);
985                 break;
986
987         default:
988                 PRINT_ERROR("Context %x is unknown, using the thread one",
989                             context);
990                 /* go through */
991         case SCST_CONTEXT_THREAD:
992                 if (check_retries)
993                         scst_check_retries(cmd->tgt);
994                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
995                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
996                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
997                         list_add(&cmd->cmd_list_entry,
998                                 &cmd->cmd_lists->active_cmd_list);
999                 else
1000                         list_add_tail(&cmd->cmd_list_entry,
1001                                 &cmd->cmd_lists->active_cmd_list);
1002                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
1003                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
1004                 break;
1005
1006         case SCST_CONTEXT_TASKLET:
1007                 if (check_retries)
1008                         scst_check_retries(cmd->tgt);
1009                 scst_schedule_tasklet(cmd);
1010                 break;
1011         }
1012
1013         TRACE_EXIT();
1014         return;
1015 }
1016
1017 void scst_rx_data(struct scst_cmd *cmd, int status, int pref_context)
1018 {
1019         TRACE_ENTRY();
1020
1021         TRACE_DBG("Preferred context: %d", pref_context);
1022         TRACE(TRACE_SCSI, "tag=%llu status=%#x",
1023               (long long unsigned int)scst_cmd_get_tag(cmd),
1024               status);
1025
1026 #ifdef CONFIG_SCST_EXTRACHECKS
1027         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
1028                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
1029                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
1030                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
1031                         cmd->tgtt->name);
1032                 pref_context = SCST_CONTEXT_TASKLET;
1033         }
1034 #endif
1035
1036         switch (status) {
1037         case SCST_RX_STATUS_SUCCESS:
1038                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1039                 /* Small context optimization */
1040                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
1041                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1042                         if (!test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC,
1043                                         &cmd->tgt_dev->tgt_dev_flags))
1044                                 pref_context = SCST_CONTEXT_THREAD;
1045                 }
1046                 break;
1047
1048         case SCST_RX_STATUS_ERROR_SENSE_SET:
1049                 scst_set_cmd_abnormal_done_state(cmd);
1050                 break;
1051
1052         case SCST_RX_STATUS_ERROR_FATAL:
1053                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1054                 /* go through */
1055         case SCST_RX_STATUS_ERROR:
1056                 scst_set_cmd_error(cmd,
1057                            SCST_LOAD_SENSE(scst_sense_hardw_error));
1058                 scst_set_cmd_abnormal_done_state(cmd);
1059                 break;
1060
1061         default:
1062                 PRINT_ERROR("scst_rx_data() received unknown status %x",
1063                         status);
1064                 scst_set_cmd_abnormal_done_state(cmd);
1065                 break;
1066         }
1067
1068         scst_proccess_redirect_cmd(cmd, pref_context, 1);
1069
1070         TRACE_EXIT();
1071         return;
1072 }
1073 EXPORT_SYMBOL(scst_rx_data);
1074
1075 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
1076 {
1077         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
1078
1079         TRACE_ENTRY();
1080
1081         cmd->state = SCST_CMD_STATE_SEND_FOR_EXEC;
1082
1083         if ((cmd->tgtt->pre_exec == NULL) || unlikely(cmd->internal))
1084                 goto out;
1085
1086         TRACE_DBG("Calling pre_exec(%p)", cmd);
1087         rc = cmd->tgtt->pre_exec(cmd);
1088         TRACE_DBG("pre_exec() returned %d", rc);
1089
1090         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
1091                 switch (rc) {
1092                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
1093                         scst_set_cmd_abnormal_done_state(cmd);
1094                         break;
1095                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
1096                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1097                         /* go through */
1098                 case SCST_PREPROCESS_STATUS_ERROR:
1099                         scst_set_cmd_error(cmd,
1100                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1101                         scst_set_cmd_abnormal_done_state(cmd);
1102                         break;
1103                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1104                         TRACE_DBG("Target driver's %s pre_exec() requested "
1105                                 "thread context, rescheduling", cmd->tgtt->name);
1106                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1107                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1108                         break;
1109                 default:
1110                         sBUG();
1111                         break;
1112                 }
1113         }
1114
1115 out:
1116         TRACE_EXIT_RES(res);
1117         return res;
1118 }
1119
1120 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1121         const uint8_t *rq_sense, int rq_sense_len, int resid)
1122 {
1123         TRACE_ENTRY();
1124
1125 #ifdef CONFIG_SCST_MEASURE_LATENCY
1126         {
1127                 struct timespec ts;
1128                 getnstimeofday(&ts);
1129                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1130                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1131                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start, ts.tv_sec,
1132                         ts.tv_nsec);
1133         }
1134 #endif
1135
1136         cmd->status = result & 0xff;
1137         cmd->msg_status = msg_byte(result);
1138         cmd->host_status = host_byte(result);
1139         cmd->driver_status = driver_byte(result);
1140         if (unlikely(resid != 0)) {
1141 #ifdef CONFIG_SCST_EXTRACHECKS
1142                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1143                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1144                                 "op %x)", resid, cmd->resp_data_len,
1145                                 cmd->cdb[0]);
1146                 } else
1147 #endif
1148                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1149         }
1150
1151         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION)) {
1152                 /* We might have double reset UA here */
1153                 cmd->dbl_ua_orig_resp_data_len = cmd->resp_data_len;
1154                 cmd->dbl_ua_orig_data_direction = cmd->data_direction;
1155
1156                 scst_alloc_set_sense(cmd, scst_is_context_gfp_atomic(),
1157                                         rq_sense, rq_sense_len);
1158         }
1159
1160         TRACE(TRACE_SCSI, "cmd%p, result=%x, cmd->status=%x, resid=%d, "
1161               "cmd->msg_status=%x, cmd->host_status=%x, "
1162               "cmd->driver_status=%x", cmd, result, cmd->status, resid,
1163               cmd->msg_status, cmd->host_status, cmd->driver_status);
1164
1165         cmd->completed = 1;
1166
1167         TRACE_EXIT();
1168         return;
1169 }
1170
1171 /* For small context optimization */
1172 static inline int scst_optimize_post_exec_context(struct scst_cmd *cmd,
1173         int context)
1174 {
1175         if ((context == SCST_CONTEXT_TASKLET) ||
1176             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1177                 if (!test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC,
1178                                 &cmd->tgt_dev->tgt_dev_flags))
1179                         context = SCST_CONTEXT_THREAD;
1180         }
1181         return context;
1182 }
1183
1184 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1185 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1186                                             struct scsi_request **req)
1187 {
1188         struct scst_cmd *cmd = NULL;
1189
1190         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1191                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1192
1193         if (cmd == NULL) {
1194                 PRINT_ERROR("%s", "Request with NULL cmd");
1195                 if (*req)
1196                         scsi_release_request(*req);
1197         }
1198
1199         return cmd;
1200 }
1201
1202 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1203 {
1204         struct scsi_request *req = NULL;
1205         struct scst_cmd *cmd;
1206
1207         TRACE_ENTRY();
1208
1209         cmd = scst_get_cmd(scsi_cmd, &req);
1210         if (cmd == NULL)
1211                 goto out;
1212
1213         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1214                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1215
1216         /* Clear out request structure */
1217         req->sr_use_sg = 0;
1218         req->sr_sglist_len = 0;
1219         req->sr_bufflen = 0;
1220         req->sr_buffer = NULL;
1221         req->sr_underflow = 0;
1222         req->sr_request->rq_disk = NULL; /* disown request blk */
1223
1224         scst_release_request(cmd);
1225
1226         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1227
1228         scst_proccess_redirect_cmd(cmd,
1229                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1230
1231 out:
1232         TRACE_EXIT();
1233         return;
1234 }
1235 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1236 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1237 {
1238         struct scst_cmd *cmd;
1239
1240         TRACE_ENTRY();
1241
1242         cmd = (struct scst_cmd *)data;
1243         if (cmd == NULL)
1244                 goto out;
1245
1246         scst_do_cmd_done(cmd, result, sense, SCST_SENSE_BUFFERSIZE, resid);
1247
1248         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1249
1250         scst_proccess_redirect_cmd(cmd,
1251                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1252
1253 out:
1254         TRACE_EXIT();
1255         return;
1256 }
1257 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1258
1259 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state)
1260 {
1261         int context;
1262
1263         TRACE_ENTRY();
1264
1265 #ifdef CONFIG_SCST_MEASURE_LATENCY
1266         {
1267                 struct timespec ts;
1268                 getnstimeofday(&ts);
1269                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1270                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1271                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start, ts.tv_sec,
1272                         ts.tv_nsec);
1273         }
1274 #endif
1275
1276         if (next_state == SCST_CMD_STATE_DEFAULT)
1277                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1278
1279 #if defined(CONFIG_SCST_DEBUG)
1280         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1281                 if (cmd->sg) {
1282                         int i;
1283                         struct scatterlist *sg = cmd->sg;
1284                         TRACE_RECV_TOP("Exec'd %d S/G(s) at %p sg[0].page at "
1285                                 "%p", cmd->sg_cnt, sg, (void *)sg_page(&sg[0]));
1286                         for (i = 0; i < cmd->sg_cnt; ++i) {
1287                                 TRACE_BUFF_FLAG(TRACE_RCV_TOP,
1288                                         "Exec'd sg", sg_virt(&sg[i]),
1289                                         sg[i].length);
1290                         }
1291                 }
1292         }
1293 #endif
1294
1295         cmd->state = next_state;
1296
1297 #ifdef CONFIG_SCST_EXTRACHECKS
1298         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1299             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1300             (next_state != SCST_CMD_STATE_FINISHED)) {
1301                 PRINT_ERROR("%s() received invalid cmd state %d (opcode %d)",
1302                         __func__, next_state, cmd->cdb[0]);
1303                 scst_set_cmd_error(cmd,
1304                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1305                 scst_set_cmd_abnormal_done_state(cmd);
1306         }
1307 #endif
1308         context = scst_optimize_post_exec_context(cmd, scst_get_context());
1309         if (cmd->context_processable)
1310                 context |= SCST_CONTEXT_PROCESSABLE;
1311         scst_proccess_redirect_cmd(cmd, context, 0);
1312
1313         TRACE_EXIT();
1314         return;
1315 }
1316
1317 static int scst_report_luns_local(struct scst_cmd *cmd)
1318 {
1319         int rc;
1320         int dev_cnt = 0;
1321         int buffer_size;
1322         int i;
1323         struct scst_tgt_dev *tgt_dev = NULL;
1324         uint8_t *buffer;
1325         int offs, overflow = 0;
1326
1327         TRACE_ENTRY();
1328
1329         rc = scst_check_local_events(cmd);
1330         if (unlikely(rc != 0))
1331                 goto out_done;
1332
1333         cmd->status = 0;
1334         cmd->msg_status = 0;
1335         cmd->host_status = DID_OK;
1336         cmd->driver_status = 0;
1337
1338         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1339                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1340                         "LUNS command", cmd->cdb[2]);
1341                 goto out_err;
1342         }
1343
1344         buffer_size = scst_get_buf_first(cmd, &buffer);
1345         if (unlikely(buffer_size == 0))
1346                 goto out_compl;
1347         else if (unlikely(buffer_size < 0))
1348                 goto out_err;
1349
1350         if (buffer_size < 16)
1351                 goto out_put_err;
1352
1353         memset(buffer, 0, buffer_size);
1354         offs = 8;
1355
1356         /* sess->sess_tgt_dev_list_hash is protected by suspended activity */
1357         for (i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1358                 struct list_head *sess_tgt_dev_list_head =
1359                         &cmd->sess->sess_tgt_dev_list_hash[i];
1360                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1361                                 sess_tgt_dev_list_entry) {
1362                         if (!overflow) {
1363                                 if (offs >= buffer_size) {
1364                                         scst_put_buf(cmd, buffer);
1365                                         buffer_size = scst_get_buf_next(cmd, &buffer);
1366                                         if (buffer_size > 0) {
1367                                                 memset(buffer, 0, buffer_size);
1368                                                 offs = 0;
1369                                         } else {
1370                                                 overflow = 1;
1371                                                 goto inc_dev_cnt;
1372                                         }
1373                                 }
1374                                 if ((buffer_size - offs) < 8) {
1375                                         PRINT_ERROR("Buffer allocated for REPORT "
1376                                                 "LUNS command doesn't allow to fit 8 "
1377                                                 "byte entry (buffer_size=%d)",
1378                                                 buffer_size);
1379                                         goto out_put_hw_err;
1380                                 }
1381                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1382                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1383                                 offs += 8;
1384                         }
1385 inc_dev_cnt:
1386                         dev_cnt++;
1387                 }
1388         }
1389         if (!overflow)
1390                 scst_put_buf(cmd, buffer);
1391
1392         /* Set the response header */
1393         buffer_size = scst_get_buf_first(cmd, &buffer);
1394         if (unlikely(buffer_size == 0))
1395                 goto out_compl;
1396         else if (unlikely(buffer_size < 0))
1397                 goto out_err;
1398
1399         dev_cnt *= 8;
1400         buffer[0] = (dev_cnt >> 24) & 0xff;
1401         buffer[1] = (dev_cnt >> 16) & 0xff;
1402         buffer[2] = (dev_cnt >> 8) & 0xff;
1403         buffer[3] = dev_cnt & 0xff;
1404
1405         scst_put_buf(cmd, buffer);
1406
1407         dev_cnt += 8;
1408         if (dev_cnt < cmd->resp_data_len)
1409                 scst_set_resp_data_len(cmd, dev_cnt);
1410
1411 out_compl:
1412         cmd->completed = 1;
1413
1414 out_done:
1415         /* Report the result */
1416         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
1417
1418         TRACE_EXIT();
1419         return SCST_EXEC_COMPLETED;
1420
1421 out_put_err:
1422         scst_put_buf(cmd, buffer);
1423
1424 out_err:
1425         scst_set_cmd_error(cmd,
1426                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1427         goto out_compl;
1428
1429 out_put_hw_err:
1430         scst_put_buf(cmd, buffer);
1431         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1432         goto out_compl;
1433 }
1434
1435 static int scst_pre_select(struct scst_cmd *cmd)
1436 {
1437         int res = SCST_EXEC_NOT_COMPLETED;
1438
1439         TRACE_ENTRY();
1440
1441         if (scst_cmd_atomic(cmd)) {
1442                 res = SCST_EXEC_NEED_THREAD;
1443                 goto out;
1444         }
1445
1446         scst_block_dev_cmd(cmd, 1);
1447
1448         /* Check for local events will be done when cmd will be executed */
1449
1450 out:
1451         TRACE_EXIT_RES(res);
1452         return res;
1453 }
1454
1455 static int scst_reserve_local(struct scst_cmd *cmd)
1456 {
1457         int res = SCST_EXEC_NOT_COMPLETED, rc;
1458         struct scst_device *dev;
1459         struct scst_tgt_dev *tgt_dev_tmp;
1460
1461         TRACE_ENTRY();
1462
1463         if (scst_cmd_atomic(cmd)) {
1464                 res = SCST_EXEC_NEED_THREAD;
1465                 goto out;
1466         }
1467
1468         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1469                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1470                      "(lun=%lld)", (long long unsigned int)cmd->lun);
1471                 scst_set_cmd_error(cmd,
1472                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1473                 goto out_done;
1474         }
1475
1476         dev = cmd->dev;
1477
1478         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1479                 scst_block_dev_cmd(cmd, 1);
1480
1481         rc = scst_check_local_events(cmd);
1482         if (unlikely(rc != 0))
1483                 goto out_done;
1484
1485         spin_lock_bh(&dev->dev_lock);
1486
1487         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1488                 spin_unlock_bh(&dev->dev_lock);
1489                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1490                 goto out_done;
1491         }
1492
1493         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1494                             dev_tgt_dev_list_entry) {
1495                 if (cmd->tgt_dev != tgt_dev_tmp)
1496                         set_bit(SCST_TGT_DEV_RESERVED,
1497                                 &tgt_dev_tmp->tgt_dev_flags);
1498         }
1499         dev->dev_reserved = 1;
1500
1501         spin_unlock_bh(&dev->dev_lock);
1502
1503 out:
1504         TRACE_EXIT_RES(res);
1505         return res;
1506
1507 out_done:
1508         /* Report the result */
1509         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
1510         res = SCST_EXEC_COMPLETED;
1511         goto out;
1512 }
1513
1514 static int scst_release_local(struct scst_cmd *cmd)
1515 {
1516         int res = SCST_EXEC_NOT_COMPLETED, rc;
1517         struct scst_tgt_dev *tgt_dev_tmp;
1518         struct scst_device *dev;
1519
1520         TRACE_ENTRY();
1521
1522         if (scst_cmd_atomic(cmd)) {
1523                 res = SCST_EXEC_NEED_THREAD;
1524                 goto out;
1525         }
1526
1527         dev = cmd->dev;
1528
1529         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1530                 scst_block_dev_cmd(cmd, 1);
1531
1532         rc = scst_check_local_events(cmd);
1533         if (unlikely(rc != 0))
1534                 goto out_done;
1535
1536         spin_lock_bh(&dev->dev_lock);
1537
1538         /*
1539          * The device could be RELEASED behind us, if RESERVING session
1540          * is closed (see scst_free_tgt_dev()), but this actually doesn't
1541          * matter, so use lock and no retest for DEV_RESERVED bits again
1542          */
1543         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1544                 res = SCST_EXEC_COMPLETED;
1545                 cmd->status = 0;
1546                 cmd->msg_status = 0;
1547                 cmd->host_status = DID_OK;
1548                 cmd->driver_status = 0;
1549                 cmd->completed = 1;
1550         } else {
1551                 list_for_each_entry(tgt_dev_tmp,
1552                                     &dev->dev_tgt_dev_list,
1553                                     dev_tgt_dev_list_entry) {
1554                         clear_bit(SCST_TGT_DEV_RESERVED,
1555                                 &tgt_dev_tmp->tgt_dev_flags);
1556                 }
1557                 dev->dev_reserved = 0;
1558         }
1559
1560         spin_unlock_bh(&dev->dev_lock);
1561
1562         if (res == SCST_EXEC_COMPLETED)
1563                 goto out_done;
1564
1565 out:
1566         TRACE_EXIT_RES(res);
1567         return res;
1568
1569 out_done:
1570         res = SCST_EXEC_COMPLETED;
1571         /* Report the result */
1572         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
1573         goto out;
1574 }
1575
1576 /* No locks, no IRQ or IRQ-safe context allowed */
1577 int scst_check_local_events(struct scst_cmd *cmd)
1578 {
1579         int res, rc;
1580         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1581         struct scst_device *dev = cmd->dev;
1582
1583         TRACE_ENTRY();
1584
1585         /*
1586          * There's no race here, because we need to trace commands sent
1587          * *after* dev_double_ua_possible flag was set.
1588          */
1589         if (unlikely(dev->dev_double_ua_possible))
1590                 cmd->double_ua_possible = 1;
1591
1592         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1593                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1594                 goto out_uncomplete;
1595         }
1596
1597         /* Reserve check before Unit Attention */
1598         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED, &tgt_dev->tgt_dev_flags))) {
1599                 if ((cmd->cdb[0] != INQUIRY) && (cmd->cdb[0] != REPORT_LUNS) &&
1600                     (cmd->cdb[0] != RELEASE) && (cmd->cdb[0] != RELEASE_10) &&
1601                     (cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER) &&
1602                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL || (cmd->cdb[4] & 3)) &&
1603                     (cmd->cdb[0] != LOG_SENSE) && (cmd->cdb[0] != REQUEST_SENSE)) {
1604                         scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1605                         goto out_complete;
1606                 }
1607         }
1608
1609         /* If we had internal bus reset, set the command error unit attention */
1610         if ((dev->scsi_dev != NULL) &&
1611             unlikely(dev->scsi_dev->was_reset)) {
1612                 if (scst_is_ua_command(cmd)) {
1613                         int done = 0;
1614                         /* Prevent more than 1 cmd to be triggered by was_reset */
1615                         spin_lock_bh(&dev->dev_lock);
1616                         barrier(); /* to reread was_reset */
1617                         if (dev->scsi_dev->was_reset) {
1618                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1619                                 scst_set_cmd_error(cmd,
1620                                            SCST_LOAD_SENSE(scst_sense_reset_UA));
1621                                 /* It looks like it is safe to clear was_reset here */
1622                                 dev->scsi_dev->was_reset = 0;
1623                                 done = 1;
1624                         }
1625                         spin_unlock_bh(&dev->dev_lock);
1626
1627                         if (done)
1628                                 goto out_complete;
1629                 }
1630         }
1631
1632         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING,
1633                         &cmd->tgt_dev->tgt_dev_flags))) {
1634                 if (scst_is_ua_command(cmd)) {
1635                         rc = scst_set_pending_UA(cmd);
1636                         if (rc == 0)
1637                                 goto out_complete;
1638                 }
1639         }
1640
1641         res = 0;
1642
1643 out:
1644         TRACE_EXIT_RES(res);
1645         return res;
1646
1647 out_complete:
1648         res = 1;
1649         sBUG_ON(!cmd->completed);
1650         goto out;
1651
1652 out_uncomplete:
1653         res = -1;
1654         goto out;
1655 }
1656 EXPORT_SYMBOL(scst_check_local_events);
1657
1658 /* No locks */
1659 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1660 {
1661         if (slot == NULL)
1662                 goto inc;
1663
1664         /* Optimized for lockless fast path */
1665
1666         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1667                 atomic_read(slot));
1668
1669         if (!atomic_dec_and_test(slot))
1670                 goto out;
1671
1672         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1673                 tgt_dev->num_free_sn_slots);
1674         if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
1675                 spin_lock_irq(&tgt_dev->sn_lock);
1676                 if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
1677                         if (tgt_dev->num_free_sn_slots < 0)
1678                                 tgt_dev->cur_sn_slot = slot;
1679                         smp_mb(); /* to be in-sync with SIMPLE case in scst_cmd_set_sn() */
1680                         tgt_dev->num_free_sn_slots++;
1681                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1682                                 tgt_dev->num_free_sn_slots);
1683
1684                 }
1685                 spin_unlock_irq(&tgt_dev->sn_lock);
1686         }
1687
1688 inc:
1689         /*
1690          * No locks is needed, because only one thread at time can
1691          * be here (serialized by sn). Also it is supposed that there
1692          * could not be half-incremented halves.
1693          */
1694         tgt_dev->expected_sn++;
1695         smp_mb(); /* write must be before def_cmd_count read */
1696         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1697
1698 out:
1699         return;
1700 }
1701
1702 /* No locks */
1703 static struct scst_cmd *scst_post_exec_sn(struct scst_cmd *cmd,
1704         bool make_active)
1705 {
1706         /* For HQ commands SN is not set */
1707         bool inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1708                                cmd->sn_set && !cmd->retry;
1709         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1710         struct scst_cmd *res;
1711
1712         TRACE_ENTRY();
1713
1714         if (inc_expected_sn)
1715                 scst_inc_expected_sn(tgt_dev, cmd->sn_slot);
1716
1717         if (make_active) {
1718                 scst_make_deferred_commands_active(tgt_dev);
1719                 res = NULL;
1720         } else
1721                 res = scst_check_deferred_commands(tgt_dev);
1722
1723         TRACE_EXIT_HRES(res);
1724         return res;
1725 }
1726
1727 /* cmd must be additionally referenced to not die inside */
1728 static int scst_do_real_exec(struct scst_cmd *cmd)
1729 {
1730         int res = SCST_EXEC_NOT_COMPLETED, rc;
1731         struct scst_device *dev = cmd->dev;
1732         struct scst_dev_type *handler = dev->handler;
1733
1734         TRACE_ENTRY();
1735
1736         cmd->state = SCST_CMD_STATE_REAL_EXECUTING;
1737
1738         if (!handler->exec_sync)
1739                 cmd->context_processable = 0;
1740
1741         if (handler->exec) {
1742                 if (unlikely(!dev->handler->exec_atomic &&
1743                              scst_cmd_atomic(cmd))) {
1744                         /*
1745                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
1746                          * optimization.
1747                          */
1748                         TRACE_DBG("Dev handler %s exec() needs thread "
1749                                 "context, rescheduling", dev->handler->name);
1750                         res = SCST_EXEC_NEED_THREAD;
1751                         goto out_restore;
1752                 }
1753
1754                 TRACE_DBG("Calling dev handler %s exec(%p)",
1755                       handler->name, cmd);
1756                 TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb,
1757                         cmd->cdb_len);
1758                 res = handler->exec(cmd);
1759                 TRACE_DBG("Dev handler %s exec() returned %d",
1760                       handler->name, res);
1761
1762                 if (res == SCST_EXEC_COMPLETED)
1763                         goto out_complete;
1764                 else if (res == SCST_EXEC_NEED_THREAD)
1765                         goto out_restore;
1766
1767                 sBUG_ON(res != SCST_EXEC_NOT_COMPLETED);
1768         }
1769
1770         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1771
1772         if (unlikely(dev->scsi_dev == NULL)) {
1773                 PRINT_ERROR("Command for virtual device must be "
1774                         "processed by device handler (lun %lld)!",
1775                         (long long unsigned int)cmd->lun);
1776                 goto out_error;
1777         }
1778
1779         res = scst_check_local_events(cmd);
1780         if (unlikely(res != 0))
1781                 goto out_done;
1782
1783 #ifndef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1784         if (unlikely(scst_cmd_atomic(cmd))) {
1785                 TRACE_DBG("Pass-through exec() can not be called in atomic "
1786                         "context, rescheduling to the thread (handler %s)",
1787                         handler->name);
1788                 res = SCST_EXEC_NEED_THREAD;
1789                 goto out_restore;
1790         }
1791 #endif
1792
1793 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1794         if (unlikely(scst_alloc_request(cmd) != 0)) {
1795                 if (scst_cmd_atomic(cmd)) {
1796                         res = SCST_EXEC_NEED_THREAD;
1797                         goto out_restore;
1798                 } else {
1799                         PRINT_INFO("%s", "Unable to allocate request, "
1800                                 "sending BUSY status");
1801                         goto out_busy;
1802                 }
1803         }
1804
1805         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1806                     (void *)cmd->scsi_req->sr_buffer,
1807                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1808                     cmd->retries);
1809 #else
1810         rc = scst_exec_req(dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1811                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1812                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1813                         scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
1814         if (unlikely(rc != 0)) {
1815                 if (scst_cmd_atomic(cmd)) {
1816                         res = SCST_EXEC_NEED_THREAD;
1817                         goto out_restore;
1818                 } else {
1819                         PRINT_ERROR("scst_exec_req() failed: %d", res);
1820                         goto out_error;
1821                 }
1822         }
1823 #endif
1824
1825 out_complete:
1826         res = SCST_EXEC_COMPLETED;
1827
1828 out:
1829         TRACE_EXIT();
1830         return res;
1831
1832 out_restore:
1833         /* Restore the state */
1834         cmd->state = SCST_CMD_STATE_REAL_EXEC;
1835         goto out;
1836
1837 out_error:
1838         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1839         goto out_done;
1840
1841 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1842 out_busy:
1843         scst_set_busy(cmd);
1844         /* go through */
1845 #endif
1846
1847 out_done:
1848         res = SCST_EXEC_COMPLETED;
1849         /* Report the result */
1850         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
1851         goto out_complete;
1852 }
1853
1854 static inline int scst_real_exec(struct scst_cmd *cmd)
1855 {
1856         int res;
1857
1858         TRACE_ENTRY();
1859
1860         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
1861         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
1862         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
1863
1864         __scst_cmd_get(cmd);
1865
1866         res = scst_do_real_exec(cmd);
1867
1868         if (likely(res == SCST_EXEC_COMPLETED)) {
1869                 scst_post_exec_sn(cmd, true);
1870                 if (cmd->dev->scsi_dev != NULL)
1871                         generic_unplug_device(cmd->dev->scsi_dev->request_queue);
1872         } else
1873                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
1874
1875         __scst_cmd_put(cmd);
1876
1877         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
1878
1879         TRACE_EXIT_RES(res);
1880         return res;
1881 }
1882
1883 static int scst_do_local_exec(struct scst_cmd *cmd)
1884 {
1885         int res;
1886         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1887
1888         TRACE_ENTRY();
1889
1890         /* Check READ_ONLY device status */
1891         if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
1892             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
1893              cmd->cdb[0] == WRITE_10 ||
1894              cmd->cdb[0] == WRITE_12 ||
1895              cmd->cdb[0] == WRITE_16 ||
1896              cmd->cdb[0] == WRITE_VERIFY ||
1897              cmd->cdb[0] == WRITE_VERIFY_12 ||
1898              cmd->cdb[0] == WRITE_VERIFY_16 ||
1899              (cmd->dev->handler->type == TYPE_TAPE &&
1900               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
1901                 scst_set_cmd_error(cmd,
1902                            SCST_LOAD_SENSE(scst_sense_data_protect));
1903                 goto out_done;
1904         }
1905
1906         /*
1907          * Adding new commands here don't forget to update
1908          * scst_is_cmd_local() in scst.h, if necessary
1909          */
1910
1911         switch (cmd->cdb[0]) {
1912         case MODE_SELECT:
1913         case MODE_SELECT_10:
1914         case LOG_SELECT:
1915                 res = scst_pre_select(cmd);
1916                 break;
1917         case RESERVE:
1918         case RESERVE_10:
1919                 res = scst_reserve_local(cmd);
1920                 break;
1921         case RELEASE:
1922         case RELEASE_10:
1923                 res = scst_release_local(cmd);
1924                 break;
1925         case REPORT_LUNS:
1926                 res = scst_report_luns_local(cmd);
1927                 break;
1928         default:
1929                 res = SCST_EXEC_NOT_COMPLETED;
1930                 break;
1931         }
1932
1933 out:
1934         TRACE_EXIT_RES(res);
1935         return res;
1936
1937 out_done:
1938         /* Report the result */
1939         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT);
1940         res = SCST_EXEC_COMPLETED;
1941         goto out;
1942 }
1943
1944 static int scst_local_exec(struct scst_cmd *cmd)
1945 {
1946         int res;
1947
1948         TRACE_ENTRY();
1949
1950         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
1951         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
1952         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
1953
1954         __scst_cmd_get(cmd);
1955
1956         res = scst_do_local_exec(cmd);
1957         if (likely(res == SCST_EXEC_NOT_COMPLETED))
1958                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
1959         else if (res == SCST_EXEC_COMPLETED)
1960                 scst_post_exec_sn(cmd, true);
1961         else
1962                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
1963
1964         __scst_cmd_put(cmd);
1965
1966         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
1967         TRACE_EXIT_RES(res);
1968         return res;
1969 }
1970
1971 static int scst_exec(struct scst_cmd **active_cmd)
1972 {
1973         struct scst_cmd *cmd = *active_cmd;
1974         struct scst_cmd *ref_cmd;
1975         struct scst_device *dev = cmd->dev;
1976         int res = SCST_CMD_STATE_RES_CONT_NEXT, count;
1977
1978         TRACE_ENTRY();
1979
1980         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
1981                 goto out;
1982
1983         /* To protect tgt_dev */
1984         ref_cmd = cmd;
1985         __scst_cmd_get(ref_cmd);
1986
1987         count = 0;
1988         while (1) {
1989                 int rc;
1990
1991                 cmd->sent_for_exec = 1;
1992                 cmd->scst_cmd_done = scst_cmd_done_local;
1993                 cmd->state = SCST_CMD_STATE_LOCAL_EXEC;
1994
1995                 rc = scst_do_local_exec(cmd);
1996                 if (likely(rc == SCST_EXEC_NOT_COMPLETED))
1997                         /* Nothing to do */;
1998                 else if (rc == SCST_EXEC_NEED_THREAD) {
1999                         TRACE_DBG("%s", "scst_do_local_exec() requested "
2000                                 "thread context, rescheduling");
2001                         scst_dec_on_dev_cmd(cmd);
2002                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2003                         break;
2004                 } else {
2005                         sBUG_ON(rc != SCST_EXEC_COMPLETED);
2006                         goto done;
2007                 }
2008
2009                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2010
2011                 rc = scst_do_real_exec(cmd);
2012                 if (likely(rc == SCST_EXEC_COMPLETED))
2013                         /* Nothing to do */;
2014                 else if (rc == SCST_EXEC_NEED_THREAD) {
2015                         TRACE_DBG("scst_real_exec() requested thread "
2016                                 "context, rescheduling (cmd %p)", cmd);
2017                         scst_dec_on_dev_cmd(cmd);
2018                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2019                         break;
2020                 } else
2021                         sBUG();
2022
2023 done:
2024                 count++;
2025
2026                 cmd = scst_post_exec_sn(cmd, false);
2027                 if (cmd == NULL)
2028                         break;
2029
2030                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2031                         break;
2032
2033                 __scst_cmd_put(ref_cmd);
2034                 ref_cmd = cmd;
2035                 __scst_cmd_get(ref_cmd);
2036         }
2037
2038         *active_cmd = cmd;
2039
2040         if (count == 0)
2041                 goto out_put;
2042
2043         if (dev->scsi_dev != NULL)
2044                 generic_unplug_device(dev->scsi_dev->request_queue);
2045
2046 out_put:
2047         __scst_cmd_put(ref_cmd);
2048         /* !! At this point sess, dev and tgt_dev can be already freed !! */
2049
2050 out:
2051         TRACE_EXIT_RES(res);
2052         return res;
2053 }
2054
2055 static int scst_send_for_exec(struct scst_cmd **active_cmd)
2056 {
2057         int res;
2058         struct scst_cmd *cmd = *active_cmd;
2059         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2060         typeof(tgt_dev->expected_sn) expected_sn;
2061
2062         TRACE_ENTRY();
2063
2064 #ifdef CONFIG_SCST_MEASURE_LATENCY
2065         if (cmd->pre_exec_finish == 0) {
2066                 struct timespec ts;
2067                 getnstimeofday(&ts);
2068                 cmd->pre_exec_finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2069                 TRACE_DBG("cmd %p (sess %p): pre_exec_finish %lld (tv_sec %ld, "
2070                         "tv_nsec %ld)", cmd, cmd->sess, cmd->pre_exec_finish,
2071                         ts.tv_sec, ts.tv_nsec);
2072         }
2073 #endif
2074
2075         if (unlikely(cmd->internal))
2076                 goto exec;
2077
2078         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2079                 goto exec;
2080
2081         sBUG_ON(!cmd->sn_set);
2082
2083         expected_sn = tgt_dev->expected_sn;
2084         /* Optimized for lockless fast path */
2085         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2086                 spin_lock_irq(&tgt_dev->sn_lock);
2087
2088                 tgt_dev->def_cmd_count++;
2089                 smp_mb();
2090
2091                 expected_sn = tgt_dev->expected_sn;
2092                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2093                         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2094                                 /* Necessary to allow aborting out of sn cmds */
2095                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p "
2096                                         "(tag %llu, sn %lu)", cmd,
2097                                         (long long unsigned)cmd->tag, cmd->sn);
2098                                 tgt_dev->def_cmd_count--;
2099                                 scst_set_cmd_abnormal_done_state(cmd);
2100                                 res = SCST_CMD_STATE_RES_CONT_SAME;
2101                         } else {
2102                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
2103                                         "expected_sn=%ld)", cmd, cmd->sn,
2104                                         cmd->sn_set, expected_sn);
2105                                 list_add_tail(&cmd->sn_cmd_list_entry,
2106                                               &tgt_dev->deferred_cmd_list);
2107                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2108                         }
2109                         spin_unlock_irq(&tgt_dev->sn_lock);
2110                         goto out;
2111                 } else {
2112                         TRACE_SN("Somebody incremented expected_sn %ld, "
2113                                 "continuing", expected_sn);
2114                         tgt_dev->def_cmd_count--;
2115                         spin_unlock_irq(&tgt_dev->sn_lock);
2116                 }
2117         }
2118
2119 exec:
2120         res = scst_exec(active_cmd);
2121
2122 out:
2123         TRACE_EXIT_HRES(res);
2124         return res;
2125 }
2126
2127 /* No locks supposed to be held */
2128 static int scst_check_sense(struct scst_cmd *cmd)
2129 {
2130         int res = 0;
2131         struct scst_device *dev = cmd->dev;
2132
2133         TRACE_ENTRY();
2134
2135         if (unlikely(cmd->ua_ignore))
2136                 goto out;
2137
2138         /* If we had internal bus reset behind us, set the command error UA */
2139         if ((dev->scsi_dev != NULL) &&
2140             unlikely(cmd->host_status == DID_RESET) &&
2141             scst_is_ua_command(cmd)) {
2142                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
2143                       dev->scsi_dev->was_reset, cmd->host_status);
2144                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_reset_UA));
2145                 /* It looks like it is safe to clear was_reset here */
2146                 dev->scsi_dev->was_reset = 0;
2147         }
2148
2149         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2150             SCST_SENSE_VALID(cmd->sense)) {
2151                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2152                         SCST_SENSE_BUFFERSIZE);
2153
2154                 /* Check Unit Attention Sense Key */
2155                 if (scst_is_ua_sense(cmd->sense)) {
2156                         if (cmd->sense[12] == SCST_SENSE_ASC_UA_RESET) {
2157                                 if (cmd->double_ua_possible) {
2158                                         TRACE(TRACE_MGMT_MINOR, "Double UA "
2159                                                 "detected for device %p", dev);
2160                                         TRACE(TRACE_MGMT_MINOR, "Retrying cmd %p "
2161                                                 "(tag %llu)", cmd,
2162                                                 (long long unsigned)cmd->tag);
2163
2164                                         cmd->status = 0;
2165                                         cmd->msg_status = 0;
2166                                         cmd->host_status = DID_OK;
2167                                         cmd->driver_status = 0;
2168
2169                                         mempool_free(cmd->sense, scst_sense_mempool);
2170                                         cmd->sense = NULL;
2171
2172                                         scst_check_restore_sg_buff(cmd);
2173
2174                                         sBUG_ON(cmd->dbl_ua_orig_resp_data_len < 0);
2175                                         cmd->data_direction =
2176                                                 cmd->dbl_ua_orig_data_direction;
2177                                         cmd->resp_data_len =
2178                                                 cmd->dbl_ua_orig_resp_data_len;
2179
2180                                         cmd->state = SCST_CMD_STATE_REAL_EXEC;
2181                                         cmd->retry = 1;
2182                                         res = 1;
2183                                         goto out;
2184                                 }
2185                         }
2186                         scst_dev_check_set_UA(dev, cmd, cmd->sense,
2187                                 SCST_SENSE_BUFFERSIZE);
2188                 }
2189         }
2190
2191         if (unlikely(cmd->double_ua_possible)) {
2192                 if (scst_is_ua_command(cmd)) {
2193                         TRACE_MGMT_DBG("Clearing dbl_ua_possible flag (dev %p, "
2194                                 "cmd %p)", dev, cmd);
2195                         /*
2196                          * Lock used to protect other flags in the bitfield
2197                          * (just in case, actually). Those 2 flags can't be
2198                          * changed in parallel, because the device is
2199                          * serialized.
2200                          */
2201                         spin_lock_bh(&dev->dev_lock);
2202                         dev->dev_double_ua_possible = 0;
2203                         dev->dev_serialized = 0;
2204                         spin_unlock_bh(&dev->dev_lock);
2205                 }
2206         }
2207
2208 out:
2209         TRACE_EXIT_RES(res);
2210         return res;
2211 }
2212
2213 static int scst_check_auto_sense(struct scst_cmd *cmd)
2214 {
2215         int res = 0;
2216
2217         TRACE_ENTRY();
2218
2219         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2220             (!SCST_SENSE_VALID(cmd->sense) ||
2221              SCST_NO_SENSE(cmd->sense))) {
2222                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2223                       "cmd->status=%x, cmd->msg_status=%x, "
2224                       "cmd->host_status=%x, cmd->driver_status=%x", cmd->status,
2225                       cmd->msg_status, cmd->host_status, cmd->driver_status);
2226                 res = 1;
2227         } else if (unlikely(cmd->host_status)) {
2228                 if ((cmd->host_status == DID_REQUEUE) ||
2229                     (cmd->host_status == DID_IMM_RETRY) ||
2230                     (cmd->host_status == DID_SOFT_ERROR) ||
2231                     (cmd->host_status == DID_ABORT)) {
2232                         scst_set_busy(cmd);
2233                 } else {
2234                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2235                                 "received, returning HARDWARE ERROR instead",
2236                                 cmd->host_status);
2237                         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2238                 }
2239         }
2240
2241         TRACE_EXIT_RES(res);
2242         return res;
2243 }
2244
2245 static int scst_pre_dev_done(struct scst_cmd *cmd)
2246 {
2247         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2248
2249         TRACE_ENTRY();
2250
2251         if (unlikely(scst_check_auto_sense(cmd))) {
2252                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2253                             "without sense data (opcode 0x%x), issuing "
2254                             "REQUEST SENSE", cmd->cdb[0]);
2255                 rc = scst_prepare_request_sense(cmd);
2256                 if (rc == 0)
2257                         res = SCST_CMD_STATE_RES_CONT_NEXT;
2258                 else {
2259                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2260                                     "returning HARDWARE ERROR");
2261                         scst_set_cmd_error(cmd,
2262                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2263                 }
2264                 goto out;
2265         } else if (unlikely(scst_check_sense(cmd)))
2266                 goto out;
2267
2268         if (likely(scsi_status_is_good(cmd->status))) {
2269                 unsigned char type = cmd->dev->handler->type;
2270                 if (unlikely((cmd->cdb[0] == MODE_SENSE ||
2271                               cmd->cdb[0] == MODE_SENSE_10)) &&
2272                     cmd->tgt_dev->acg_dev->rd_only_flag &&
2273                     (type == TYPE_DISK || type == TYPE_WORM || type == TYPE_MOD ||
2274                      type == TYPE_TAPE)) {
2275                         int32_t length;
2276                         uint8_t *address;
2277                         bool err = false;
2278
2279                         length = scst_get_buf_first(cmd, &address);
2280                         if (length < 0) {
2281                                 PRINT_ERROR("%s", "Unable to get "
2282                                         "MODE_SENSE buffer");
2283                                 scst_set_cmd_error(cmd,
2284                                         SCST_LOAD_SENSE(
2285                                                 scst_sense_hardw_error));
2286                                 err = true;
2287                         } else if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2288                                 address[2] |= 0x80;   /* Write Protect*/
2289                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2290                                 address[3] |= 0x80;   /* Write Protect*/
2291                         scst_put_buf(cmd, address);
2292
2293                         if (err)
2294                                 goto out;
2295                 }
2296
2297                 /*
2298                  * Check and clear NormACA option for the device, if necessary,
2299                  * since we don't support ACA
2300                  */
2301                 if (unlikely((cmd->cdb[0] == INQUIRY)) &&
2302                     !(cmd->cdb[1] & SCST_INQ_EVPD/* Std INQUIRY data (no EVPD) */) &&
2303                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2304                         uint8_t *buffer;
2305                         int buflen;
2306                         bool err = false;
2307
2308                         /* ToDo: all pages ?? */
2309                         buflen = scst_get_buf_first(cmd, &buffer);
2310                         if (buflen > SCST_INQ_BYTE3) {
2311 #ifdef CONFIG_SCST_EXTRACHECKS
2312                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2313                                         PRINT_INFO("NormACA set for device: "
2314                                             "lun=%lld, type 0x%02x. Clear it, "
2315                                             "since it's unsupported.",
2316                                             (long long unsigned int)cmd->lun,
2317                                             buffer[0]);
2318                                 }
2319 #endif
2320                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2321                         } else if (buflen != 0) {
2322                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2323                                     "buffer");
2324                                 scst_set_cmd_error(cmd,
2325                                         SCST_LOAD_SENSE(scst_sense_hardw_error));
2326                                 err = true;
2327                         }
2328                         if (buflen > 0)
2329                                 scst_put_buf(cmd, buffer);
2330
2331                         if (err)
2332                                 goto out;
2333                 }
2334
2335                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2336                     (cmd->cdb[0] == MODE_SELECT_10) ||
2337                     (cmd->cdb[0] == LOG_SELECT))) {
2338                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded (LUN %lld)",
2339                                 (long long unsigned int)cmd->lun);
2340                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2341                         goto out;
2342                 }
2343         } else {
2344                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2345                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2346                                         &cmd->tgt_dev->tgt_dev_flags)) {
2347                                 struct scst_tgt_dev *tgt_dev_tmp;
2348                                 struct scst_device *dev = cmd->dev;
2349
2350                                 TRACE(TRACE_SCSI, "Real RESERVE failed lun=%lld, "
2351                                         "status=%x",
2352                                         (long long unsigned int)cmd->lun,
2353                                         cmd->status);
2354                                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2355                                         SCST_SENSE_BUFFERSIZE);
2356
2357                                 /* Clearing the reservation */
2358                                 spin_lock_bh(&dev->dev_lock);
2359                                 list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
2360                                                     dev_tgt_dev_list_entry) {
2361                                         clear_bit(SCST_TGT_DEV_RESERVED,
2362                                                 &tgt_dev_tmp->tgt_dev_flags);
2363                                 }
2364                                 dev->dev_reserved = 0;
2365                                 spin_unlock_bh(&dev->dev_lock);
2366                         }
2367                 }
2368
2369                 /* Check for MODE PARAMETERS CHANGED UA */
2370                 if ((cmd->dev->scsi_dev != NULL) &&
2371                     (cmd->status == SAM_STAT_CHECK_CONDITION) &&
2372                     SCST_SENSE_VALID(cmd->sense) &&
2373                     scst_is_ua_sense(cmd->sense) &&
2374                     (cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) {
2375                         TRACE(TRACE_SCSI,
2376                               "MODE PARAMETERS CHANGED UA (lun %lld)",
2377                               (long long unsigned int)cmd->lun);
2378                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2379                         goto out;
2380                 }
2381         }
2382
2383         cmd->state = SCST_CMD_STATE_DEV_DONE;
2384
2385 out:
2386         TRACE_EXIT_RES(res);
2387         return res;
2388 }
2389
2390 static int scst_mode_select_checks(struct scst_cmd *cmd)
2391 {
2392         int res = SCST_CMD_STATE_RES_CONT_SAME;
2393         int atomic = scst_cmd_atomic(cmd);
2394
2395         TRACE_ENTRY();
2396
2397         if (likely(scsi_status_is_good(cmd->status))) {
2398                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2399                     (cmd->cdb[0] == MODE_SELECT_10) ||
2400                     (cmd->cdb[0] == LOG_SELECT))) {
2401                         struct scst_device *dev = cmd->dev;
2402                         if (atomic && (dev->scsi_dev != NULL)) {
2403                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2404                                         "context required");
2405                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2406                                 goto out;
2407                         }
2408
2409                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2410                                 "setting the SELECT UA (lun=%lld)",
2411                                 (long long unsigned int)cmd->lun);
2412
2413                         spin_lock_bh(&dev->dev_lock);
2414                         spin_lock(&scst_temp_UA_lock);
2415                         if (cmd->cdb[0] == LOG_SELECT) {
2416                                 scst_set_sense(scst_temp_UA,
2417                                         sizeof(scst_temp_UA),
2418                                         UNIT_ATTENTION, 0x2a, 0x02);
2419                         } else {
2420                                 scst_set_sense(scst_temp_UA,
2421                                         sizeof(scst_temp_UA),
2422                                         UNIT_ATTENTION, 0x2a, 0x01);
2423                         }
2424                         scst_dev_check_set_local_UA(dev, cmd, scst_temp_UA,
2425                                 sizeof(scst_temp_UA));
2426                         spin_unlock(&scst_temp_UA_lock);
2427                         spin_unlock_bh(&dev->dev_lock);
2428
2429                         if (dev->scsi_dev != NULL)
2430                                 scst_obtain_device_parameters(dev);
2431                 }
2432         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) &&
2433                     SCST_SENSE_VALID(cmd->sense) &&
2434                     scst_is_ua_sense(cmd->sense) &&
2435                     (((cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) ||
2436                      (cmd->sense[12] == 0x29) /* reset */ ||
2437                      (cmd->sense[12] == 0x28) /* medium changed */ ||
2438                      (cmd->sense[12] == 0x2F) /* cleared by another ini (just in case) */)) {
2439                 if (atomic) {
2440                         TRACE_DBG("Possible parameters changed UA %x: "
2441                                 "thread context required", cmd->sense[12]);
2442                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2443                         goto out;
2444                 }
2445
2446                 TRACE(TRACE_SCSI, "Possible parameters changed UA %x "
2447                         "(lun %lld): getting new parameters", cmd->sense[12],
2448                         (long long unsigned int)cmd->lun);
2449
2450                 scst_obtain_device_parameters(cmd->dev);
2451         } else
2452                 sBUG();
2453
2454         cmd->state = SCST_CMD_STATE_DEV_DONE;
2455
2456 out:
2457         TRACE_EXIT_HRES(res);
2458         return res;
2459 }
2460
2461 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2462 {
2463         if (likely(cmd->sn_set))
2464                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2465
2466         scst_make_deferred_commands_active(cmd->tgt_dev);
2467 }
2468
2469 static int scst_dev_done(struct scst_cmd **pcmd)
2470 {
2471         int res = SCST_CMD_STATE_RES_CONT_SAME;
2472         struct scst_cmd *cmd = *pcmd;
2473         int state;
2474         struct scst_device *dev = cmd->dev;
2475
2476         TRACE_ENTRY();
2477
2478         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2479
2480         if (likely(!scst_is_cmd_local(cmd)) &&
2481             likely(dev->handler->dev_done != NULL)) {
2482                 int rc;
2483
2484                 if (unlikely(!dev->handler->dev_done_atomic &&
2485                              scst_cmd_atomic(cmd))) {
2486                         /*
2487                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2488                          * optimization.
2489                          */
2490                         TRACE_DBG("Dev handler %s dev_done() needs thread "
2491                               "context, rescheduling", dev->handler->name);
2492                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2493                         goto out;
2494                 }
2495
2496                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2497                       dev->handler->name, cmd);
2498                 rc = dev->handler->dev_done(cmd);
2499                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2500                       dev->handler->name, rc);
2501                 if (rc != SCST_CMD_STATE_DEFAULT)
2502                         state = rc;
2503         }
2504
2505         switch (state) {
2506         case SCST_CMD_STATE_PRE_XMIT_RESP:
2507         case SCST_CMD_STATE_DEV_PARSE:
2508         case SCST_CMD_STATE_PRE_PARSE:
2509         case SCST_CMD_STATE_PREPARE_SPACE:
2510         case SCST_CMD_STATE_RDY_TO_XFER:
2511         case SCST_CMD_STATE_TGT_PRE_EXEC:
2512         case SCST_CMD_STATE_SEND_FOR_EXEC:
2513         case SCST_CMD_STATE_LOCAL_EXEC:
2514         case SCST_CMD_STATE_REAL_EXEC:
2515         case SCST_CMD_STATE_PRE_DEV_DONE:
2516         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2517         case SCST_CMD_STATE_DEV_DONE:
2518         case SCST_CMD_STATE_XMIT_RESP:
2519         case SCST_CMD_STATE_FINISHED:
2520                 cmd->state = state;
2521                 break;
2522
2523         case SCST_CMD_STATE_NEED_THREAD_CTX:
2524                 TRACE_DBG("Dev handler %s dev_done() requested "
2525                       "thread context, rescheduling",
2526                       dev->handler->name);
2527                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2528                 break;
2529
2530         default:
2531                 if (state >= 0) {
2532                         PRINT_ERROR("Dev handler %s dev_done() returned "
2533                                 "invalid cmd state %d",
2534                                 dev->handler->name, state);
2535                 } else {
2536                         PRINT_ERROR("Dev handler %s dev_done() returned "
2537                                 "error %d", dev->handler->name,
2538                                 state);
2539                 }
2540                 scst_set_cmd_error(cmd,
2541                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2542                 scst_set_cmd_abnormal_done_state(cmd);
2543                 break;
2544         }
2545
2546         if (cmd->needs_unblocking)
2547                 scst_unblock_dev_cmd(cmd);
2548
2549         if (likely(cmd->dec_on_dev_needed))
2550                 scst_dec_on_dev_cmd(cmd);
2551
2552         if (cmd->inc_expected_sn_on_done && cmd->sent_for_exec)
2553                 scst_inc_check_expected_sn(cmd);
2554
2555         if (unlikely(cmd->cdb[0] == REQUEST_SENSE) && (cmd->internal))
2556                 *pcmd = scst_complete_request_sense(cmd);
2557
2558 out:
2559         TRACE_EXIT_HRES(res);
2560         return res;
2561 }
2562
2563 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2564 {
2565         int res;
2566
2567         TRACE_ENTRY();
2568
2569 #ifdef CONFIG_SCST_DEBUG_TM
2570         if (cmd->tm_dbg_delayed && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2571                 if (scst_cmd_atomic(cmd)) {
2572                         TRACE_MGMT_DBG("%s", "DEBUG_TM delayed cmd needs a thread");
2573                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2574                         return res;
2575                 }
2576                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2577                         cmd, cmd->tag);
2578                 schedule_timeout_uninterruptible(HZ);
2579         }
2580 #endif
2581
2582         if (likely(cmd->tgt_dev != NULL)) {
2583                 atomic_dec(&cmd->tgt_dev->tgt_dev_cmd_count);
2584                 atomic_dec(&cmd->dev->dev_cmd_count);
2585                 /* If expected values not set, expected direction is UNKNOWN */
2586                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
2587                         atomic_dec(&cmd->dev->write_cmd_count);
2588
2589                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2590                         scst_on_hq_cmd_response(cmd);
2591
2592                 if (unlikely(!cmd->sent_for_exec)) {
2593                         TRACE_SN("cmd %p was not sent to mid-lev (sn %ld, set %d)",
2594                                 cmd, cmd->sn, cmd->sn_set);
2595                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2596                         cmd->sent_for_exec = 1;
2597                 }
2598         }
2599
2600         /*
2601          * If we don't remove cmd from the search list here, before
2602          * submitting it for transmittion, we will have a race, when for
2603          * some reason cmd's release is delayed after transmittion and
2604          * initiator sends cmd with the same tag => it is possible that
2605          * a wrong cmd will be found by find() functions.
2606          */
2607         spin_lock_irq(&cmd->sess->sess_list_lock);
2608         list_del(&cmd->search_cmd_list_entry);
2609         spin_unlock_irq(&cmd->sess->sess_list_lock);
2610
2611         cmd->done = 1;
2612         smp_mb(); /* to sync with scst_abort_cmd() */
2613
2614         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2615                 scst_xmit_process_aborted_cmd(cmd);
2616
2617         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2618                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu), skipping",
2619                                cmd, (long long unsigned int)cmd->tag);
2620                 cmd->state = SCST_CMD_STATE_FINISHED;
2621                 res = SCST_CMD_STATE_RES_CONT_SAME;
2622                 goto out;
2623         }
2624
2625         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2626         res = SCST_CMD_STATE_RES_CONT_SAME;
2627
2628 out:
2629 #ifdef CONFIG_SCST_MEASURE_LATENCY
2630         {
2631                 struct timespec ts;
2632                 uint64_t finish, scst_time, proc_time;
2633                 struct scst_session *sess = cmd->sess;
2634
2635                 getnstimeofday(&ts);
2636                 finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2637
2638                 spin_lock_bh(&sess->meas_lock);
2639
2640                 scst_time = cmd->pre_exec_finish - cmd->start;
2641                 scst_time += finish - cmd->post_exec_start;
2642                 proc_time = finish - cmd->start;
2643
2644                 sess->scst_time += scst_time;
2645                 sess->processing_time += proc_time;
2646                 sess->processed_cmds++;
2647
2648                 spin_unlock_bh(&sess->meas_lock);
2649
2650                 TRACE_DBG("cmd %p (sess %p): finish %lld (tv_sec %ld, "
2651                         "tv_nsec %ld), scst_time %lld, proc_time %lld", cmd, sess,
2652                         finish, ts.tv_sec, ts.tv_nsec, scst_time, proc_time);
2653         }
2654 #endif
2655         TRACE_EXIT_HRES(res);
2656         return res;
2657 }
2658
2659 static int scst_xmit_response(struct scst_cmd *cmd)
2660 {
2661         int res, rc;
2662
2663         TRACE_ENTRY();
2664
2665         if (unlikely(!cmd->tgtt->xmit_response_atomic &&
2666                      scst_cmd_atomic(cmd))) {
2667                 /*
2668                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2669                  * optimization.
2670                  */
2671                 TRACE_DBG("Target driver %s xmit_response() needs thread "
2672                               "context, rescheduling", cmd->tgtt->name);
2673                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2674                 goto out;
2675         }
2676
2677         while (1) {
2678                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2679
2680                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2681                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2682
2683                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2684
2685 #if defined(CONFIG_SCST_DEBUG)
2686                 if (cmd->sg) {
2687                         int i;
2688                         struct scatterlist *sg = cmd->sg;
2689                         TRACE_SEND_BOT("Xmitting %d S/G(s) at %p sg[0].page at %p",
2690                               cmd->sg_cnt, sg, (void *)sg_page(&sg[0]));
2691                         for (i = 0; i < cmd->sg_cnt; ++i) {
2692                                 TRACE_BUFF_FLAG(TRACE_SND_BOT,
2693                                     "Xmitting sg", sg_virt(&sg[i]),
2694                                     sg[i].length);
2695                         }
2696                 }
2697 #endif
2698
2699 #ifdef CONFIG_SCST_DEBUG_RETRY
2700                 if (((scst_random() % 100) == 77))
2701                         rc = SCST_TGT_RES_QUEUE_FULL;
2702                 else
2703 #endif
2704                         rc = cmd->tgtt->xmit_response(cmd);
2705                 TRACE_DBG("xmit_response() returned %d", rc);
2706
2707                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2708                         goto out;
2709
2710                 /* Restore the previous state */
2711                 cmd->state = SCST_CMD_STATE_XMIT_RESP;
2712
2713                 switch (rc) {
2714                 case SCST_TGT_RES_QUEUE_FULL:
2715                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2716                                 break;
2717                         else
2718                                 continue;
2719
2720                 case SCST_TGT_RES_NEED_THREAD_CTX:
2721                         TRACE_DBG("Target driver %s xmit_response() "
2722                               "requested thread context, rescheduling",
2723                               cmd->tgtt->name);
2724                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2725                         break;
2726
2727                 default:
2728                         goto out_error;
2729                 }
2730                 break;
2731         }
2732
2733 out:
2734         /* Caution: cmd can be already dead here */
2735         TRACE_EXIT_HRES(res);
2736         return res;
2737
2738 out_error:
2739         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2740                 PRINT_ERROR("Target driver %s xmit_response() returned "
2741                         "fatal error", cmd->tgtt->name);
2742         } else {
2743                 PRINT_ERROR("Target driver %s xmit_response() returned "
2744                         "invalid value %d", cmd->tgtt->name, rc);
2745         }
2746         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2747         cmd->state = SCST_CMD_STATE_FINISHED;
2748         res = SCST_CMD_STATE_RES_CONT_SAME;
2749         goto out;
2750 }
2751
2752 void scst_tgt_cmd_done(struct scst_cmd *cmd)
2753 {
2754         TRACE_ENTRY();
2755
2756         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2757
2758         cmd->state = SCST_CMD_STATE_FINISHED;
2759         scst_proccess_redirect_cmd(cmd, scst_get_context(), 1);
2760
2761         TRACE_EXIT();
2762         return;
2763 }
2764 EXPORT_SYMBOL(scst_tgt_cmd_done);
2765
2766 static int scst_finish_cmd(struct scst_cmd *cmd)
2767 {
2768         int res;
2769
2770         TRACE_ENTRY();
2771
2772         atomic_dec(&cmd->sess->sess_cmd_count);
2773
2774         cmd->finished = 1;
2775         smp_mb(); /* to sync with scst_abort_cmd() */
2776
2777         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2778                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
2779                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
2780                         atomic_read(&scst_cmd_count));
2781
2782                 scst_finish_cmd_mgmt(cmd);
2783         }
2784
2785         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
2786                 if ((cmd->tgt_dev != NULL) &&
2787                     scst_is_ua_sense(cmd->sense)) {
2788                         /* This UA delivery failed, so requeue it */
2789                         TRACE_MGMT_DBG("Requeuing UA for delivery failed cmd "
2790                                 "%p", cmd);
2791                         scst_check_set_UA(cmd->tgt_dev, cmd->sense,
2792                                         SCST_SENSE_BUFFERSIZE, 1);
2793                 }
2794         }
2795
2796         __scst_cmd_put(cmd);
2797
2798         res = SCST_CMD_STATE_RES_CONT_NEXT;
2799
2800         TRACE_EXIT_HRES(res);
2801         return res;
2802 }
2803
2804 /*
2805  * No locks, but it must be externally serialized (see comment for
2806  * scst_cmd_init_done() in scst.h)
2807  */
2808 static void scst_cmd_set_sn(struct scst_cmd *cmd)
2809 {
2810         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2811         unsigned long flags;
2812
2813         TRACE_ENTRY();
2814
2815         if (scst_is_implicit_hq(cmd)) {
2816                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "Implicit HQ cmd %p", cmd);
2817                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
2818         }
2819
2820         EXTRACHECKS_BUG_ON(cmd->sn_set || cmd->hq_cmd_inced);
2821
2822         /* Optimized for lockless fast path */
2823
2824         scst_check_debug_sn(cmd);
2825
2826         if (cmd->dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) {
2827                 /*
2828                  * Not the best way, but well enough until there will be a
2829                  * possibility to specify queue type during pass-through
2830                  * commands submission.
2831                  */
2832                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
2833         }
2834
2835         switch (cmd->queue_type) {
2836         case SCST_CMD_QUEUE_SIMPLE:
2837         case SCST_CMD_QUEUE_UNTAGGED:
2838 #if 1 /* temporary, ToDo */
2839                 if (scst_cmd_is_expected_set(cmd)) {
2840                         if ((cmd->expected_data_direction == SCST_DATA_READ) &&
2841                             (atomic_read(&cmd->dev->write_cmd_count) == 0))
2842                                 goto ordered;
2843                 } else
2844                         goto ordered;
2845 #endif
2846                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
2847                         /*
2848                          * atomic_inc_return() implies memory barrier to sync
2849                          * with scst_inc_expected_sn()
2850                          */
2851                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
2852                                 tgt_dev->curr_sn++;
2853                                 TRACE_SN("Incremented curr_sn %ld",
2854                                         tgt_dev->curr_sn);
2855                         }
2856                         cmd->sn_slot = tgt_dev->cur_sn_slot;
2857                         cmd->sn = tgt_dev->curr_sn;
2858
2859                         tgt_dev->prev_cmd_ordered = 0;
2860                 } else {
2861                         TRACE(TRACE_MINOR, "***WARNING*** Not enough SN slots "
2862                                 "%zd", ARRAY_SIZE(tgt_dev->sn_slots));
2863                         goto ordered;
2864                 }
2865                 break;
2866
2867         case SCST_CMD_QUEUE_ORDERED:
2868                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "ORDERED cmd %p "
2869                         "(op %x)", cmd, cmd->cdb[0]);
2870 ordered:
2871                 if (!tgt_dev->prev_cmd_ordered) {
2872                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2873                         if (tgt_dev->num_free_sn_slots >= 0) {
2874                                 tgt_dev->num_free_sn_slots--;
2875                                 if (tgt_dev->num_free_sn_slots >= 0) {
2876                                         int i = 0;
2877                                         /*
2878                                          * Commands can finish in any order, so we don't
2879                                          * know, which slot is empty.
2880                                          */
2881                                         while (1) {
2882                                                 tgt_dev->cur_sn_slot++;
2883                                                 if (tgt_dev->cur_sn_slot == tgt_dev->sn_slots +
2884                                                                 ARRAY_SIZE(tgt_dev->sn_slots))
2885                                                         tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
2886
2887                                                 if (atomic_read(tgt_dev->cur_sn_slot) == 0)
2888                                                         break;
2889
2890                                                 i++;
2891                                                 sBUG_ON(i == ARRAY_SIZE(tgt_dev->sn_slots));
2892                                         }
2893                                         TRACE_SN("New cur SN slot %zd",
2894                                                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
2895                                 }
2896                         }
2897                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2898                 }
2899                 tgt_dev->prev_cmd_ordered = 1;
2900                 tgt_dev->curr_sn++;
2901                 cmd->sn = tgt_dev->curr_sn;
2902                 break;
2903
2904         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
2905                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "HQ cmd %p "
2906                         "(op %x)", cmd, cmd->cdb[0]);
2907                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2908                 tgt_dev->hq_cmd_count++;
2909                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2910                 cmd->hq_cmd_inced = 1;
2911                 goto out;
2912
2913         default:
2914                 sBUG();
2915         }
2916
2917         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
2918                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
2919                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
2920                 atomic_read(tgt_dev->cur_sn_slot),
2921                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
2922                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
2923
2924         cmd->sn_set = 1;
2925
2926 out:
2927         TRACE_EXIT();
2928         return;
2929 }
2930
2931 /*
2932  * Returns 0 on success, > 0 when we need to wait for unblock,
2933  * < 0 if there is no device (lun) or device type handler.
2934  *
2935  * No locks, but might be on IRQ, protection is done by the
2936  * suspended activity.
2937  */
2938 static int scst_translate_lun(struct scst_cmd *cmd)
2939 {
2940         struct scst_tgt_dev *tgt_dev = NULL;
2941         int res;
2942
2943         TRACE_ENTRY();
2944
2945         __scst_get(1);
2946
2947         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
2948                 struct list_head *sess_tgt_dev_list_head =
2949                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
2950                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %lld)", cmd,
2951                         (long long unsigned int)cmd->lun);
2952                 res = -1;
2953                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
2954                                 sess_tgt_dev_list_entry) {
2955                         if (tgt_dev->lun == cmd->lun) {
2956                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
2957
2958                                 if (unlikely(tgt_dev->dev->handler == &scst_null_devtype)) {
2959                                         PRINT_INFO("Dev handler for device "
2960                                           "%lld is NULL, the device will not be "
2961                                           "visible remotely",
2962                                            (long long unsigned int)cmd->lun);
2963                                         break;
2964                                 }
2965
2966                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
2967                                 cmd->tgt_dev = tgt_dev;
2968                                 cmd->dev = tgt_dev->dev;
2969
2970                                 res = 0;
2971                                 break;
2972                         }
2973                 }
2974                 if (res != 0) {
2975                         TRACE(TRACE_MINOR, "tgt_dev for lun %lld not found, command to "
2976                                 "unexisting LU?", (long long unsigned int)cmd->lun);
2977                         __scst_put();
2978                 }
2979         } else {
2980                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
2981                 __scst_put();
2982                 res = 1;
2983         }
2984
2985         TRACE_EXIT_RES(res);
2986         return res;
2987 }
2988
2989 /*
2990  * No locks, but might be on IRQ
2991  *
2992  * Returns 0 on success, > 0 when we need to wait for unblock,
2993  * < 0 if there is no device (lun) or device type handler.
2994  */
2995 static int __scst_init_cmd(struct scst_cmd *cmd)
2996 {
2997         int res = 0;
2998
2999         TRACE_ENTRY();
3000
3001         res = scst_translate_lun(cmd);
3002         if (likely(res == 0)) {
3003                 int cnt;
3004                 bool failure = false;
3005
3006                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
3007
3008                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
3009                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
3010                         TRACE(TRACE_MGMT_MINOR, "Too many pending commands (%d) in "
3011                                 "session, returning BUSY to initiator \"%s\"",
3012                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
3013                                   "Anonymous" : cmd->sess->initiator_name);
3014                         failure = true;
3015                 }
3016
3017                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
3018                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
3019                         if (!failure) {
3020                                 TRACE(TRACE_MGMT_MINOR, "Too many pending device "
3021                                         "commands (%d), returning BUSY to "
3022                                         "initiator \"%s\"", cnt,
3023                                         (cmd->sess->initiator_name[0] == '\0') ?
3024                                           "Anonymous" : cmd->sess->initiator_name);
3025                                 failure = true;
3026                         }
3027                 }
3028
3029                 /* If expected values not set, expected direction is UNKNOWN */
3030                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
3031                         atomic_inc(&cmd->dev->write_cmd_count);
3032
3033                 if (unlikely(failure))
3034                         goto out_busy;
3035
3036                 if (!cmd->set_sn_on_restart_cmd)
3037                         scst_cmd_set_sn(cmd);
3038         } else if (res < 0) {
3039                 TRACE_DBG("Finishing cmd %p", cmd);
3040                 scst_set_cmd_error(cmd,
3041                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
3042                 scst_set_cmd_abnormal_done_state(cmd);
3043         } else
3044                 goto out;
3045
3046 out:
3047         TRACE_EXIT_RES(res);
3048         return res;
3049
3050 out_busy:
3051         scst_set_busy(cmd);
3052         scst_set_cmd_abnormal_done_state(cmd);
3053         goto out;
3054 }
3055
3056 /* Called under scst_init_lock and IRQs disabled */
3057 static void scst_do_job_init(void)
3058 {
3059         struct scst_cmd *cmd;
3060         int susp;
3061
3062         TRACE_ENTRY();
3063
3064 restart:
3065         /*
3066          * There is no need for read barrier here, because we don't care where
3067          * this check will be done.
3068          */
3069         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
3070         if (scst_init_poll_cnt > 0)
3071                 scst_init_poll_cnt--;
3072
3073         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
3074                 int rc;
3075                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
3076                         continue;
3077                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
3078                         spin_unlock_irq(&scst_init_lock);
3079                         rc = __scst_init_cmd(cmd);
3080                         spin_lock_irq(&scst_init_lock);
3081                         if (rc > 0) {
3082                                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, restarting");
3083                                 goto restart;
3084                         }
3085                 } else {
3086                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
3087                                        cmd, (long long unsigned int)cmd->tag);
3088                         scst_set_cmd_abnormal_done_state(cmd);
3089                 }
3090
3091                 /*
3092                  * Deleting cmd from init cmd list after __scst_init_cmd()
3093                  * is necessary to keep the check in scst_init_cmd() correct
3094                  * to preserve the commands order.
3095                  *
3096                  * We don't care about the race, when init cmd list is empty
3097                  * and one command detected that it just was not empty, so
3098                  * it's inserting to it, but another command at the same time
3099                  * seeing init cmd list empty and goes directly, because it
3100                  * could affect only commands from the same initiator to the
3101                  * same tgt_dev, but init_cmd_done() doesn't guarantee the order
3102                  * in case of simultaneous such calls anyway.
3103                  */
3104                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
3105                 smp_wmb();
3106                 list_del(&cmd->cmd_list_entry);
3107                 spin_unlock(&scst_init_lock);
3108
3109                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
3110                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
3111                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
3112                         list_add(&cmd->cmd_list_entry,
3113                                 &cmd->cmd_lists->active_cmd_list);
3114                 else
3115                         list_add_tail(&cmd->cmd_list_entry,
3116                                 &cmd->cmd_lists->active_cmd_list);
3117                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3118                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
3119
3120                 spin_lock(&scst_init_lock);
3121                 goto restart;
3122         }
3123
3124         /* It isn't really needed, but let's keep it */
3125         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
3126                 goto restart;
3127
3128         TRACE_EXIT();
3129         return;
3130 }
3131
3132 static inline int test_init_cmd_list(void)
3133 {
3134         int res = (!list_empty(&scst_init_cmd_list) &&
3135                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
3136                   unlikely(kthread_should_stop()) ||
3137                   (scst_init_poll_cnt > 0);
3138         return res;
3139 }
3140
3141 int scst_init_cmd_thread(void *arg)
3142 {
3143         TRACE_ENTRY();
3144
3145         PRINT_INFO("Init thread started, PID %d", current->pid);
3146
3147         current->flags |= PF_NOFREEZE;
3148
3149         set_user_nice(current, -10);
3150
3151         spin_lock_irq(&scst_init_lock);
3152         while (!kthread_should_stop()) {
3153                 wait_queue_t wait;
3154                 init_waitqueue_entry(&wait, current);
3155
3156                 if (!test_init_cmd_list()) {
3157                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
3158                                                  &wait);
3159                         for (;;) {
3160                                 set_current_state(TASK_INTERRUPTIBLE);
3161                                 if (test_init_cmd_list())
3162                                         break;
3163                                 spin_unlock_irq(&scst_init_lock);
3164                                 schedule();
3165                                 spin_lock_irq(&scst_init_lock);
3166                         }
3167                         set_current_state(TASK_RUNNING);
3168                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
3169                 }
3170                 scst_do_job_init();
3171         }
3172         spin_unlock_irq(&scst_init_lock);
3173
3174         /*
3175          * If kthread_should_stop() is true, we are guaranteed to be
3176          * on the module unload, so scst_init_cmd_list must be empty.
3177          */
3178         sBUG_ON(!list_empty(&scst_init_cmd_list));
3179
3180         PRINT_INFO("Init thread PID %d finished", current->pid);
3181
3182         TRACE_EXIT();
3183         return 0;
3184 }
3185
3186 /* Called with no locks held */
3187 void scst_process_active_cmd(struct scst_cmd *cmd, int context)
3188 {
3189         int res;
3190
3191         TRACE_ENTRY();
3192
3193         EXTRACHECKS_BUG_ON(in_irq());
3194
3195         cmd->context_processable = context | SCST_CONTEXT_PROCESSABLE;
3196         context &= ~SCST_CONTEXT_PROCESSABLE;
3197         cmd->atomic = (context == SCST_CONTEXT_DIRECT_ATOMIC);
3198
3199         TRACE_DBG("cmd %p, context_processable %d, atomic %d", cmd,
3200                 cmd->context_processable, cmd->atomic);
3201
3202         do {
3203                 switch (cmd->state) {
3204                 case SCST_CMD_STATE_PRE_PARSE:
3205                         res = scst_pre_parse(cmd);
3206                         EXTRACHECKS_BUG_ON(res ==
3207                                 SCST_CMD_STATE_RES_NEED_THREAD);
3208                         break;
3209
3210                 case SCST_CMD_STATE_DEV_PARSE:
3211                         res = scst_parse_cmd(cmd);
3212                         break;
3213
3214                 case SCST_CMD_STATE_PREPARE_SPACE:
3215                         res = scst_prepare_space(cmd);
3216                         break;
3217
3218                 case SCST_CMD_STATE_RDY_TO_XFER:
3219                         res = scst_rdy_to_xfer(cmd);
3220                         break;
3221
3222                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3223                         res = scst_tgt_pre_exec(cmd);
3224                         break;
3225
3226                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3227                         if (tm_dbg_check_cmd(cmd) != 0) {
3228                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
3229                                 TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
3230                                         "because of TM DBG delay", cmd,
3231                                         (long long unsigned int)cmd->tag);
3232                                 break;
3233                         }
3234                         res = scst_send_for_exec(&cmd);
3235                         /* !! At this point cmd, sess & tgt_dev can be already freed !! */
3236                         break;
3237
3238                 case SCST_CMD_STATE_LOCAL_EXEC:
3239                         res = scst_local_exec(cmd);
3240                         /* !! At this point cmd, sess & tgt_dev can be already freed !! */
3241                         break;
3242
3243                 case SCST_CMD_STATE_REAL_EXEC:
3244                         res = scst_real_exec(cmd);
3245                         /* !! At this point cmd, sess & tgt_dev can be already freed !! */
3246                         break;
3247
3248                 case SCST_CMD_STATE_PRE_DEV_DONE:
3249                         res = scst_pre_dev_done(cmd);
3250                         EXTRACHECKS_BUG_ON(res ==
3251                                 SCST_CMD_STATE_RES_NEED_THREAD);
3252                         break;
3253
3254                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3255                         res = scst_mode_select_checks(cmd);
3256                         break;
3257
3258                 case SCST_CMD_STATE_DEV_DONE:
3259                         res = scst_dev_done(&cmd);
3260                         break;
3261
3262                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3263                         res = scst_pre_xmit_response(cmd);
3264                         EXTRACHECKS_BUG_ON(res ==
3265                                 SCST_CMD_STATE_RES_NEED_THREAD);
3266                         break;
3267
3268                 case SCST_CMD_STATE_XMIT_RESP:
3269                         res = scst_xmit_response(cmd);
3270                         break;
3271
3272                 case SCST_CMD_STATE_FINISHED:
3273                         res = scst_finish_cmd(cmd);
3274                         EXTRACHECKS_BUG_ON(res ==
3275                                 SCST_CMD_STATE_RES_NEED_THREAD);
3276                         break;
3277
3278                 default:
3279                         PRINT_CRIT_ERROR("cmd (%p) in state %d, but shouldn't "
3280                                 "be", cmd, cmd->state);
3281                         sBUG();
3282                         res = SCST_CMD_STATE_RES_CONT_NEXT;
3283                         break;
3284                 }
3285         } while (res == SCST_CMD_STATE_RES_CONT_SAME);
3286
3287         if (res == SCST_CMD_STATE_RES_CONT_NEXT) {
3288                 /* None */
3289         } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
3290                 spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3291                 switch (cmd->state) {
3292                 case SCST_CMD_STATE_PRE_PARSE:
3293                 case SCST_CMD_STATE_DEV_PARSE:
3294                 case SCST_CMD_STATE_PREPARE_SPACE:
3295                 case SCST_CMD_STATE_RDY_TO_XFER:
3296                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3297                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3298                 case SCST_CMD_STATE_LOCAL_EXEC:
3299                 case SCST_CMD_STATE_REAL_EXEC:
3300                 case SCST_CMD_STATE_PRE_DEV_DONE:
3301                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3302                 case SCST_CMD_STATE_DEV_DONE:
3303                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3304                 case SCST_CMD_STATE_XMIT_RESP:
3305                 case SCST_CMD_STATE_FINISHED:
3306                         TRACE_DBG("Adding cmd %p to head of active cmd list", cmd);
3307                         list_add(&cmd->cmd_list_entry,
3308                                 &cmd->cmd_lists->active_cmd_list);
3309                         break;
3310 #ifdef CONFIG_SCST_EXTRACHECKS
3311                 /* not very valid commands */
3312                 case SCST_CMD_STATE_DEFAULT:
3313                 case SCST_CMD_STATE_NEED_THREAD_CTX:
3314                         PRINT_CRIT_ERROR("cmd %p is in state %d, not putting on "
3315                                 "useful list (left on scst cmd list)", cmd,
3316                                 cmd->state);
3317                         spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3318                         sBUG();
3319                         spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3320                         break;
3321 #endif
3322                 default:
3323                         break;
3324                 }
3325                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3326                 spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3327         } else
3328                 sBUG();
3329
3330         TRACE_EXIT();
3331         return;
3332 }
3333 EXPORT_SYMBOL(scst_process_active_cmd);
3334
3335 /* Called under cmd_list_lock and IRQs disabled */
3336 static void scst_do_job_active(struct list_head *cmd_list,
3337         spinlock_t *cmd_list_lock, int context)
3338 {
3339         TRACE_ENTRY();
3340
3341 #ifdef CONFIG_SCST_EXTRACHECKS
3342         {
3343                 int c = context & ~SCST_CONTEXT_PROCESSABLE;
3344                 sBUG_ON((c != SCST_CONTEXT_DIRECT_ATOMIC) &&
3345                         (c != SCST_CONTEXT_DIRECT));
3346         }
3347 #endif
3348
3349         while (!list_empty(cmd_list)) {
3350                 struct scst_cmd *cmd = list_entry(cmd_list->next, typeof(*cmd),
3351                                         cmd_list_entry);
3352                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
3353                 list_del(&cmd->cmd_list_entry);
3354                 spin_unlock_irq(cmd_list_lock);
3355                 scst_process_active_cmd(cmd, context);
3356                 spin_lock_irq(cmd_list_lock);
3357         }
3358
3359         TRACE_EXIT();
3360         return;
3361 }
3362
3363 static inline int test_cmd_lists(struct scst_cmd_lists *p_cmd_lists)
3364 {
3365         int res = !list_empty(&p_cmd_lists->active_cmd_list) ||
3366             unlikely(kthread_should_stop()) ||
3367             tm_dbg_is_release();
3368         return res;
3369 }
3370
3371 int scst_cmd_thread(void *arg)
3372 {
3373         struct scst_cmd_lists *p_cmd_lists = (struct scst_cmd_lists *)arg;
3374
3375         TRACE_ENTRY();
3376
3377         PRINT_INFO("Processing thread started, PID %d", current->pid);
3378
3379 #if 0
3380         set_user_nice(current, 10);
3381 #endif
3382         current->flags |= PF_NOFREEZE;
3383
3384         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3385         while (!kthread_should_stop()) {
3386                 wait_queue_t wait;
3387                 init_waitqueue_entry(&wait, current);
3388
3389                 if (!test_cmd_lists(p_cmd_lists)) {
3390                         add_wait_queue_exclusive(&p_cmd_lists->cmd_list_waitQ,
3391                                 &wait);
3392                         for (;;) {
3393                                 set_current_state(TASK_INTERRUPTIBLE);
3394                                 if (test_cmd_lists(p_cmd_lists))
3395                                         break;
3396                                 spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3397                                 schedule();
3398                                 spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3399                         }
3400                         set_current_state(TASK_RUNNING);
3401                         remove_wait_queue(&p_cmd_lists->cmd_list_waitQ, &wait);
3402                 }
3403
3404                 if (tm_dbg_is_release()) {
3405                         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3406                         tm_dbg_check_released_cmds();
3407                         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3408                 }
3409
3410                 scst_do_job_active(&p_cmd_lists->active_cmd_list,
3411                         &p_cmd_lists->cmd_list_lock, SCST_CONTEXT_DIRECT |
3412                                                 SCST_CONTEXT_PROCESSABLE);
3413         }
3414         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3415
3416 #ifdef CONFIG_SCST_EXTRACHECKS
3417         /*
3418          * If kthread_should_stop() is true, we are guaranteed to be either
3419          * on the module unload, or there must be at least one other thread to
3420          * process the commands lists.
3421          */
3422         if (p_cmd_lists == &scst_main_cmd_lists) {
3423                 sBUG_ON((scst_threads_info.nr_cmd_threads == 1) &&
3424                          !list_empty(&scst_main_cmd_lists.active_cmd_list));
3425         }
3426 #endif
3427
3428         PRINT_INFO("Processing thread PID %d finished", current->pid);
3429
3430         TRACE_EXIT();
3431         return 0;
3432 }
3433
3434 void scst_cmd_tasklet(long p)
3435 {
3436         struct scst_tasklet *t = (struct scst_tasklet *)p;
3437
3438         TRACE_ENTRY();
3439
3440         spin_lock_irq(&t->tasklet_lock);
3441         scst_do_job_active(&t->tasklet_cmd_list, &t->tasklet_lock,
3442                 SCST_CONTEXT_DIRECT_ATOMIC);
3443         spin_unlock_irq(&t->tasklet_lock);
3444
3445         TRACE_EXIT();
3446         return;
3447 }
3448
3449 /*
3450  * Returns 0 on success, < 0 if there is no device handler or
3451  * > 0 if SCST_FLAG_SUSPENDED set and SCST_FLAG_SUSPENDING - not.
3452  * No locks, protection is done by the suspended activity.
3453  */
3454 static int scst_mgmt_translate_lun(struct scst_mgmt_cmd *mcmd)
3455 {
3456         struct scst_tgt_dev *tgt_dev = NULL;
3457         struct list_head *sess_tgt_dev_list_head;
3458         int res = -1;
3459
3460         TRACE_ENTRY();
3461
3462         TRACE_DBG("Finding tgt_dev for mgmt cmd %p (lun %lld)", mcmd,
3463               (long long unsigned int)mcmd->lun);
3464
3465         __scst_get(1);
3466
3467         if (unlikely(test_bit(SCST_FLAG_SUSPENDED, &scst_flags) &&
3468                      !test_bit(SCST_FLAG_SUSPENDING, &scst_flags))) {
3469                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3470                 __scst_put();
3471                 res = 1;
3472                 goto out;
3473         }
3474
3475         sess_tgt_dev_list_head =
3476                 &mcmd->sess->sess_tgt_dev_list_hash[HASH_VAL(mcmd->lun)];
3477         list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3478                         sess_tgt_dev_list_entry) {
3479                 if (tgt_dev->lun == mcmd->lun) {
3480                         TRACE_DBG("tgt_dev %p found", tgt_dev);
3481                         mcmd->mcmd_tgt_dev = tgt_dev;
3482                         res = 0;
3483                         break;
3484                 }
3485         }
3486         if (mcmd->mcmd_tgt_dev == NULL)
3487                 __scst_put();
3488
3489 out:
3490         TRACE_EXIT_HRES(res);
3491         return res;
3492 }
3493
3494 /* No locks */
3495 void scst_done_cmd_mgmt(struct scst_cmd *cmd)
3496 {
3497         struct scst_mgmt_cmd_stub *mstb;
3498         bool wake = 0;
3499         unsigned long flags;
3500
3501         TRACE_ENTRY();
3502
3503         TRACE_MGMT_DBG("cmd %p done (tag %llu)",
3504                        cmd, (long long unsigned int)cmd->tag);
3505
3506         spin_lock_irqsave(&scst_mcmd_lock, flags);
3507
3508         list_for_each_entry(mstb, &cmd->mgmt_cmd_list,
3509                         cmd_mgmt_cmd_list_entry) {
3510                 struct scst_mgmt_cmd *mcmd = mstb->mcmd;
3511
3512                 TRACE_MGMT_DBG("mcmd %p, mcmd->cmd_done_wait_count %d",
3513                         mcmd, mcmd->cmd_done_wait_count);
3514
3515                 mcmd->cmd_done_wait_count--;
3516                 if (mcmd->cmd_done_wait_count > 0) {
3517                         TRACE_MGMT_DBG("cmd_done_wait_count(%d) not 0, "
3518                                 "skipping", mcmd->cmd_done_wait_count);
3519                         continue;
3520                 }
3521
3522                 if (mcmd->co