The major TM processing cleanup in scst_user module which was possible after the...
[mirror/scst/.git] / scst / src / scst_targ.c
1 /*
2  *  scst_targ.c
3  *
4  *  Copyright (C) 2004-2007 Vladislav Bolkhovitin <vst@vlnb.net>
5  *                 and Leonid Stoljar
6  *
7  *  This program is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU General Public License
9  *  as published by the Free Software Foundation, version 2
10  *  of the License.
11  *
12  *  This program is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  *  GNU General Public License for more details.
16  */
17
18 #include <linux/init.h>
19 #include <linux/kernel.h>
20 #include <linux/errno.h>
21 #include <linux/list.h>
22 #include <linux/spinlock.h>
23 #include <linux/slab.h>
24 #include <linux/sched.h>
25 #include <linux/smp_lock.h>
26 #include <linux/unistd.h>
27 #include <linux/string.h>
28 #include <linux/kthread.h>
29 #include <linux/delay.h>
30
31 #include "scst.h"
32 #include "scst_priv.h"
33
34 static void scst_cmd_set_sn(struct scst_cmd *cmd);
35 static int __scst_init_cmd(struct scst_cmd *cmd);
36 static void scst_finish_cmd_mgmt(struct scst_cmd *cmd);
37
38 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
39 {
40         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
41         unsigned long flags;
42
43         spin_lock_irqsave(&t->tasklet_lock, flags);
44         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
45                 smp_processor_id());
46         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
47         spin_unlock_irqrestore(&t->tasklet_lock, flags);
48
49         tasklet_schedule(&t->tasklet);
50 }
51
52 /*
53  * Must not be called in parallel with scst_unregister_session_ex() for the
54  * same sess
55  */
56 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
57                              const uint8_t *lun, int lun_len,
58                              const uint8_t *cdb, int cdb_len, int atomic)
59 {
60         struct scst_cmd *cmd;
61
62         TRACE_ENTRY();
63
64 #ifdef EXTRACHECKS
65         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
66                 PRINT_CRIT_ERROR("%s", "New cmd while shutting down the session");
67                 sBUG();
68         }
69 #endif
70
71         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
72         if (cmd == NULL)
73                 goto out;
74
75         cmd->sess = sess;
76         cmd->tgt = sess->tgt;
77         cmd->tgtt = sess->tgt->tgtt;
78
79         cmd->start_time = jiffies;
80
81         /*
82          * For both wrong lun and CDB defer the error reporting for
83          * scst_cmd_init_done()
84          */
85
86         cmd->lun = scst_unpack_lun(lun, lun_len);
87
88         if (cdb_len <= SCST_MAX_CDB_SIZE) {
89                 memcpy(cmd->cdb, cdb, cdb_len);
90                 cmd->cdb_len = cdb_len;
91         }
92
93         TRACE_DBG("cmd %p, sess %p", cmd, sess);
94         scst_sess_get(sess);
95
96 out:
97         TRACE_EXIT();
98         return cmd;
99 }
100 EXPORT_SYMBOL(scst_rx_cmd);
101
102 static int scst_init_cmd(struct scst_cmd *cmd, int context)
103 {
104         int rc;
105
106         TRACE_ENTRY();
107
108         /* See the comment in scst_do_job_init() */
109         if (unlikely(!list_empty(&scst_init_cmd_list))) {
110                 TRACE_MGMT_DBG("%s", "init cmd list busy");
111                 goto out_redirect;
112         }
113         /*
114          * Memory barrier isn't necessary here, because CPU appears to
115          * be self-consistent
116          */
117
118         rc = __scst_init_cmd(cmd);
119         if (unlikely(rc > 0))
120                 goto out_redirect;
121         else if (unlikely(rc != 0))
122                 goto out;
123
124         /* Small context optimization */
125         if (((context == SCST_CONTEXT_TASKLET) ||
126              (context == SCST_CONTEXT_DIRECT_ATOMIC)) &&
127             scst_cmd_is_expected_set(cmd)) {
128                 if (cmd->expected_data_direction == SCST_DATA_WRITE) {
129                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
130                                         &cmd->tgt_dev->tgt_dev_flags))
131                                 context = SCST_CONTEXT_THREAD;
132                 } else {
133                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
134                                         &cmd->tgt_dev->tgt_dev_flags))
135                                 context = SCST_CONTEXT_THREAD;
136                 }
137         }
138
139 out:
140         TRACE_EXIT_RES(context);
141         return context;
142
143 out_redirect:
144         if (cmd->preprocessing_only) {
145                 /*
146                  * Poor man solution for single threaded targets, where
147                  * blocking receiver at least sometimes means blocking all.
148                  */
149                 sBUG_ON(context != SCST_CONTEXT_DIRECT);
150                 scst_set_busy(cmd);
151                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
152                 /* Keep initiator away from too many BUSY commands */
153                 if (!in_interrupt() && !in_atomic())
154                         msleep(50);
155                 else
156                         WARN_ON_ONCE(1);
157         } else {
158                 unsigned long flags;
159                 spin_lock_irqsave(&scst_init_lock, flags);
160                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
161                         "%d)", cmd, atomic_read(&scst_cmd_count));
162                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
163                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
164                         scst_init_poll_cnt++;
165                 spin_unlock_irqrestore(&scst_init_lock, flags);
166                 wake_up(&scst_init_cmd_list_waitQ);
167                 context = -1;
168         }
169         goto out;
170 }
171
172 #ifdef MEASURE_LATENCY
173 static inline uint64_t scst_sec_to_nsec(time_t sec)
174 {
175         return (uint64_t)sec * 1000000000;
176 }
177 #endif
178
179 void scst_cmd_init_done(struct scst_cmd *cmd, int pref_context)
180 {
181         unsigned long flags;
182         struct scst_session *sess = cmd->sess;
183
184         TRACE_ENTRY();
185
186 #ifdef MEASURE_LATENCY
187         {
188                 struct timespec ts;
189                 getnstimeofday(&ts);
190                 cmd->start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
191                 TRACE_DBG("cmd %p (sess %p): start %Ld (tv_sec %ld, "
192                         "tv_nsec %ld)", cmd, sess, cmd->start, ts.tv_sec,
193                         ts.tv_nsec);
194         }
195 #endif
196
197         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
198         TRACE(TRACE_SCSI, "tag=%llu, lun=%Ld, CDB len=%d",
199               (long long unsigned int)cmd->tag,
200               (long long unsigned int)cmd->lun,
201               cmd->cdb_len);
202         PRINT_BUFF_FLAG(TRACE_SCSI|TRACE_RCV_BOT, "Recieving CDB",
203                 cmd->cdb, cmd->cdb_len);
204
205 #ifdef EXTRACHECKS
206         if (unlikely(in_irq()) && ((pref_context == SCST_CONTEXT_DIRECT) ||
207                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
208                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
209                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
210                         cmd->tgtt->name);
211                 pref_context = SCST_CONTEXT_TASKLET;
212         }
213 #endif
214
215         atomic_inc(&sess->sess_cmd_count);
216
217         spin_lock_irqsave(&sess->sess_list_lock, flags);
218
219         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
220                 /*
221                  * We have to always keep command in the search list from the
222                  * very beginning, because otherwise it can be missed during
223                  * TM processing. This check is needed because there might be
224                  * old, i.e. deferred, commands and new, i.e. just coming, ones.
225                  */
226                 if (cmd->search_cmd_list_entry.next == NULL)
227                         list_add_tail(&cmd->search_cmd_list_entry,
228                                 &sess->search_cmd_list);
229                 switch (sess->init_phase) {
230                 case SCST_SESS_IPH_SUCCESS:
231                         break;
232                 case SCST_SESS_IPH_INITING:
233                         TRACE_DBG("Adding cmd %p to init deferred cmd list", cmd);
234                         list_add_tail(&cmd->cmd_list_entry,
235                                 &sess->init_deferred_cmd_list);
236                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
237                         goto out;
238                 case SCST_SESS_IPH_FAILED:
239                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
240                         scst_set_busy(cmd);
241                         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
242                         goto active;
243                 default:
244                         sBUG();
245                 }
246         } else
247                 list_add_tail(&cmd->search_cmd_list_entry, &sess->search_cmd_list);
248
249         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
250
251         if (unlikely(cmd->lun == (lun_t)-1)) {
252                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
253                 scst_set_cmd_error(cmd,
254                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
255                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
256                 goto active;
257         }
258
259         if (unlikely(cmd->cdb_len == 0)) {
260                 PRINT_ERROR("Wrong CDB len %d, finishing cmd", 0);
261                 scst_set_cmd_error(cmd,
262                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
263                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
264                 goto active;
265         }
266
267         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
268                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
269                 scst_set_cmd_error(cmd,
270                         SCST_LOAD_SENSE(scst_sense_invalid_message));
271                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
272                 goto active;
273         }
274
275         cmd->state = SCST_CMD_STATE_INIT;
276         /* cmd must be inited here to keep the order */
277         pref_context = scst_init_cmd(cmd, pref_context);
278         if (unlikely(pref_context < 0))
279                 goto out;
280
281 active:
282         /* Here cmd must not be in any cmd list, no locks */
283         switch (pref_context) {
284         case SCST_CONTEXT_TASKLET:
285                 scst_schedule_tasklet(cmd);
286                 break;
287
288         case SCST_CONTEXT_DIRECT:
289         case SCST_CONTEXT_DIRECT_ATOMIC:
290                 scst_process_active_cmd(cmd, pref_context);
291                 /* For *NEED_THREAD wake_up() is already done */
292                 break;
293
294         default:
295                 PRINT_ERROR("Context %x is undefined, using the thread one",
296                         pref_context);
297                 /* go through */
298         case SCST_CONTEXT_THREAD:
299                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
300                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
301                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
302                         list_add(&cmd->cmd_list_entry,
303                                 &cmd->cmd_lists->active_cmd_list);
304                 else
305                         list_add_tail(&cmd->cmd_list_entry,
306                                 &cmd->cmd_lists->active_cmd_list);
307                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
308                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
309                 break;
310         }
311
312 out:
313         TRACE_EXIT();
314         return;
315 }
316 EXPORT_SYMBOL(scst_cmd_init_done);
317
318 static int scst_pre_parse(struct scst_cmd *cmd)
319 {
320         int res = SCST_CMD_STATE_RES_CONT_SAME;
321         struct scst_device *dev = cmd->dev;
322         int rc;
323
324         TRACE_ENTRY();
325
326         cmd->inc_expected_sn_on_done = dev->handler->exec_sync ||
327                 (!dev->has_own_order_mgmt &&
328                  ((dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) ||
329                   (cmd->queue_type == SCST_CMD_QUEUE_ORDERED)));
330
331         sBUG_ON(cmd->internal);
332
333         /*
334          * Expected transfer data supplied by the SCSI transport via the
335          * target driver are untrusted, so we prefer to fetch them from CDB.
336          * Additionally, not all transports support supplying the expected
337          * transfer data.
338          */
339
340         rc = scst_get_cdb_info(cmd);
341         if (unlikely(rc != 0)) {
342                 if (rc > 0) {
343                         PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
344                         goto out_xmit;
345                 }
346                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
347                         "Should you update scst_scsi_op_table?",
348                         cmd->cdb[0], dev->handler->name);
349                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
350 #ifdef USE_EXPECTED_VALUES
351                 if (scst_cmd_is_expected_set(cmd)) {
352                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
353                                 "direction %d, transfer_len %d",
354                                 cmd->expected_data_direction,
355                                 cmd->expected_transfer_len);
356                         cmd->data_direction = cmd->expected_data_direction;
357
358                         cmd->bufflen = cmd->expected_transfer_len;
359                         /* Restore (likely) lost CDB length */
360                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
361                         if (cmd->cdb_len == -1) {
362                                 PRINT_ERROR("Unable to get CDB length for "
363                                         "opcode 0x%02x. Returning INVALID "
364                                         "OPCODE", cmd->cdb[0]);
365                                 scst_set_cmd_error(cmd,
366                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
367                                 goto out_xmit;
368                         }
369                 } else {
370                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
371                              "target %s not supplied expected values",
372                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
373                         scst_set_cmd_error(cmd,
374                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
375                         goto out_xmit;
376                 }
377 #else
378                 scst_set_cmd_error(cmd,
379                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
380                 goto out_xmit;
381 #endif
382         } else {
383                 TRACE(TRACE_SCSI, "op_name <%s>, direction=%d (expected %d, "
384                         "set %s), transfer_len=%d (expected len %d), flags=%d",
385                         cmd->op_name, cmd->data_direction,
386                         cmd->expected_data_direction,
387                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
388                         cmd->bufflen, cmd->expected_transfer_len, cmd->op_flags);
389
390                 if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
391                         if (scst_cmd_is_expected_set(cmd)) {
392                                 /*
393                                  * Command data length can't be easily
394                                  * determined from the CDB. ToDo, all such
395                                  * commands should be fixed. Until they are
396                                  * fixed, get it from the supplied expected
397                                  * value, but limit it to some reasonable
398                                  * value (15MB).
399                                  */
400                                 cmd->bufflen = min(cmd->expected_transfer_len,
401                                                         15*1024*1024);
402                                 cmd->op_flags &= ~SCST_UNKNOWN_LENGTH;
403                         } else
404                                 cmd->bufflen = 0;
405                 }
406         }
407
408         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
409                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
410                             "(opcode 0x%02x)", cmd->cdb[0]);
411                 scst_set_cmd_error(cmd,
412                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
413                 goto out_xmit;
414         }
415
416         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
417                 PRINT_ERROR("Linked commands are not supported "
418                             "(opcode 0x%02x)", cmd->cdb[0]);
419                 scst_set_cmd_error(cmd,
420                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
421                 goto out_xmit;
422         }
423
424         cmd->state = SCST_CMD_STATE_DEV_PARSE;
425
426 out:
427         TRACE_EXIT_RES(res);
428         return res;
429
430 out_xmit:
431         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
432         res = SCST_CMD_STATE_RES_CONT_SAME;
433         goto out;
434 }
435
436 static int scst_parse_cmd(struct scst_cmd *cmd)
437 {
438         int res = SCST_CMD_STATE_RES_CONT_SAME;
439         int state;
440         struct scst_device *dev = cmd->dev;
441         int orig_bufflen = cmd->bufflen;
442
443         TRACE_ENTRY();
444
445         if (likely(!scst_is_cmd_local(cmd))) {
446                 TRACE_DBG("Calling dev handler %s parse(%p)",
447                       dev->handler->name, cmd);
448                 TRACE_BUFF_FLAG(TRACE_SND_BOT, "Parsing: ", cmd->cdb, cmd->cdb_len);
449                 state = dev->handler->parse(cmd);
450                 /* Caution: cmd can be already dead here */
451                 TRACE_DBG("Dev handler %s parse() returned %d",
452                         dev->handler->name, state);
453
454                 switch (state) {
455                 case SCST_CMD_STATE_NEED_THREAD_CTX:
456                         TRACE_DBG("Dev handler %s parse() requested thread "
457                               "context, rescheduling", dev->handler->name);
458                         res = SCST_CMD_STATE_RES_NEED_THREAD;
459                         goto out;
460
461                 case SCST_CMD_STATE_STOP:
462                         TRACE_DBG("Dev handler %s parse() requested stop "
463                                 "processing", dev->handler->name);
464                         res = SCST_CMD_STATE_RES_CONT_NEXT;
465                         goto out;
466                 }
467
468                 if (state == SCST_CMD_STATE_DEFAULT)
469                         state = SCST_CMD_STATE_PREPARE_SPACE;
470         } else
471                 state = SCST_CMD_STATE_PREPARE_SPACE;
472
473         if (cmd->data_len == -1)
474                 cmd->data_len = cmd->bufflen;
475
476         if (cmd->data_buf_alloced && unlikely((orig_bufflen > cmd->bufflen))) {
477                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
478                         "is less, than required (size %d)", cmd->bufflen,
479                         orig_bufflen);
480                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
481                 goto out_error;
482         }
483
484         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
485                 goto set_res;
486
487         if (unlikely((cmd->bufflen == 0) &&
488                      (cmd->op_flags & SCST_UNKNOWN_LENGTH))) {
489                 PRINT_ERROR("Unknown data transfer length for opcode 0x%x "
490                         "(handler %s, target %s)", cmd->cdb[0],
491                         dev->handler->name, cmd->tgtt->name);
492                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
493                 goto out_error;
494         }
495
496 #ifdef EXTRACHECKS
497         if ((cmd->bufflen != 0) &&
498             ((cmd->data_direction == SCST_DATA_NONE) ||
499              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
500                 PRINT_ERROR("Dev handler %s parse() returned "
501                         "invalid cmd data_direction %d, bufflen %d, state %d "
502                         "or sg %p (opcode 0x%x)", dev->handler->name,
503                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
504                         cmd->cdb[0]);
505                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
506                 goto out_error;
507         }
508 #endif
509
510         if (scst_cmd_is_expected_set(cmd)) {
511 #ifdef USE_EXPECTED_VALUES
512 #       ifdef EXTRACHECKS
513                 if ((cmd->data_direction != cmd->expected_data_direction) ||
514                     (cmd->bufflen != cmd->expected_transfer_len)) {
515                         PRINT_ERROR("Expected values don't match decoded ones: "
516                                 "data_direction %d, expected_data_direction %d, "
517                                 "bufflen %d, expected_transfer_len %d",
518                                 cmd->data_direction, cmd->expected_data_direction,
519                                 cmd->bufflen, cmd->expected_transfer_len);
520                         PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
521                 }
522 #       endif
523                 cmd->data_direction = cmd->expected_data_direction;
524                 cmd->bufflen = cmd->expected_transfer_len;
525 #else
526                 if (unlikely(cmd->data_direction != cmd->expected_data_direction)) {
527                         if (((cmd->expected_data_direction != SCST_DATA_NONE) ||
528                             (cmd->bufflen != 0)) &&
529                             /* Crazy VMware people sometimes do TUR with READ direction */
530                             !(cmd->cdb[0] == TEST_UNIT_READY)) {
531                                 PRINT_ERROR("Expected data direction %d for opcode "
532                                         "0x%02x (handler %s, target %s) doesn't match "
533                                         "decoded value %d", cmd->expected_data_direction,
534                                         cmd->cdb[0], dev->handler->name,
535                                         cmd->tgtt->name, cmd->data_direction);
536                                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
537                                 scst_set_cmd_error(cmd,
538                                         SCST_LOAD_SENSE(scst_sense_invalid_message));
539                                 goto out_dev_done;
540                         }
541                 }
542                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
543                         static int repd;
544                         
545                         if (repd < 100) {
546                                 /*
547                                  * Intentionally unlocked. Few messages more
548                                  * or less don't matter.
549                                  */
550                                 repd++;
551                                 TRACE(TRACE_MINOR, "Warning: expected transfer "
552                                         "length %d for opcode 0x%02x (handler "
553                                         "%s, target %s) doesn't match decoded "
554                                         "value %d. Faulty initiator (e.g. "
555                                         "VMware is known to be such) or "
556                                         "scst_scsi_op_table should be updated?",
557                                         cmd->expected_transfer_len, cmd->cdb[0],
558                                         dev->handler->name, cmd->tgtt->name,
559                                         cmd->bufflen);
560                                 PRINT_BUFF_FLAG(TRACE_MINOR, "Suspicious CDB",
561                                         cmd->cdb, cmd->cdb_len);
562                         }
563                 }
564 #endif
565         }
566
567         if (unlikely(cmd->data_direction == SCST_DATA_UNKNOWN)) {
568                 PRINT_ERROR("Unknown data direction. Opcode 0x%x, handler %s, "
569                         "target %s", cmd->cdb[0], dev->handler->name,
570                         cmd->tgtt->name);
571                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
572                 goto out_error;
573         }
574
575 set_res:
576         switch (state) {
577         case SCST_CMD_STATE_PREPARE_SPACE:
578         case SCST_CMD_STATE_PRE_PARSE:
579         case SCST_CMD_STATE_DEV_PARSE:
580         case SCST_CMD_STATE_RDY_TO_XFER:
581         case SCST_CMD_STATE_TGT_PRE_EXEC:
582         case SCST_CMD_STATE_SEND_TO_MIDLEV:
583         case SCST_CMD_STATE_PRE_DEV_DONE:
584         case SCST_CMD_STATE_DEV_DONE:
585         case SCST_CMD_STATE_PRE_XMIT_RESP:
586         case SCST_CMD_STATE_XMIT_RESP:
587         case SCST_CMD_STATE_FINISHED:
588                 cmd->state = state;
589                 res = SCST_CMD_STATE_RES_CONT_SAME;
590                 break;
591
592         default:
593                 if (state >= 0) {
594                         PRINT_ERROR("Dev handler %s parse() returned "
595                              "invalid cmd state %d (opcode %d)",
596                              dev->handler->name, state, cmd->cdb[0]);
597                 } else {
598                         PRINT_ERROR("Dev handler %s parse() returned "
599                                 "error %d (opcode %d)", dev->handler->name,
600                                 state, cmd->cdb[0]);
601                 }
602                 goto out_error;
603         }
604
605         if (cmd->resp_data_len == -1) {
606                 if (cmd->data_direction == SCST_DATA_READ)
607                         cmd->resp_data_len = cmd->bufflen;
608                 else
609                          cmd->resp_data_len = 0;
610         }
611
612 out:
613         TRACE_EXIT_HRES(res);
614         return res;
615
616 out_error:
617         /* dev_done() will be called as part of the regular cmd's finish */
618         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
619
620 #ifndef USE_EXPECTED_VALUES
621 out_dev_done:
622 #endif
623         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
624         res = SCST_CMD_STATE_RES_CONT_SAME;
625         goto out;
626 }
627
628 static int scst_prepare_space(struct scst_cmd *cmd)
629 {
630         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
631
632         TRACE_ENTRY();
633
634         if (cmd->data_direction == SCST_DATA_NONE)
635                 goto prep_done;
636
637         if (cmd->data_buf_tgt_alloc) {
638                 int orig_bufflen = cmd->bufflen;
639
640                 TRACE_MEM("%s", "Custom tgt data buf allocation requested");
641
642                 r = cmd->tgtt->alloc_data_buf(cmd);
643                 if (r > 0)
644                         goto alloc;
645                 else if (r == 0) {
646                         if (unlikely(cmd->bufflen == 0)) {
647                                 /* See comment in scst_alloc_space() */
648                                 if (cmd->sg == NULL)
649                                         goto alloc;
650                         }
651                         cmd->data_buf_alloced = 1;
652                         if (unlikely(orig_bufflen < cmd->bufflen)) {
653                                 PRINT_ERROR("Target driver allocated data "
654                                         "buffer (size %d), is less, than "
655                                         "required (size %d)", orig_bufflen,
656                                         cmd->bufflen);
657                                 goto out_error;
658                         }
659                         TRACE_MEM("%s", "data_buf_alloced, returning");
660                 }
661                 goto check;
662         }
663
664 alloc:
665         if (!cmd->data_buf_alloced)
666                 r = scst_alloc_space(cmd);
667         else
668                 TRACE_MEM("%s", "data_buf_alloced set, returning");
669
670 check:
671         if (r != 0) {
672                 if (scst_cmd_atomic(cmd)) {
673                         TRACE_MEM("%s", "Atomic memory allocation failed, "
674                               "rescheduling to the thread");
675                         res = SCST_CMD_STATE_RES_NEED_THREAD;
676                         goto out;
677                 } else
678                         goto out_no_space;
679         }
680
681 prep_done:
682         if (cmd->preprocessing_only) {
683                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
684                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
685                                 "cmd %p", cmd);
686                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
687                         res = SCST_CMD_STATE_RES_CONT_SAME;
688                         goto out;
689                 }
690
691                 res = SCST_CMD_STATE_RES_CONT_NEXT;
692                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
693
694                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
695                 cmd->tgtt->preprocessing_done(cmd);
696                 TRACE_DBG("%s", "preprocessing_done() returned");
697                 goto out;
698
699         }
700
701         switch (cmd->data_direction) {
702         case SCST_DATA_WRITE:
703                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
704                 break;
705
706         default:
707                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
708                 break;
709         }
710
711 out:
712         TRACE_EXIT_HRES(res);
713         return res;
714
715 out_no_space:
716         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
717                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
718         scst_set_busy(cmd);
719         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
720         res = SCST_CMD_STATE_RES_CONT_SAME;
721         goto out;
722
723 out_error:
724         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
725         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
726         res = SCST_CMD_STATE_RES_CONT_SAME;
727         goto out;
728 }
729
730 void scst_restart_cmd(struct scst_cmd *cmd, int status, int pref_context)
731 {
732         TRACE_ENTRY();
733
734         TRACE_DBG("Preferred context: %d", pref_context);
735         TRACE_DBG("tag=%llu, status=%#x",
736                   (long long unsigned int)scst_cmd_get_tag(cmd),
737                   status);
738
739 #ifdef EXTRACHECKS
740         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
741                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
742                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
743                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
744                         cmd->tgtt->name);
745                 pref_context = SCST_CONTEXT_TASKLET;
746         }
747 #endif
748
749         switch (status) {
750         case SCST_PREPROCESS_STATUS_SUCCESS:
751                 switch (cmd->data_direction) {
752                 case SCST_DATA_WRITE:
753                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
754                         break;
755                 default:
756                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
757                         break;
758                 }
759                 if (cmd->set_sn_on_restart_cmd)
760                         scst_cmd_set_sn(cmd);
761                 /* Small context optimization */
762                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
763                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) {
764                         if (cmd->data_direction == SCST_DATA_WRITE) {
765                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
766                                                 &cmd->tgt_dev->tgt_dev_flags))
767                                         pref_context = SCST_CONTEXT_THREAD;
768                         } else {
769                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
770                                                 &cmd->tgt_dev->tgt_dev_flags))
771                                         pref_context = SCST_CONTEXT_THREAD;
772                         }
773                 }
774                 break;
775
776         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
777                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
778                 break;
779
780         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
781                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
782                 /* go through */
783         case SCST_PREPROCESS_STATUS_ERROR:
784                 scst_set_cmd_error(cmd,
785                            SCST_LOAD_SENSE(scst_sense_hardw_error));
786                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
787                 break;
788
789         default:
790                 PRINT_ERROR("%s() received unknown status %x", __func__,
791                         status);
792                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
793                 break;
794         }
795
796         scst_proccess_redirect_cmd(cmd, pref_context, 1);
797
798         TRACE_EXIT();
799         return;
800 }
801 EXPORT_SYMBOL(scst_restart_cmd);
802
803 /* No locks */
804 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
805 {
806         struct scst_tgt *tgt = cmd->sess->tgt;
807         int res = 0;
808         unsigned long flags;
809
810         TRACE_ENTRY();
811
812         spin_lock_irqsave(&tgt->tgt_lock, flags);
813         tgt->retry_cmds++;
814         smp_mb();
815         TRACE_RETRY("TGT QUEUE FULL: incrementing retry_cmds %d",
816               tgt->retry_cmds);
817         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
818                 /* At least one cmd finished, so try again */
819                 tgt->retry_cmds--;
820                 TRACE_RETRY("Some command(s) finished, direct retry "
821                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
822                       "retry_cmds=%d)", finished_cmds,
823                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
824                 res = -1;
825                 goto out_unlock_tgt;
826         }
827
828         TRACE_RETRY("Adding cmd %p to retry cmd list", cmd);
829         list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
830
831         if (!tgt->retry_timer_active) {
832                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
833                 add_timer(&tgt->retry_timer);
834                 tgt->retry_timer_active = 1;
835         }
836
837 out_unlock_tgt:
838         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
839
840         TRACE_EXIT_RES(res);
841         return res;
842 }
843
844 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
845 {
846         int res, rc;
847
848         TRACE_ENTRY();
849
850         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
851                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
852                 goto out_dev_done;
853         }
854
855         if (cmd->tgtt->rdy_to_xfer == NULL) {
856                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
857                 res = SCST_CMD_STATE_RES_CONT_SAME;
858                 goto out;
859         }
860
861         while (1) {
862                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
863
864                 res = SCST_CMD_STATE_RES_CONT_NEXT;
865                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
866
867                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
868 #ifdef DEBUG_RETRY
869                 if (((scst_random() % 100) == 75))
870                         rc = SCST_TGT_RES_QUEUE_FULL;
871                 else
872 #endif
873                         rc = cmd->tgtt->rdy_to_xfer(cmd);
874                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
875
876                 if (likely(rc == SCST_TGT_RES_SUCCESS))
877                         goto out;
878
879                 /* Restore the previous state */
880                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
881
882                 switch (rc) {
883                 case SCST_TGT_RES_QUEUE_FULL:
884                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
885                                 break;
886                         else
887                                 continue;
888
889                 case SCST_TGT_RES_NEED_THREAD_CTX:
890                         TRACE_DBG("Target driver %s "
891                               "rdy_to_xfer() requested thread "
892                               "context, rescheduling", cmd->tgtt->name);
893                         res = SCST_CMD_STATE_RES_NEED_THREAD;
894                         break;
895
896                 default:
897                         goto out_error_rc;
898                 }
899                 break;
900         }
901
902 out:
903         TRACE_EXIT_HRES(res);
904         return res;
905
906 out_error_rc:
907         if (rc == SCST_TGT_RES_FATAL_ERROR) {
908                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
909                      "fatal error", cmd->tgtt->name);
910         } else {
911                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
912                             "value %d", cmd->tgtt->name, rc);
913         }
914         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
915
916 out_dev_done:
917         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
918         res = SCST_CMD_STATE_RES_CONT_SAME;
919         goto out;
920 }
921
922 /* No locks, but might be in IRQ */
923 void scst_proccess_redirect_cmd(struct scst_cmd *cmd, int context,
924         int check_retries)
925 {
926         unsigned long flags;
927
928         TRACE_ENTRY();
929
930         TRACE_DBG("Context: %x", context);
931
932         switch (context & ~SCST_CONTEXT_PROCESSABLE) {
933         case SCST_CONTEXT_DIRECT_ATOMIC:
934                 context &= ~SCST_CONTEXT_PROCESSABLE;
935                 /* go through */
936         case SCST_CONTEXT_DIRECT:
937                 if (check_retries)
938                         scst_check_retries(cmd->tgt);
939                 scst_process_active_cmd(cmd, context);
940                 break;
941
942         default:
943                 PRINT_ERROR("Context %x is unknown, using the thread one",
944                             context);
945                 /* go through */
946         case SCST_CONTEXT_THREAD:
947                 if (check_retries)
948                         scst_check_retries(cmd->tgt);
949                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
950                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
951                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
952                         list_add(&cmd->cmd_list_entry,
953                                 &cmd->cmd_lists->active_cmd_list);
954                 else
955                         list_add_tail(&cmd->cmd_list_entry,
956                                 &cmd->cmd_lists->active_cmd_list);
957                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
958                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
959                 break;
960
961         case SCST_CONTEXT_TASKLET:
962                 if (check_retries)
963                         scst_check_retries(cmd->tgt);
964                 scst_schedule_tasklet(cmd);
965                 break;
966         }
967
968         TRACE_EXIT();
969         return;
970 }
971
972 void scst_rx_data(struct scst_cmd *cmd, int status, int pref_context)
973 {
974         TRACE_ENTRY();
975
976         TRACE_DBG("Preferred context: %d", pref_context);
977         TRACE(TRACE_SCSI, "tag=%llu status=%#x",
978               (long long unsigned int)scst_cmd_get_tag(cmd),
979               status);
980
981 #ifdef EXTRACHECKS
982         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
983                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
984                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
985                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
986                         cmd->tgtt->name);
987                 pref_context = SCST_CONTEXT_TASKLET;
988         }
989 #endif
990
991         switch (status) {
992         case SCST_RX_STATUS_SUCCESS:
993                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
994                 /* Small context optimization */
995                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
996                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) {
997                         if (!test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC,
998                                         &cmd->tgt_dev->tgt_dev_flags))
999                                 pref_context = SCST_CONTEXT_THREAD;
1000                 }
1001                 break;
1002
1003         case SCST_RX_STATUS_ERROR_SENSE_SET:
1004                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1005                 break;
1006
1007         case SCST_RX_STATUS_ERROR_FATAL:
1008                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1009                 /* go through */
1010         case SCST_RX_STATUS_ERROR:
1011                 scst_set_cmd_error(cmd,
1012                            SCST_LOAD_SENSE(scst_sense_hardw_error));
1013                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1014                 break;
1015
1016         default:
1017                 PRINT_ERROR("scst_rx_data() received unknown status %x",
1018                         status);
1019                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1020                 break;
1021         }
1022
1023         scst_proccess_redirect_cmd(cmd, pref_context, 1);
1024
1025         TRACE_EXIT();
1026         return;
1027 }
1028 EXPORT_SYMBOL(scst_rx_data);
1029
1030 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
1031 {
1032         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
1033
1034         TRACE_ENTRY();
1035
1036         cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
1037
1038         if (cmd->tgtt->pre_exec == NULL)
1039                 goto out;
1040
1041         TRACE_DBG("Calling pre_exec(%p)", cmd);
1042         rc = cmd->tgtt->pre_exec(cmd);
1043         TRACE_DBG("pre_exec() returned %d", rc);
1044
1045         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
1046                 switch (rc) {
1047                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
1048                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1049                         break;
1050                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
1051                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1052                         /* go through */
1053                 case SCST_PREPROCESS_STATUS_ERROR:
1054                         scst_set_cmd_error(cmd,
1055                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1056                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1057                         break;
1058                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1059                         TRACE_DBG("Target driver's %s pre_exec() requested "
1060                                 "thread context, rescheduling", cmd->tgtt->name);
1061                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1062                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1063                         break;
1064                 default:
1065                         sBUG();
1066                         break;
1067                 }
1068         }
1069
1070 out:
1071         TRACE_EXIT_RES(res);
1072         return res;
1073 }
1074
1075 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1076         const uint8_t *rq_sense, int rq_sense_len, int resid)
1077 {
1078         TRACE_ENTRY();
1079
1080 #ifdef MEASURE_LATENCY
1081         {
1082                 struct timespec ts;
1083                 getnstimeofday(&ts);
1084                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1085                 TRACE_DBG("cmd %p (sess %p): post_exec_start %Ld (tv_sec %ld, "
1086                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start, ts.tv_sec,
1087                         ts.tv_nsec);
1088         }
1089 #endif
1090
1091         cmd->status = result & 0xff;
1092         cmd->msg_status = msg_byte(result);
1093         cmd->host_status = host_byte(result);
1094         cmd->driver_status = driver_byte(result);
1095         if (unlikely(resid != 0)) {
1096 #ifdef EXTRACHECKS
1097                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1098                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1099                                 "op %x)", resid, cmd->resp_data_len,
1100                                 cmd->cdb[0]);
1101                 } else
1102 #endif
1103                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1104         }
1105
1106         if (cmd->status == SAM_STAT_CHECK_CONDITION)
1107                 scst_alloc_set_sense(cmd, scst_is_context_gfp_atomic(),
1108                                         rq_sense, rq_sense_len);
1109
1110         TRACE(TRACE_SCSI, "result=%x, cmd->status=%x, resid=%d, "
1111               "cmd->msg_status=%x, cmd->host_status=%x, "
1112               "cmd->driver_status=%x", result, cmd->status, resid,
1113               cmd->msg_status, cmd->host_status, cmd->driver_status);
1114
1115         cmd->completed = 1;
1116
1117         TRACE_EXIT();
1118         return;
1119 }
1120
1121 /* For small context optimization */
1122 static inline int scst_optimize_post_exec_context(struct scst_cmd *cmd,
1123         int context)
1124 {
1125         if ((context == SCST_CONTEXT_TASKLET) ||
1126             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1127                 if (!test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC,
1128                                 &cmd->tgt_dev->tgt_dev_flags))
1129                         context = SCST_CONTEXT_THREAD;
1130         }
1131         return context;
1132 }
1133
1134 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1135 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1136                                             struct scsi_request **req)
1137 {
1138         struct scst_cmd *cmd = NULL;
1139
1140         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1141                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1142
1143         if (cmd == NULL) {
1144                 PRINT_ERROR("%s", "Request with NULL cmd");
1145                 if (*req)
1146                         scsi_release_request(*req);
1147         }
1148
1149         return cmd;
1150 }
1151
1152 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1153 {
1154         struct scsi_request *req = NULL;
1155         struct scst_cmd *cmd;
1156
1157         TRACE_ENTRY();
1158
1159         cmd = scst_get_cmd(scsi_cmd, &req);
1160         if (cmd == NULL)
1161                 goto out;
1162
1163         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1164                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1165
1166         /* Clear out request structure */
1167         req->sr_use_sg = 0;
1168         req->sr_sglist_len = 0;
1169         req->sr_bufflen = 0;
1170         req->sr_buffer = NULL;
1171         req->sr_underflow = 0;
1172         req->sr_request->rq_disk = NULL; /* disown request blk */
1173
1174         scst_release_request(cmd);
1175
1176         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1177
1178         scst_proccess_redirect_cmd(cmd,
1179                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1180
1181 out:
1182         TRACE_EXIT();
1183         return;
1184 }
1185 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1186 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1187 {
1188         struct scst_cmd *cmd;
1189
1190         TRACE_ENTRY();
1191
1192         cmd = (struct scst_cmd *)data;
1193         if (cmd == NULL)
1194                 goto out;
1195
1196         scst_do_cmd_done(cmd, result, sense, SCST_SENSE_BUFFERSIZE, resid);
1197
1198         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1199
1200         scst_proccess_redirect_cmd(cmd,
1201                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1202
1203 out:
1204         TRACE_EXIT();
1205         return;
1206 }
1207 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1208
1209 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state)
1210 {
1211         int context;
1212
1213         TRACE_ENTRY();
1214
1215 #ifdef MEASURE_LATENCY
1216         {
1217                 struct timespec ts;
1218                 getnstimeofday(&ts);
1219                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1220                 TRACE_DBG("cmd %p (sess %p): post_exec_start %Ld (tv_sec %ld, "
1221                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start, ts.tv_sec,
1222                         ts.tv_nsec);
1223         }
1224 #endif
1225
1226         if (next_state == SCST_CMD_STATE_DEFAULT)
1227                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1228
1229 #if defined(DEBUG)
1230         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1231                 if (cmd->sg) {
1232                         int i;
1233                         struct scatterlist *sg = cmd->sg;
1234                         TRACE_RECV_TOP("Exec'd %d S/G(s) at %p sg[0].page at "
1235                                 "%p", cmd->sg_cnt, sg, (void *)sg_page(&sg[0]));
1236                         for (i = 0; i < cmd->sg_cnt; ++i) {
1237                                 TRACE_BUFF_FLAG(TRACE_RCV_TOP,
1238                                         "Exec'd sg", sg_virt(&sg[i]),
1239                                         sg[i].length);
1240                         }
1241                 }
1242         }
1243 #endif
1244
1245
1246 #ifdef EXTRACHECKS
1247         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1248             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1249             (next_state != SCST_CMD_STATE_FINISHED)) {
1250                 PRINT_ERROR("scst_cmd_done_local() received invalid cmd "
1251                             "state %d (opcode %d)", next_state, cmd->cdb[0]);
1252                 scst_set_cmd_error(cmd,
1253                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1254                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1255         }
1256 #endif
1257         cmd->state = next_state;
1258
1259         context = scst_optimize_post_exec_context(cmd, scst_get_context());
1260         if (cmd->context_processable)
1261                 context |= SCST_CONTEXT_PROCESSABLE;
1262         scst_proccess_redirect_cmd(cmd, context, 0);
1263
1264         TRACE_EXIT();
1265         return;
1266 }
1267
1268 static int scst_report_luns_local(struct scst_cmd *cmd)
1269 {
1270         int rc;
1271         int dev_cnt = 0;
1272         int buffer_size;
1273         int i;
1274         struct scst_tgt_dev *tgt_dev = NULL;
1275         uint8_t *buffer;
1276         int offs, overflow = 0;
1277
1278         TRACE_ENTRY();
1279
1280         rc = scst_check_local_events(cmd);
1281         if (unlikely(rc != 0))
1282                 goto out_done;
1283
1284         cmd->status = 0;
1285         cmd->msg_status = 0;
1286         cmd->host_status = DID_OK;
1287         cmd->driver_status = 0;
1288
1289         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1290                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1291                         "LUNS command", cmd->cdb[2]);
1292                 goto out_err;
1293         }
1294
1295         buffer_size = scst_get_buf_first(cmd, &buffer);
1296         if (unlikely(buffer_size <= 0))
1297                 goto out_err;
1298
1299         if (buffer_size < 16)
1300                 goto out_put_err;
1301
1302         memset(buffer, 0, buffer_size);
1303         offs = 8;
1304
1305         /* sess->sess_tgt_dev_list_hash is protected by suspended activity */
1306         for (i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1307                 struct list_head *sess_tgt_dev_list_head =
1308                         &cmd->sess->sess_tgt_dev_list_hash[i];
1309                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1310                                 sess_tgt_dev_list_entry) {
1311                         if (!overflow) {
1312                                 if (offs >= buffer_size) {
1313                                         scst_put_buf(cmd, buffer);
1314                                         buffer_size = scst_get_buf_next(cmd, &buffer);
1315                                         if (buffer_size > 0) {
1316                                                 memset(buffer, 0, buffer_size);
1317                                                 offs = 0;
1318                                         } else {
1319                                                 overflow = 1;
1320                                                 goto inc_dev_cnt;
1321                                         }
1322                                 }
1323                                 if ((buffer_size - offs) < 8) {
1324                                         PRINT_ERROR("Buffer allocated for REPORT "
1325                                                 "LUNS command doesn't allow to fit 8 "
1326                                                 "byte entry (buffer_size=%d)",
1327                                                 buffer_size);
1328                                         goto out_put_hw_err;
1329                                 }
1330                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1331                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1332                                 offs += 8;
1333                         }
1334 inc_dev_cnt:
1335                         dev_cnt++;
1336                 }
1337         }
1338         if (!overflow)
1339                 scst_put_buf(cmd, buffer);
1340
1341         /* Set the response header */
1342         buffer_size = scst_get_buf_first(cmd, &buffer);
1343         if (unlikely(buffer_size <= 0))
1344                 goto out_err;
1345         dev_cnt *= 8;
1346         buffer[0] = (dev_cnt >> 24) & 0xff;
1347         buffer[1] = (dev_cnt >> 16) & 0xff;
1348         buffer[2] = (dev_cnt >> 8) & 0xff;
1349         buffer[3] = dev_cnt & 0xff;
1350         scst_put_buf(cmd, buffer);
1351
1352         dev_cnt += 8;
1353         if (dev_cnt < cmd->resp_data_len)
1354                 scst_set_resp_data_len(cmd, dev_cnt);
1355
1356 out_compl:
1357         cmd->completed = 1;
1358
1359 out_done:
1360         /* Report the result */
1361         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1362
1363         TRACE_EXIT();
1364         return SCST_EXEC_COMPLETED;
1365
1366 out_put_err:
1367         scst_put_buf(cmd, buffer);
1368
1369 out_err:
1370         scst_set_cmd_error(cmd,
1371                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1372         goto out_compl;
1373
1374 out_put_hw_err:
1375         scst_put_buf(cmd, buffer);
1376         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1377         goto out_compl;
1378 }
1379
1380 static int scst_pre_select(struct scst_cmd *cmd)
1381 {
1382         int res = SCST_EXEC_NOT_COMPLETED;
1383
1384         TRACE_ENTRY();
1385
1386         if (scst_cmd_atomic(cmd)) {
1387                 res = SCST_EXEC_NEED_THREAD;
1388                 goto out;
1389         }
1390
1391         if (cmd->local_exec_done)
1392                 goto out;
1393
1394         cmd->local_exec_done = 1;
1395
1396         scst_block_dev_cmd(cmd, 1);
1397
1398         /* Check for local events will be done when cmd will be executed */
1399
1400 out:
1401         TRACE_EXIT_RES(res);
1402         return res;
1403 }
1404
1405 static int scst_reserve_local(struct scst_cmd *cmd)
1406 {
1407         int res = SCST_EXEC_NOT_COMPLETED, rc;
1408         struct scst_device *dev;
1409         struct scst_tgt_dev *tgt_dev_tmp;
1410
1411         TRACE_ENTRY();
1412
1413         if (scst_cmd_atomic(cmd)) {
1414                 res = SCST_EXEC_NEED_THREAD;
1415                 goto out;
1416         }
1417
1418         if (cmd->local_exec_done)
1419                 goto out;
1420
1421         cmd->local_exec_done = 1;
1422
1423         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1424                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1425                      "(lun=%Ld)", (long long unsigned int)cmd->lun);
1426                 scst_set_cmd_error(cmd,
1427                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1428                 goto out_done;
1429         }
1430
1431         dev = cmd->dev;
1432
1433         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1434                 scst_block_dev_cmd(cmd, 1);
1435
1436         rc = scst_check_local_events(cmd);
1437         if (unlikely(rc != 0))
1438                 goto out_done;
1439
1440         spin_lock_bh(&dev->dev_lock);
1441
1442         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1443                 spin_unlock_bh(&dev->dev_lock);
1444                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1445                 goto out_done;
1446         }
1447
1448         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1449                             dev_tgt_dev_list_entry) {
1450                 if (cmd->tgt_dev != tgt_dev_tmp)
1451                         set_bit(SCST_TGT_DEV_RESERVED,
1452                                 &tgt_dev_tmp->tgt_dev_flags);
1453         }
1454         dev->dev_reserved = 1;
1455
1456         spin_unlock_bh(&dev->dev_lock);
1457
1458 out:
1459         TRACE_EXIT_RES(res);
1460         return res;
1461
1462 out_done:
1463         /* Report the result */
1464         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1465         res = SCST_EXEC_COMPLETED;
1466         goto out;
1467 }
1468
1469 static int scst_release_local(struct scst_cmd *cmd)
1470 {
1471         int res = SCST_EXEC_NOT_COMPLETED, rc;
1472         struct scst_tgt_dev *tgt_dev_tmp;
1473         struct scst_device *dev;
1474
1475         TRACE_ENTRY();
1476
1477         if (scst_cmd_atomic(cmd)) {
1478                 res = SCST_EXEC_NEED_THREAD;
1479                 goto out;
1480         }
1481
1482         if (cmd->local_exec_done)
1483                 goto out;
1484
1485         cmd->local_exec_done = 1;
1486
1487         dev = cmd->dev;
1488
1489         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1490                 scst_block_dev_cmd(cmd, 1);
1491
1492         rc = scst_check_local_events(cmd);
1493         if (unlikely(rc != 0))
1494                 goto out_done;
1495
1496         spin_lock_bh(&dev->dev_lock);
1497
1498         /*
1499          * The device could be RELEASED behind us, if RESERVING session
1500          * is closed (see scst_free_tgt_dev()), but this actually doesn't
1501          * matter, so use lock and no retest for DEV_RESERVED bits again
1502          */
1503         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1504                 res = SCST_EXEC_COMPLETED;
1505                 cmd->status = 0;
1506                 cmd->msg_status = 0;
1507                 cmd->host_status = DID_OK;
1508                 cmd->driver_status = 0;
1509                 cmd->completed = 1;
1510         } else {
1511                 list_for_each_entry(tgt_dev_tmp,
1512                                     &dev->dev_tgt_dev_list,
1513                                     dev_tgt_dev_list_entry) {
1514                         clear_bit(SCST_TGT_DEV_RESERVED,
1515                                 &tgt_dev_tmp->tgt_dev_flags);
1516                 }
1517                 dev->dev_reserved = 0;
1518         }
1519
1520         spin_unlock_bh(&dev->dev_lock);
1521
1522         if (res == SCST_EXEC_COMPLETED)
1523                 goto out_done;
1524
1525 out:
1526         TRACE_EXIT_RES(res);
1527         return res;
1528
1529 out_done:
1530         res = SCST_EXEC_COMPLETED;
1531         /* Report the result */
1532         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1533         goto out;
1534 }
1535
1536 /* No locks, no IRQ or IRQ-safe context allowed */
1537 int scst_check_local_events(struct scst_cmd *cmd)
1538 {
1539         int res, rc;
1540         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1541
1542         TRACE_ENTRY();
1543
1544         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1545                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1546                 goto out_uncomplete;
1547         }
1548
1549         /* Reserve check before Unit Attention */
1550         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED, &tgt_dev->tgt_dev_flags))) {
1551                 if ((cmd->cdb[0] != INQUIRY) && (cmd->cdb[0] != REPORT_LUNS) &&
1552                     (cmd->cdb[0] != RELEASE) && (cmd->cdb[0] != RELEASE_10) &&
1553                     (cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER) &&
1554                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL || (cmd->cdb[4] & 3)) &&
1555                     (cmd->cdb[0] != LOG_SENSE) && (cmd->cdb[0] != REQUEST_SENSE)) {
1556                         scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1557                         goto out_complete;
1558                 }
1559         }
1560
1561         /* If we had internal bus reset, set the command error unit attention */
1562         if ((cmd->dev->scsi_dev != NULL) &&
1563             unlikely(cmd->dev->scsi_dev->was_reset)) {
1564                 if (scst_is_ua_command(cmd)) {
1565                         struct scst_device *dev = cmd->dev;
1566                         int done = 0;
1567                         /* Prevent more than 1 cmd to be triggered by was_reset */
1568                         spin_lock_bh(&dev->dev_lock);
1569                         barrier(); /* to reread was_reset */
1570                         if (dev->scsi_dev->was_reset) {
1571                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1572                                 scst_set_cmd_error(cmd,
1573                                            SCST_LOAD_SENSE(scst_sense_reset_UA));
1574                                 /* It looks like it is safe to clear was_reset here */
1575                                 dev->scsi_dev->was_reset = 0;
1576                                 done = 1;
1577                         }
1578                         spin_unlock_bh(&dev->dev_lock);
1579
1580                         if (done)
1581                                 goto out_complete;
1582                 }
1583         }
1584
1585         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING,
1586                         &cmd->tgt_dev->tgt_dev_flags))) {
1587                 if (scst_is_ua_command(cmd)) {
1588                         rc = scst_set_pending_UA(cmd);
1589                         if (rc == 0)
1590                                 goto out_complete;
1591                 }
1592         }
1593
1594         res = 0;
1595
1596 out:
1597         TRACE_EXIT_RES(res);
1598         return res;
1599
1600 out_complete:
1601         res = 1;
1602         sBUG_ON(!cmd->completed);
1603         goto out;
1604
1605 out_uncomplete:
1606         res = -1;
1607         goto out;
1608 }
1609 EXPORT_SYMBOL(scst_check_local_events);
1610
1611 /*
1612  * The result of cmd execution, if any, should be reported
1613  * via scst_cmd_done_local()
1614  */
1615 static int scst_pre_exec(struct scst_cmd *cmd)
1616 {
1617         int res = SCST_EXEC_NOT_COMPLETED;
1618         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1619
1620         TRACE_ENTRY();
1621
1622         /*
1623          * This function can be called several times for the same cmd, so it
1624          * can't change any state in a non-reentrable way or use something
1625          * like local_exec_done!!
1626          */
1627
1628         /* Check READ_ONLY device status */
1629         if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
1630             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
1631              cmd->cdb[0] == WRITE_10 ||
1632              cmd->cdb[0] == WRITE_12 ||
1633              cmd->cdb[0] == WRITE_16 ||
1634              cmd->cdb[0] == WRITE_VERIFY ||
1635              cmd->cdb[0] == WRITE_VERIFY_12 ||
1636              cmd->cdb[0] == WRITE_VERIFY_16 ||
1637              (cmd->dev->handler->type == TYPE_TAPE &&
1638               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
1639                 scst_set_cmd_error(cmd,
1640                            SCST_LOAD_SENSE(scst_sense_data_protect));
1641                 goto out_done;
1642         }
1643
1644 out:
1645         TRACE_EXIT_RES(res);
1646         return res;
1647
1648 out_done:
1649         res = SCST_EXEC_COMPLETED;
1650         /* Report the result */
1651         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1652         goto out;
1653 }
1654
1655 /*
1656  * The result of cmd execution, if any, should be reported
1657  * via scst_cmd_done_local()
1658  */
1659 static inline int scst_local_exec(struct scst_cmd *cmd)
1660 {
1661         int res = SCST_EXEC_NOT_COMPLETED;
1662
1663         TRACE_ENTRY();
1664
1665         /*
1666          * Adding new commands here don't forget to update
1667          * scst_is_cmd_local() in scst.h, if necessary
1668          */
1669
1670         switch (cmd->cdb[0]) {
1671         case MODE_SELECT:
1672         case MODE_SELECT_10:
1673         case LOG_SELECT:
1674                 res = scst_pre_select(cmd);
1675                 break;
1676         case RESERVE:
1677         case RESERVE_10:
1678                 res = scst_reserve_local(cmd);
1679                 break;
1680         case RELEASE:
1681         case RELEASE_10:
1682                 res = scst_release_local(cmd);
1683                 break;
1684         case REPORT_LUNS:
1685                 res = scst_report_luns_local(cmd);
1686                 break;
1687         }
1688
1689         TRACE_EXIT_RES(res);
1690         return res;
1691 }
1692
1693 /* cmd must be additionally referenced to not die inside */
1694 static int scst_do_send_to_midlev(struct scst_cmd *cmd)
1695 {
1696         int rc = SCST_EXEC_NOT_COMPLETED;
1697         struct scst_device *dev = cmd->dev;
1698         struct scst_dev_type *handler = dev->handler;
1699
1700         TRACE_ENTRY();
1701
1702         cmd->sent_to_midlev = 1;
1703         cmd->state = SCST_CMD_STATE_EXECUTING;
1704         cmd->scst_cmd_done = scst_cmd_done_local;
1705
1706         rc = scst_pre_exec(cmd);
1707         if (rc != SCST_EXEC_NOT_COMPLETED) {
1708                 if (rc == SCST_EXEC_COMPLETED)
1709                         goto out;
1710                 else if (rc == SCST_EXEC_NEED_THREAD)
1711                         goto out_clear;
1712                 else
1713                         goto out_rc_error;
1714         }
1715
1716         rc = scst_local_exec(cmd);
1717         if (rc != SCST_EXEC_NOT_COMPLETED) {
1718                 if (rc == SCST_EXEC_COMPLETED)
1719                         goto out;
1720                 else if (rc == SCST_EXEC_NEED_THREAD)
1721                         goto out_clear;
1722                 else
1723                         goto out_rc_error;
1724         }
1725
1726         if (!handler->exec_sync)
1727                 cmd->context_processable = 0;
1728
1729         if (handler->exec) {
1730                 TRACE_DBG("Calling dev handler %s exec(%p)",
1731                       handler->name, cmd);
1732                 TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb, cmd->cdb_len);
1733                 cmd->scst_cmd_done = scst_cmd_done_local;
1734                 rc = handler->exec(cmd);
1735                 TRACE_DBG("Dev handler %s exec() returned %d",
1736                       handler->name, rc);
1737                 if (rc == SCST_EXEC_COMPLETED)
1738                         goto out;
1739                 else if (rc == SCST_EXEC_NEED_THREAD)
1740                         goto out_clear;
1741                 else if (rc != SCST_EXEC_NOT_COMPLETED)
1742                         goto out_rc_error;
1743         }
1744
1745         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1746
1747         if (unlikely(dev->scsi_dev == NULL)) {
1748                 PRINT_ERROR("Command for virtual device must be "
1749                         "processed by device handler (lun %Ld)!",
1750                         (long long unsigned int)cmd->lun);
1751                 goto out_error;
1752         }
1753
1754         rc = scst_check_local_events(cmd);
1755         if (unlikely(rc != 0))
1756                 goto out_done;
1757
1758 #ifndef ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1759         if (scst_cmd_atomic(cmd)) {
1760                 TRACE_DBG("Pass-through exec() can not be called in atomic "
1761                         "context, rescheduling to the thread (handler %s)",
1762                         handler->name);
1763                 rc = SCST_EXEC_NEED_THREAD;
1764                 goto out_clear;
1765         }
1766 #endif
1767
1768 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1769         if (unlikely(scst_alloc_request(cmd) != 0)) {
1770                 if (scst_cmd_atomic(cmd)) {
1771                         rc = SCST_EXEC_NEED_THREAD;
1772                         goto out_clear;
1773                 } else {
1774                         PRINT_INFO("%s", "Unable to allocate request, "
1775                                 "sending BUSY status");
1776                         goto out_busy;
1777                 }
1778         }
1779
1780         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1781                     (void *)cmd->scsi_req->sr_buffer,
1782                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1783                     cmd->retries);
1784 #else
1785         rc = scst_exec_req(dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1786                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1787                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1788                         scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
1789         if (unlikely(rc != 0)) {
1790                 if (scst_cmd_atomic(cmd)) {
1791                         rc = SCST_EXEC_NEED_THREAD;
1792                         goto out_clear;
1793                 } else {
1794                         PRINT_INFO("scst_exec_req() failed: %d", rc);
1795                         goto out_error;
1796                 }
1797         }
1798 #endif
1799
1800         rc = SCST_EXEC_COMPLETED;
1801
1802 out:
1803         TRACE_EXIT();
1804         return rc;
1805
1806 out_clear:
1807         /* Restore the state */
1808         cmd->sent_to_midlev = 0;
1809         cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
1810         goto out;
1811
1812 out_rc_error:
1813         PRINT_ERROR("Dev handler %s exec() or scst_local_exec() returned "
1814                     "invalid code %d", handler->name, rc);
1815         /* go through */
1816
1817 out_error:
1818         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1819
1820 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1821 out_busy:
1822         scst_set_busy(cmd);
1823         /* go through */
1824 #endif
1825
1826 out_done:
1827         rc = SCST_EXEC_COMPLETED;
1828         /* Report the result */
1829         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1830         goto out;
1831 }
1832
1833 /* No locks */
1834 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1835 {
1836         if (slot == NULL)
1837                 goto inc;
1838
1839         /* Optimized for lockless fast path */
1840
1841         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1842                 atomic_read(slot));
1843
1844         if (!atomic_dec_and_test(slot))
1845                 goto out;
1846
1847         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1848                 tgt_dev->num_free_sn_slots);
1849         if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
1850                 spin_lock_irq(&tgt_dev->sn_lock);
1851                 if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
1852                         if (tgt_dev->num_free_sn_slots < 0)
1853                                 tgt_dev->cur_sn_slot = slot;
1854                         smp_mb(); /* to be in-sync with SIMPLE case in scst_cmd_set_sn() */
1855                         tgt_dev->num_free_sn_slots++;
1856                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1857                                 tgt_dev->num_free_sn_slots);
1858
1859                 }
1860                 spin_unlock_irq(&tgt_dev->sn_lock);
1861         }
1862
1863 inc:
1864         /*
1865          * No locks is needed, because only one thread at time can
1866          * be here (serialized by sn). Also it is supposed that there
1867          * could not be half-incremented halves.
1868          */
1869         tgt_dev->expected_sn++;
1870         smp_mb(); /* write must be before def_cmd_count read */
1871         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1872
1873 out:
1874         return;
1875 }
1876
1877 static int scst_process_internal_cmd(struct scst_cmd *cmd)
1878 {
1879         int res = SCST_CMD_STATE_RES_CONT_NEXT, rc;
1880
1881         TRACE_ENTRY();
1882
1883         __scst_cmd_get(cmd);
1884
1885         rc = scst_do_send_to_midlev(cmd);
1886         if (rc == SCST_EXEC_NEED_THREAD) {
1887                 TRACE_DBG("%s", "scst_do_send_to_midlev() requested "
1888                       "thread context, rescheduling");
1889                 res = SCST_CMD_STATE_RES_NEED_THREAD;
1890         } else {
1891                 struct scst_device *dev = cmd->dev;
1892                 sBUG_ON(rc != SCST_EXEC_COMPLETED);
1893                 if (dev->scsi_dev != NULL)
1894                         generic_unplug_device(dev->scsi_dev->request_queue);
1895         }
1896
1897         __scst_cmd_put(cmd);
1898
1899         TRACE_EXIT_RES(res);
1900         return res;
1901 }
1902
1903 static int scst_send_to_midlev(struct scst_cmd **active_cmd)
1904 {
1905         int res, rc;
1906         struct scst_cmd *cmd = *active_cmd;
1907         struct scst_cmd *ref_cmd;
1908         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1909         struct scst_device *dev = cmd->dev;
1910         typeof(tgt_dev->expected_sn) expected_sn;
1911         int count;
1912
1913         TRACE_ENTRY();
1914
1915         res = SCST_CMD_STATE_RES_CONT_NEXT;
1916
1917         if (unlikely(cmd->internal || cmd->retry)) {
1918                 res = scst_process_internal_cmd(cmd);
1919                 goto out;
1920         }
1921
1922 #ifdef MEASURE_LATENCY
1923         if (cmd->pre_exec_finish == 0) {
1924                 struct timespec ts;
1925                 getnstimeofday(&ts);
1926                 cmd->pre_exec_finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1927                 TRACE_DBG("cmd %p (sess %p): pre_exec_finish %Ld (tv_sec %ld, "
1928                         "tv_nsec %ld)", cmd, sess, cmd->pre_exec_finish, ts.tv_sec,
1929                         ts.tv_nsec);
1930         }
1931 #endif
1932
1933         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
1934                 goto out;
1935
1936         ref_cmd = cmd;
1937         __scst_cmd_get(ref_cmd);
1938
1939         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
1940                 goto exec;
1941
1942         sBUG_ON(!cmd->sn_set);
1943
1944         expected_sn = tgt_dev->expected_sn;
1945         /* Optimized for lockless fast path */
1946         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
1947                 spin_lock_irq(&tgt_dev->sn_lock);
1948
1949                 tgt_dev->def_cmd_count++;
1950                 smp_mb();
1951
1952                 expected_sn = tgt_dev->expected_sn;
1953                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
1954                         /* We are under IRQ lock, but dev->dev_lock is BH one */
1955                         int cmd_blocking = scst_pre_dec_on_dev_cmd(cmd);
1956                         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1957                                 /* Necessary to allow aborting out of sn cmds */
1958                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p (tag %llu)",
1959                                                cmd,
1960                                                (long long unsigned)cmd->tag);
1961                                 tgt_dev->def_cmd_count--;
1962                                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1963                                 res = SCST_CMD_STATE_RES_CONT_SAME;
1964                         } else {
1965                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
1966                                         "expected_sn=%ld)", cmd, cmd->sn,
1967                                         cmd->sn_set, expected_sn);
1968                                 list_add_tail(&cmd->sn_cmd_list_entry,
1969                                               &tgt_dev->deferred_cmd_list);
1970                         }
1971                         spin_unlock_irq(&tgt_dev->sn_lock);
1972
1973                         __scst_dec_on_dev_cmd(dev, cmd_blocking);
1974
1975                         goto out_put;
1976                 } else {
1977                         TRACE_SN("Somebody incremented expected_sn %ld, "
1978                                 "continuing", expected_sn);
1979                         tgt_dev->def_cmd_count--;
1980                         spin_unlock_irq(&tgt_dev->sn_lock);
1981                 }
1982         }
1983
1984 exec:
1985         count = 0;
1986         while (1) {
1987                 atomic_t *slot = cmd->sn_slot;
1988                 /* For HQ commands SN is not set */
1989                 int inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1990                                       cmd->sn_set;
1991
1992                 rc = scst_do_send_to_midlev(cmd);
1993                 if (rc == SCST_EXEC_NEED_THREAD) {
1994                         TRACE_DBG("%s", "scst_do_send_to_midlev() requested "
1995                               "thread context, rescheduling");
1996                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1997                         scst_dec_on_dev_cmd(cmd);
1998                         *active_cmd = cmd;
1999                         if (count != 0)
2000                                 goto out_unplug;
2001                         else
2002                                 goto out_put;
2003                 }
2004                 sBUG_ON(rc != SCST_EXEC_COMPLETED);
2005
2006                 count++;
2007
2008                 if (inc_expected_sn)
2009                         scst_inc_expected_sn(tgt_dev, slot);
2010
2011                 cmd = scst_check_deferred_commands(tgt_dev);
2012                 if (cmd == NULL)
2013                         break;
2014
2015                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2016                         break;
2017
2018                 __scst_cmd_put(ref_cmd);
2019                 ref_cmd = cmd;
2020                 __scst_cmd_get(ref_cmd);
2021         }
2022
2023 out_unplug:
2024         if (dev->scsi_dev != NULL)
2025                 generic_unplug_device(dev->scsi_dev->request_queue);
2026
2027 out_put:
2028         __scst_cmd_put(ref_cmd);
2029         /* !! At this point sess, dev and tgt_dev can be already freed !! */
2030
2031 out:
2032         TRACE_EXIT_HRES(res);
2033         return res;
2034 }
2035
2036 /* No locks supposed to be held */
2037 static int scst_check_sense(struct scst_cmd *cmd)
2038 {
2039         int res = 0;
2040         struct scst_device *dev = cmd->dev;
2041         int dbl_ua_possible, ua_sent = 0;
2042
2043         TRACE_ENTRY();
2044
2045         /* If we had internal bus reset behind us, set the command error UA */
2046         if ((dev->scsi_dev != NULL) &&
2047             unlikely(cmd->host_status == DID_RESET) &&
2048             scst_is_ua_command(cmd)) {
2049                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
2050                       dev->scsi_dev->was_reset, cmd->host_status);
2051                 scst_set_cmd_error(cmd,
2052                    SCST_LOAD_SENSE(scst_sense_reset_UA));
2053                 /* just in case */
2054                 cmd->ua_ignore = 0;
2055                 /* It looks like it is safe to clear was_reset here */
2056                 dev->scsi_dev->was_reset = 0;
2057         }
2058
2059         dbl_ua_possible = dev->dev_double_ua_possible;
2060         TRACE_DBG("cmd %p dbl_ua_possible %d", cmd, dbl_ua_possible);
2061         if (unlikely(dbl_ua_possible)) {
2062                 spin_lock_bh(&dev->dev_lock);
2063                 barrier(); /* to reread dev_double_ua_possible */
2064                 dbl_ua_possible = dev->dev_double_ua_possible;
2065                 if (dbl_ua_possible)
2066                         ua_sent = dev->dev_reset_ua_sent;
2067                 else
2068                         spin_unlock_bh(&dev->dev_lock);
2069         }
2070
2071         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2072             SCST_SENSE_VALID(cmd->sense)) {
2073                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2074                         SCST_SENSE_BUFFERSIZE);
2075                 /* Check Unit Attention Sense Key */
2076                 if (scst_is_ua_sense(cmd->sense)) {
2077                         if (cmd->sense[12] == SCST_SENSE_ASC_UA_RESET) {
2078                                 if (dbl_ua_possible) {
2079                                         if (ua_sent) {
2080                                                 TRACE(TRACE_MGMT_MINOR, "%s",
2081                                                         "Double UA detected");
2082                                                 /* Do retry */
2083                                                 TRACE(TRACE_MGMT_MINOR, "Retrying cmd %p "
2084                                                         "(tag %llu)", cmd,
2085                                                         (long long unsigned)cmd->tag);
2086                                                 cmd->status = 0;
2087                                                 cmd->msg_status = 0;
2088                                                 cmd->host_status = DID_OK;
2089                                                 cmd->driver_status = 0;
2090                                                 mempool_free(cmd->sense, scst_sense_mempool);
2091                                                 cmd->sense = NULL;
2092                                                 sBUG_ON(cmd->dbl_ua_orig_resp_data_len < 0);
2093                                                 sBUG_ON(cmd->sg_buff_modified);
2094                                                 cmd->data_direction =
2095                                                         cmd->dbl_ua_orig_data_direction;
2096                                                 cmd->resp_data_len =
2097                                                         cmd->dbl_ua_orig_resp_data_len;
2098                                                 cmd->retry = 1;
2099                                                 cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
2100                                                 res = 1;
2101                                                 /*
2102                                                  * Dev is still blocked by this cmd, so
2103                                                  * it's OK to clear SCST_DEV_SERIALIZED
2104                                                  * here.
2105                                                  */
2106                                                 dev->dev_double_ua_possible = 0;
2107                                                 dev->dev_serialized = 0;
2108                                                 dev->dev_reset_ua_sent = 0;
2109                                                 goto out_unlock;
2110                                         } else
2111                                                 dev->dev_reset_ua_sent = 1;
2112                                 }
2113                         }
2114                         if (cmd->ua_ignore == 0) {
2115                                 if (unlikely(dbl_ua_possible)) {
2116                                         __scst_dev_check_set_UA(dev, cmd,
2117                                                 cmd->sense,
2118                                                 SCST_SENSE_BUFFERSIZE);
2119                                 } else {
2120                                         scst_dev_check_set_UA(dev, cmd,
2121                                                 cmd->sense,
2122                                                 SCST_SENSE_BUFFERSIZE);
2123                                 }
2124                         }
2125                 }
2126         }
2127
2128         if (unlikely(dbl_ua_possible)) {
2129                 if (ua_sent && scst_is_ua_command(cmd)) {
2130                         TRACE_MGMT_DBG("%s", "Clearing dbl_ua_possible flag");
2131                         dev->dev_double_ua_possible = 0;
2132                         dev->dev_serialized = 0;
2133                         dev->dev_reset_ua_sent = 0;
2134                 }
2135                 spin_unlock_bh(&dev->dev_lock);
2136         }
2137
2138 out:
2139         TRACE_EXIT_RES(res);
2140         return res;
2141
2142 out_unlock:
2143         spin_unlock_bh(&dev->dev_lock);
2144         goto out;
2145 }
2146
2147 static int scst_check_auto_sense(struct scst_cmd *cmd)
2148 {
2149         int res = 0;
2150
2151         TRACE_ENTRY();
2152
2153         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2154             (!SCST_SENSE_VALID(cmd->sense) ||
2155              SCST_NO_SENSE(cmd->sense))) {
2156                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2157                       "cmd->status=%x, cmd->msg_status=%x, "
2158                       "cmd->host_status=%x, cmd->driver_status=%x", cmd->status,
2159                       cmd->msg_status, cmd->host_status, cmd->driver_status);
2160                 res = 1;
2161         } else if (unlikely(cmd->host_status)) {
2162                 if ((cmd->host_status == DID_REQUEUE) ||
2163                     (cmd->host_status == DID_IMM_RETRY) ||
2164                     (cmd->host_status == DID_SOFT_ERROR) ||
2165                     (cmd->host_status == DID_ABORT)) {
2166                         scst_set_busy(cmd);
2167                 } else {
2168                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2169                                 "received, returning HARDWARE ERROR instead",
2170                                 cmd->host_status);
2171                         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2172                 }
2173         }
2174
2175         TRACE_EXIT_RES(res);
2176         return res;
2177 }
2178
2179 static int scst_done_cmd_check(struct scst_cmd *cmd, int *pres)
2180 {
2181         int res = 0, rc;
2182
2183         TRACE_ENTRY();
2184
2185         if (unlikely(cmd->cdb[0] == REQUEST_SENSE)) {
2186                 if (cmd->internal)
2187                         cmd = scst_complete_request_sense(cmd);
2188         } else if (unlikely(scst_check_auto_sense(cmd))) {
2189                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2190                             "without sense data (opcode 0x%x), issuing "
2191                             "REQUEST SENSE", cmd->cdb[0]);
2192                 rc = scst_prepare_request_sense(cmd);
2193                 if (rc > 0) {
2194                         *pres = rc;
2195                         res = 1;
2196                         goto out;
2197                 } else {
2198                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2199                                     "returning HARDWARE ERROR");
2200                         scst_set_cmd_error(cmd,
2201                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2202                 }
2203         } else if (unlikely(scst_check_sense(cmd))) {
2204                 *pres = SCST_CMD_STATE_RES_CONT_SAME;
2205                 res = 1;
2206                 goto out;
2207         }
2208
2209         if (likely(scsi_status_is_good(cmd->status))) {
2210                 unsigned char type = cmd->dev->handler->type;
2211                 if (unlikely((cmd->cdb[0] == MODE_SENSE ||
2212                               cmd->cdb[0] == MODE_SENSE_10)) &&
2213                     cmd->tgt_dev->acg_dev->rd_only_flag &&
2214                     (type == TYPE_DISK || type == TYPE_WORM || type == TYPE_MOD ||
2215                      type == TYPE_TAPE)) {
2216                         int32_t length;
2217                         uint8_t *address;
2218
2219                         length = scst_get_buf_first(cmd, &address);
2220                         if (length <= 0)
2221                                 goto out;
2222                         if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2223                                 address[2] |= 0x80;   /* Write Protect*/
2224                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2225                                 address[3] |= 0x80;   /* Write Protect*/
2226                         scst_put_buf(cmd, address);
2227                 }
2228
2229                 /*
2230                  * Check and clear NormACA option for the device, if necessary,
2231                  * since we don't support ACA
2232                  */
2233                 if (unlikely((cmd->cdb[0] == INQUIRY)) &&
2234                     !(cmd->cdb[1] & SCST_INQ_EVPD/* Std INQUIRY data (no EVPD) */) &&
2235                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2236                         uint8_t *buffer;
2237                         int buflen;
2238
2239                         /* ToDo: all pages ?? */
2240                         buflen = scst_get_buf_first(cmd, &buffer);
2241                         if (buflen > SCST_INQ_BYTE3) {
2242 #ifdef EXTRACHECKS
2243                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2244                                         PRINT_INFO("NormACA set for device: "
2245                                             "lun=%Ld, type 0x%02x. Clear it, "
2246                                             "since it's unsupported.",
2247                                             (long long unsigned int)cmd->lun,
2248                                             buffer[0]);
2249                                 }
2250 #endif
2251                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2252                         } else {
2253                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2254                                     "buffer");
2255                                 scst_set_cmd_error(cmd,
2256                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
2257                         }
2258                         if (buflen > 0)
2259                                 scst_put_buf(cmd, buffer);
2260                 }
2261
2262                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2263                     (cmd->cdb[0] == MODE_SELECT_10) ||
2264                     (cmd->cdb[0] == LOG_SELECT))) {
2265                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded (LUN %Ld)",
2266                                 (long long unsigned int)cmd->lun);
2267                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2268                         *pres = SCST_CMD_STATE_RES_CONT_SAME;
2269                         res = 1;
2270                         goto out;
2271                 }
2272         } else {
2273                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2274                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2275                                         &cmd->tgt_dev->tgt_dev_flags)) {
2276                                 struct scst_tgt_dev *tgt_dev_tmp;
2277                                 struct scst_device *dev = cmd->dev;
2278
2279                                 TRACE(TRACE_SCSI, "Real RESERVE failed lun=%Ld, "
2280                                         "status=%x",
2281                                         (long long unsigned int)cmd->lun,
2282                                         cmd->status);
2283                                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2284                                         SCST_SENSE_BUFFERSIZE);
2285
2286                                 /* Clearing the reservation */
2287                                 spin_lock_bh(&dev->dev_lock);
2288                                 list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
2289                                                     dev_tgt_dev_list_entry) {
2290                                         clear_bit(SCST_TGT_DEV_RESERVED,
2291                                                 &tgt_dev_tmp->tgt_dev_flags);
2292                                 }
2293                                 dev->dev_reserved = 0;
2294                                 spin_unlock_bh(&dev->dev_lock);
2295                         }
2296                 }
2297
2298                 /* Check for MODE PARAMETERS CHANGED UA */
2299                 if ((cmd->dev->scsi_dev != NULL) &&
2300                     (cmd->status == SAM_STAT_CHECK_CONDITION) &&
2301                     SCST_SENSE_VALID(cmd->sense) &&
2302                     scst_is_ua_sense(cmd->sense) &&
2303                     (cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) {
2304                         TRACE(TRACE_SCSI,
2305                               "MODE PARAMETERS CHANGED UA (lun %Ld)",
2306                               (long long unsigned int)cmd->lun);
2307                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2308                         *pres = SCST_CMD_STATE_RES_CONT_SAME;
2309                         res = 1;
2310                         goto out;
2311                 }
2312         }
2313
2314 out:
2315         TRACE_EXIT_RES(res);
2316         return res;
2317 }
2318
2319 static int scst_pre_dev_done(struct scst_cmd *cmd)
2320 {
2321         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2322
2323         TRACE_ENTRY();
2324
2325         rc = scst_done_cmd_check(cmd, &res);
2326         if (rc)
2327                 goto out;
2328
2329         cmd->state = SCST_CMD_STATE_DEV_DONE;
2330
2331 out:
2332         TRACE_EXIT_HRES(res);
2333         return res;
2334 }
2335
2336 static int scst_mode_select_checks(struct scst_cmd *cmd)
2337 {
2338         int res = SCST_CMD_STATE_RES_CONT_SAME;
2339         int atomic = scst_cmd_atomic(cmd);
2340
2341         TRACE_ENTRY();
2342
2343         if (likely(scsi_status_is_good(cmd->status))) {
2344                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2345                     (cmd->cdb[0] == MODE_SELECT_10) ||
2346                     (cmd->cdb[0] == LOG_SELECT))) {
2347                         struct scst_device *dev = cmd->dev;
2348                         if (atomic && (dev->scsi_dev != NULL)) {
2349                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2350                                         "context required");
2351                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2352                                 goto out;
2353                         }
2354
2355                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2356                                 "setting the SELECT UA (lun=%Ld)",
2357                                 (long long unsigned int)cmd->lun);
2358
2359                         spin_lock_bh(&dev->dev_lock);
2360                         spin_lock(&scst_temp_UA_lock);
2361                         if (cmd->cdb[0] == LOG_SELECT) {
2362                                 scst_set_sense(scst_temp_UA,
2363                                         sizeof(scst_temp_UA),
2364                                         UNIT_ATTENTION, 0x2a, 0x02);
2365                         } else {
2366                                 scst_set_sense(scst_temp_UA,
2367                                         sizeof(scst_temp_UA),
2368                                         UNIT_ATTENTION, 0x2a, 0x01);
2369                         }
2370                         scst_dev_check_set_local_UA(dev, cmd, scst_temp_UA,
2371                                 sizeof(scst_temp_UA));
2372                         spin_unlock(&scst_temp_UA_lock);
2373                         spin_unlock_bh(&dev->dev_lock);
2374
2375                         if (dev->scsi_dev != NULL)
2376                                 scst_obtain_device_parameters(dev);
2377                 }
2378         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) &&
2379                     SCST_SENSE_VALID(cmd->sense) &&
2380                     scst_is_ua_sense(cmd->sense) &&
2381                     (((cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) ||
2382                      (cmd->sense[12] == 0x29) /* reset */ ||
2383                      (cmd->sense[12] == 0x28) /* medium changed */ ||
2384                      (cmd->sense[12] == 0x2F) /* cleared by another ini (just in case) */)) {
2385                 if (atomic) {
2386                         TRACE_DBG("Possible parameters changed UA %x: "
2387                                 "thread context required", cmd->sense[12]);
2388                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2389                         goto out;
2390                 }
2391
2392                 TRACE(TRACE_SCSI, "Possible parameters changed UA %x "
2393                         "(lun %Ld): getting new parameters", cmd->sense[12],
2394                         (long long unsigned int)cmd->lun);
2395
2396                 scst_obtain_device_parameters(cmd->dev);
2397         } else
2398                 sBUG();
2399
2400         cmd->state = SCST_CMD_STATE_DEV_DONE;
2401
2402 out:
2403         TRACE_EXIT_HRES(res);
2404         return res;
2405 }
2406
2407 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2408 {
2409         if (likely(cmd->sn_set))
2410                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2411
2412         scst_make_deferred_commands_active(cmd->tgt_dev, cmd);
2413 }
2414
2415 static int scst_dev_done(struct scst_cmd *cmd)
2416 {
2417         int res = SCST_CMD_STATE_RES_CONT_SAME;
2418         int state;
2419
2420         TRACE_ENTRY();
2421
2422         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2423         if (likely(!scst_is_cmd_local(cmd)) &&
2424             likely(cmd->dev->handler->dev_done != NULL)) {
2425                 int rc;
2426                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2427                       cmd->dev->handler->name, cmd);
2428                 rc = cmd->dev->handler->dev_done(cmd);
2429                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2430                       cmd->dev->handler->name, rc);
2431                 if (rc != SCST_CMD_STATE_DEFAULT)
2432                         state = rc;
2433         }
2434
2435         switch (state) {
2436         case SCST_CMD_STATE_PRE_XMIT_RESP:
2437         case SCST_CMD_STATE_DEV_PARSE:
2438         case SCST_CMD_STATE_PRE_PARSE:
2439         case SCST_CMD_STATE_PREPARE_SPACE:
2440         case SCST_CMD_STATE_RDY_TO_XFER:
2441         case SCST_CMD_STATE_TGT_PRE_EXEC:
2442         case SCST_CMD_STATE_SEND_TO_MIDLEV:
2443         case SCST_CMD_STATE_PRE_DEV_DONE:
2444         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2445         case SCST_CMD_STATE_DEV_DONE:
2446         case SCST_CMD_STATE_XMIT_RESP:
2447         case SCST_CMD_STATE_FINISHED:
2448                 cmd->state = state;
2449                 res = SCST_CMD_STATE_RES_CONT_SAME;
2450                 break;
2451
2452         case SCST_CMD_STATE_NEED_THREAD_CTX:
2453                 TRACE_DBG("Dev handler %s dev_done() requested "
2454                       "thread context, rescheduling",
2455                       cmd->dev->handler->name);
2456                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2457                 break;
2458
2459         default:
2460                 if (state >= 0) {
2461                         PRINT_ERROR("Dev handler %s dev_done() returned "
2462                                 "invalid cmd state %d",
2463                                 cmd->dev->handler->name, state);
2464                 } else {
2465                         PRINT_ERROR("Dev handler %s dev_done() returned "
2466                                 "error %d", cmd->dev->handler->name,
2467                                 state);
2468                 }
2469                 scst_set_cmd_error(cmd,
2470                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2471                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2472                 res = SCST_CMD_STATE_RES_CONT_SAME;
2473                 break;
2474         }
2475
2476         if (cmd->needs_unblocking)
2477                 scst_unblock_dev_cmd(cmd);
2478
2479         if (likely(cmd->dec_on_dev_needed))
2480                 scst_dec_on_dev_cmd(cmd);
2481
2482         if (cmd->inc_expected_sn_on_done && cmd->sent_to_midlev)
2483                 scst_inc_check_expected_sn(cmd);
2484
2485         TRACE_EXIT_HRES(res);
2486         return res;
2487 }
2488
2489 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2490 {
2491         int res;
2492
2493         TRACE_ENTRY();
2494
2495 #ifdef DEBUG_TM
2496         if (cmd->tm_dbg_delayed && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2497                 if (scst_cmd_atomic(cmd)) {
2498                         TRACE_MGMT_DBG("%s", "DEBUG_TM delayed cmd needs a thread");
2499                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2500                         return res;
2501                 }
2502                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2503                         cmd, cmd->tag);
2504                 schedule_timeout_uninterruptible(HZ);
2505         }
2506 #endif
2507
2508         if (likely(cmd->tgt_dev != NULL)) {
2509                 atomic_dec(&cmd->tgt_dev->tgt_dev_cmd_count);
2510                 atomic_dec(&cmd->dev->dev_cmd_count);
2511                 /* If expected values not set, expected direction is UNKNOWN */
2512                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
2513                         atomic_dec(&cmd->dev->write_cmd_count);
2514
2515                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2516                         scst_on_hq_cmd_response(cmd);
2517
2518                 if (unlikely(!cmd->sent_to_midlev)) {
2519                         TRACE_SN("cmd %p was not sent to mid-lev (sn %ld, set %d)",
2520                                 cmd, cmd->sn, cmd->sn_set);
2521                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2522                         cmd->sent_to_midlev = 1;
2523                 }
2524         }
2525
2526         /*
2527          * If we don't remove cmd from the search list here, before
2528          * submitting it for transmittion, we will have a race, when for
2529          * some reason cmd's release is delayed after transmittion and
2530          * initiator sends cmd with the same tag => it is possible that
2531          * a wrong cmd will be found by find() functions.
2532          */
2533         spin_lock_irq(&cmd->sess->sess_list_lock);
2534         list_del(&cmd->search_cmd_list_entry);
2535         spin_unlock_irq(&cmd->sess->sess_list_lock);
2536
2537         cmd->done = 1;
2538         smp_mb(); /* to sync with scst_abort_cmd() */
2539
2540         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2541                 scst_xmit_process_aborted_cmd(cmd);
2542
2543         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2544                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu), skipping",
2545                                cmd, (long long unsigned int)cmd->tag);
2546                 cmd->state = SCST_CMD_STATE_FINISHED;
2547                 res = SCST_CMD_STATE_RES_CONT_SAME;
2548                 goto out;
2549         }
2550
2551         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2552         res = SCST_CMD_STATE_RES_CONT_SAME;
2553
2554 out:
2555 #ifdef MEASURE_LATENCY
2556         {
2557                 struct timespec ts;
2558                 uint64_t finish, scst_time, proc_time;
2559                 struct scst_session *sess = cmd->sess;
2560
2561                 getnstimeofday(&ts);
2562                 finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2563
2564                 spin_lock_bh(&sess->meas_lock);
2565
2566                 scst_time = cmd->pre_exec_finish - cmd->start;
2567                 scst_time += finish - cmd->post_exec_start;
2568                 proc_time = finish - cmd->start;
2569
2570                 sess->scst_time += scst_time;
2571                 sess->processing_time += proc_time;
2572                 sess->processed_cmds++;
2573
2574                 spin_unlock_bh(&sess->meas_lock);
2575
2576                 TRACE_DBG("cmd %p (sess %p): finish %Ld (tv_sec %ld, "
2577                         "tv_nsec %ld), scst_time %Ld, proc_time %Ld", cmd, sess,
2578                         finish, ts.tv_sec, ts.tv_nsec, scst_time, proc_time);
2579         }
2580 #endif
2581         TRACE_EXIT_HRES(res);
2582         return res;
2583 }
2584
2585 static int scst_xmit_response(struct scst_cmd *cmd)
2586 {
2587         int res, rc;
2588
2589         TRACE_ENTRY();
2590
2591         while (1) {
2592                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2593
2594                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2595                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2596
2597                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2598
2599 #if defined(DEBUG)
2600                 if (cmd->sg) {
2601                         int i;
2602                         struct scatterlist *sg = cmd->sg;
2603                         TRACE_SEND_BOT("Xmitting %d S/G(s) at %p sg[0].page at %p",
2604                               cmd->sg_cnt, sg, (void *)sg_page(&sg[0]));
2605                         for (i = 0; i < cmd->sg_cnt; ++i) {
2606                                 TRACE_BUFF_FLAG(TRACE_SND_BOT,
2607                                     "Xmitting sg", sg_virt(&sg[i]),
2608                                     sg[i].length);
2609                         }
2610                 }
2611 #endif
2612
2613 #ifdef DEBUG_RETRY
2614                 if (((scst_random() % 100) == 77))
2615                         rc = SCST_TGT_RES_QUEUE_FULL;
2616                 else
2617 #endif
2618                         rc = cmd->tgtt->xmit_response(cmd);
2619                 TRACE_DBG("xmit_response() returned %d", rc);
2620
2621                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2622                         goto out;
2623
2624                 /* Restore the previous state */
2625                 cmd->state = SCST_CMD_STATE_XMIT_RESP;
2626
2627                 switch (rc) {
2628                 case SCST_TGT_RES_QUEUE_FULL:
2629                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2630                                 break;
2631                         else
2632                                 continue;
2633
2634                 case SCST_TGT_RES_NEED_THREAD_CTX:
2635                         TRACE_DBG("Target driver %s xmit_response() "
2636                               "requested thread context, rescheduling",
2637                               cmd->tgtt->name);
2638                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2639                         break;
2640
2641                 default:
2642                         goto out_error;
2643                 }
2644                 break;
2645         }
2646
2647 out:
2648         /* Caution: cmd can be already dead here */
2649         TRACE_EXIT_HRES(res);
2650         return res;
2651
2652 out_error:
2653         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2654                 PRINT_ERROR("Target driver %s xmit_response() returned "
2655                         "fatal error", cmd->tgtt->name);
2656         } else {
2657                 PRINT_ERROR("Target driver %s xmit_response() returned "
2658                         "invalid value %d", cmd->tgtt->name, rc);
2659         }
2660         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2661         cmd->state = SCST_CMD_STATE_FINISHED;
2662         res = SCST_CMD_STATE_RES_CONT_SAME;
2663         goto out;
2664 }
2665
2666 void scst_tgt_cmd_done(struct scst_cmd *cmd)
2667 {
2668         TRACE_ENTRY();
2669
2670         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2671
2672         cmd->state = SCST_CMD_STATE_FINISHED;
2673         scst_proccess_redirect_cmd(cmd, scst_get_context(), 1);
2674
2675         TRACE_EXIT();
2676         return;
2677 }
2678 EXPORT_SYMBOL(scst_tgt_cmd_done);
2679
2680 static int scst_finish_cmd(struct scst_cmd *cmd)
2681 {
2682         int res;
2683
2684         TRACE_ENTRY();
2685
2686         atomic_dec(&cmd->sess->sess_cmd_count);
2687
2688         cmd->finished = 1;
2689         smp_mb(); /* to sync with scst_abort_cmd() */
2690
2691         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2692                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
2693                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
2694                         atomic_read(&scst_cmd_count));
2695
2696                 scst_finish_cmd_mgmt(cmd);
2697         }
2698
2699         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
2700                 if ((cmd->tgt_dev != NULL) &&
2701                     scst_is_ua_sense(cmd->sense)) {
2702                         /* This UA delivery failed, so requeue it */
2703                         TRACE_MGMT_DBG("Requeuing UA for delivery failed cmd "
2704                                 "%p", cmd);
2705                         scst_check_set_UA(cmd->tgt_dev, cmd->sense,
2706                                         SCST_SENSE_BUFFERSIZE, 1);
2707                 }
2708         }
2709
2710         __scst_cmd_put(cmd);
2711
2712         res = SCST_CMD_STATE_RES_CONT_NEXT;
2713
2714         TRACE_EXIT_HRES(res);
2715         return res;
2716 }
2717
2718 /*
2719  * No locks, but it must be externally serialized (see comment for
2720  * scst_cmd_init_done() in scst.h)
2721  */
2722 static void scst_cmd_set_sn(struct scst_cmd *cmd)
2723 {
2724         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2725         unsigned long flags;
2726
2727         TRACE_ENTRY();
2728
2729         if (scst_is_implicit_hq(cmd)) {
2730                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "Implicit HQ cmd %p", cmd);
2731                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
2732         }
2733
2734         EXTRACHECKS_BUG_ON(cmd->sn_set || cmd->hq_cmd_inced);
2735
2736         /* Optimized for lockless fast path */
2737
2738         scst_check_debug_sn(cmd);
2739
2740         if (cmd->dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) {
2741                 /*
2742                  * Not the best way, but well enough until there will be a
2743                  * possibility to specify queue type during pass-through
2744                  * commands submission.
2745                  */
2746                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
2747         }
2748
2749         switch (cmd->queue_type) {
2750         case SCST_CMD_QUEUE_SIMPLE:
2751         case SCST_CMD_QUEUE_UNTAGGED:
2752 #if 1 /* temporary, ToDo */
2753                 if (scst_cmd_is_expected_set(cmd)) {
2754                         if ((cmd->expected_data_direction == SCST_DATA_READ) &&
2755                             (atomic_read(&cmd->dev->write_cmd_count) == 0))
2756                                 goto ordered;
2757                 } else
2758                         goto ordered;
2759 #endif
2760                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
2761                         /*
2762                          * atomic_inc_return() implies memory barrier to sync
2763                          * with scst_inc_expected_sn()
2764                          */
2765                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
2766                                 tgt_dev->curr_sn++;
2767                                 TRACE_SN("Incremented curr_sn %ld",
2768                                         tgt_dev->curr_sn);
2769                         }
2770                         cmd->sn_slot = tgt_dev->cur_sn_slot;
2771                         cmd->sn = tgt_dev->curr_sn;
2772
2773                         tgt_dev->prev_cmd_ordered = 0;
2774                 } else {
2775                         TRACE(TRACE_MINOR, "***WARNING*** Not enough SN slots "
2776                                 "%zd", ARRAY_SIZE(tgt_dev->sn_slots));
2777                         goto ordered;
2778                 }
2779                 break;
2780
2781         case SCST_CMD_QUEUE_ORDERED:
2782                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "ORDERED cmd %p "
2783                         "(op %x)", cmd, cmd->cdb[0]);
2784 ordered:
2785                 if (!tgt_dev->prev_cmd_ordered) {
2786                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2787                         if (tgt_dev->num_free_sn_slots >= 0) {
2788                                 tgt_dev->num_free_sn_slots--;
2789                                 if (tgt_dev->num_free_sn_slots >= 0) {
2790                                         int i = 0;
2791                                         /*
2792                                          * Commands can finish in any order, so we don't
2793                                          * know, which slot is empty.
2794                                          */
2795                                         while (1) {
2796                                                 tgt_dev->cur_sn_slot++;
2797                                                 if (tgt_dev->cur_sn_slot == tgt_dev->sn_slots +
2798                                                                 ARRAY_SIZE(tgt_dev->sn_slots))
2799                                                         tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
2800
2801                                                 if (atomic_read(tgt_dev->cur_sn_slot) == 0)
2802                                                         break;
2803
2804                                                 i++;
2805                                                 sBUG_ON(i == ARRAY_SIZE(tgt_dev->sn_slots));
2806                                         }
2807                                         TRACE_SN("New cur SN slot %zd",
2808                                                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
2809                                 }
2810                         }
2811                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2812                 }
2813                 tgt_dev->prev_cmd_ordered = 1;
2814                 tgt_dev->curr_sn++;
2815                 cmd->sn = tgt_dev->curr_sn;
2816                 break;
2817
2818         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
2819                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "HQ cmd %p "
2820                         "(op %x)", cmd, cmd->cdb[0]);
2821                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2822                 tgt_dev->hq_cmd_count++;
2823                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2824                 cmd->hq_cmd_inced = 1;
2825                 goto out;
2826
2827         default:
2828                 sBUG();
2829         }
2830
2831         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
2832                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
2833                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
2834                 atomic_read(tgt_dev->cur_sn_slot),
2835                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
2836                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
2837
2838         cmd->sn_set = 1;
2839
2840 out:
2841         TRACE_EXIT();
2842         return;
2843 }
2844
2845 /*
2846  * Returns 0 on success, > 0 when we need to wait for unblock,
2847  * < 0 if there is no device (lun) or device type handler.
2848  *
2849  * No locks, but might be on IRQ, protection is done by the
2850  * suspended activity.
2851  */
2852 static int scst_translate_lun(struct scst_cmd *cmd)
2853 {
2854         struct scst_tgt_dev *tgt_dev = NULL;
2855         int res;
2856
2857         TRACE_ENTRY();
2858
2859         __scst_get(1);
2860
2861         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
2862                 struct list_head *sess_tgt_dev_list_head =
2863                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
2864                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %Ld)", cmd,
2865                         (long long unsigned int)cmd->lun);
2866                 res = -1;
2867                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
2868                                 sess_tgt_dev_list_entry) {
2869                         if (tgt_dev->lun == cmd->lun) {
2870                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
2871
2872                                 if (unlikely(tgt_dev->dev->handler == &scst_null_devtype)) {
2873                                         PRINT_INFO("Dev handler for device "
2874                                           "%Ld is NULL, the device will not be "
2875                                           "visible remotely",
2876                                            (long long unsigned int)cmd->lun);
2877                                         break;
2878                                 }
2879
2880                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
2881                                 cmd->tgt_dev = tgt_dev;
2882                                 cmd->dev = tgt_dev->dev;
2883
2884                                 res = 0;
2885                                 break;
2886                         }
2887                 }
2888                 if (res != 0) {
2889                         TRACE(TRACE_MINOR, "tgt_dev for lun %Ld not found, command to "
2890                                 "unexisting LU?", (long long unsigned int)cmd->lun);
2891                         __scst_put();
2892                 }
2893         } else {
2894                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
2895                 __scst_put();
2896                 res = 1;
2897         }
2898
2899         TRACE_EXIT_RES(res);
2900         return res;
2901 }
2902
2903 /*
2904  * No locks, but might be on IRQ
2905  *
2906  * Returns 0 on success, > 0 when we need to wait for unblock,
2907  * < 0 if there is no device (lun) or device type handler.
2908  */
2909 static int __scst_init_cmd(struct scst_cmd *cmd)
2910 {
2911         int res = 0;
2912
2913         TRACE_ENTRY();
2914
2915         res = scst_translate_lun(cmd);
2916         if (likely(res == 0)) {
2917                 int cnt;
2918                 bool failure = false;
2919
2920                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
2921
2922                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
2923                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
2924                         TRACE(TRACE_MGMT_MINOR, "Too many pending commands (%d) in "
2925                                 "session, returning BUSY to initiator \"%s\"",
2926                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
2927                                   "Anonymous" : cmd->sess->initiator_name);
2928                         failure = true;
2929                 }
2930
2931                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
2932                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
2933                         if (!failure) {
2934                                 TRACE(TRACE_MGMT_MINOR, "Too many pending device "
2935                                         "commands (%d), returning BUSY to "
2936                                         "initiator \"%s\"", cnt,
2937                                         (cmd->sess->initiator_name[0] == '\0') ?
2938                                           "Anonymous" : cmd->sess->initiator_name);
2939                                 failure = true;
2940                         }
2941                 }
2942
2943                 /* If expected values not set, expected direction is UNKNOWN */
2944                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
2945                         atomic_inc(&cmd->dev->write_cmd_count);
2946
2947                 if (unlikely(failure))
2948                         goto out_busy;
2949
2950                 if (!cmd->set_sn_on_restart_cmd)
2951                         scst_cmd_set_sn(cmd);
2952         } else if (res < 0) {
2953                 TRACE_DBG("Finishing cmd %p", cmd);
2954                 scst_set_cmd_error(cmd,
2955                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
2956                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2957         } else
2958                 goto out;
2959
2960 out:
2961         TRACE_EXIT_RES(res);
2962         return res;
2963
2964 out_busy:
2965         scst_set_busy(cmd);
2966         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2967         goto out;
2968 }
2969
2970 /* Called under scst_init_lock and IRQs disabled */
2971 static void scst_do_job_init(void)
2972 {
2973         struct scst_cmd *cmd;
2974         int susp;
2975
2976         TRACE_ENTRY();
2977
2978 restart:
2979         /*
2980          * There is no need for read barrier here, because we don't care where
2981          * this check will be done.
2982          */
2983         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
2984         if (scst_init_poll_cnt > 0)
2985                 scst_init_poll_cnt--;
2986
2987         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
2988                 int rc;
2989                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
2990                         continue;
2991                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2992                         spin_unlock_irq(&scst_init_lock);
2993                         rc = __scst_init_cmd(cmd);
2994                         spin_lock_irq(&scst_init_lock);
2995                         if (rc > 0) {
2996                                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, restarting");
2997                                 goto restart;
2998                         }
2999                 } else {
3000                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
3001                                        cmd, (long long unsigned int)cmd->tag);
3002                         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
3003                 }
3004
3005                 /*
3006                  * Deleting cmd from init cmd list after __scst_init_cmd()
3007                  * is necessary to keep the check in scst_init_cmd() correct
3008                  * to preserve the commands order.
3009                  *
3010                  * We don't care about the race, when init cmd list is empty
3011                  * and one command detected that it just was not empty, so
3012                  * it's inserting to it, but another command at the same time
3013                  * seeing init cmd list empty and goes directly, because it
3014                  * could affect only commands from the same initiator to the
3015                  * same tgt_dev, but init_cmd_done() doesn't guarantee the order
3016                  * in case of simultaneous such calls anyway.
3017                  */
3018                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
3019                 smp_wmb();
3020                 list_del(&cmd->cmd_list_entry);
3021                 spin_unlock(&scst_init_lock);
3022
3023                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
3024                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
3025                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
3026                         list_add(&cmd->cmd_list_entry,
3027                                 &cmd->cmd_lists->active_cmd_list);
3028                 else
3029                         list_add_tail(&cmd->cmd_list_entry,
3030                                 &cmd->cmd_lists->active_cmd_list);
3031                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3032                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
3033
3034                 spin_lock(&scst_init_lock);
3035                 goto restart;
3036         }
3037
3038         /* It isn't really needed, but let's keep it */
3039         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
3040                 goto restart;
3041
3042         TRACE_EXIT();
3043         return;
3044 }
3045
3046 static inline int test_init_cmd_list(void)
3047 {
3048         int res = (!list_empty(&scst_init_cmd_list) &&
3049                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
3050                   unlikely(kthread_should_stop()) ||
3051                   (scst_init_poll_cnt > 0);
3052         return res;
3053 }
3054
3055 int scst_init_cmd_thread(void *arg)
3056 {
3057         TRACE_ENTRY();
3058
3059         PRINT_INFO("Init thread started, PID %d", current->pid);
3060
3061         current->flags |= PF_NOFREEZE;
3062
3063         set_user_nice(current, -10);
3064
3065         spin_lock_irq(&scst_init_lock);
3066         while (!kthread_should_stop()) {
3067                 wait_queue_t wait;
3068                 init_waitqueue_entry(&wait, current);
3069
3070                 if (!test_init_cmd_list()) {
3071                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
3072                                                  &wait);
3073                         for (;;) {
3074                                 set_current_state(TASK_INTERRUPTIBLE);
3075                                 if (test_init_cmd_list())
3076                                         break;
3077                                 spin_unlock_irq(&scst_init_lock);
3078                                 schedule();
3079                                 spin_lock_irq(&scst_init_lock);
3080                         }
3081                         set_current_state(TASK_RUNNING);
3082                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
3083                 }
3084                 scst_do_job_init();
3085         }
3086         spin_unlock_irq(&scst_init_lock);
3087
3088         /*
3089          * If kthread_should_stop() is true, we are guaranteed to be
3090          * on the module unload, so scst_init_cmd_list must be empty.
3091          */
3092         sBUG_ON(!list_empty(&scst_init_cmd_list));
3093
3094         PRINT_INFO("Init thread PID %d finished", current->pid);
3095
3096         TRACE_EXIT();
3097         return 0;
3098 }
3099
3100 /* Called with no locks held */
3101 void scst_process_active_cmd(struct scst_cmd *cmd, int context)
3102 {
3103         int res;
3104
3105         TRACE_ENTRY();
3106
3107         EXTRACHECKS_BUG_ON(in_irq());
3108
3109         cmd->context_processable = context | SCST_CONTEXT_PROCESSABLE;
3110         context &= ~SCST_CONTEXT_PROCESSABLE;
3111         cmd->atomic = (context == SCST_CONTEXT_DIRECT_ATOMIC);
3112
3113         TRACE_DBG("cmd %p, context_processable %d, atomic %d", cmd,
3114                 cmd->context_processable, cmd->atomic);
3115
3116         do {
3117                 switch (cmd->state) {
3118                 case SCST_CMD_STATE_PRE_PARSE:
3119                         res = scst_pre_parse(cmd);
3120                         EXTRACHECKS_BUG_ON(res ==
3121                                 SCST_CMD_STATE_RES_NEED_THREAD);
3122                         break;
3123
3124                 case SCST_CMD_STATE_DEV_PARSE:
3125                         res = scst_parse_cmd(cmd);
3126                         break;
3127
3128                 case SCST_CMD_STATE_PREPARE_SPACE:
3129                         res = scst_prepare_space(cmd);
3130                         break;
3131
3132                 case SCST_CMD_STATE_RDY_TO_XFER:
3133                         res = scst_rdy_to_xfer(cmd);
3134                         break;
3135
3136                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3137                         res = scst_tgt_pre_exec(cmd);
3138                         break;
3139
3140                 case SCST_CMD_STATE_SEND_TO_MIDLEV:
3141                         if (tm_dbg_check_cmd(cmd) != 0) {
3142                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
3143                                 TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
3144                                         "because of TM DBG delay", cmd,
3145                                         (long long unsigned int)cmd->tag);
3146                                 break;
3147                         }
3148                         res = scst_send_to_midlev(&cmd);
3149                         /* !! At this point cmd, sess & tgt_dev can be already freed !! */
3150                         break;
3151
3152                 case SCST_CMD_STATE_PRE_DEV_DONE:
3153                         res = scst_pre_dev_done(cmd);
3154                         EXTRACHECKS_BUG_ON(res ==
3155                                 SCST_CMD_STATE_RES_NEED_THREAD);
3156                         break;
3157
3158                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3159                         res = scst_mode_select_checks(cmd);
3160                         break;
3161
3162                 case SCST_CMD_STATE_DEV_DONE:
3163                         res = scst_dev_done(cmd);
3164                         break;
3165
3166                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3167                         res = scst_pre_xmit_response(cmd);
3168                         EXTRACHECKS_BUG_ON(res ==
3169                                 SCST_CMD_STATE_RES_NEED_THREAD);
3170                         break;
3171
3172                 case SCST_CMD_STATE_XMIT_RESP:
3173                         res = scst_xmit_response(cmd);
3174                         break;
3175
3176                 case SCST_CMD_STATE_FINISHED:
3177                         res = scst_finish_cmd(cmd);
3178                         EXTRACHECKS_BUG_ON(res ==
3179                                 SCST_CMD_STATE_RES_NEED_THREAD);
3180                         break;
3181
3182                 default:
3183                         PRINT_CRIT_ERROR("cmd (%p) in state %d, but shouldn't "
3184                                 "be", cmd, cmd->state);
3185                         sBUG();
3186                         res = SCST_CMD_STATE_RES_CONT_NEXT;
3187                         break;
3188                 }
3189         } while (res == SCST_CMD_STATE_RES_CONT_SAME);
3190
3191         if (res == SCST_CMD_STATE_RES_CONT_NEXT) {
3192                 /* None */
3193         } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
3194                 spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3195                 switch (cmd->state) {
3196                 case SCST_CMD_STATE_PRE_PARSE:
3197                 case SCST_CMD_STATE_DEV_PARSE:
3198                 case SCST_CMD_STATE_PREPARE_SPACE:
3199                 case SCST_CMD_STATE_RDY_TO_XFER:
3200                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3201                 case SCST_CMD_STATE_SEND_TO_MIDLEV:
3202                 case SCST_CMD_STATE_PRE_DEV_DONE:
3203                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3204                 case SCST_CMD_STATE_DEV_DONE:
3205                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3206                 case SCST_CMD_STATE_XMIT_RESP:
3207                 case SCST_CMD_STATE_FINISHED:
3208                         TRACE_DBG("Adding cmd %p to head of active cmd list", cmd);
3209                         list_add(&cmd->cmd_list_entry,
3210                                 &cmd->cmd_lists->active_cmd_list);
3211                         break;
3212 #ifdef EXTRACHECKS
3213                 /* not very valid commands */
3214                 case SCST_CMD_STATE_DEFAULT:
3215                 case SCST_CMD_STATE_NEED_THREAD_CTX:
3216                         PRINT_CRIT_ERROR("cmd %p is in state %d, not putting on "
3217                                 "useful list (left on scst cmd list)", cmd,
3218                                 cmd->state);
3219                         spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3220                         sBUG();
3221                         spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3222                         break;
3223 #endif
3224                 default:
3225                         break;
3226                 }
3227                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3228                 spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3229         } else
3230                 sBUG();
3231
3232         TRACE_EXIT();
3233         return;
3234 }
3235 EXPORT_SYMBOL(scst_process_active_cmd);
3236
3237 /* Called under cmd_list_lock and IRQs disabled */
3238 static void scst_do_job_active(struct list_head *cmd_list,
3239         spinlock_t *cmd_list_lock, int context)
3240 {
3241         TRACE_ENTRY();
3242
3243 #ifdef EXTRACHECKS
3244         {
3245                 int c = context & ~SCST_CONTEXT_PROCESSABLE;
3246                 sBUG_ON((c != SCST_CONTEXT_DIRECT_ATOMIC) &&
3247                         (c != SCST_CONTEXT_DIRECT));
3248         }
3249 #endif
3250
3251         while (!list_empty(cmd_list)) {
3252                 struct scst_cmd *cmd = list_entry(cmd_list->next, typeof(*cmd),
3253                                         cmd_list_entry);
3254                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
3255                 list_del(&cmd->cmd_list_entry);
3256                 spin_unlock_irq(cmd_list_lock);
3257                 scst_process_active_cmd(cmd, context);
3258                 spin_lock_irq(cmd_list_lock);
3259         }
3260
3261         TRACE_EXIT();
3262         return;
3263 }
3264
3265 static inline int test_cmd_lists(struct scst_cmd_lists *p_cmd_lists)
3266 {
3267         int res = !list_empty(&p_cmd_lists->active_cmd_list) ||
3268             unlikely(kthread_should_stop()) ||
3269             tm_dbg_is_release();
3270         return res;
3271 }
3272
3273 int scst_cmd_thread(void *arg)
3274 {
3275         struct scst_cmd_lists *p_cmd_lists = (struct scst_cmd_lists *)arg;
3276
3277         TRACE_ENTRY();
3278
3279         PRINT_INFO("Processing thread started, PID %d", current->pid);
3280
3281 #if 0
3282         set_user_nice(current, 10);
3283 #endif
3284         current->flags |= PF_NOFREEZE;
3285
3286         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3287         while (!kthread_should_stop()) {
3288                 wait_queue_t wait;
3289                 init_waitqueue_entry(&wait, current);
3290
3291                 if (!test_cmd_lists(p_cmd_lists)) {
3292                         add_wait_queue_exclusive(&p_cmd_lists->cmd_list_waitQ,
3293                                 &wait);
3294                         for (;;) {
3295                                 set_current_state(TASK_INTERRUPTIBLE);
3296                                 if (test_cmd_lists(p_cmd_lists))
3297                                         break;
3298                                 spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3299                                 schedule();
3300                                 spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3301                         }
3302                         set_current_state(TASK_RUNNING);
3303                         remove_wait_queue(&p_cmd_lists->cmd_list_waitQ, &wait);
3304                 }
3305
3306                 if (tm_dbg_is_release()) {
3307                         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3308                         tm_dbg_check_released_cmds();
3309                         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3310                 }
3311
3312                 scst_do_job_active(&p_cmd_lists->active_cmd_list,
3313                         &p_cmd_lists->cmd_list_lock, SCST_CONTEXT_DIRECT |
3314                                                 SCST_CONTEXT_PROCESSABLE);
3315         }
3316         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3317
3318 #ifdef EXTRACHECKS
3319         /*
3320          * If kthread_should_stop() is true, we are guaranteed to be either
3321          * on the module unload, or there must be at least one other thread to
3322          * process the commands lists.
3323          */
3324         if (p_cmd_lists == &scst_main_cmd_lists) {
3325                 sBUG_ON((scst_threads_info.nr_cmd_threads == 1) &&
3326                          !list_empty(&scst_main_cmd_lists.active_cmd_list));
3327         }
3328 #endif
3329
3330         PRINT_INFO("Processing thread PID %d finished", current->pid);
3331
3332         TRACE_EXIT();
3333         return 0;
3334 }
3335
3336 void scst_cmd_tasklet(long p)
3337 {
3338         struct scst_tasklet *t = (struct scst_tasklet *)p;
3339
3340         TRACE_ENTRY();
3341
3342         spin_lock_irq(&t->tasklet_lock);
3343         scst_do_job_active(&t->tasklet_cmd_list, &t->tasklet_lock,
3344                 SCST_CONTEXT_DIRECT_ATOMIC);
3345         spin_unlock_irq(&t->tasklet_lock);
3346
3347         TRACE_EXIT();
3348         return;
3349 }
3350
3351 /*
3352  * Returns 0 on success, < 0 if there is no device handler or
3353  * > 0 if SCST_FLAG_SUSPENDED set and SCST_FLAG_SUSPENDING - not.
3354  * No locks, protection is done by the suspended activity.
3355  */
3356 static int scst_mgmt_translate_lun(struct scst_mgmt_cmd *mcmd)
3357 {
3358         struct scst_tgt_dev *tgt_dev = NULL;
3359         struct list_head *sess_tgt_dev_list_head;
3360         int res = -1;
3361
3362         TRACE_ENTRY();
3363
3364         TRACE_DBG("Finding tgt_dev for mgmt cmd %p (lun %Ld)", mcmd,
3365               (long long unsigned int)mcmd->lun);
3366
3367         __scst_get(1);
3368
3369         if (unlikely(test_bit(SCST_FLAG_SUSPENDED, &scst_flags) &&
3370                      !test_bit(SCST_FLAG_SUSPENDING, &scst_flags))) {
3371                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3372                 __scst_put();
3373                 res = 1;
3374                 goto out;
3375         }
3376
3377         sess_tgt_dev_list_head =
3378                 &mcmd->sess->sess_tgt_dev_list_hash[HASH_VAL(mcmd->lun)];
3379         list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3380                         sess_tgt_dev_list_entry) {
3381                 if (tgt_dev->lun == mcmd->lun) {
3382                         TRACE_DBG("tgt_dev %p found", tgt_dev);
3383                         mcmd->mcmd_tgt_dev = tgt_dev;
3384                         res = 0;
3385                         break;
3386                 }
3387         }
3388         if (mcmd->mcmd_tgt_dev == NULL)
3389                 __scst_put();
3390
3391 out:
3392         TRACE_EXIT_HRES(res);
3393         return res;
3394 }
3395
3396 /* No locks */
3397 void scst_done_cmd_mgmt(struct scst_cmd *cmd)
3398 {
3399         struct scst_mgmt_cmd_stub *mstb;
3400         bool wake = 0;
3401         unsigned long flags;
3402
3403         TRACE_ENTRY();
3404
3405         TRACE_MGMT_DBG("cmd %p done (tag %llu)",
3406                        cmd, (long long unsigned int)cmd->tag);
3407
3408         spin_lock_irqsave(&scst_mcmd_lock, flags);
3409
3410         list_for_each_entry(mstb, &cmd->mgmt_cmd_list,
3411                         cmd_mgmt_cmd_list_entry) {
3412                 struct scst_mgmt_cmd *mcmd = mstb->mcmd;
3413
3414                 TRACE_MGMT_DBG("mcmd %p, mcmd->cmd_done_wait_count %d",
3415                         mcmd, mcmd->cmd_done_wait_count);
3416
3417                 mcmd->cmd_done_wait_count--;
3418                 if (mcmd->cmd_done_wait_count > 0) {
3419                         TRACE_MGMT_DBG("cmd_done_wait_count(%d) not 0, "
3420                                 "skipping", mcmd->cmd_done_wait_count);
3421                         continue;
3422                 }
3423
3424                 if (mcmd->completed) {
3425                         sBUG_ON(mcmd->nexus_loss_check_done);
3426                         mcmd->completed = 0;
3427                         mcmd->state = SCST_MGMT_CMD_STATE_CHECK_NEXUS_LOSS;
3428                         TRACE_MGMT_DBG("Adding mgmt cmd %p to active mgmt cmd "
3429                                 "list", mcmd);
3430                         list_add_tail(&mcmd->mgmt_cmd_list_entry,
3431                                 &scst_active_mgmt_cmd_list);
3432                         wake = 1;
3433                 }
3434         }
3435
3436         spin_unlock_irqrestore(&scst_mcmd_lock, flags);
3437
3438         if (wake)
3439                 wake_up(&scst_mgmt_cmd_list_waitQ);
3440
3441         TRACE_EXIT();
3442         return;
3443 }
3444
3445 /* Called under scst_mcmd_lock and IRQs disabled */
3446 static int __scst_dec_finish_wait_count(struct scst_mgmt_cmd *mcmd, bool *wake)
3447 {
3448         TRACE_ENTRY();
3449
3450         mcmd->cmd_finish_wait_count--;
3451         if (mcmd->cmd_finish_wait_count > 0) {
3452                 TRACE_MGMT_DBG("cmd_finish_wait_count(%d) not 0, "
3453                         "skipping", mcmd->cmd_finish_wait_count);
3454                 goto out;
3455         }
3456
3457         if (mcmd->completed) {
3458                 mcmd->state = SCST_MGMT_CMD_STATE_DONE;
3459                 TRACE_MGMT_DBG("Adding mgmt cmd %p to active mgmt cmd "
3460                         "list", mcmd);
3461                 list_add_tail(&mcmd->mgmt_cmd_list_entry,
3462                         &scst_active_mgmt_cmd_list);
3463                 *wake = true;
3464         }
3465
3466 out:
3467         TRACE_EXIT_RES(mcmd->cmd_finish_wait_count);
3468         return mcmd->cmd_finish_wait_count;
3469 }
3470
3471 /* No locks */
3472 void scst_prepare_async_mcmd(struct scst_mgmt_cmd *mcmd)
3473 {
3474         unsigned long flags;
3475
3476         TRACE_ENTRY();
3477
3478         TRACE_MGMT_DBG("Preparing mcmd %p for async execution", mcmd);
3479
3480         spin_lock_irqsave(&scst_mcmd_lock, flags);
3481          mcmd->cmd_finish_wait_count++;
3482         spin_unlock_irqrestore(&scst_mcmd_lock, flags);
3483
3484         TRACE_EXIT();
3485         return;
3486 }
3487 EXPORT_SYMBOL(scst_prepare_async_mcmd);
3488
3489 /* No locks */
3490 void scst_async_mcmd_completed(struct scst_mgmt_cmd *mcmd, int status)
3491 {
3492         unsigned long flags;
3493         bool wake = false;
3494
3495         TRACE_ENTRY();
3496
3497         TRACE_MGMT_DBG("Async mcmd %p completed (status %d)", mcmd, status);
3498
3499         spin_lock_irqsave(&scst_mcmd_lock, flags);
3500
3501         if (status != SCST_MGMT_STATUS_SUCCESS)
3502                 mcmd->status = status;
3503
3504         __scst_dec_finish_wait_count(mcmd, &wake);
3505
3506         spin_unlock_irqrestore(&scst_mcmd_lock, flags);
3507
3508         if (wake)
3509                 wake_up(&scst_mgmt_cmd_list_waitQ);
3510
3511         TRACE_EXIT();
3512         return;
3513 }
3514 EXPORT_SYMBOL(scst_async_mcmd_completed);
3515
3516 /* No locks */
3517 static void scst_finish_cmd_mgmt(struct scst_cmd *cmd)
3518 {
3519         struct scst_mgmt_cmd_stub *mstb, *t;
3520         bool wake = false;
3521         unsigned long flags;
3522
3523         TRACE_ENTRY();
3524
3525         TRACE_MGMT_DBG("cmd %p finished (tag %llu)",
3526                        cmd, (long long unsigned int)cmd->tag);
3527
3528         spin_lock_irqsave(&scst_mcmd_lock, flags);
3529
3530         list_for_each_entry_safe(mstb, t, &cmd->mgmt_cmd_list,
3531                         cmd_mgmt_cmd_list_entry) {