1c5a9f5d09c76f6f2c7352da59baf82d05c7b137
[mirror/scst/.git] / scst / src / scst_targ.c
1 /*
2  *  scst_targ.c
3  *  
4  *  Copyright (C) 2004-2007 Vladislav Bolkhovitin <vst@vlnb.net>
5  *                 and Leonid Stoljar
6  *  
7  *  This program is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU General Public License
9  *  as published by the Free Software Foundation, version 2
10  *  of the License.
11  * 
12  *  This program is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  *  GNU General Public License for more details.
16  */
17
18 #include <linux/init.h>
19 #include <linux/kernel.h>
20 #include <linux/errno.h>
21 #include <linux/list.h>
22 #include <linux/spinlock.h>
23 #include <linux/slab.h>
24 #include <linux/sched.h>
25 #include <linux/smp_lock.h>
26 #include <asm/unistd.h>
27 #include <asm/string.h>
28 #include <linux/kthread.h>
29 #include <linux/delay.h>
30
31 #include "scsi_tgt.h"
32 #include "scst_priv.h"
33
34 static void scst_cmd_set_sn(struct scst_cmd *cmd);
35 static int __scst_init_cmd(struct scst_cmd *cmd);
36
37 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
38 {
39         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
40         unsigned long flags;
41
42         spin_lock_irqsave(&t->tasklet_lock, flags);
43         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
44                 smp_processor_id());
45         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
46         spin_unlock_irqrestore(&t->tasklet_lock, flags);
47
48         tasklet_schedule(&t->tasklet);
49 }
50
51 /* 
52  * Must not be called in parallel with scst_unregister_session() for the 
53  * same sess
54  */
55 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
56                              const uint8_t *lun, int lun_len,
57                              const uint8_t *cdb, int cdb_len, int atomic)
58 {
59         struct scst_cmd *cmd;
60
61         TRACE_ENTRY();
62
63 #ifdef EXTRACHECKS
64         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
65                 PRINT_ERROR("%s", "New cmd while shutting down the session");
66                 sBUG();
67         }
68 #endif
69
70         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
71         if (cmd == NULL)
72                 goto out;
73
74         cmd->sess = sess;
75         cmd->tgt = sess->tgt;
76         cmd->tgtt = sess->tgt->tgtt;
77
78         /* 
79          * For both wrong lun and CDB defer the error reporting for
80          * scst_cmd_init_done()
81          */
82
83         cmd->lun = scst_unpack_lun(lun, lun_len);
84
85         if (cdb_len <= SCST_MAX_CDB_SIZE) {
86                 memcpy(cmd->cdb, cdb, cdb_len);
87                 cmd->cdb_len = cdb_len;
88         }
89
90         TRACE_DBG("cmd %p, sess %p", cmd, sess);
91         scst_sess_get(sess);
92
93 out:
94         TRACE_EXIT();
95         return cmd;
96 }
97
98 static int scst_init_cmd(struct scst_cmd *cmd, int context)
99 {
100         int rc;
101
102         TRACE_ENTRY();
103
104         /* See the comment in scst_do_job_init() */
105         if (unlikely(!list_empty(&scst_init_cmd_list))) {
106                 TRACE_MGMT_DBG("%s", "init cmd list busy");
107                 goto out_redirect;
108         }
109         /*
110          * Memory barrier isn't necessary here, because CPU appears to
111          * be self-consistent
112          */
113
114         rc = __scst_init_cmd(cmd);
115         if (unlikely(rc > 0))
116                 goto out_redirect;
117         else if (unlikely(rc != 0))
118                 goto out;
119
120         /* Small context optimization */
121         if (((context == SCST_CONTEXT_TASKLET) ||
122              (context == SCST_CONTEXT_DIRECT_ATOMIC)) && 
123             scst_cmd_is_expected_set(cmd)) {
124                 if (cmd->expected_data_direction == SCST_DATA_WRITE) {
125                         if ( !test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
126                                         &cmd->tgt_dev->tgt_dev_flags))
127                                 context = SCST_CONTEXT_THREAD;
128                 } else {
129                         if ( !test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
130                                         &cmd->tgt_dev->tgt_dev_flags))
131                                 context = SCST_CONTEXT_THREAD;
132                 }
133         }
134
135 out:
136         TRACE_EXIT_RES(context);
137         return context;
138
139 out_redirect:
140         if (cmd->preprocessing_only) {
141                 /*
142                  * Poor man solution for single threaded targets, where 
143                  * blocking receiver at least sometimes means blocking all.
144                  */
145                 sBUG_ON(context != SCST_CONTEXT_DIRECT);
146                 scst_set_busy(cmd);
147                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
148                 /* Keep initiator away from too many BUSY commands */
149                 if (!in_interrupt() && !in_atomic())
150                         msleep(50);
151                 else
152                         WARN_ON_ONCE(1);
153         } else {
154                 unsigned long flags;
155                 spin_lock_irqsave(&scst_init_lock, flags);
156                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
157                         "%d)", cmd, atomic_read(&scst_cmd_count));
158                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
159                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
160                         scst_init_poll_cnt++;
161                 spin_unlock_irqrestore(&scst_init_lock, flags);
162                 wake_up(&scst_init_cmd_list_waitQ);
163                 context = -1;
164         }
165         goto out;
166 }
167
168 #ifdef MEASURE_LATENCY
169 static inline uint64_t scst_sec_to_nsec(time_t sec)
170 {
171         return (uint64_t)sec * 1000000000;
172 }
173 #endif
174
175 void scst_cmd_init_done(struct scst_cmd *cmd, int pref_context)
176 {
177         unsigned long flags;
178         struct scst_session *sess = cmd->sess;
179
180         TRACE_ENTRY();
181
182 #ifdef MEASURE_LATENCY
183         {
184                 struct timespec ts;
185                 getnstimeofday(&ts);
186                 cmd->start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
187                 TRACE_DBG("cmd %p (sess %p): start %Ld (tv_sec %ld, "
188                         "tv_nsec %ld)", cmd, sess, cmd->start, ts.tv_sec,
189                         ts.tv_nsec);
190         }
191 #endif
192
193         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
194         TRACE(TRACE_SCSI, "tag=%llu, lun=%Ld, CDB len=%d", cmd->tag, 
195                 (uint64_t)cmd->lun, cmd->cdb_len);
196         PRINT_BUFF_FLAG(TRACE_SCSI|TRACE_RCV_BOT, "Recieving CDB",
197                 cmd->cdb, cmd->cdb_len);
198
199 #ifdef EXTRACHECKS
200         if (unlikely(in_irq()) && ((pref_context == SCST_CONTEXT_DIRECT) ||
201                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
202         {
203                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
204                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
205                         cmd->tgtt->name);
206                 pref_context = SCST_CONTEXT_TASKLET;
207         }
208 #endif
209
210         atomic_inc(&sess->sess_cmd_count);
211
212         spin_lock_irqsave(&sess->sess_list_lock, flags);
213
214         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
215                 /*
216                  * We have to always keep command in the search list from the
217                  * very beginning, because otherwise it can be missed during
218                  * TM processing. This check is needed because there might be
219                  * old, i.e. deferred, commands and new, i.e. just coming, ones.
220                  */
221                 if (cmd->search_cmd_list_entry.next == NULL)
222                         list_add_tail(&cmd->search_cmd_list_entry,
223                                 &sess->search_cmd_list);
224                 switch(sess->init_phase) {
225                 case SCST_SESS_IPH_SUCCESS:
226                         break;
227                 case SCST_SESS_IPH_INITING:
228                         TRACE_DBG("Adding cmd %p to init deferred cmd list", cmd);
229                         list_add_tail(&cmd->cmd_list_entry, 
230                                 &sess->init_deferred_cmd_list);
231                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
232                         goto out;
233                 case SCST_SESS_IPH_FAILED:
234                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
235                         scst_set_busy(cmd);
236                         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
237                         goto active;
238                 default:
239                         sBUG();
240                 }
241         } else
242                 list_add_tail(&cmd->search_cmd_list_entry, &sess->search_cmd_list);
243
244         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
245
246         if (unlikely(cmd->lun == (lun_t)-1)) {
247                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
248                 scst_set_cmd_error(cmd,
249                         SCST_LOAD_SENSE(scst_sense_lun_not_supported));
250                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
251                 goto active;
252         }
253
254         if (unlikely(cmd->cdb_len == 0)) {
255                 PRINT_ERROR("Wrong CDB len %d, finishing cmd", 0);
256                 scst_set_cmd_error(cmd,
257                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
258                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
259                 goto active;
260         }
261
262         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
263                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
264                 scst_set_cmd_error(cmd,
265                         SCST_LOAD_SENSE(scst_sense_invalid_message));
266                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
267                 goto active;
268         }
269
270         cmd->state = SCST_CMD_STATE_INIT;
271         /* cmd must be inited here to keep the order */
272         pref_context = scst_init_cmd(cmd, pref_context);
273         if (unlikely(pref_context < 0))
274                 goto out;
275
276 active:
277         /* Here cmd must not be in any cmd list, no locks */
278         switch (pref_context) {
279         case SCST_CONTEXT_TASKLET:
280                 scst_schedule_tasklet(cmd);
281                 break;
282
283         case SCST_CONTEXT_DIRECT:
284         case SCST_CONTEXT_DIRECT_ATOMIC:
285                 scst_process_active_cmd(cmd, pref_context);
286                 /* For *NEED_THREAD wake_up() is already done */
287                 break;
288
289         default:
290                 PRINT_ERROR("Context %x is undefined, using the thread one",
291                         pref_context);
292                 /* go through */
293         case SCST_CONTEXT_THREAD:
294                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
295                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
296                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
297                         list_add(&cmd->cmd_list_entry,
298                                 &cmd->cmd_lists->active_cmd_list);
299                 else
300                         list_add_tail(&cmd->cmd_list_entry,
301                                 &cmd->cmd_lists->active_cmd_list);
302                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
303                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
304                 break;
305         }
306
307 out:
308         TRACE_EXIT();
309         return;
310 }
311
312 static int scst_pre_parse(struct scst_cmd *cmd)
313 {
314         int res = SCST_CMD_STATE_RES_CONT_SAME;
315         struct scst_device *dev = cmd->dev;
316
317         TRACE_ENTRY();
318
319         cmd->inc_expected_sn_on_done = !dev->has_own_order_mgmt &&
320                 (dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER);
321
322         sBUG_ON(cmd->internal);
323
324         /*
325          * Expected transfer data supplied by the SCSI transport via the
326          * target driver are untrusted, so we prefer to fetch them from CDB.
327          * Additionally, not all transports support supplying the expected
328          * transfer data.
329          */
330
331         if (unlikely(scst_get_cdb_info(cmd->cdb, dev->handler->type, 
332                         &cmd->op_flags, &cmd->data_direction,
333                         &cmd->bufflen, &cmd->cdb_len, &cmd->op_name) != 0)) {
334                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
335                         "Should you update scst_scsi_op_table?",
336                         cmd->cdb[0], dev->handler->name);
337 #ifdef USE_EXPECTED_VALUES
338                 if (scst_cmd_is_expected_set(cmd)) {
339                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
340                                 "direction %d, transfer_len %d",
341                                 cmd->expected_data_direction,
342                                 cmd->expected_transfer_len);
343                         cmd->data_direction = cmd->expected_data_direction;
344                         
345                         cmd->bufflen = cmd->expected_transfer_len;
346                         /* Restore (likely) lost CDB length */
347                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
348                         if (cmd->cdb_len == -1) {
349                                 PRINT_ERROR("Unable to get CDB length for "
350                                         "opcode 0x%02x. Returning INVALID "
351                                         "OPCODE", cmd->cdb[0]);
352                                 scst_set_cmd_error(cmd,
353                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
354                                 goto out_xmit;
355                         }
356                 } else {
357                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
358                              "target %s not supplied expected values",
359                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
360                         scst_set_cmd_error(cmd,
361                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
362                         goto out_xmit;
363                 }
364 #else
365                 scst_set_cmd_error(cmd,
366                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
367                 goto out_xmit;
368 #endif
369         } else {
370                 TRACE(TRACE_SCSI, "op_name <%s>, direction=%d (expected %d, "
371                         "set %s), transfer_len=%d (expected len %d), flags=%d",
372                         cmd->op_name, cmd->data_direction,
373                         cmd->expected_data_direction,
374                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
375                         cmd->bufflen, cmd->expected_transfer_len, cmd->op_flags);
376
377                 if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
378                         if (scst_cmd_is_expected_set(cmd)) {
379                                 /*
380                                  * Command data length can't be easily
381                                  * determined from the CDB. ToDo, that should 
382                                  * be fixed. Until it's fixed, get it from
383                                  * the supplied expected value, but
384                                  * limit it to some reasonable value (15MB).
385                                  */
386                                 cmd->bufflen = min(cmd->expected_transfer_len,
387                                                         15*1024*1024);
388                         } else
389                                 cmd->bufflen = 0;
390                 }
391         }
392
393         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
394                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
395                             "(opcode 0x%02x)", cmd->cdb[0]);
396                 scst_set_cmd_error(cmd,
397                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
398                 goto out_xmit;
399         }
400
401         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
402                 PRINT_ERROR("Linked commands are not supported "
403                             "(opcode 0x%02x)", cmd->cdb[0]);
404                 scst_set_cmd_error(cmd,
405                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
406                 goto out_xmit;
407         }
408
409         cmd->state = SCST_CMD_STATE_DEV_PARSE;
410
411 out:
412         TRACE_EXIT_RES(res);
413         return res;
414
415 out_xmit:
416         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
417         res = SCST_CMD_STATE_RES_CONT_SAME;
418         goto out;
419 }
420
421 static int scst_parse_cmd(struct scst_cmd *cmd)
422 {
423         int res = SCST_CMD_STATE_RES_CONT_SAME;
424         int state;
425         struct scst_device *dev = cmd->dev;
426         int atomic = scst_cmd_atomic(cmd);
427         int orig_bufflen = cmd->bufflen;
428         scst_data_direction orig_data_direction = cmd->data_direction;
429
430         TRACE_ENTRY();
431
432         if (atomic && !dev->handler->parse_atomic) {
433                 TRACE_DBG("Dev handler %s parse() can not be "
434                       "called in atomic context, rescheduling to the thread",
435                       dev->handler->name);
436                 res = SCST_CMD_STATE_RES_NEED_THREAD;
437                 goto out;
438         }
439
440         if (likely(!scst_is_cmd_local(cmd))) {
441                 TRACE_DBG("Calling dev handler %s parse(%p)",
442                       dev->handler->name, cmd);
443                 TRACE_BUFF_FLAG(TRACE_SND_BOT, "Parsing: ", cmd->cdb, cmd->cdb_len);
444                 state = dev->handler->parse(cmd);
445                 /* Caution: cmd can be already dead here */
446                 TRACE_DBG("Dev handler %s parse() returned %d",
447                         dev->handler->name, state);
448
449                 switch (state) {
450                 case SCST_CMD_STATE_NEED_THREAD_CTX:
451                         TRACE_DBG("Dev handler %s parse() requested thread "
452                               "context, rescheduling", dev->handler->name);
453                         res = SCST_CMD_STATE_RES_NEED_THREAD;
454                         goto out;
455
456                 case SCST_CMD_STATE_STOP:
457                         TRACE_DBG("Dev handler %s parse() requested stop "
458                                 "processing", dev->handler->name);
459                         res = SCST_CMD_STATE_RES_CONT_NEXT;
460                         goto out;
461                 }
462
463                 if (state == SCST_CMD_STATE_DEFAULT)
464                         state = SCST_CMD_STATE_PREPARE_SPACE;
465         } else
466                 state = SCST_CMD_STATE_PREPARE_SPACE;
467
468         if (cmd->data_len == -1)
469                 cmd->data_len = cmd->bufflen;
470
471         if (cmd->data_buf_alloced && unlikely((orig_bufflen > cmd->bufflen))) {
472                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
473                         "is less, than required (size %d)", cmd->bufflen,
474                         orig_bufflen);
475                 goto out_error;
476         }
477
478         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
479                 goto set_res;
480
481         if (unlikely((cmd->bufflen == 0) &&
482                      (cmd->op_flags & SCST_UNKNOWN_LENGTH))) {
483                 PRINT_ERROR("Unknown data transfer length for opcode 0x%x "
484                         "(handler %s, target %s)", cmd->cdb[0],
485                         dev->handler->name, cmd->tgtt->name);
486                 goto out_error;
487         }
488
489 #ifdef EXTRACHECKS
490         if ((cmd->bufflen != 0) &&
491             ((cmd->data_direction == SCST_DATA_NONE) ||
492              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
493                 PRINT_ERROR("Dev handler %s parse() returned "
494                         "invalid cmd data_direction %d, bufflen %d, state %d "
495                         "or sg %p (opcode 0x%x)", dev->handler->name, 
496                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
497                         cmd->cdb[0]);
498                 goto out_error;
499         }
500 #endif
501
502         if (scst_cmd_is_expected_set(cmd)) {
503 #ifdef USE_EXPECTED_VALUES
504 #       ifdef EXTRACHECKS
505                 if ((cmd->data_direction != cmd->expected_data_direction) ||
506                     (cmd->bufflen != cmd->expected_transfer_len)) {
507                         PRINT_ERROR("Expected values don't match decoded ones: "
508                                 "data_direction %d, expected_data_direction %d, "
509                                 "bufflen %d, expected_transfer_len %d",
510                                 cmd->data_direction, cmd->expected_data_direction,
511                                 cmd->bufflen, cmd->expected_transfer_len);
512                 }
513 #       endif
514                 cmd->data_direction = cmd->expected_data_direction;
515                 cmd->bufflen = cmd->expected_transfer_len;
516 #else
517                 if (unlikely(cmd->data_direction != orig_data_direction)) {
518                         PRINT_ERROR("Expected data direction %d for opcode "
519                                 "0x%02x (handler %s, target %s) doesn't match "
520                                 "decoded value %d", cmd->data_direction,
521                                 cmd->cdb[0], dev->handler->name,
522                                 cmd->tgtt->name, orig_data_direction);
523                         scst_set_cmd_error(cmd,
524                                 SCST_LOAD_SENSE(scst_sense_invalid_message));
525                         goto out_dev_done;
526                 }
527                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
528                         PRINT_INFO("Warning: expected transfer length %d for "
529                                 "opcode 0x%02x (handler %s, target %s) doesn't "
530                                 "match decoded value %d. Faulty initiator "
531                                 "(e.g. VMware is known to be such) or "
532                                 "scst_scsi_op_table should be updated?",
533                                 cmd->expected_transfer_len, cmd->cdb[0],
534                                 dev->handler->name, cmd->tgtt->name,
535                                 cmd->bufflen);
536                         PRINT_BUFFER("Suspicious CDB", cmd->cdb, cmd->cdb_len);
537                 }
538 #endif
539         }
540
541         if ((cmd->data_direction == SCST_DATA_UNKNOWN) ||
542             ((cmd->bufflen == 0) && (cmd->data_direction != SCST_DATA_NONE))) {
543                 PRINT_ERROR("Wrong data direction (%d) or/and buffer "
544                         "length (%d). Opcode 0x%x, handler %s, target %s",
545                         cmd->data_direction, cmd->bufflen, cmd->cdb[0],
546                         dev->handler->name, cmd->tgtt->name);
547                 goto out_error;
548         }
549
550 set_res:
551         switch (state) {
552         case SCST_CMD_STATE_PREPARE_SPACE:
553         case SCST_CMD_STATE_PRE_PARSE:
554         case SCST_CMD_STATE_DEV_PARSE:
555         case SCST_CMD_STATE_RDY_TO_XFER:
556         case SCST_CMD_STATE_TGT_PRE_EXEC:
557         case SCST_CMD_STATE_SEND_TO_MIDLEV:
558         case SCST_CMD_STATE_PRE_DEV_DONE:
559         case SCST_CMD_STATE_DEV_DONE:
560         case SCST_CMD_STATE_PRE_XMIT_RESP:
561         case SCST_CMD_STATE_XMIT_RESP:
562         case SCST_CMD_STATE_FINISHED:
563                 cmd->state = state;
564                 res = SCST_CMD_STATE_RES_CONT_SAME;
565                 break;
566
567         default:
568                 if (state >= 0) {
569                         PRINT_ERROR("Dev handler %s parse() returned "
570                              "invalid cmd state %d (opcode %d)", 
571                              dev->handler->name, state, cmd->cdb[0]);
572                 } else {
573                         PRINT_ERROR("Dev handler %s parse() returned "
574                                 "error %d (opcode %d)", dev->handler->name, 
575                                 state, cmd->cdb[0]);
576                 }
577                 goto out_error;
578         }
579
580         if (cmd->resp_data_len == -1) {
581                 if (cmd->data_direction == SCST_DATA_READ)
582                         cmd->resp_data_len = cmd->bufflen;
583                 else
584                          cmd->resp_data_len = 0;
585         }
586         
587 out:
588         TRACE_EXIT_HRES(res);
589         return res;
590
591 out_error:
592         /* dev_done() will be called as part of the regular cmd's finish */
593         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
594
595 #ifndef USE_EXPECTED_VALUES
596 out_dev_done:
597 #endif
598         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
599         res = SCST_CMD_STATE_RES_CONT_SAME;
600         goto out;
601 }
602
603 static int scst_prepare_space(struct scst_cmd *cmd)
604 {
605         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
606
607         TRACE_ENTRY();
608
609         if (cmd->data_direction == SCST_DATA_NONE)
610                 goto prep_done;
611
612         if (cmd->data_buf_tgt_alloc) {
613                 int orig_bufflen = cmd->bufflen;
614
615                 TRACE_MEM("%s", "Custom tgt data buf allocation requested");
616
617                 r = cmd->tgtt->alloc_data_buf(cmd);
618                 if (r > 0)
619                         goto alloc;
620                 else if (r == 0) {
621                         cmd->data_buf_alloced = 1;
622                         if (unlikely(orig_bufflen < cmd->bufflen)) {
623                                 PRINT_ERROR("Target driver allocated data "
624                                         "buffer (size %d), is less, than "
625                                         "required (size %d)", orig_bufflen,
626                                         cmd->bufflen);
627                                 goto out_error;
628                         }
629                 } else
630                         goto check;
631         }
632
633 alloc:
634         if (!cmd->data_buf_alloced) {
635                 r = scst_alloc_space(cmd);
636         } else {
637                 TRACE_MEM("%s", "data_buf_alloced set, returning");
638         }
639         
640 check:
641         if (r != 0) {
642                 if (scst_cmd_atomic(cmd)) {
643                         TRACE_MEM("%s", "Atomic memory allocation failed, "
644                               "rescheduling to the thread");
645                         res = SCST_CMD_STATE_RES_NEED_THREAD;
646                         goto out;
647                 } else
648                         goto out_no_space;
649         }
650
651 prep_done:
652         if (cmd->preprocessing_only) {
653                 if (scst_cmd_atomic(cmd) && 
654                     !cmd->tgtt->preprocessing_done_atomic) {
655                         TRACE_DBG("%s", "preprocessing_done() can not be "
656                               "called in atomic context, rescheduling to "
657                               "the thread");
658                         res = SCST_CMD_STATE_RES_NEED_THREAD;
659                         goto out;
660                 }
661
662                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
663                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
664                                 "cmd %p", cmd);
665                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
666                         res = SCST_CMD_STATE_RES_CONT_SAME;
667                         goto out;
668                 }
669
670                 res = SCST_CMD_STATE_RES_CONT_NEXT;
671                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
672
673                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
674                 cmd->tgtt->preprocessing_done(cmd);
675                 TRACE_DBG("%s", "preprocessing_done() returned");
676                 goto out;
677
678         }
679
680         switch (cmd->data_direction) {
681         case SCST_DATA_WRITE:
682                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
683                 break;
684
685         default:
686                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
687                 break;
688         }
689
690 out:
691         TRACE_EXIT_HRES(res);
692         return res;
693
694 out_no_space:
695         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
696                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
697         scst_set_busy(cmd);
698         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
699         res = SCST_CMD_STATE_RES_CONT_SAME;
700         goto out;
701
702 out_error:
703         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
704         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
705         res = SCST_CMD_STATE_RES_CONT_SAME;
706         goto out;
707 }
708
709 void scst_restart_cmd(struct scst_cmd *cmd, int status, int pref_context)
710 {
711         TRACE_ENTRY();
712
713         TRACE_DBG("Preferred context: %d", pref_context);
714         TRACE_DBG("tag=%llu, status=%#x", scst_cmd_get_tag(cmd), status);
715
716 #ifdef EXTRACHECKS
717         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
718                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
719         {
720                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
721                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
722                         cmd->tgtt->name);
723                 pref_context = SCST_CONTEXT_TASKLET;
724         }
725 #endif
726
727         switch (status) {
728         case SCST_PREPROCESS_STATUS_SUCCESS:
729                 switch (cmd->data_direction) {
730                 case SCST_DATA_WRITE:
731                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
732                         break;
733                 default:
734                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
735                         break;
736                 }
737                 if (cmd->set_sn_on_restart_cmd)
738                         scst_cmd_set_sn(cmd);
739                 /* Small context optimization */
740                 if ((pref_context == SCST_CONTEXT_TASKLET) || 
741                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) {
742                         if (cmd->data_direction == SCST_DATA_WRITE) {
743                                 if ( !test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
744                                                 &cmd->tgt_dev->tgt_dev_flags))
745                                         pref_context = SCST_CONTEXT_THREAD;
746                         } else {
747                                 if ( !test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
748                                                 &cmd->tgt_dev->tgt_dev_flags))
749                                         pref_context = SCST_CONTEXT_THREAD;
750                         }
751                 }
752                 break;
753
754         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
755                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
756                 break;
757
758         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
759                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
760                 /* go through */
761         case SCST_PREPROCESS_STATUS_ERROR:
762                 scst_set_cmd_error(cmd,
763                            SCST_LOAD_SENSE(scst_sense_hardw_error));
764                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
765                 break;
766
767         default:
768                 PRINT_ERROR("%s() received unknown status %x", __func__,
769                         status);
770                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
771                 break;
772         }
773
774         scst_proccess_redirect_cmd(cmd, pref_context, 1);
775
776         TRACE_EXIT();
777         return;
778 }
779
780 /* No locks */
781 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
782 {
783         struct scst_tgt *tgt = cmd->sess->tgt;
784         int res = 0;
785         unsigned long flags;
786
787         TRACE_ENTRY();
788
789         spin_lock_irqsave(&tgt->tgt_lock, flags);
790         tgt->retry_cmds++;
791         smp_mb();
792         TRACE_RETRY("TGT QUEUE FULL: incrementing retry_cmds %d",
793               tgt->retry_cmds);
794         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
795                 /* At least one cmd finished, so try again */
796                 tgt->retry_cmds--;
797                 TRACE_RETRY("Some command(s) finished, direct retry "
798                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
799                       "retry_cmds=%d)", finished_cmds,
800                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
801                 res = -1;
802                 goto out_unlock_tgt;
803         }
804
805         TRACE_RETRY("Adding cmd %p to retry cmd list", cmd);
806         list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
807
808         if (!tgt->retry_timer_active) {
809                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
810                 add_timer(&tgt->retry_timer);
811                 tgt->retry_timer_active = 1;
812         }
813
814 out_unlock_tgt:
815         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
816
817         TRACE_EXIT_RES(res);
818         return res;
819 }
820
821 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
822 {
823         int res, rc;
824         int atomic = scst_cmd_atomic(cmd);
825
826         TRACE_ENTRY();
827
828         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
829                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
830                 goto out_dev_done;
831         }
832
833         if (cmd->tgtt->rdy_to_xfer == NULL) {
834                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
835                 res = SCST_CMD_STATE_RES_CONT_SAME;
836                 goto out;
837         }
838
839         if (atomic && !cmd->tgtt->rdy_to_xfer_atomic) {
840                 TRACE_DBG("%s", "rdy_to_xfer() can not be "
841                       "called in atomic context, rescheduling to the thread");
842                 res = SCST_CMD_STATE_RES_NEED_THREAD;
843                 goto out;
844         }
845
846         while (1) {
847                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
848
849                 res = SCST_CMD_STATE_RES_CONT_NEXT;
850                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
851
852                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
853 #ifdef DEBUG_RETRY
854                 if (((scst_random() % 100) == 75))
855                         rc = SCST_TGT_RES_QUEUE_FULL;
856                 else
857 #endif
858                         rc = cmd->tgtt->rdy_to_xfer(cmd);
859                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
860
861                 if (likely(rc == SCST_TGT_RES_SUCCESS))
862                         goto out;
863
864                 /* Restore the previous state */
865                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
866
867                 switch (rc) {
868                 case SCST_TGT_RES_QUEUE_FULL:
869                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
870                                 break;
871                         else
872                                 continue;
873
874                 case SCST_TGT_RES_NEED_THREAD_CTX:
875                         TRACE_DBG("Target driver %s "
876                               "rdy_to_xfer() requested thread "
877                               "context, rescheduling", cmd->tgtt->name);
878                         res = SCST_CMD_STATE_RES_NEED_THREAD;
879                         break;
880
881                 default:
882                         goto out_error_rc;
883                 }
884                 break;
885         }
886
887 out:
888         TRACE_EXIT_HRES(res);
889         return res;
890
891 out_error_rc:
892         if (rc == SCST_TGT_RES_FATAL_ERROR) {
893                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
894                      "fatal error", cmd->tgtt->name);
895         } else {
896                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
897                             "value %d", cmd->tgtt->name, rc);
898         }
899         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
900
901 out_dev_done:
902         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
903         res = SCST_CMD_STATE_RES_CONT_SAME;
904         goto out;
905 }
906
907 /* No locks, but might be in IRQ */
908 void scst_proccess_redirect_cmd(struct scst_cmd *cmd, int context,
909         int check_retries)
910 {
911         unsigned long flags;
912
913         TRACE_ENTRY();
914
915         TRACE_DBG("Context: %d", context);
916
917         switch(context) {
918         case SCST_CONTEXT_DIRECT:
919         case SCST_CONTEXT_DIRECT_ATOMIC:
920                 if (check_retries)
921                         scst_check_retries(cmd->tgt);
922                 scst_process_active_cmd(cmd, context);
923                 break;
924
925         default:
926                 PRINT_ERROR("Context %x is unknown, using the thread one",
927                             context);
928                 /* go through */
929         case SCST_CONTEXT_THREAD:
930                 if (check_retries)
931                         scst_check_retries(cmd->tgt);
932                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
933                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
934                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
935                         list_add(&cmd->cmd_list_entry,
936                                 &cmd->cmd_lists->active_cmd_list);
937                 else
938                         list_add_tail(&cmd->cmd_list_entry,
939                                 &cmd->cmd_lists->active_cmd_list);
940                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
941                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
942                 break;
943
944         case SCST_CONTEXT_TASKLET:
945                 if (check_retries)
946                         scst_check_retries(cmd->tgt);
947                 scst_schedule_tasklet(cmd);
948                 break;
949         }
950
951         TRACE_EXIT();
952         return;
953 }
954
955 void scst_rx_data(struct scst_cmd *cmd, int status, int pref_context)
956 {
957         TRACE_ENTRY();
958
959         TRACE_DBG("Preferred context: %d", pref_context);
960         TRACE(TRACE_SCSI, "tag=%llu status=%#x", scst_cmd_get_tag(cmd), status);
961
962 #ifdef EXTRACHECKS
963         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
964                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
965         {
966                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
967                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
968                         cmd->tgtt->name);
969                 pref_context = SCST_CONTEXT_TASKLET;
970         }
971 #endif
972
973         switch (status) {
974         case SCST_RX_STATUS_SUCCESS:
975                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
976                 /* Small context optimization */
977                 if ((pref_context == SCST_CONTEXT_TASKLET) || 
978                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) {
979                         if ( !test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC, 
980                                         &cmd->tgt_dev->tgt_dev_flags))
981                                 pref_context = SCST_CONTEXT_THREAD;
982                 }
983                 break;
984
985         case SCST_RX_STATUS_ERROR_SENSE_SET:
986                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
987                 break;
988
989         case SCST_RX_STATUS_ERROR_FATAL:
990                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
991                 /* go through */
992         case SCST_RX_STATUS_ERROR:
993                 scst_set_cmd_error(cmd,
994                            SCST_LOAD_SENSE(scst_sense_hardw_error));
995                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
996                 break;
997
998         default:
999                 PRINT_ERROR("scst_rx_data() received unknown status %x",
1000                         status);
1001                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1002                 break;
1003         }
1004
1005         scst_proccess_redirect_cmd(cmd, pref_context, 1);
1006
1007         TRACE_EXIT();
1008         return;
1009 }
1010
1011 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
1012 {
1013         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
1014
1015         TRACE_ENTRY();
1016
1017         cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
1018
1019         if (cmd->tgtt->pre_exec == NULL)
1020                 goto out;
1021
1022         TRACE_DBG("Calling pre_exec(%p)", cmd);
1023         rc = cmd->tgtt->pre_exec(cmd);
1024         TRACE_DBG("pre_exec() returned %d", rc);
1025
1026         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
1027                 switch(rc) {
1028                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
1029                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1030                         break;
1031                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
1032                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1033                         /* go through */
1034                 case SCST_PREPROCESS_STATUS_ERROR:
1035                         scst_set_cmd_error(cmd,
1036                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1037                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1038                         break;
1039                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1040                         TRACE_DBG("Target driver's %s pre_exec() requested "
1041                                 "thread context, rescheduling", cmd->tgtt->name);
1042                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1043                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1044                         break;
1045                 default:
1046                         sBUG();
1047                         break;
1048                 }
1049         }
1050
1051 out:
1052         TRACE_EXIT_RES(res);
1053         return res;
1054 }
1055
1056 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1057         const uint8_t *rq_sense, int rq_sense_len, int resid)
1058 {
1059         TRACE_ENTRY();
1060
1061 #ifdef MEASURE_LATENCY
1062         {
1063                 struct timespec ts;
1064                 getnstimeofday(&ts);
1065                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1066                 TRACE_DBG("cmd %p (sess %p): post_exec_start %Ld (tv_sec %ld, "
1067                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start, ts.tv_sec,
1068                         ts.tv_nsec);
1069         }
1070 #endif
1071
1072         cmd->status = result & 0xff;
1073         cmd->msg_status = msg_byte(result);
1074         cmd->host_status = host_byte(result);
1075         cmd->driver_status = driver_byte(result);
1076         if (unlikely(resid != 0)) {
1077 #ifdef EXTRACHECKS
1078                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1079                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1080                                 "op %x)", resid, cmd->resp_data_len,
1081                                 cmd->cdb[0]);
1082                 } else
1083 #endif
1084                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1085         }
1086
1087         if (cmd->status == SAM_STAT_CHECK_CONDITION)
1088                 scst_alloc_set_sense(cmd, in_irq(), rq_sense, rq_sense_len);
1089
1090         TRACE(TRACE_SCSI, "result=%x, cmd->status=%x, resid=%d, "
1091               "cmd->msg_status=%x, cmd->host_status=%x, "
1092               "cmd->driver_status=%x", result, cmd->status, resid,
1093               cmd->msg_status, cmd->host_status, cmd->driver_status);
1094
1095         cmd->completed = 1;
1096
1097         TRACE_EXIT();
1098         return;
1099 }
1100
1101 /* For small context optimization */
1102 static inline int scst_optimize_post_exec_context(struct scst_cmd *cmd,
1103         int context)
1104 {
1105         if ((context == SCST_CONTEXT_TASKLET) || 
1106             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1107                 if ( !test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC, 
1108                                 &cmd->tgt_dev->tgt_dev_flags))
1109                         context = SCST_CONTEXT_THREAD;
1110         }
1111         return context;
1112 }
1113
1114 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18)
1115 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1116                                             struct scsi_request **req)
1117 {
1118         struct scst_cmd *cmd = NULL;
1119
1120         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1121                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1122
1123         if (cmd == NULL) {
1124                 PRINT_ERROR("%s", "Request with NULL cmd");
1125                 if (*req)
1126                         scsi_release_request(*req);
1127         }
1128
1129         return cmd;
1130 }
1131
1132 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1133 {
1134         struct scsi_request *req = NULL;
1135         struct scst_cmd *cmd;
1136
1137         TRACE_ENTRY();
1138
1139         cmd = scst_get_cmd(scsi_cmd, &req);
1140         if (cmd == NULL)
1141                 goto out;
1142
1143         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1144                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1145
1146         /* Clear out request structure */
1147         req->sr_use_sg = 0;
1148         req->sr_sglist_len = 0;
1149         req->sr_bufflen = 0;
1150         req->sr_buffer = NULL;
1151         req->sr_underflow = 0;
1152         req->sr_request->rq_disk = NULL; /* disown request blk */
1153
1154         scst_release_request(cmd);
1155
1156         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1157
1158         scst_proccess_redirect_cmd(cmd,
1159                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1160
1161 out:
1162         TRACE_EXIT();
1163         return;
1164 }
1165 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) */
1166 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1167 {
1168         struct scst_cmd *cmd;
1169
1170         TRACE_ENTRY();
1171
1172         cmd = (struct scst_cmd *)data;
1173         if (cmd == NULL)
1174                 goto out;
1175
1176         scst_do_cmd_done(cmd, result, sense, SCST_SENSE_BUFFERSIZE, resid);
1177
1178         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1179
1180         scst_proccess_redirect_cmd(cmd,
1181                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1182
1183 out:
1184         TRACE_EXIT();
1185         return;
1186 }
1187 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) */
1188
1189 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state)
1190 {
1191         TRACE_ENTRY();
1192
1193 #ifdef MEASURE_LATENCY
1194         {
1195                 struct timespec ts;
1196                 getnstimeofday(&ts);
1197                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1198                 TRACE_DBG("cmd %p (sess %p): post_exec_start %Ld (tv_sec %ld, "
1199                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start, ts.tv_sec,
1200                         ts.tv_nsec);
1201         }
1202 #endif
1203
1204         if (next_state == SCST_CMD_STATE_DEFAULT)
1205                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1206
1207 #if defined(DEBUG)
1208         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1209                 if (cmd->sg) {
1210                         int i;
1211                         struct scatterlist *sg = cmd->sg;
1212                         TRACE_RECV_TOP("Exec'd %d S/G(s) at %p sg[0].page at "
1213                                 "%p", cmd->sg_cnt, sg, (void*)sg_page(&sg[0]));
1214                         for(i = 0; i < cmd->sg_cnt; ++i) {
1215                                 TRACE_BUFF_FLAG(TRACE_RCV_TOP, 
1216                                         "Exec'd sg", sg_virt(&sg[i]),
1217                                         sg[i].length);
1218                         }
1219                 }
1220         }
1221 #endif
1222
1223
1224 #ifdef EXTRACHECKS
1225         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1226             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1227             (next_state != SCST_CMD_STATE_FINISHED)) 
1228         {
1229                 PRINT_ERROR("scst_cmd_done_local() received invalid cmd "
1230                             "state %d (opcode %d)", next_state, cmd->cdb[0]);
1231                 scst_set_cmd_error(cmd,
1232                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1233                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1234         }
1235 #endif
1236         cmd->state = next_state;
1237
1238         scst_proccess_redirect_cmd(cmd,
1239                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1240
1241         TRACE_EXIT();
1242         return;
1243 }
1244
1245 static int scst_report_luns_local(struct scst_cmd *cmd)
1246 {
1247         int rc;
1248         int dev_cnt = 0;
1249         int buffer_size;
1250         int i;
1251         struct scst_tgt_dev *tgt_dev = NULL;
1252         uint8_t *buffer;
1253         int offs, overflow = 0;
1254
1255         TRACE_ENTRY();
1256
1257         rc = scst_check_local_events(cmd);
1258         if (unlikely(rc != 0))
1259                 goto out_done;
1260
1261         cmd->status = 0;
1262         cmd->msg_status = 0;
1263         cmd->host_status = DID_OK;
1264         cmd->driver_status = 0;
1265
1266         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1267                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1268                         "LUNS command", cmd->cdb[2]);
1269                 goto out_err;
1270         }
1271
1272         buffer_size = scst_get_buf_first(cmd, &buffer);
1273         if (unlikely(buffer_size <= 0))
1274                 goto out_err;
1275
1276         if (buffer_size < 16)
1277                 goto out_put_err;
1278
1279         memset(buffer, 0, buffer_size);
1280         offs = 8;
1281
1282         /* sess->sess_tgt_dev_list_hash is protected by suspended activity */
1283         for(i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1284                 struct list_head *sess_tgt_dev_list_head =
1285                         &cmd->sess->sess_tgt_dev_list_hash[i];
1286                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1287                                 sess_tgt_dev_list_entry) {
1288                         if (!overflow) {
1289                                 if (offs >= buffer_size) {
1290                                         scst_put_buf(cmd, buffer);
1291                                         buffer_size = scst_get_buf_next(cmd, &buffer);
1292                                         if (buffer_size > 0) {
1293                                                 memset(buffer, 0, buffer_size);
1294                                                 offs = 0;
1295                                         } else {
1296                                                 overflow = 1;
1297                                                 goto inc_dev_cnt;
1298                                         }
1299                                 }
1300                                 if ((buffer_size - offs) < 8) {
1301                                         PRINT_ERROR("Buffer allocated for REPORT "
1302                                                 "LUNS command doesn't allow to fit 8 "
1303                                                 "byte entry (buffer_size=%d)",
1304                                                 buffer_size);
1305                                         goto out_put_hw_err;
1306                                 }
1307                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1308                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1309                                 offs += 8;
1310                         }
1311 inc_dev_cnt:
1312                         dev_cnt++;
1313                 }
1314         }
1315         if (!overflow)
1316                 scst_put_buf(cmd, buffer);
1317
1318         /* Set the response header */
1319         buffer_size = scst_get_buf_first(cmd, &buffer);
1320         if (unlikely(buffer_size <= 0))
1321                 goto out_err;
1322         dev_cnt *= 8;
1323         buffer[0] = (dev_cnt >> 24) & 0xff;
1324         buffer[1] = (dev_cnt >> 16) & 0xff;
1325         buffer[2] = (dev_cnt >> 8) & 0xff;
1326         buffer[3] = dev_cnt & 0xff;
1327         scst_put_buf(cmd, buffer);
1328
1329         dev_cnt += 8;
1330         if (dev_cnt < cmd->resp_data_len)
1331                 scst_set_resp_data_len(cmd, dev_cnt);
1332
1333 out_compl:
1334         cmd->completed = 1;
1335
1336 out_done:
1337         /* Report the result */
1338         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1339
1340         TRACE_EXIT();
1341         return SCST_EXEC_COMPLETED;
1342         
1343 out_put_err:
1344         scst_put_buf(cmd, buffer);
1345
1346 out_err:
1347         scst_set_cmd_error(cmd,
1348                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1349         goto out_compl;
1350
1351 out_put_hw_err:
1352         scst_put_buf(cmd, buffer);
1353         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1354         goto out_compl;
1355 }
1356
1357 static int scst_pre_select(struct scst_cmd *cmd)
1358 {
1359         int res = SCST_EXEC_NOT_COMPLETED;
1360
1361         TRACE_ENTRY();
1362
1363         if (scst_cmd_atomic(cmd)) {
1364                 res = SCST_EXEC_NEED_THREAD;
1365                 goto out;
1366         }
1367
1368         if (cmd->local_exec_done)
1369                 goto out;
1370
1371         cmd->local_exec_done = 1;
1372
1373         scst_block_dev_cmd(cmd, 1);
1374
1375         /* Check for local events will be done when cmd will be executed */
1376
1377 out:
1378         TRACE_EXIT_RES(res);
1379         return res;
1380 }
1381
1382 static int scst_reserve_local(struct scst_cmd *cmd)
1383 {
1384         int res = SCST_EXEC_NOT_COMPLETED, rc;
1385         struct scst_device *dev;
1386         struct scst_tgt_dev *tgt_dev_tmp;
1387
1388         TRACE_ENTRY();
1389
1390         if (scst_cmd_atomic(cmd)) {
1391                 res = SCST_EXEC_NEED_THREAD;
1392                 goto out;
1393         }
1394
1395         if (cmd->local_exec_done)
1396                 goto out;
1397
1398         cmd->local_exec_done = 1;
1399
1400         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1401                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1402                      "(lun=%Ld)", (uint64_t)cmd->lun);
1403                 scst_set_cmd_error(cmd,
1404                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1405                 goto out_done;
1406         }
1407
1408         dev = cmd->dev;
1409
1410         scst_block_dev_cmd(cmd, 1);
1411
1412         rc = scst_check_local_events(cmd);
1413         if (unlikely(rc != 0))
1414                 goto out_done;
1415
1416         spin_lock_bh(&dev->dev_lock);
1417
1418         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1419                 spin_unlock_bh(&dev->dev_lock);
1420                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1421                 goto out_done;
1422         }
1423
1424         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1425                             dev_tgt_dev_list_entry) {
1426                 if (cmd->tgt_dev != tgt_dev_tmp)
1427                         set_bit(SCST_TGT_DEV_RESERVED, 
1428                                 &tgt_dev_tmp->tgt_dev_flags);
1429         }
1430         dev->dev_reserved = 1;
1431
1432         spin_unlock_bh(&dev->dev_lock);
1433         
1434 out:
1435         TRACE_EXIT_RES(res);
1436         return res;
1437
1438 out_done:
1439         /* Report the result */
1440         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1441         res = SCST_EXEC_COMPLETED;
1442         goto out;
1443 }
1444
1445 static int scst_release_local(struct scst_cmd *cmd)
1446 {
1447         int res = SCST_EXEC_NOT_COMPLETED, rc;
1448         struct scst_tgt_dev *tgt_dev_tmp;
1449         struct scst_device *dev;
1450
1451         TRACE_ENTRY();
1452
1453         if (scst_cmd_atomic(cmd)) {
1454                 res = SCST_EXEC_NEED_THREAD;
1455                 goto out;
1456         }
1457
1458         if (cmd->local_exec_done)
1459                 goto out;
1460
1461         cmd->local_exec_done = 1;
1462
1463         dev = cmd->dev;
1464
1465         scst_block_dev_cmd(cmd, 1);
1466
1467         rc = scst_check_local_events(cmd);
1468         if (unlikely(rc != 0))
1469                 goto out_done;
1470
1471         spin_lock_bh(&dev->dev_lock);
1472
1473         /* 
1474          * The device could be RELEASED behind us, if RESERVING session 
1475          * is closed (see scst_free_tgt_dev()), but this actually doesn't 
1476          * matter, so use lock and no retest for DEV_RESERVED bits again
1477          */
1478         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1479                 res = SCST_EXEC_COMPLETED;
1480                 cmd->status = 0;
1481                 cmd->msg_status = 0;
1482                 cmd->host_status = DID_OK;
1483                 cmd->driver_status = 0;
1484                 cmd->completed = 1;
1485         } else {
1486                 list_for_each_entry(tgt_dev_tmp,
1487                                     &dev->dev_tgt_dev_list,
1488                                     dev_tgt_dev_list_entry) {
1489                         clear_bit(SCST_TGT_DEV_RESERVED, 
1490                                 &tgt_dev_tmp->tgt_dev_flags);
1491                 }
1492                 dev->dev_reserved = 0;
1493         }
1494
1495         spin_unlock_bh(&dev->dev_lock);
1496
1497         if (res == SCST_EXEC_COMPLETED)
1498                 goto out_done;
1499
1500 out:
1501         TRACE_EXIT_RES(res);
1502         return res;
1503
1504 out_done:
1505         res = SCST_EXEC_COMPLETED;
1506         /* Report the result */
1507         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1508         goto out;
1509 }
1510
1511 /* No locks, no IRQ or IRQ-safe context allowed */
1512 int scst_check_local_events(struct scst_cmd *cmd)
1513 {
1514         int res, rc;
1515         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1516
1517         TRACE_ENTRY();
1518
1519         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1520                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1521                 goto out_uncomplete;
1522         }
1523
1524         /* Reserve check before Unit Attention */
1525         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED, &tgt_dev->tgt_dev_flags))) {
1526                 if ((cmd->cdb[0] != INQUIRY) && (cmd->cdb[0] != REPORT_LUNS) &&
1527                     (cmd->cdb[0] != RELEASE) && (cmd->cdb[0] != RELEASE_10) &&
1528                     (cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER) &&
1529                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL || (cmd->cdb[4] & 3)) &&
1530                     (cmd->cdb[0] != LOG_SENSE) && (cmd->cdb[0] != REQUEST_SENSE)) {
1531                         scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1532                         goto out_complete;
1533                 }
1534         }
1535
1536         /* If we had internal bus reset, set the command error unit attention */
1537         if ((cmd->dev->scsi_dev != NULL) &&
1538             unlikely(cmd->dev->scsi_dev->was_reset)) {
1539                 if (scst_is_ua_command(cmd)) {
1540                         struct scst_device *dev = cmd->dev;
1541                         int done = 0;
1542                         /* Prevent more than 1 cmd to be triggered by was_reset */
1543                         spin_lock_bh(&dev->dev_lock);
1544                         barrier(); /* to reread was_reset */
1545                         if (dev->scsi_dev->was_reset) {
1546                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1547                                 scst_set_cmd_error(cmd,
1548                                            SCST_LOAD_SENSE(scst_sense_reset_UA));
1549                                 /* It looks like it is safe to clear was_reset here */
1550                                 dev->scsi_dev->was_reset = 0;
1551                                 done = 1;
1552                         }
1553                         spin_unlock_bh(&dev->dev_lock);
1554
1555                         if (done)
1556                                 goto out_complete;
1557                 }
1558         }
1559
1560         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING, 
1561                         &cmd->tgt_dev->tgt_dev_flags))) {
1562                 if (scst_is_ua_command(cmd)) {
1563                         rc = scst_set_pending_UA(cmd);
1564                         if (rc == 0)
1565                                 goto out_complete;
1566                 }
1567         }
1568
1569         res = 0;
1570
1571 out:
1572         TRACE_EXIT_RES(res);
1573         return res;
1574
1575 out_complete:
1576         res = 1;
1577         sBUG_ON(!cmd->completed);
1578         goto out;
1579
1580 out_uncomplete:
1581         res = -1;
1582         goto out;
1583 }
1584
1585 /* 
1586  * The result of cmd execution, if any, should be reported 
1587  * via scst_cmd_done_local() 
1588  */
1589 static int scst_pre_exec(struct scst_cmd *cmd)
1590 {
1591         int res = SCST_EXEC_NOT_COMPLETED;
1592         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1593
1594         TRACE_ENTRY();
1595
1596         /*
1597          * This function can be called several times for the same cmd, so it
1598          * can't change any state in a non-reentrable way or use something
1599          * like local_exec_done!!
1600          */
1601
1602         /* Check READ_ONLY device status */
1603         if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
1604             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
1605              cmd->cdb[0] == WRITE_10 ||
1606              cmd->cdb[0] == WRITE_12 ||
1607              cmd->cdb[0] == WRITE_16 ||
1608              cmd->cdb[0] == WRITE_VERIFY ||
1609              cmd->cdb[0] == WRITE_VERIFY_12 ||
1610              cmd->cdb[0] == WRITE_VERIFY_16 ||
1611              (cmd->dev->handler->type == TYPE_TAPE &&
1612               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
1613                 scst_set_cmd_error(cmd,
1614                            SCST_LOAD_SENSE(scst_sense_data_protect));
1615                 goto out_done;
1616         }
1617
1618 out:
1619         TRACE_EXIT_RES(res);
1620         return res;
1621
1622 out_done:
1623         res = SCST_EXEC_COMPLETED;
1624         /* Report the result */
1625         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1626         goto out;
1627 }
1628
1629 /* 
1630  * The result of cmd execution, if any, should be reported 
1631  * via scst_cmd_done_local() 
1632  */
1633 static inline int scst_local_exec(struct scst_cmd *cmd)
1634 {
1635         int res = SCST_EXEC_NOT_COMPLETED;
1636
1637         TRACE_ENTRY();
1638
1639         /*
1640          * Adding new commands here don't forget to update
1641          * scst_is_cmd_local() in scsi_tgt.h, if necessary
1642          */
1643
1644         switch (cmd->cdb[0]) {
1645         case MODE_SELECT:
1646         case MODE_SELECT_10:
1647         case LOG_SELECT:
1648                 res = scst_pre_select(cmd);
1649                 break;
1650         case RESERVE:
1651         case RESERVE_10:
1652                 res = scst_reserve_local(cmd);
1653                 break;
1654         case RELEASE:
1655         case RELEASE_10:
1656                 res = scst_release_local(cmd);
1657                 break;
1658         case REPORT_LUNS:
1659                 res = scst_report_luns_local(cmd);
1660                 break;
1661         }
1662
1663         TRACE_EXIT_RES(res);
1664         return res;
1665 }
1666
1667 /* cmd must be additionally referenced to not die inside */
1668 static int scst_do_send_to_midlev(struct scst_cmd *cmd)
1669 {
1670         int rc = SCST_EXEC_NOT_COMPLETED;
1671
1672         TRACE_ENTRY();
1673
1674         /* Check here to let an out of SN cmd be queued w/o context switch */
1675         if (scst_cmd_atomic(cmd) && !cmd->dev->handler->exec_atomic) {
1676                 TRACE_DBG("Dev handler %s exec() can not be "
1677                       "called in atomic context, rescheduling to the thread",
1678                       cmd->dev->handler->name);
1679                 rc = SCST_EXEC_NEED_THREAD;
1680                 goto out;
1681         }
1682
1683         cmd->sent_to_midlev = 1;
1684         cmd->state = SCST_CMD_STATE_EXECUTING;
1685         cmd->scst_cmd_done = scst_cmd_done_local;
1686
1687         rc = scst_pre_exec(cmd);
1688         if (rc != SCST_EXEC_NOT_COMPLETED) {
1689                 if (rc == SCST_EXEC_COMPLETED)
1690                         goto out;
1691                 else if (rc == SCST_EXEC_NEED_THREAD)
1692                         goto out_clear;
1693                 else
1694                         goto out_rc_error;
1695         }
1696
1697         rc = scst_local_exec(cmd);
1698         if (rc != SCST_EXEC_NOT_COMPLETED) {
1699                 if (rc == SCST_EXEC_COMPLETED)
1700                         goto out;
1701                 else if (rc == SCST_EXEC_NEED_THREAD)
1702                         goto out_clear;
1703                 else
1704                         goto out_rc_error;
1705         }
1706
1707         if (cmd->dev->handler->exec) {
1708                 struct scst_device *dev = cmd->dev;
1709                 TRACE_DBG("Calling dev handler %s exec(%p)",
1710                       dev->handler->name, cmd);
1711                 TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb, cmd->cdb_len);
1712                 cmd->scst_cmd_done = scst_cmd_done_local;
1713                 rc = dev->handler->exec(cmd);
1714                 TRACE_DBG("Dev handler %s exec() returned %d",
1715                       dev->handler->name, rc);
1716                 if (rc == SCST_EXEC_COMPLETED)
1717                         goto out;
1718                 else if (rc == SCST_EXEC_NEED_THREAD)
1719                         goto out_clear;
1720                 else if (rc != SCST_EXEC_NOT_COMPLETED)
1721                         goto out_rc_error;
1722         }
1723
1724         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1725         
1726         if (unlikely(cmd->dev->scsi_dev == NULL)) {
1727                 PRINT_ERROR("Command for virtual device must be "
1728                         "processed by device handler (lun %Ld)!",
1729                         (uint64_t)cmd->lun);
1730                 goto out_error;
1731         }
1732
1733         rc = scst_check_local_events(cmd);
1734         if (unlikely(rc != 0))
1735                 goto out_done;
1736
1737 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) 
1738         if (unlikely(scst_alloc_request(cmd) != 0)) {
1739                 if (scst_cmd_atomic(cmd)) {
1740                         rc = SCST_EXEC_NEED_THREAD;
1741                         goto out_clear;
1742                 } else {
1743                         PRINT_INFO("%s", "Unable to allocate request, "
1744                                 "sending BUSY status");
1745                         goto out_busy;
1746                 }
1747         }
1748         
1749         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1750                     (void *)cmd->scsi_req->sr_buffer,
1751                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1752                     cmd->retries);
1753 #else
1754         rc = scst_exec_req(cmd->dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1755                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1756                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1757                         scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
1758         if (unlikely(rc != 0)) {
1759                 if (scst_cmd_atomic(cmd)) {
1760                         rc = SCST_EXEC_NEED_THREAD;
1761                         goto out_clear;
1762                 } else {
1763                         PRINT_INFO("scst_exec_req() failed: %d", rc);
1764                         goto out_error;
1765                 }
1766         }
1767 #endif
1768
1769         rc = SCST_EXEC_COMPLETED;
1770
1771 out:
1772         TRACE_EXIT();
1773         return rc;
1774
1775 out_clear:
1776         /* Restore the state */
1777         cmd->sent_to_midlev = 0;
1778         cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
1779         goto out;
1780
1781 out_rc_error:
1782         PRINT_ERROR("Dev handler %s exec() or scst_local_exec() returned "
1783                     "invalid code %d", cmd->dev->handler->name, rc);
1784         /* go through */
1785
1786 out_error:
1787         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1788
1789 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) 
1790 out_busy:
1791         scst_set_busy(cmd);
1792         /* go through */
1793 #endif
1794
1795 out_done:
1796         rc = SCST_EXEC_COMPLETED;
1797         /* Report the result */
1798         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1799         goto out;
1800 }
1801
1802 /* No locks */
1803 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1804 {
1805         if (slot == NULL)
1806                 goto inc;
1807
1808         /* Optimized for lockless fast path */
1809
1810         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1811                 atomic_read(slot));
1812
1813         if (!atomic_dec_and_test(slot))
1814                 goto out;
1815
1816         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1817                 tgt_dev->num_free_sn_slots);
1818         if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
1819                 spin_lock_irq(&tgt_dev->sn_lock);
1820                 if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
1821                         if (tgt_dev->num_free_sn_slots < 0)
1822                                 tgt_dev->cur_sn_slot = slot;
1823                         smp_mb(); /* to be in-sync with SIMPLE case in scst_cmd_set_sn() */
1824                         tgt_dev->num_free_sn_slots++;
1825                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1826                                 tgt_dev->num_free_sn_slots);
1827                         
1828                 }
1829                 spin_unlock_irq(&tgt_dev->sn_lock);
1830         }
1831
1832 inc:
1833         /*
1834          * No locks is needed, because only one thread at time can 
1835          * be here (serialized by sn). Also it is supposed that there
1836          * could not be half-incremented halves.
1837          */
1838         tgt_dev->expected_sn++;
1839         smp_mb(); /* write must be before def_cmd_count read */
1840         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1841
1842 out:
1843         return;
1844 }
1845
1846 static int scst_process_internal_cmd(struct scst_cmd *cmd)
1847 {
1848         int res = SCST_CMD_STATE_RES_CONT_NEXT, rc;
1849
1850         TRACE_ENTRY();
1851
1852         __scst_cmd_get(cmd);
1853
1854         rc = scst_do_send_to_midlev(cmd);
1855         if (rc == SCST_EXEC_NEED_THREAD) {
1856                 TRACE_DBG("%s", "scst_do_send_to_midlev() requested "
1857                       "thread context, rescheduling");
1858                 res = SCST_CMD_STATE_RES_NEED_THREAD;
1859         } else {
1860                 struct scst_device *dev = cmd->dev;
1861                 sBUG_ON(rc != SCST_EXEC_COMPLETED);
1862                 if (dev->scsi_dev != NULL)
1863                         generic_unplug_device(dev->scsi_dev->request_queue);
1864         }
1865
1866         __scst_cmd_put(cmd);
1867
1868         TRACE_EXIT_RES(res);
1869         return res;
1870 }
1871
1872 static int scst_send_to_midlev(struct scst_cmd **active_cmd)
1873 {
1874         int res, rc;
1875         struct scst_cmd *cmd = *active_cmd;
1876         struct scst_cmd *ref_cmd;
1877         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1878         struct scst_device *dev = cmd->dev;
1879         typeof(tgt_dev->expected_sn) expected_sn;
1880         int count;
1881
1882         TRACE_ENTRY();
1883
1884         res = SCST_CMD_STATE_RES_CONT_NEXT;
1885
1886         if (unlikely(cmd->internal || cmd->retry)) {
1887                 res = scst_process_internal_cmd(cmd);
1888                 goto out;
1889         }
1890
1891 #ifdef MEASURE_LATENCY
1892         if (cmd->pre_exec_finish == 0) {
1893                 struct timespec ts;
1894                 getnstimeofday(&ts);
1895                 cmd->pre_exec_finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1896                 TRACE_DBG("cmd %p (sess %p): pre_exec_finish %Ld (tv_sec %ld, "
1897                         "tv_nsec %ld)", cmd, sess, cmd->pre_exec_finish, ts.tv_sec,
1898                         ts.tv_nsec);
1899         }
1900 #endif
1901
1902         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
1903                 goto out;
1904
1905         ref_cmd = cmd;
1906         __scst_cmd_get(ref_cmd);
1907
1908         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
1909                 goto exec;
1910
1911         sBUG_ON(!cmd->sn_set);
1912
1913         expected_sn = tgt_dev->expected_sn;
1914         /* Optimized for lockless fast path */
1915         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
1916                 spin_lock_irq(&tgt_dev->sn_lock);
1917
1918                 tgt_dev->def_cmd_count++;
1919                 smp_mb();
1920
1921                 expected_sn = tgt_dev->expected_sn;
1922                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
1923                         /* We are under IRQ lock, but dev->dev_lock is BH one */
1924                         int cmd_blocking = scst_pre_dec_on_dev_cmd(cmd);
1925                         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1926                                 /* Necessary to allow aborting out of sn cmds */
1927                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p (tag %llu)",
1928                                         cmd, cmd->tag);
1929                                 tgt_dev->def_cmd_count--;
1930                                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1931                                 res = SCST_CMD_STATE_RES_CONT_SAME;
1932                         } else {
1933                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
1934                                         "expected_sn=%ld)", cmd, cmd->sn,
1935                                         cmd->sn_set, expected_sn);
1936                                 list_add_tail(&cmd->sn_cmd_list_entry,
1937                                               &tgt_dev->deferred_cmd_list);
1938                         }
1939                         spin_unlock_irq(&tgt_dev->sn_lock);
1940
1941                         __scst_dec_on_dev_cmd(dev, cmd_blocking);
1942
1943                         goto out_put;
1944                 } else {
1945                         TRACE_SN("Somebody incremented expected_sn %ld, "
1946                                 "continuing", expected_sn);
1947                         tgt_dev->def_cmd_count--;
1948                         spin_unlock_irq(&tgt_dev->sn_lock);
1949                 }
1950         }
1951
1952 exec:
1953         count = 0;
1954         while(1) {
1955                 atomic_t *slot = cmd->sn_slot;
1956                 int inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1957                                       cmd->sn_set;
1958
1959                 rc = scst_do_send_to_midlev(cmd);
1960                 if (rc == SCST_EXEC_NEED_THREAD) {
1961                         TRACE_DBG("%s", "scst_do_send_to_midlev() requested "
1962                               "thread context, rescheduling");
1963                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1964                         scst_dec_on_dev_cmd(cmd);
1965                         *active_cmd = cmd;
1966                         if (count != 0)
1967                                 goto out_unplug;
1968                         else
1969                                 goto out_put;
1970                 }
1971                 sBUG_ON(rc != SCST_EXEC_COMPLETED);
1972
1973                 count++;
1974
1975                 if (inc_expected_sn)
1976                         scst_inc_expected_sn(tgt_dev, slot);
1977
1978                 cmd = scst_check_deferred_commands(tgt_dev);
1979                 if (cmd == NULL)
1980                         break;
1981
1982                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
1983                         break;
1984
1985                 __scst_cmd_put(ref_cmd);
1986                 ref_cmd = cmd;
1987                 __scst_cmd_get(ref_cmd);
1988         }
1989
1990 out_unplug:
1991         if (dev->scsi_dev != NULL)
1992                 generic_unplug_device(dev->scsi_dev->request_queue);
1993
1994 out_put:
1995         __scst_cmd_put(ref_cmd);
1996         /* !! At this point sess, dev and tgt_dev can be already freed !! */
1997
1998 out:
1999         TRACE_EXIT_HRES(res);
2000         return res;
2001 }
2002
2003 /* No locks supposed to be held */
2004 static int scst_check_sense(struct scst_cmd *cmd)
2005 {
2006         int res = 0;
2007         struct scst_device *dev = cmd->dev;
2008         int dbl_ua_possible, ua_sent = 0;
2009
2010         TRACE_ENTRY();
2011
2012         /* If we had internal bus reset behind us, set the command error UA */
2013         if ((dev->scsi_dev != NULL) &&
2014             unlikely(cmd->host_status == DID_RESET) &&
2015             scst_is_ua_command(cmd)) {
2016                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
2017                       dev->scsi_dev->was_reset, cmd->host_status);
2018                 scst_set_cmd_error(cmd,
2019                    SCST_LOAD_SENSE(scst_sense_reset_UA));
2020                 /* just in case */
2021                 cmd->ua_ignore = 0;
2022                 /* It looks like it is safe to clear was_reset here */
2023                 dev->scsi_dev->was_reset = 0;
2024         }
2025
2026         dbl_ua_possible = dev->dev_double_ua_possible;
2027         TRACE_DBG("cmd %p dbl_ua_possible %d", cmd, dbl_ua_possible);
2028         if (unlikely(dbl_ua_possible)) {
2029                 spin_lock_bh(&dev->dev_lock);
2030                 barrier(); /* to reread dev_double_ua_possible */
2031                 dbl_ua_possible = dev->dev_double_ua_possible;
2032                 if (dbl_ua_possible)
2033                         ua_sent = dev->dev_reset_ua_sent;
2034                 else
2035                         spin_unlock_bh(&dev->dev_lock);
2036         }
2037
2038         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) && 
2039             SCST_SENSE_VALID(cmd->sense)) {
2040                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2041                         SCST_SENSE_BUFFERSIZE);
2042                 /* Check Unit Attention Sense Key */
2043                 if (scst_is_ua_sense(cmd->sense)) {
2044                         if (cmd->sense[12] == SCST_SENSE_ASC_UA_RESET) {
2045                                 if (dbl_ua_possible) {
2046                                         if (ua_sent) {
2047                                                 TRACE(TRACE_MGMT_MINOR, "%s", 
2048                                                         "Double UA detected");
2049                                                 /* Do retry */
2050                                                 TRACE(TRACE_MGMT_MINOR, "Retrying cmd %p "
2051                                                         "(tag %llu)", cmd, cmd->tag);
2052                                                 cmd->status = 0;
2053                                                 cmd->msg_status = 0;
2054                                                 cmd->host_status = DID_OK;
2055                                                 cmd->driver_status = 0;
2056                                                 mempool_free(cmd->sense, scst_sense_mempool);
2057                                                 cmd->sense = NULL;
2058                                                 cmd->retry = 1;
2059                                                 cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
2060                                                 res = 1;
2061                                                 /* 
2062                                                  * Dev is still blocked by this cmd, so
2063                                                  * it's OK to clear SCST_DEV_SERIALIZED
2064                                                  * here.
2065                                                  */
2066                                                 dev->dev_double_ua_possible = 0;
2067                                                 dev->dev_serialized = 0;
2068                                                 dev->dev_reset_ua_sent = 0;
2069                                                 goto out_unlock;
2070                                         } else
2071                                                 dev->dev_reset_ua_sent = 1;
2072                                 }
2073                         }
2074                         if (cmd->ua_ignore == 0) {
2075                                 if (unlikely(dbl_ua_possible)) {
2076                                         __scst_dev_check_set_UA(dev, cmd,
2077                                                 cmd->sense,
2078                                                 SCST_SENSE_BUFFERSIZE);
2079                                 } else {
2080                                         scst_dev_check_set_UA(dev, cmd,
2081                                                 cmd->sense,
2082                                                 SCST_SENSE_BUFFERSIZE);
2083                                 }
2084                         }
2085                 }
2086         }
2087
2088         if (unlikely(dbl_ua_possible)) {
2089                 if (ua_sent && scst_is_ua_command(cmd)) {
2090                         TRACE_MGMT_DBG("%s", "Clearing dbl_ua_possible flag");
2091                         dev->dev_double_ua_possible = 0;
2092                         dev->dev_serialized = 0;
2093                         dev->dev_reset_ua_sent = 0;
2094                 }
2095                 spin_unlock_bh(&dev->dev_lock);
2096         }
2097
2098 out:
2099         TRACE_EXIT_RES(res);
2100         return res;
2101
2102 out_unlock:
2103         spin_unlock_bh(&dev->dev_lock);
2104         goto out;
2105 }
2106
2107 static int scst_check_auto_sense(struct scst_cmd *cmd)
2108 {
2109         int res = 0;
2110
2111         TRACE_ENTRY();
2112
2113         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2114             (!SCST_SENSE_VALID(cmd->sense) ||
2115              SCST_NO_SENSE(cmd->sense))) {
2116                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2117                       "cmd->status=%x, cmd->msg_status=%x, "
2118                       "cmd->host_status=%x, cmd->driver_status=%x", cmd->status,
2119                       cmd->msg_status, cmd->host_status, cmd->driver_status);
2120                 res = 1;
2121         } else if (unlikely(cmd->host_status)) {
2122                 if ((cmd->host_status == DID_REQUEUE) ||
2123                     (cmd->host_status == DID_IMM_RETRY) ||
2124                     (cmd->host_status == DID_SOFT_ERROR) ||
2125                     (cmd->host_status == DID_ABORT)) {
2126                         scst_set_busy(cmd);
2127                 } else {
2128                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2129                                 "received, returning HARDWARE ERROR instead",
2130                                 cmd->host_status);
2131                         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2132                 }
2133         }
2134
2135         TRACE_EXIT_RES(res);
2136         return res;
2137 }
2138
2139 static int scst_done_cmd_check(struct scst_cmd *cmd, int *pres)
2140 {
2141         int res = 0, rc;
2142
2143         TRACE_ENTRY();
2144
2145         if (unlikely(cmd->cdb[0] == REQUEST_SENSE)) {
2146                 if (cmd->internal)
2147                         cmd = scst_complete_request_sense(cmd);
2148         } else if (unlikely(scst_check_auto_sense(cmd))) {
2149                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2150                             "without sense data (opcode 0x%x), issuing "
2151                             "REQUEST SENSE", cmd->cdb[0]);
2152                 rc = scst_prepare_request_sense(cmd);
2153                 if (rc > 0) {
2154                         *pres = rc;
2155                         res = 1;
2156                         goto out;
2157                 } else {
2158                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2159                                     "returning HARDWARE ERROR");
2160                         scst_set_cmd_error(cmd,
2161                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2162                 }
2163         } else if (unlikely(scst_check_sense(cmd))) {
2164                 *pres = SCST_CMD_STATE_RES_CONT_SAME;
2165                 res = 1;
2166                 goto out;
2167         }
2168
2169         if (likely(scsi_status_is_good(cmd->status))) {
2170                 unsigned char type = cmd->dev->handler->type;
2171                 if ((cmd->cdb[0] == MODE_SENSE || cmd->cdb[0] == MODE_SENSE_10) &&
2172                     cmd->tgt_dev->acg_dev->rd_only_flag &&
2173                     (type == TYPE_DISK || type == TYPE_WORM || type == TYPE_MOD ||
2174                      type == TYPE_TAPE)) {
2175                         int32_t length;
2176                         uint8_t *address;
2177
2178                         length = scst_get_buf_first(cmd, &address);
2179                         if (length <= 0)
2180                                 goto out;
2181                         if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2182                                 address[2] |= 0x80;   /* Write Protect*/
2183                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2184                                 address[3] |= 0x80;   /* Write Protect*/
2185                         scst_put_buf(cmd, address);
2186                 }
2187
2188                 /* 
2189                  * Check and clear NormACA option for the device, if necessary,
2190                  * since we don't support ACA
2191                  */
2192                 if ((cmd->cdb[0] == INQUIRY) &&
2193                     !(cmd->cdb[1] & SCST_INQ_EVPD/* Std INQUIRY data (no EVPD) */) &&
2194                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2195                         uint8_t *buffer;
2196                         int buflen;
2197
2198                         /* ToDo: all pages ?? */
2199                         buflen = scst_get_buf_first(cmd, &buffer);
2200                         if (buflen > SCST_INQ_BYTE3) {
2201 #ifdef EXTRACHECKS
2202                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2203                                         PRINT_INFO("NormACA set for device: "
2204                                             "lun=%Ld, type 0x%02x", 
2205                                             (uint64_t)cmd->lun, buffer[0]);
2206                                 }
2207 #endif
2208                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2209                         } else {
2210                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2211                                     "buffer");
2212                                 scst_set_cmd_error(cmd,
2213                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
2214                         }
2215                         if (buflen > 0)
2216                                 scst_put_buf(cmd, buffer);
2217                 }
2218
2219                 if (unlikely((cmd->cdb[0] == MODE_SELECT) || 
2220                     (cmd->cdb[0] == MODE_SELECT_10) ||
2221                     (cmd->cdb[0] == LOG_SELECT))) {
2222                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded (LUN %Ld)",
2223                                 (uint64_t)cmd->lun);
2224                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2225                         *pres = SCST_CMD_STATE_RES_CONT_SAME;
2226                         res = 1;
2227                         goto out;
2228                 }
2229         } else {
2230                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2231                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2232                                         &cmd->tgt_dev->tgt_dev_flags)) {
2233                                 struct scst_tgt_dev *tgt_dev_tmp;
2234                                 struct scst_device *dev = cmd->dev;
2235
2236                                 TRACE(TRACE_SCSI, "Real RESERVE failed lun=%Ld, "
2237                                         "status=%x", (uint64_t)cmd->lun, cmd->status);
2238                                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2239                                         SCST_SENSE_BUFFERSIZE);
2240
2241                                 /* Clearing the reservation */
2242                                 spin_lock_bh(&dev->dev_lock);
2243                                 list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
2244                                                     dev_tgt_dev_list_entry) {
2245                                         clear_bit(SCST_TGT_DEV_RESERVED, 
2246                                                 &tgt_dev_tmp->tgt_dev_flags);
2247                                 }
2248                                 dev->dev_reserved = 0;
2249                                 spin_unlock_bh(&dev->dev_lock);
2250                         }
2251                 }
2252
2253                 /* Check for MODE PARAMETERS CHANGED UA */
2254                 if ((cmd->dev->scsi_dev != NULL) &&
2255                     (cmd->status == SAM_STAT_CHECK_CONDITION) && 
2256                     SCST_SENSE_VALID(cmd->sense) &&
2257                     scst_is_ua_sense(cmd->sense) &&
2258                     (cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) {
2259                         TRACE(TRACE_SCSI, "MODE PARAMETERS CHANGED UA (lun %Ld)",
2260                                 (uint64_t)cmd->lun);
2261                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2262                         *pres = SCST_CMD_STATE_RES_CONT_SAME;
2263                         res = 1;
2264                         goto out;
2265                 }
2266         }
2267
2268 out:
2269         TRACE_EXIT_RES(res);
2270         return res;
2271 }
2272
2273 static int scst_pre_dev_done(struct scst_cmd *cmd)
2274 {
2275         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2276
2277         TRACE_ENTRY();
2278
2279         atomic_dec(&cmd->tgt_dev->tgt_dev_cmd_count);
2280         atomic_dec(&cmd->dev->dev_cmd_count);
2281
2282         rc = scst_done_cmd_check(cmd, &res);
2283         if (rc)
2284                 goto out;
2285
2286         cmd->state = SCST_CMD_STATE_DEV_DONE;
2287
2288 out:
2289         TRACE_EXIT_HRES(res);
2290         return res;
2291 }
2292
2293 static int scst_mode_select_checks(struct scst_cmd *cmd)
2294 {
2295         int res = SCST_CMD_STATE_RES_CONT_SAME;
2296         int atomic = scst_cmd_atomic(cmd);
2297
2298         TRACE_ENTRY();
2299
2300         if (likely(scsi_status_is_good(cmd->status))) {
2301                 if (unlikely((cmd->cdb[0] == MODE_SELECT) || 
2302                     (cmd->cdb[0] == MODE_SELECT_10) ||
2303                     (cmd->cdb[0] == LOG_SELECT))) {
2304                         struct scst_device *dev = cmd->dev;
2305                         if (atomic && (dev->scsi_dev != NULL)) {
2306                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2307                                         "context required");
2308                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2309                                 goto out;
2310                         }
2311
2312                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2313                                 "setting the SELECT UA (lun=%Ld)", 
2314                                 (uint64_t)cmd->lun);
2315
2316                         spin_lock_bh(&dev->dev_lock);
2317                         spin_lock(&scst_temp_UA_lock);
2318                         if (cmd->cdb[0] == LOG_SELECT) {
2319                                 scst_set_sense(scst_temp_UA,
2320                                         sizeof(scst_temp_UA),
2321                                         UNIT_ATTENTION, 0x2a, 0x02);
2322                         } else {
2323                                 scst_set_sense(scst_temp_UA,
2324                                         sizeof(scst_temp_UA),
2325                                         UNIT_ATTENTION, 0x2a, 0x01);
2326                         }
2327                         scst_dev_check_set_local_UA(dev, cmd, scst_temp_UA,
2328                                 sizeof(scst_temp_UA));
2329                         spin_unlock(&scst_temp_UA_lock);
2330                         spin_unlock_bh(&dev->dev_lock);
2331
2332                         if (dev->scsi_dev != NULL)
2333                                 scst_obtain_device_parameters(dev);
2334                 }
2335         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) && 
2336                     SCST_SENSE_VALID(cmd->sense) &&
2337                     scst_is_ua_sense(cmd->sense) &&
2338                     (cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) {
2339                 if (atomic) {
2340                         TRACE_DBG("%s", "MODE PARAMETERS CHANGED UA: thread "
2341                                 "context required");
2342                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2343                         goto out;
2344                 }
2345
2346                 TRACE(TRACE_SCSI, "MODE PARAMETERS CHANGED UA (lun %Ld): "
2347                         "getting new parameters", (uint64_t)cmd->lun);
2348
2349                 scst_obtain_device_parameters(cmd->dev);
2350         } else
2351                 sBUG();
2352
2353         cmd->state = SCST_CMD_STATE_DEV_DONE;
2354
2355 out:
2356         TRACE_EXIT_HRES(res);
2357         return res;
2358 }
2359
2360 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2361 {
2362         struct scst_cmd *c;
2363
2364         if (likely(cmd->sn_set))
2365                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2366
2367         c = scst_check_deferred_commands(cmd->tgt_dev);
2368         if (c != NULL) {
2369                 unsigned long flags;
2370                 spin_lock_irqsave(&c->cmd_lists->cmd_list_lock, flags);
2371                 TRACE_SN("Adding cmd %p to active cmd list", c);
2372                 list_add_tail(&c->cmd_list_entry,
2373                         &c->cmd_lists->active_cmd_list);
2374                 wake_up(&c->cmd_lists->cmd_list_waitQ);
2375                 spin_unlock_irqrestore(&c->cmd_lists->cmd_list_lock, flags);
2376         }
2377 }
2378
2379 static int scst_dev_done(struct scst_cmd *cmd)
2380 {
2381         int res = SCST_CMD_STATE_RES_CONT_SAME;
2382         int state;
2383         int atomic = scst_cmd_atomic(cmd);
2384
2385         TRACE_ENTRY();
2386
2387         if (atomic && !cmd->dev->handler->dev_done_atomic) {
2388                 TRACE_DBG("Dev handler %s dev_done() can not be "
2389                       "called in atomic context, rescheduling to the thread",
2390                       cmd->dev->handler->name);
2391                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2392                 goto out;
2393         }
2394
2395         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2396         if (likely(!scst_is_cmd_local(cmd)) && 
2397             likely(cmd->dev->handler->dev_done != NULL))
2398         {
2399                 int rc;
2400                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2401                       cmd->dev->handler->name, cmd);
2402                 rc = cmd->dev->handler->dev_done(cmd);
2403                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2404                       cmd->dev->handler->name, rc);
2405                 if (rc != SCST_CMD_STATE_DEFAULT)
2406                         state = rc;
2407         }
2408
2409         switch (state) {
2410         case SCST_CMD_STATE_PRE_XMIT_RESP:
2411         case SCST_CMD_STATE_DEV_PARSE:
2412         case SCST_CMD_STATE_PRE_PARSE:
2413         case SCST_CMD_STATE_PREPARE_SPACE:
2414         case SCST_CMD_STATE_RDY_TO_XFER:
2415         case SCST_CMD_STATE_TGT_PRE_EXEC:
2416         case SCST_CMD_STATE_SEND_TO_MIDLEV:
2417         case SCST_CMD_STATE_PRE_DEV_DONE:
2418         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2419         case SCST_CMD_STATE_DEV_DONE:
2420         case SCST_CMD_STATE_XMIT_RESP:
2421         case SCST_CMD_STATE_FINISHED:
2422                 cmd->state = state;
2423                 res = SCST_CMD_STATE_RES_CONT_SAME;
2424                 break;
2425
2426         case SCST_CMD_STATE_NEED_THREAD_CTX:
2427                 TRACE_DBG("Dev handler %s dev_done() requested "
2428                       "thread context, rescheduling",
2429                       cmd->dev->handler->name);
2430                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2431                 break;
2432
2433         default:
2434                 if (state >= 0) {
2435                         PRINT_ERROR("Dev handler %s dev_done() returned "
2436                                 "invalid cmd state %d", 
2437                                 cmd->dev->handler->name, state);
2438                 } else {
2439                         PRINT_ERROR("Dev handler %s dev_done() returned "
2440                                 "error %d", cmd->dev->handler->name, 
2441                                 state);
2442                 }
2443                 scst_set_cmd_error(cmd,
2444                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2445                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2446                 res = SCST_CMD_STATE_RES_CONT_SAME;
2447                 break;
2448         }
2449
2450         if (cmd->needs_unblocking)
2451                 scst_unblock_dev_cmd(cmd);
2452
2453         if (likely(cmd->dec_on_dev_needed))
2454                 scst_dec_on_dev_cmd(cmd);
2455
2456         if (cmd->inc_expected_sn_on_done && cmd->sent_to_midlev)
2457                 scst_inc_check_expected_sn(cmd);
2458
2459 out:
2460         TRACE_EXIT_HRES(res);
2461         return res;
2462 }
2463
2464 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2465 {
2466         int res;
2467
2468         TRACE_ENTRY();
2469
2470 #ifdef DEBUG_TM
2471         if (cmd->tm_dbg_delayed && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2472                 if (scst_cmd_atomic(cmd)) {
2473                         TRACE_MGMT_DBG("%s", "DEBUG_TM delayed cmd needs a thread");
2474                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2475                         return res;
2476                 }
2477                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2478                         cmd, cmd->tag);
2479                 schedule_timeout_uninterruptible(HZ);
2480         }
2481 #endif
2482
2483         if (cmd->tgt_dev != NULL) {
2484                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2485                         scst_on_hq_cmd_response(cmd);
2486
2487                 if (unlikely(!cmd->sent_to_midlev)) {
2488                         TRACE_SN("cmd %p was not sent to mid-lev (sn %ld, set %d)",
2489                                 cmd, cmd->sn, cmd->sn_set);
2490                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2491                         cmd->sent_to_midlev = 1;
2492                 }
2493         }
2494
2495         /*
2496          * If we don't remove cmd from the search list here, before
2497          * submitting it for transmittion, we will have a race, when for
2498          * some reason cmd's release is delayed after transmittion and
2499          * initiator sends cmd with the same tag => it is possible that
2500          * a wrong cmd will be found by find() functions.
2501          */
2502         spin_lock_irq(&cmd->sess->sess_list_lock);
2503         list_del(&cmd->search_cmd_list_entry);
2504         spin_unlock_irq(&cmd->sess->sess_list_lock);
2505
2506         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2507                 scst_xmit_process_aborted_cmd(cmd);
2508
2509         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2510                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu), skipping",
2511                         cmd, cmd->tag);
2512                 cmd->state = SCST_CMD_STATE_FINISHED;
2513                 res = SCST_CMD_STATE_RES_CONT_SAME;
2514                 goto out;
2515         }
2516
2517         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2518         res = SCST_CMD_STATE_RES_CONT_SAME;
2519
2520 out:
2521 #ifdef MEASURE_LATENCY
2522         {
2523                 struct timespec ts;
2524                 uint64_t finish, scst_time, proc_time;
2525                 struct scst_session *sess = cmd->sess;
2526
2527                 getnstimeofday(&ts);
2528                 finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2529
2530                 spin_lock_bh(&sess->meas_lock);
2531
2532                 scst_time = cmd->pre_exec_finish - cmd->start;
2533                 scst_time += finish - cmd->post_exec_start;
2534                 proc_time = finish - cmd->start;
2535
2536                 sess->scst_time += scst_time;
2537                 sess->processing_time += proc_time;
2538                 sess->processed_cmds++;
2539
2540                 spin_unlock_bh(&sess->meas_lock);
2541
2542                 TRACE_DBG("cmd %p (sess %p): finish %Ld (tv_sec %ld, "
2543                         "tv_nsec %ld), scst_time %Ld, proc_time %Ld", cmd, sess,
2544                         finish, ts.tv_sec, ts.tv_nsec, scst_time, proc_time);
2545         }
2546 #endif
2547         TRACE_EXIT_HRES(res);
2548         return res;
2549 }
2550
2551 static int scst_xmit_response(struct scst_cmd *cmd)
2552 {
2553         int res, rc;
2554         int atomic = scst_cmd_atomic(cmd);
2555
2556         TRACE_ENTRY();
2557
2558         if (atomic && !cmd->tgtt->xmit_response_atomic) {
2559                 TRACE_DBG("%s", "xmit_response() can not be "
2560                       "called in atomic context, rescheduling to the thread");
2561                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2562                 goto out;
2563         }
2564
2565         while (1) {
2566                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2567
2568                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2569                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2570
2571                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2572
2573 #if defined(DEBUG)
2574                 if (cmd->sg) {
2575                         int i;
2576                         struct scatterlist *sg = cmd->sg;
2577                         TRACE_SEND_BOT("Xmitting %d S/G(s) at %p sg[0].page at %p",
2578                               cmd->sg_cnt, sg, (void*)sg_page(&sg[0]));
2579                         for(i = 0; i < cmd->sg_cnt; ++i) {
2580                                 TRACE_BUFF_FLAG(TRACE_SND_BOT,
2581                                     "Xmitting sg", sg_virt(&sg[i]),
2582                                     sg[i].length);
2583                         }
2584                 }
2585 #endif
2586
2587 #ifdef DEBUG_RETRY
2588                 if (((scst_random() % 100) == 77))
2589                         rc = SCST_TGT_RES_QUEUE_FULL;
2590                 else
2591 #endif
2592                         rc = cmd->tgtt->xmit_response(cmd);
2593                 TRACE_DBG("xmit_response() returned %d", rc);
2594
2595                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2596                         goto out;
2597
2598                 /* Restore the previous state */
2599                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2600
2601                 switch (rc) {
2602                 case SCST_TGT_RES_QUEUE_FULL:
2603                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2604                                 break;
2605                         else
2606                                 continue;
2607
2608                 case SCST_TGT_RES_NEED_THREAD_CTX:
2609                         TRACE_DBG("Target driver %s xmit_response() "
2610                               "requested thread context, rescheduling",
2611                               cmd->tgtt->name);
2612                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2613                         break;
2614
2615                 default:
2616                         goto out_error;
2617                 }
2618                 break;
2619         }
2620
2621 out:
2622         /* Caution: cmd can be already dead here */
2623         TRACE_EXIT_HRES(res);
2624         return res;
2625
2626 out_error:
2627         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2628                 PRINT_ERROR("Target driver %s xmit_response() returned "
2629                         "fatal error", cmd->tgtt->name);
2630         } else {
2631                 PRINT_ERROR("Target driver %s xmit_response() returned "
2632                         "invalid value %d", cmd->tgtt->name, rc);
2633         }
2634         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2635         cmd->state = SCST_CMD_STATE_FINISHED;
2636         res = SCST_CMD_STATE_RES_CONT_SAME;
2637         goto out;
2638 }
2639
2640 void scst_tgt_cmd_done(struct scst_cmd *cmd)
2641 {
2642         TRACE_ENTRY();
2643
2644         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2645
2646         cmd->state = SCST_CMD_STATE_FINISHED;
2647         scst_proccess_redirect_cmd(cmd, scst_get_context(), 1);
2648
2649         TRACE_EXIT();
2650         return;
2651 }
2652
2653 static int scst_finish_cmd(struct scst_cmd *cmd)
2654 {
2655         int res;
2656
2657         TRACE_ENTRY();
2658
2659         atomic_dec(&cmd->sess->sess_cmd_count);
2660
2661         cmd->finished = 1;
2662         smp_mb(); /* to sync with scst_abort_cmd() */
2663
2664         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2665                 unsigned long flags;
2666
2667                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
2668                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
2669                         atomic_read(&scst_cmd_count));
2670
2671                 spin_lock_irqsave(&scst_mcmd_lock, flags);
2672                 if (cmd->mgmt_cmnd)
2673                         scst_complete_cmd_mgmt(cmd, cmd->mgmt_cmnd);
2674                 spin_unlock_irqrestore(&scst_mcmd_lock, flags);
2675         }
2676
2677         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
2678                 if ((cmd->tgt_dev != NULL) &&
2679                     scst_is_ua_sense(cmd->sense)) {
2680                         /* This UA delivery failed, so requeue it */
2681                         TRACE_MGMT_DBG("Requeuing UA for delivery failed cmd "
2682                                 "%p", cmd);
2683                         scst_check_set_UA(cmd->tgt_dev, cmd->sense,
2684                                         SCST_SENSE_BUFFERSIZE, 1);
2685                 }
2686         }
2687
2688         __scst_cmd_put(cmd);
2689
2690         res = SCST_CMD_STATE_RES_CONT_NEXT;
2691
2692         TRACE_EXIT_HRES(res);
2693         return res;
2694 }
2695
2696 /*
2697  * No locks, but it must be externally serialized (see comment for
2698  * scst_cmd_init_done() in scsi_tgt.h)
2699  */
2700 static void scst_cmd_set_sn(struct scst_cmd *cmd)
2701 {
2702         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2703         unsigned long flags;
2704
2705         if (scst_is_implicit_hq(cmd)) {
2706                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "Implicit HQ cmd %p", cmd);
2707                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
2708         }
2709
2710         /* Optimized for lockless fast path */
2711
2712         scst_check_debug_sn(cmd);
2713
2714         if (cmd->dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) {
2715                 /*
2716                  * Not the best way, but well enough until there will be a
2717                  * possibility to specify queue type during pass-through
2718                  * commands submission.
2719                  */
2720                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
2721         }
2722
2723         switch(cmd->queue_type) {
2724         case SCST_CMD_QUEUE_SIMPLE:
2725         case SCST_CMD_QUEUE_UNTAGGED:
2726 #if 1 /* temporary, ToDo */
2727                 if (scst_cmd_is_expected_set(cmd)) {
2728                         if (cmd->expected_data_direction == SCST_DATA_READ)
2729                                 goto ordered;
2730                 } else
2731                         goto ordered;
2732 #endif
2733                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
2734                         /*
2735                          * atomic_inc_return() implies memory barrier to sync
2736                          * with scst_inc_expected_sn()
2737                          */
2738                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
2739                                 tgt_dev->curr_sn++;
2740                                 TRACE_SN("Incremented curr_sn %ld",
2741                                         tgt_dev->curr_sn);
2742                         }
2743                         cmd->sn_slot = tgt_dev->cur_sn_slot;
2744                         cmd->sn = tgt_dev->curr_sn;
2745                         
2746                         tgt_dev->prev_cmd_ordered = 0;
2747                 } else {
2748                         TRACE(TRACE_MINOR, "***WARNING*** Not enough SN slots "
2749                                 "%d", ARRAY_SIZE(tgt_dev->sn_slots));
2750                         goto ordered;
2751                 }
2752                 break;
2753
2754         case SCST_CMD_QUEUE_ORDERED:
2755                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "ORDERED cmd %p "
2756                         "(op %x)", cmd, cmd->cdb[0]);
2757 ordered:
2758                 if (!tgt_dev->prev_cmd_ordered) {
2759                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2760                         if (tgt_dev->num_free_sn_slots >= 0) {
2761                                 tgt_dev->num_free_sn_slots--;
2762                                 if (tgt_dev->num_free_sn_slots >= 0) {
2763                                         int i = 0;
2764                                         /*
2765                                          * Commands can finish in any order, so we don't
2766                                          * know, which slot is empty.
2767                                          */
2768                                         while(1) {
2769                                                 tgt_dev->cur_sn_slot++;
2770                                                 if (tgt_dev->cur_sn_slot == tgt_dev->sn_slots +
2771                                                                 ARRAY_SIZE(tgt_dev->sn_slots))
2772                                                         tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
2773
2774                                                 if (atomic_read(tgt_dev->cur_sn_slot) == 0)
2775                                                         break;
2776
2777                                                 i++;
2778                                                 sBUG_ON(i == ARRAY_SIZE(tgt_dev->sn_slots));
2779                                         }
2780                                         TRACE_SN("New cur SN slot %zd",
2781                                                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
2782                                 }
2783                         }
2784                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2785                 }
2786                 tgt_dev->prev_cmd_ordered = 1;
2787                 tgt_dev->curr_sn++;
2788                 cmd->sn = tgt_dev->curr_sn;
2789                 break;
2790
2791         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
2792                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "HQ cmd %p "
2793                         "(op %x)", cmd, cmd->cdb[0]);
2794                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2795                 tgt_dev->hq_cmd_count++;
2796                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2797                 goto out;
2798
2799         default:
2800                 sBUG();
2801         }
2802
2803         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
2804                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
2805                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
2806                 atomic_read(tgt_dev->cur_sn_slot), 
2807                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
2808                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
2809
2810         cmd->sn_set = 1;
2811 out:
2812         return;
2813 }
2814
2815 /*
2816  * Returns 0 on success, > 0 when we need to wait for unblock,
2817  * < 0 if there is no device (lun) or device type handler.
2818  *
2819  * No locks, but might be on IRQ, protection is done by the
2820  * suspended activity.
2821  */
2822 static int scst_translate_lun(struct scst_cmd *cmd)
2823 {
2824         struct scst_tgt_dev *tgt_dev = NULL;
2825         int res;
2826
2827         TRACE_ENTRY();
2828
2829         __scst_get(1);
2830
2831         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
2832                 struct list_head *sess_tgt_dev_list_head =
2833                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
2834                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %Ld)", cmd,
2835                         (uint64_t)cmd->lun);
2836                 res = -1;
2837                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
2838                                 sess_tgt_dev_list_entry) {
2839                         if (tgt_dev->lun == cmd->lun) {
2840                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
2841
2842                                 if (unlikely(tgt_dev->dev->handler == &scst_null_devtype)) {
2843                                         PRINT_INFO("Dev handler for device "
2844                                           "%Ld is NULL, the device will not be "
2845                                           "visible remotely", (uint64_t)cmd->lun);
2846                                         break;
2847                                 }
2848                                 
2849                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
2850                                 cmd->tgt_dev = tgt_dev;
2851                                 cmd->dev = tgt_dev->dev;
2852
2853                                 res = 0;
2854                                 break;
2855                         }
2856                 }
2857                 if (res != 0) {
2858                         TRACE(TRACE_MINOR, "tgt_dev for lun %Ld not found, command to "
2859                                 "unexisting LU?", (uint64_t)cmd->lun);
2860                         __scst_put();
2861                 }
2862         } else {
2863                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
2864                 __scst_put();
2865                 res = 1;
2866         }
2867
2868         TRACE_EXIT_RES(res);
2869         return res;
2870 }
2871
2872 /*
2873  * No locks, but might be on IRQ
2874  *
2875  * Returns 0 on success, > 0 when we need to wait for unblock,
2876  * < 0 if there is no device (lun) or device type handler.
2877  */
2878 static int __scst_init_cmd(struct scst_cmd *cmd)
2879 {
2880         int res = 0;
2881
2882         TRACE_ENTRY();
2883
2884         res = scst_translate_lun(cmd);
2885         if (likely(res == 0)) {
2886                 int cnt;
2887                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
2888                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
2889                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
2890                         TRACE(TRACE_MGMT_MINOR, "Too many pending commands (%d) in "
2891                                 "session, returning BUSY to initiator \"%s\"",
2892                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
2893                                   "Anonymous" : cmd->sess->initiator_name);
2894                         goto out_busy;
2895                 }
2896                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
2897                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
2898                         TRACE(TRACE_MGMT_MINOR, "Too many pending device commands "
2899                                 "(%d), returning BUSY to initiator \"%s\"",
2900                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
2901                                   "Anonymous" : cmd->sess->initiator_name);
2902                         goto out_busy;
2903                 }
2904                 if (!cmd->set_sn_on_restart_cmd)
2905                         scst_cmd_set_sn(cmd);
2906         } else if (res < 0) {
2907                 TRACE_DBG("Finishing cmd %p", cmd);
2908                 scst_set_cmd_error(cmd,
2909                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
2910                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2911         } else
2912                 goto out;
2913
2914 out:
2915         TRACE_EXIT_RES(res);
2916         return res;
2917
2918 out_busy:
2919         scst_set_busy(cmd);
2920         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2921         goto out;
2922 }
2923
2924 /* Called under scst_init_lock and IRQs disabled */
2925 static void scst_do_job_init(void)
2926 {
2927         struct scst_cmd *cmd;
2928         int susp;
2929
2930         TRACE_ENTRY();
2931
2932 restart:
2933         /*
2934          * There is no need for read barrier here, because we don't care where
2935          * this check will be done.
2936          */
2937         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
2938         if (scst_init_poll_cnt > 0)
2939                 scst_init_poll_cnt--;
2940
2941         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
2942                 int rc;
2943                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
2944                         continue;
2945                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2946                         spin_unlock_irq(&scst_init_lock);
2947                         rc = __scst_init_cmd(cmd);
2948                         spin_lock_irq(&scst_init_lock);
2949                         if (rc > 0) {
2950                                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, restarting");
2951                                 goto restart;
2952                         }
2953                 } else {
2954                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
2955                                 cmd, cmd->tag);
2956                         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2957                 }
2958
2959                 /*
2960                  * Deleting cmd from init cmd list after __scst_init_cmd()
2961                  * is necessary to keep the check in scst_init_cmd() correct
2962                  * to preserve the commands order.
2963                  *
2964                  * We don't care about the race, when init cmd list is empty
2965                  * and one command detected that it just was not empty, so
2966                  * it's inserting to it, but another command at the same time
2967                  * seeing init cmd list empty and goes directly, because it
2968                  * could affect only commands from the same initiator to the
2969                  * same tgt_dev, but init_cmd_done() doesn't guarantee the order
2970                  * in case of simultaneous such calls anyway.
2971                  */
2972                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
2973                 smp_wmb();
2974                 list_del(&cmd->cmd_list_entry);
2975                 spin_unlock(&scst_init_lock);
2976
2977                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
2978                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
2979                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2980                         list_add(&cmd->cmd_list_entry,
2981                                 &cmd->cmd_lists->active_cmd_list);
2982                 else
2983                         list_add_tail(&cmd->cmd_list_entry,
2984                                 &cmd->cmd_lists->active_cmd_list);
2985                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
2986                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
2987
2988                 spin_lock(&scst_init_lock);
2989                 goto restart;
2990         }
2991
2992         /* It isn't really needed, but let's keep it */
2993         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
2994                 goto restart;
2995
2996         TRACE_EXIT();
2997         return;
2998 }
2999
3000 static inline int test_init_cmd_list(void)
3001 {
3002         int res = (!list_empty(&scst_init_cmd_list) &&
3003                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
3004                   unlikely(kthread_should_stop()) ||
3005                   (scst_init_poll_cnt > 0);
3006         return res;
3007 }
3008
3009 int scst_init_cmd_thread(void *arg)
3010 {
3011         TRACE_ENTRY();
3012
3013         PRINT_INFO("Init thread started, PID %d", current->pid);
3014
3015         current->flags |= PF_NOFREEZE;
3016
3017         set_user_nice(current, -10);
3018
3019         spin_lock_irq(&scst_init_lock);
3020         while(!kthread_should_stop()) {
3021                 wait_queue_t wait;
3022                 init_waitqueue_entry(&wait, current);
3023
3024                 if (!test_init_cmd_list()) {
3025                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
3026                                                  &wait);
3027                         for (;;) {
3028                                 set_current_state(TASK_INTERRUPTIBLE);
3029                                 if (test_init_cmd_list())
3030                                         break;
3031                                 spin_unlock_irq(&scst_init_lock);
3032                                 schedule();
3033                                 spin_lock_irq(&scst_init_lock);
3034                         }
3035                         set_current_state(TASK_RUNNING);
3036                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
3037                 }
3038                 scst_do_job_init();
3039         }
3040         spin_unlock_irq(&scst_init_lock);
3041
3042         /*
3043          * If kthread_should_stop() is true, we are guaranteed to be
3044          * on the module unload, so scst_init_cmd_list must be empty.
3045          */
3046         sBUG_ON(!list_empty(&scst_init_cmd_list));
3047
3048         PRINT_INFO("Init thread PID %d finished", current->pid);
3049
3050         TRACE_EXIT();
3051         return 0;
3052 }
3053
3054 /* Called with no locks held */
3055 void scst_process_active_cmd(struct scst_cmd *cmd, int context)
3056 {
3057         int res;
3058
3059         TRACE_ENTRY();
3060
3061         EXTRACHECKS_BUG_ON(in_irq());
3062
3063         cmd->atomic = (context == SCST_CONTEXT_DIRECT_ATOMIC);
3064
3065         do {
3066                 switch (cmd->state) {
3067                 case SCST_CMD_STATE_PRE_PARSE:
3068                         res = scst_pre_parse(cmd);
3069                         EXTRACHECKS_BUG_ON(res ==
3070                                 SCST_CMD_STATE_RES_NEED_THREAD);
3071                         break;
3072
3073                 case SCST_CMD_STATE_DEV_PARSE:
3074                         res = scst_parse_cmd(cmd);
3075                         break;
3076
3077                 case SCST_CMD_STATE_PREPARE_SPACE:
3078                         res = scst_prepare_space(cmd);
3079                         break;
3080
3081                 case SCST_CMD_STATE_RDY_TO_XFER:
3082                         res = scst_rdy_to_xfer(cmd);
3083                         break;
3084
3085                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3086                         res = scst_tgt_pre_exec(cmd);
3087                         break;
3088
3089                 case SCST_CMD_STATE_SEND_TO_MIDLEV:
3090                         if (tm_dbg_check_cmd(cmd) != 0) {
3091                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
3092                                 TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
3093                                         "because of TM DBG delay", cmd,
3094                                         cmd->tag);
3095                                 break;
3096                         }
3097                         res = scst_send_to_midlev(&cmd);
3098                         /* !! At this point cmd, sess & tgt_dev can be already freed !! */
3099                         break;
3100
3101                 case SCST_CMD_STATE_PRE_DEV_DONE:
3102                         res = scst_pre_dev_done(cmd);
3103                         EXTRACHECKS_BUG_ON(res ==
3104                                 SCST_CMD_STATE_RES_NEED_THREAD);
3105                         break;
3106
3107                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3108                         res = scst_mode_select_checks(cmd);
3109                         break;
3110
3111                 case SCST_CMD_STATE_DEV_DONE:
3112                         res = scst_dev_done(cmd);
3113                         break;
3114
3115                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3116                         res = scst_pre_xmit_response(cmd);
3117                         EXTRACHECKS_BUG_ON(res ==
3118                                 SCST_CMD_STATE_RES_NEED_THREAD);
3119                         break;
3120
3121                 case SCST_CMD_STATE_XMIT_RESP:
3122                         res = scst_xmit_response(cmd);
3123                         break;
3124
3125                 case SCST_CMD_STATE_FINISHED:
3126                         res = scst_finish_cmd(cmd);
3127                         EXTRACHECKS_BUG_ON(res ==
3128                                 SCST_CMD_STATE_RES_NEED_THREAD);
3129                         break;
3130
3131                 default:
3132                         PRINT_ERROR("cmd (%p) in state %d, but shouldn't be",
3133                                cmd, cmd->state);
3134                         sBUG();
3135                         res = SCST_CMD_STATE_RES_CONT_NEXT;
3136                         break;
3137                 }
3138         } while(res == SCST_CMD_STATE_RES_CONT_SAME);
3139
3140         if (res == SCST_CMD_STATE_RES_CONT_NEXT) {
3141                 /* None */
3142         } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
3143                 spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3144                 switch (cmd->state) {
3145                 case SCST_CMD_STATE_PRE_PARSE:
3146                 case SCST_CMD_STATE_DEV_PARSE:
3147                 case SCST_CMD_STATE_PREPARE_SPACE:
3148                 case SCST_CMD_STATE_RDY_TO_XFER:
3149                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3150                 case SCST_CMD_STATE_SEND_TO_MIDLEV:
3151                 case SCST_CMD_STATE_PRE_DEV_DONE:
3152                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3153                 case SCST_CMD_STATE_DEV_DONE:
3154                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3155                 case SCST_CMD_STATE_XMIT_RESP:
3156                 case SCST_CMD_STATE_FINISHED:
3157                         TRACE_DBG("Adding cmd %p to head of active cmd list", cmd);
3158                         list_add(&cmd->cmd_list_entry,
3159                                 &cmd->cmd_lists->active_cmd_list);
3160                         break;
3161 #ifdef EXTRACHECKS
3162                 /* not very valid commands */
3163                 case SCST_CMD_STATE_DEFAULT:
3164                 case SCST_CMD_STATE_NEED_THREAD_CTX:
3165                         PRINT_ERROR("cmd %p is in state %d, not putting on "
3166                                 "useful list (left on scst cmd list)", cmd, 
3167                                 cmd->state);
3168                         spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3169                         sBUG();
3170                         spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3171                         break;
3172 #endif
3173                 default:
3174                         break;
3175                 }
3176                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3177                 spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3178         } else
3179                 sBUG();
3180
3181         TRACE_EXIT();
3182         return;
3183 }
3184
3185 /* Called under cmd_list_lock and IRQs disabled */
3186 static void scst_do_job_active(struct list_head *cmd_list,
3187         spinlock_t *cmd_list_lock, int context)
3188 {
3189         TRACE_ENTRY();
3190
3191 #ifdef EXTRACHECKS
3192         WARN_ON((context != SCST_CONTEXT_DIRECT_ATOMIC) && 
3193                 (context != SCST_CONTEXT_DIRECT));
3194 #endif
3195
3196         while (!list_empty(cmd_list)) {
3197                 struct scst_cmd *cmd = list_entry(cmd_list->next, typeof(*cmd),
3198                                         cmd_list_entry);
3199                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
3200                 list_del(&cmd->cmd_list_entry);
3201                 spin_unlock_irq(cmd_list_lock);
3202                 scst_process_active_cmd(cmd, context);
3203                 spin_lock_irq(cmd_list_lock);
3204         }
3205
3206         TRACE_EXIT();
3207         return;
3208 }
3209
3210 static inline int test_cmd_lists(struct scst_cmd_lists *p_cmd_lists)
3211 {
3212         int res = !list_empty(&p_cmd_lists->active_cmd_list) ||
3213             unlikely(kthread_should_stop()) ||
3214             tm_dbg_is_release();
3215         return res;
3216 }
3217
3218 int scst_cmd_thread(void *arg)
3219 {
3220         struct scst_cmd_lists *p_cmd_lists = (struct scst_cmd_lists*)arg;
3221
3222         TRACE_ENTRY();
3223
3224         PRINT_INFO("Processing thread started, PID %d", current->pid);
3225
3226 #if 0
3227         set_user_nice(current, 10);
3228 #endif
3229         current->flags |= PF_NOFREEZE;
3230
3231         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3232         while (!kthread_should_stop()) {
3233                 wait_queue_t wait;
3234                 init_waitqueue_entry(&wait, current);
3235
3236                 if (!test_cmd_lists(p_cmd_lists)) {
3237                         add_wait_queue_exclusive(&p_cmd_lists->cmd_list_waitQ,
3238                                 &wait);
3239                         for (;;) {
3240                                 set_current_state(TASK_INTERRUPTIBLE);
3241                                 if (test_cmd_lists(p_cmd_lists))
3242                                         break;
3243                                 spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3244                                 schedule();
3245                                 spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3246                         }
3247                         set_current_state(TASK_RUNNING);
3248                         remove_wait_queue(&p_cmd_lists->cmd_list_waitQ, &wait);
3249                 }
3250
3251                 if (tm_dbg_is_release()) {
3252                         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3253                         tm_dbg_check_released_cmds();
3254                         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3255                 }
3256
3257                 scst_do_job_active(&p_cmd_lists->active_cmd_list,
3258                         &p_cmd_lists->cmd_list_lock, SCST_CONTEXT_DIRECT);
3259         }
3260         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3261
3262 #ifdef EXTRACHECKS
3263         /*
3264          * If kthread_should_stop() is true, we are guaranteed to be either
3265          * on the module unload, or there must be at least one other thread to
3266          * process the commands lists.
3267          */
3268         if (p_cmd_lists == &scst_main_cmd_lists) {
3269                 sBUG_ON((scst_threads_info.nr_cmd_threads == 1) &&
3270                          !list_empty(&scst_main_cmd_lists.active_cmd_list));
3271         }
3272 #endif
3273
3274         PRINT_INFO("Processing thread PID %d finished", current->pid);
3275
3276         TRACE_EXIT();
3277         return 0;
3278 }
3279
3280 void scst_cmd_tasklet(long p)
3281 {
3282         struct scst_tasklet *t = (struct scst_tasklet*)p;
3283
3284         TRACE_ENTRY();
3285
3286         spin_lock_irq(&t->tasklet_lock);
3287         scst_do_job_active(&t->tasklet_cmd_list, &t->tasklet_lock,
3288                 SCST_CONTEXT_DIRECT_ATOMIC);
3289         spin_unlock_irq(&t->tasklet_lock);
3290
3291         TRACE_EXIT();
3292         return;
3293 }
3294
3295 /*
3296  * Returns 0 on success, < 0 if there is no device handler or
3297  * > 0 if SCST_FLAG_SUSPENDED set and SCST_FLAG_SUSPENDING - not.
3298  * No locks, protection is done by the suspended activity.
3299  */
3300 static int scst_mgmt_translate_lun(struct scst_mgmt_cmd *mcmd)
3301 {
3302         struct scst_tgt_dev *tgt_dev = NULL;
3303         struct list_head *sess_tgt_dev_list_head;
3304         int res = -1;
3305
3306         TRACE_ENTRY();
3307
3308         TRACE_DBG("Finding tgt_dev for mgmt cmd %p (lun %Ld)", mcmd,
3309               (uint64_t)mcmd->lun);
3310
3311         __scst_get(1);
3312
3313         if (unlikely(test_bit(SCST_FLAG_SUSPENDED, &scst_flags) &&
3314                      !test_bit(SCST_FLAG_SUSPENDING, &scst_flags))) {
3315                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3316                 __scst_put();
3317                 res = 1;
3318                 goto out;
3319         }
3320
3321         sess_tgt_dev_list_head =
3322                 &mcmd->sess->sess_tgt_dev_list_hash[HASH_VAL(mcmd->lun)];
3323         list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3324                         sess_tgt_dev_list_entry) {
3325                 if (tgt_dev->lun == mcmd->lun) {
3326                         TRACE_DBG("tgt_dev %p found", tgt_dev);
3327                         mcmd->mcmd_tgt_dev = tgt_dev;
3328                         res = 0;
3329                         break;
3330                 }
3331         }
3332         if (mcmd->mcmd_tgt_dev == NULL)
3333                 __scst_put();
3334
3335 out:
3336         TRACE_EXIT_HRES(res);
3337         return res;
3338 }
3339
3340 /* scst_mcmd_lock supposed to be held and IRQ off */
3341 void scst_complete_cmd_mgmt(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd)
3342 {
3343         TRACE_ENTRY();
3344
3345         TRACE_MGMT_DBG("cmd %p completed (tag %llu, mcmd %p, "
3346                 "mcmd->cmd_wait_count %d)", cmd, cmd->tag, mcmd,
3347                 mcmd->cmd_wait_count);
3348
3349         cmd->mgmt_cmnd = NULL;
3350
3351         if (cmd->completed)
3352                 mcmd->completed_cmd_count++;
3353
3354         mcmd->cmd_wait_count--;
3355         if (mcmd->cmd_wait_count > 0) {
3356                 TRACE_MGMT_DBG("cmd_wait_count(%d) not 0, skipping",
3357                         mcmd->cmd_wait_count);
3358                 goto out_unlock;
3359         }
3360
3361         mcmd->state = SCST_MGMT_CMD_STATE_DONE;
3362
3363         if (mcmd->completed) {
3364                 TRACE_MGMT_DBG("Adding mgmt cmd %p to active mgmt cmd list",
3365                         mcmd);
3366                 list_add_tail(&mcmd->mgmt_cmd_list_entry,
3367                         &scst_active_mgmt_cmd_list);
3368         }
3369
3370         wake_up(&scst_mgmt_cmd_list_waitQ);
3371
3372 out:
3373         TRACE_EXIT();
3374         return;
3375
3376 out_unlock:
3377         spin_unlock_irq(&scst_mcmd_lock);
3378         goto out;
3379 }
3380
3381 static int scst_call_dev_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
3382         struct scst_tgt_dev *tgt_dev, int set_status)
3383 {
3384         int res = SCST_DEV_TM_NOT_COMPLETED;
3385         struct scst_dev_type *h = tgt_dev->dev->handler;
3386
3387         if (h->task_mgmt_fn) {
3388                 TRACE_MGMT_DBG("Calling dev handler %s task_mgmt_fn(fn=%d)",
3389                         h->name, mcmd->fn);
3390                 EXTRACHECKS_BUG_ON(in_irq());
3391                 res = h->task_mgmt_fn(mcmd, tgt_dev);
3392                 TRACE_MGMT_DBG("Dev handler %s task_mgmt_fn() returned %d",
3393                       h->name, res);
3394                 if (set_status && (res != SCST_DEV_TM_NOT_COMPLETED))
3395                         mcmd->status = res;
3396         }
3397         return res;
3398 }
3399
3400 static inline int scst_is_strict_mgmt_fn(int mgmt_fn)
3401 {
3402         switch(mgmt_fn) {
3403 #ifdef ABORT_CONSIDER_FINISHED_TASKS_AS_NOT_EXISTING
3404                 case SCST_ABORT_TASK:
3405 #endif
3406 #if 0
3407                 case SCST_ABORT_TASK_SET:
3408                 case SCST_CLEAR_TASK_SET:
3409 #endif
3410                         return 1;
3411                 default:
3412                         return 0;
3413         }
3414 }
3415
3416 /* 
3417  * Might be called under sess_list_lock and IRQ off + BHs also off
3418  * Returns -1 if command is being executed (ABORT failed), 0 otherwise
3419  */
3420 void scst_abort_cmd(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd,
3421         int other_ini, int call_dev_task_mgmt_fn)
3422 {
3423         unsigned long flags;
3424
3425         TRACE_ENTRY();
3426
3427         TRACE(((mcmd != NULL) && (mcmd->fn == SCST_ABORT_TASK)) ? TRACE_MGMT_MINOR : TRACE_MGMT,
3428                 "Aborting cmd %p (tag %llu)", cmd, cmd->tag);
3429
3430         if (other_ini) {
3431                 set_bit(SCST_CMD_ABORTED_OTHER, &cmd->cmd_flags);
3432                 smp_mb__after_set_bit();
3433         } else {
3434                 /* Might be necessary if command aborted several times */
3435                 clear_bit(SCST_CMD_ABORTED_OTHER, &cmd->cmd_flags);
3436         }
3437         set_bit(SCST_CMD_ABORTED, &cmd->cmd_flags);
3438         /* To sync with cmd->finished set in scst_finish_cmd() */
3439         smp_mb__after_set_bit(); 
3440
3441         if (cmd->tgt_dev == NULL) {
3442                 unsigned long flags;
3443                 spin_lock_irqsave(&scst_init_lock, flags);
3444                 scst_init_poll_cnt++;
3445                 spin_unlock_irqrestore(&scst_init_lock, flags);
3446                 wake_up(&scst_init_cmd_list_waitQ);
3447         }
3448
3449         if (call_dev_task_mgmt_fn && (cmd->tgt_dev != NULL)) {
3450                 EXTRACHECKS_BUG_ON(irqs_disabled());
3451                 scst_call_dev_task_mgmt_fn(mcmd, cmd->tgt_dev, 1);
3452         }
3453
3454         spin_lock_irqsave(&scst_mcmd_lock, flags);
3455         if ((mcmd != NULL) && !cmd->finished) {
3456                 /*
3457                  * Delay the response until the command's finish in
3458                  * order to guarantee that "no further responses from
3459                  * the task are sent to the SCSI initiator port" after
3460                  * response from the TM function is sent (SAM). Plus,
3461                  * we must wait here to be sure that we won't receive
3462                  * double commands with the same tag.
3463                  */
3464                 TRACE_MGMT_DBG("cmd %p (tag %llu) being executed/xmitted "
3465                         "(state %d), deferring ABORT...", cmd, cmd->tag,
3466                         cmd->state);
3467 #ifdef EXTRACHECKS
3468                 if (cmd->mgmt_cmnd) {
3469                         printk(KERN_ALERT "cmd %p (tag %llu, state %d) "
3470                                 "has non-NULL mgmt_cmnd %p!!! Current "
3471                                 "mcmd %p\n", cmd, cmd->tag, cmd->state,
3472                                 cmd->mgmt_cmnd, mcmd);
3473                 }
3474 #endif
3475                 sBUG_ON(cmd->mgmt_cmnd);
3476
3477                 mcmd->cmd_wait_count++;
3478
3479                 /*
3480                  * cmd can't die here or sess_list_lock already taken and cmd is
3481                  * in the search list
3482                  */
3483                 cmd->mgmt_cmnd = mcmd;
3484         }
3485         spin_unlock_irqrestore(&scst_mcmd_lock, flags);
3486
3487         tm_dbg_release_cmd(cmd);
3488
3489         TRACE_EXIT();
3490         return;
3491 }
3492
3493 /* No locks */
3494 static int scst_set_mcmd_next_state(struct scst_mgmt_cmd *mcmd)
3495 {
3496         int res;
3497         spin_lock_irq(&scst_mcmd_lock);
3498         if (mcmd->cmd_wait_count != 0) {
3499                 TRACE_MGMT_DBG("cmd_wait_count(%d) not 0, preparing to "
3500                         "wait", mcmd->cmd_wait_count);
3501                 mcmd->state = SCST_MGMT_CMD_STATE_EXECUTING;
3502                 res = -1;
3503         } else {
3504                 mcmd->state = SCST_MGMT_CMD_STATE_DONE;
3505                 res = 0;
3506         }
3507         mcmd->completed = 1;
3508         spin_unlock_irq(&scst_mcmd_lock);
3509         return res;
3510 }
3511
3512 static bool __scst_check_unblock_aborted_cmd(struct scst_cmd *cmd,
3513         struct list_head *list_entry)
3514 {
3515         bool res;
3516         if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
3517                 list_del(list_entry);
3518                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
3519                 list_add_tail(&cmd->cmd_list_entry,
3520                         &cmd->cmd_lists->active_cmd_list);
3521                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3522                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
3523                 res = 1;
3524         } else
3525                 res = 0;
3526         return res;
3527 }
3528
3529 static void scst_unblock_aborted_cmds(int scst_mutex_held)
3530 {
3531         struct scst_device *dev;
3532
3533         TRACE_ENTRY();
3534
3535         if (!scst_mutex_held)
3536                 mutex_lock(&scst_mutex);
3537
3538         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
3539                 struct scst_cmd *cmd, *tc