36c57e8899a61eb78ff4658720b93c92eb205324
[mirror/scst/.git] / scst / src / scst_targ.c
1 /*
2  *  scst_targ.c
3  *  
4  *  Copyright (C) 2004-2007 Vladislav Bolkhovitin <vst@vlnb.net>
5  *                 and Leonid Stoljar
6  *  
7  *  This program is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU General Public License
9  *  as published by the Free Software Foundation, version 2
10  *  of the License.
11  * 
12  *  This program is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  *  GNU General Public License for more details.
16  */
17
18 #include <linux/init.h>
19 #include <linux/kernel.h>
20 #include <linux/errno.h>
21 #include <linux/list.h>
22 #include <linux/spinlock.h>
23 #include <linux/slab.h>
24 #include <linux/sched.h>
25 #include <linux/smp_lock.h>
26 #include <asm/unistd.h>
27 #include <asm/string.h>
28 #include <linux/kthread.h>
29 #include <linux/delay.h>
30
31 #include "scsi_tgt.h"
32 #include "scst_priv.h"
33
34 static void scst_cmd_set_sn(struct scst_cmd *cmd);
35 static int __scst_init_cmd(struct scst_cmd *cmd);
36 static void scst_finish_cmd_mgmt(struct scst_cmd *cmd);
37
38 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
39 {
40         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
41         unsigned long flags;
42
43         spin_lock_irqsave(&t->tasklet_lock, flags);
44         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
45                 smp_processor_id());
46         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
47         spin_unlock_irqrestore(&t->tasklet_lock, flags);
48
49         tasklet_schedule(&t->tasklet);
50 }
51
52 /* 
53  * Must not be called in parallel with scst_unregister_session_ex() for the 
54  * same sess
55  */
56 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
57                              const uint8_t *lun, int lun_len,
58                              const uint8_t *cdb, int cdb_len, int atomic)
59 {
60         struct scst_cmd *cmd;
61
62         TRACE_ENTRY();
63
64 #ifdef EXTRACHECKS
65         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
66                 PRINT_CRIT_ERROR("%s", "New cmd while shutting down the session");
67                 sBUG();
68         }
69 #endif
70
71         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
72         if (cmd == NULL)
73                 goto out;
74
75         cmd->sess = sess;
76         cmd->tgt = sess->tgt;
77         cmd->tgtt = sess->tgt->tgtt;
78
79         cmd->start_time = jiffies;
80
81         /* 
82          * For both wrong lun and CDB defer the error reporting for
83          * scst_cmd_init_done()
84          */
85
86         cmd->lun = scst_unpack_lun(lun, lun_len);
87
88         if (cdb_len <= SCST_MAX_CDB_SIZE) {
89                 memcpy(cmd->cdb, cdb, cdb_len);
90                 cmd->cdb_len = cdb_len;
91         }
92
93         TRACE_DBG("cmd %p, sess %p", cmd, sess);
94         scst_sess_get(sess);
95
96 out:
97         TRACE_EXIT();
98         return cmd;
99 }
100
101 static int scst_init_cmd(struct scst_cmd *cmd, int context)
102 {
103         int rc;
104
105         TRACE_ENTRY();
106
107         /* See the comment in scst_do_job_init() */
108         if (unlikely(!list_empty(&scst_init_cmd_list))) {
109                 TRACE_MGMT_DBG("%s", "init cmd list busy");
110                 goto out_redirect;
111         }
112         /*
113          * Memory barrier isn't necessary here, because CPU appears to
114          * be self-consistent
115          */
116
117         rc = __scst_init_cmd(cmd);
118         if (unlikely(rc > 0))
119                 goto out_redirect;
120         else if (unlikely(rc != 0))
121                 goto out;
122
123         /* Small context optimization */
124         if (((context == SCST_CONTEXT_TASKLET) ||
125              (context == SCST_CONTEXT_DIRECT_ATOMIC)) && 
126             scst_cmd_is_expected_set(cmd)) {
127                 if (cmd->expected_data_direction == SCST_DATA_WRITE) {
128                         if ( !test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
129                                         &cmd->tgt_dev->tgt_dev_flags))
130                                 context = SCST_CONTEXT_THREAD;
131                 } else {
132                         if ( !test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
133                                         &cmd->tgt_dev->tgt_dev_flags))
134                                 context = SCST_CONTEXT_THREAD;
135                 }
136         }
137
138 out:
139         TRACE_EXIT_RES(context);
140         return context;
141
142 out_redirect:
143         if (cmd->preprocessing_only) {
144                 /*
145                  * Poor man solution for single threaded targets, where 
146                  * blocking receiver at least sometimes means blocking all.
147                  */
148                 sBUG_ON(context != SCST_CONTEXT_DIRECT);
149                 scst_set_busy(cmd);
150                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
151                 /* Keep initiator away from too many BUSY commands */
152                 if (!in_interrupt() && !in_atomic())
153                         msleep(50);
154                 else
155                         WARN_ON_ONCE(1);
156         } else {
157                 unsigned long flags;
158                 spin_lock_irqsave(&scst_init_lock, flags);
159                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
160                         "%d)", cmd, atomic_read(&scst_cmd_count));
161                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
162                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
163                         scst_init_poll_cnt++;
164                 spin_unlock_irqrestore(&scst_init_lock, flags);
165                 wake_up(&scst_init_cmd_list_waitQ);
166                 context = -1;
167         }
168         goto out;
169 }
170
171 #ifdef MEASURE_LATENCY
172 static inline uint64_t scst_sec_to_nsec(time_t sec)
173 {
174         return (uint64_t)sec * 1000000000;
175 }
176 #endif
177
178 void scst_cmd_init_done(struct scst_cmd *cmd, int pref_context)
179 {
180         unsigned long flags;
181         struct scst_session *sess = cmd->sess;
182
183         TRACE_ENTRY();
184
185 #ifdef MEASURE_LATENCY
186         {
187                 struct timespec ts;
188                 getnstimeofday(&ts);
189                 cmd->start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
190                 TRACE_DBG("cmd %p (sess %p): start %Ld (tv_sec %ld, "
191                         "tv_nsec %ld)", cmd, sess, cmd->start, ts.tv_sec,
192                         ts.tv_nsec);
193         }
194 #endif
195
196         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
197         TRACE(TRACE_SCSI, "tag=%llu, lun=%Ld, CDB len=%d", cmd->tag, 
198                 (uint64_t)cmd->lun, cmd->cdb_len);
199         PRINT_BUFF_FLAG(TRACE_SCSI|TRACE_RCV_BOT, "Recieving CDB",
200                 cmd->cdb, cmd->cdb_len);
201
202 #ifdef EXTRACHECKS
203         if (unlikely(in_irq()) && ((pref_context == SCST_CONTEXT_DIRECT) ||
204                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
205         {
206                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
207                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
208                         cmd->tgtt->name);
209                 pref_context = SCST_CONTEXT_TASKLET;
210         }
211 #endif
212
213         atomic_inc(&sess->sess_cmd_count);
214
215         spin_lock_irqsave(&sess->sess_list_lock, flags);
216
217         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
218                 /*
219                  * We have to always keep command in the search list from the
220                  * very beginning, because otherwise it can be missed during
221                  * TM processing. This check is needed because there might be
222                  * old, i.e. deferred, commands and new, i.e. just coming, ones.
223                  */
224                 if (cmd->search_cmd_list_entry.next == NULL)
225                         list_add_tail(&cmd->search_cmd_list_entry,
226                                 &sess->search_cmd_list);
227                 switch(sess->init_phase) {
228                 case SCST_SESS_IPH_SUCCESS:
229                         break;
230                 case SCST_SESS_IPH_INITING:
231                         TRACE_DBG("Adding cmd %p to init deferred cmd list", cmd);
232                         list_add_tail(&cmd->cmd_list_entry, 
233                                 &sess->init_deferred_cmd_list);
234                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
235                         goto out;
236                 case SCST_SESS_IPH_FAILED:
237                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
238                         scst_set_busy(cmd);
239                         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
240                         goto active;
241                 default:
242                         sBUG();
243                 }
244         } else
245                 list_add_tail(&cmd->search_cmd_list_entry, &sess->search_cmd_list);
246
247         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
248
249         if (unlikely(cmd->lun == (lun_t)-1)) {
250                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
251                 scst_set_cmd_error(cmd,
252                         SCST_LOAD_SENSE(scst_sense_lun_not_supported));
253                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
254                 goto active;
255         }
256
257         if (unlikely(cmd->cdb_len == 0)) {
258                 PRINT_ERROR("Wrong CDB len %d, finishing cmd", 0);
259                 scst_set_cmd_error(cmd,
260                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
261                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
262                 goto active;
263         }
264
265         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
266                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
267                 scst_set_cmd_error(cmd,
268                         SCST_LOAD_SENSE(scst_sense_invalid_message));
269                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
270                 goto active;
271         }
272
273         cmd->state = SCST_CMD_STATE_INIT;
274         /* cmd must be inited here to keep the order */
275         pref_context = scst_init_cmd(cmd, pref_context);
276         if (unlikely(pref_context < 0))
277                 goto out;
278
279 active:
280         /* Here cmd must not be in any cmd list, no locks */
281         switch (pref_context) {
282         case SCST_CONTEXT_TASKLET:
283                 scst_schedule_tasklet(cmd);
284                 break;
285
286         case SCST_CONTEXT_DIRECT:
287         case SCST_CONTEXT_DIRECT_ATOMIC:
288                 scst_process_active_cmd(cmd, pref_context);
289                 /* For *NEED_THREAD wake_up() is already done */
290                 break;
291
292         default:
293                 PRINT_ERROR("Context %x is undefined, using the thread one",
294                         pref_context);
295                 /* go through */
296         case SCST_CONTEXT_THREAD:
297                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
298                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
299                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
300                         list_add(&cmd->cmd_list_entry,
301                                 &cmd->cmd_lists->active_cmd_list);
302                 else
303                         list_add_tail(&cmd->cmd_list_entry,
304                                 &cmd->cmd_lists->active_cmd_list);
305                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
306                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
307                 break;
308         }
309
310 out:
311         TRACE_EXIT();
312         return;
313 }
314
315 static int scst_pre_parse(struct scst_cmd *cmd)
316 {
317         int res = SCST_CMD_STATE_RES_CONT_SAME;
318         struct scst_device *dev = cmd->dev;
319         int rc;
320
321         TRACE_ENTRY();
322
323         cmd->inc_expected_sn_on_done = dev->handler->exec_sync ||
324                 (!dev->has_own_order_mgmt &&
325                  ((dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) ||
326                   (cmd->queue_type == SCST_CMD_QUEUE_ORDERED)));
327
328         sBUG_ON(cmd->internal);
329
330         /*
331          * Expected transfer data supplied by the SCSI transport via the
332          * target driver are untrusted, so we prefer to fetch them from CDB.
333          * Additionally, not all transports support supplying the expected
334          * transfer data.
335          */
336
337         rc = scst_get_cdb_info(cmd);
338         if (unlikely(rc != 0)) {
339                 if (rc > 0) {
340                         PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
341                         goto out_xmit;
342                 }
343                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
344                         "Should you update scst_scsi_op_table?",
345                         cmd->cdb[0], dev->handler->name);
346                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
347 #ifdef USE_EXPECTED_VALUES
348                 if (scst_cmd_is_expected_set(cmd)) {
349                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
350                                 "direction %d, transfer_len %d",
351                                 cmd->expected_data_direction,
352                                 cmd->expected_transfer_len);
353                         cmd->data_direction = cmd->expected_data_direction;
354                         
355                         cmd->bufflen = cmd->expected_transfer_len;
356                         /* Restore (likely) lost CDB length */
357                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
358                         if (cmd->cdb_len == -1) {
359                                 PRINT_ERROR("Unable to get CDB length for "
360                                         "opcode 0x%02x. Returning INVALID "
361                                         "OPCODE", cmd->cdb[0]);
362                                 scst_set_cmd_error(cmd,
363                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
364                                 goto out_xmit;
365                         }
366                 } else {
367                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
368                              "target %s not supplied expected values",
369                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
370                         scst_set_cmd_error(cmd,
371                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
372                         goto out_xmit;
373                 }
374 #else
375                 scst_set_cmd_error(cmd,
376                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
377                 goto out_xmit;
378 #endif
379         } else {
380                 TRACE(TRACE_SCSI, "op_name <%s>, direction=%d (expected %d, "
381                         "set %s), transfer_len=%d (expected len %d), flags=%d",
382                         cmd->op_name, cmd->data_direction,
383                         cmd->expected_data_direction,
384                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
385                         cmd->bufflen, cmd->expected_transfer_len, cmd->op_flags);
386
387                 if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
388                         if (scst_cmd_is_expected_set(cmd)) {
389                                 /*
390                                  * Command data length can't be easily
391                                  * determined from the CDB. ToDo, all such
392                                  * commands should be fixed. Until they are
393                                  * fixed, get it from the supplied expected
394                                  * value, but limit it to some reasonable
395                                  * value (15MB).
396                                  */
397                                 cmd->bufflen = min(cmd->expected_transfer_len,
398                                                         15*1024*1024);
399                                 cmd->op_flags &= ~SCST_UNKNOWN_LENGTH;
400                         } else
401                                 cmd->bufflen = 0;
402                 }
403         }
404
405         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
406                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
407                             "(opcode 0x%02x)", cmd->cdb[0]);
408                 scst_set_cmd_error(cmd,
409                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
410                 goto out_xmit;
411         }
412
413         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
414                 PRINT_ERROR("Linked commands are not supported "
415                             "(opcode 0x%02x)", cmd->cdb[0]);
416                 scst_set_cmd_error(cmd,
417                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
418                 goto out_xmit;
419         }
420
421         cmd->state = SCST_CMD_STATE_DEV_PARSE;
422
423 out:
424         TRACE_EXIT_RES(res);
425         return res;
426
427 out_xmit:
428         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
429         res = SCST_CMD_STATE_RES_CONT_SAME;
430         goto out;
431 }
432
433 static int scst_parse_cmd(struct scst_cmd *cmd)
434 {
435         int res = SCST_CMD_STATE_RES_CONT_SAME;
436         int state;
437         struct scst_device *dev = cmd->dev;
438         int atomic = scst_cmd_atomic(cmd);
439         int orig_bufflen = cmd->bufflen;
440
441         TRACE_ENTRY();
442
443         if (atomic && !dev->handler->parse_atomic) {
444                 TRACE_DBG("Dev handler %s parse() can not be "
445                       "called in atomic context, rescheduling to the thread",
446                       dev->handler->name);
447                 res = SCST_CMD_STATE_RES_NEED_THREAD;
448                 goto out;
449         }
450
451         if (likely(!scst_is_cmd_local(cmd))) {
452                 TRACE_DBG("Calling dev handler %s parse(%p)",
453                       dev->handler->name, cmd);
454                 TRACE_BUFF_FLAG(TRACE_SND_BOT, "Parsing: ", cmd->cdb, cmd->cdb_len);
455                 state = dev->handler->parse(cmd);
456                 /* Caution: cmd can be already dead here */
457                 TRACE_DBG("Dev handler %s parse() returned %d",
458                         dev->handler->name, state);
459
460                 switch (state) {
461                 case SCST_CMD_STATE_NEED_THREAD_CTX:
462                         TRACE_DBG("Dev handler %s parse() requested thread "
463                               "context, rescheduling", dev->handler->name);
464                         res = SCST_CMD_STATE_RES_NEED_THREAD;
465                         goto out;
466
467                 case SCST_CMD_STATE_STOP:
468                         TRACE_DBG("Dev handler %s parse() requested stop "
469                                 "processing", dev->handler->name);
470                         res = SCST_CMD_STATE_RES_CONT_NEXT;
471                         goto out;
472                 }
473
474                 if (state == SCST_CMD_STATE_DEFAULT)
475                         state = SCST_CMD_STATE_PREPARE_SPACE;
476         } else
477                 state = SCST_CMD_STATE_PREPARE_SPACE;
478
479         if (cmd->data_len == -1)
480                 cmd->data_len = cmd->bufflen;
481
482         if (cmd->data_buf_alloced && unlikely((orig_bufflen > cmd->bufflen))) {
483                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
484                         "is less, than required (size %d)", cmd->bufflen,
485                         orig_bufflen);
486                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
487                 goto out_error;
488         }
489
490         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
491                 goto set_res;
492
493         if (unlikely((cmd->bufflen == 0) &&
494                      (cmd->op_flags & SCST_UNKNOWN_LENGTH))) {
495                 PRINT_ERROR("Unknown data transfer length for opcode 0x%x "
496                         "(handler %s, target %s)", cmd->cdb[0],
497                         dev->handler->name, cmd->tgtt->name);
498                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
499                 goto out_error;
500         }
501
502 #ifdef EXTRACHECKS
503         if ((cmd->bufflen != 0) &&
504             ((cmd->data_direction == SCST_DATA_NONE) ||
505              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
506                 PRINT_ERROR("Dev handler %s parse() returned "
507                         "invalid cmd data_direction %d, bufflen %d, state %d "
508                         "or sg %p (opcode 0x%x)", dev->handler->name, 
509                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
510                         cmd->cdb[0]);
511                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
512                 goto out_error;
513         }
514 #endif
515
516         if (scst_cmd_is_expected_set(cmd)) {
517 #ifdef USE_EXPECTED_VALUES
518 #       ifdef EXTRACHECKS
519                 if ((cmd->data_direction != cmd->expected_data_direction) ||
520                     (cmd->bufflen != cmd->expected_transfer_len)) {
521                         PRINT_ERROR("Expected values don't match decoded ones: "
522                                 "data_direction %d, expected_data_direction %d, "
523                                 "bufflen %d, expected_transfer_len %d",
524                                 cmd->data_direction, cmd->expected_data_direction,
525                                 cmd->bufflen, cmd->expected_transfer_len);
526                         PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
527                 }
528 #       endif
529                 cmd->data_direction = cmd->expected_data_direction;
530                 cmd->bufflen = cmd->expected_transfer_len;
531 #else
532                 if (unlikely(cmd->data_direction != cmd->expected_data_direction)) {
533                         if (((cmd->expected_data_direction != SCST_DATA_NONE) ||
534                             (cmd->bufflen != 0)) &&
535                             /* Crazy VMware people sometimes do TUR with READ direction */
536                             !(cmd->cdb[0] == TEST_UNIT_READY)) {
537                                 PRINT_ERROR("Expected data direction %d for opcode "
538                                         "0x%02x (handler %s, target %s) doesn't match "
539                                         "decoded value %d", cmd->expected_data_direction,
540                                         cmd->cdb[0], dev->handler->name,
541                                         cmd->tgtt->name, cmd->data_direction);
542                                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
543                                 scst_set_cmd_error(cmd,
544                                         SCST_LOAD_SENSE(scst_sense_invalid_message));
545                                 goto out_dev_done;
546                         }
547                 }
548                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
549                         TRACE(TRACE_MINOR, "Warning: expected transfer length "
550                                 "%d for opcode 0x%02x (handler %s, target %s) "
551                                 "doesn't match decoded value %d. Faulty "
552                                 "initiator (e.g. VMware is known to be such) or "
553                                 "scst_scsi_op_table should be updated?",
554                                 cmd->expected_transfer_len, cmd->cdb[0],
555                                 dev->handler->name, cmd->tgtt->name,
556                                 cmd->bufflen);
557                         PRINT_BUFF_FLAG(TRACE_MINOR, "Suspicious CDB", cmd->cdb,
558                                 cmd->cdb_len);
559                 }
560 #endif
561         }
562
563         if (unlikely(cmd->data_direction == SCST_DATA_UNKNOWN)) {
564                 PRINT_ERROR("Unknown data direction. Opcode 0x%x, handler %s, "
565                         "target %s", cmd->cdb[0], dev->handler->name,
566                         cmd->tgtt->name);
567                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
568                 goto out_error;
569         }
570
571 set_res:
572         switch (state) {
573         case SCST_CMD_STATE_PREPARE_SPACE:
574         case SCST_CMD_STATE_PRE_PARSE:
575         case SCST_CMD_STATE_DEV_PARSE:
576         case SCST_CMD_STATE_RDY_TO_XFER:
577         case SCST_CMD_STATE_TGT_PRE_EXEC:
578         case SCST_CMD_STATE_SEND_TO_MIDLEV:
579         case SCST_CMD_STATE_PRE_DEV_DONE:
580         case SCST_CMD_STATE_DEV_DONE:
581         case SCST_CMD_STATE_PRE_XMIT_RESP:
582         case SCST_CMD_STATE_XMIT_RESP:
583         case SCST_CMD_STATE_FINISHED:
584                 cmd->state = state;
585                 res = SCST_CMD_STATE_RES_CONT_SAME;
586                 break;
587
588         default:
589                 if (state >= 0) {
590                         PRINT_ERROR("Dev handler %s parse() returned "
591                              "invalid cmd state %d (opcode %d)", 
592                              dev->handler->name, state, cmd->cdb[0]);
593                 } else {
594                         PRINT_ERROR("Dev handler %s parse() returned "
595                                 "error %d (opcode %d)", dev->handler->name, 
596                                 state, cmd->cdb[0]);
597                 }
598                 goto out_error;
599         }
600
601         if (cmd->resp_data_len == -1) {
602                 if (cmd->data_direction == SCST_DATA_READ)
603                         cmd->resp_data_len = cmd->bufflen;
604                 else
605                          cmd->resp_data_len = 0;
606         }
607         
608 out:
609         TRACE_EXIT_HRES(res);
610         return res;
611
612 out_error:
613         /* dev_done() will be called as part of the regular cmd's finish */
614         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
615
616 #ifndef USE_EXPECTED_VALUES
617 out_dev_done:
618 #endif
619         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
620         res = SCST_CMD_STATE_RES_CONT_SAME;
621         goto out;
622 }
623
624 static int scst_prepare_space(struct scst_cmd *cmd)
625 {
626         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
627
628         TRACE_ENTRY();
629
630         if (cmd->data_direction == SCST_DATA_NONE)
631                 goto prep_done;
632
633         if (cmd->data_buf_tgt_alloc) {
634                 int orig_bufflen = cmd->bufflen;
635
636                 TRACE_MEM("%s", "Custom tgt data buf allocation requested");
637
638                 r = cmd->tgtt->alloc_data_buf(cmd);
639                 if (r > 0)
640                         goto alloc;
641                 else if (r == 0) {
642                         if (unlikely(cmd->bufflen == 0)) {
643                                 /* See comment in scst_alloc_space() */
644                                 if (cmd->sg == NULL)
645                                         goto alloc;
646                         }
647                         cmd->data_buf_alloced = 1;
648                         if (unlikely(orig_bufflen < cmd->bufflen)) {
649                                 PRINT_ERROR("Target driver allocated data "
650                                         "buffer (size %d), is less, than "
651                                         "required (size %d)", orig_bufflen,
652                                         cmd->bufflen);
653                                 goto out_error;
654                         }
655                         TRACE_MEM("%s", "data_buf_alloced, returning");
656                 }
657                 goto check;
658         }
659
660 alloc:
661         r = scst_alloc_space(cmd);
662         
663 check:
664         if (r != 0) {
665                 if (scst_cmd_atomic(cmd)) {
666                         TRACE_MEM("%s", "Atomic memory allocation failed, "
667                               "rescheduling to the thread");
668                         res = SCST_CMD_STATE_RES_NEED_THREAD;
669                         goto out;
670                 } else
671                         goto out_no_space;
672         }
673
674 prep_done:
675         if (cmd->preprocessing_only) {
676                 if (scst_cmd_atomic(cmd) && 
677                     !cmd->tgtt->preprocessing_done_atomic) {
678                         TRACE_DBG("%s", "preprocessing_done() can not be "
679                               "called in atomic context, rescheduling to "
680                               "the thread");
681                         res = SCST_CMD_STATE_RES_NEED_THREAD;
682                         goto out;
683                 }
684
685                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
686                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
687                                 "cmd %p", cmd);
688                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
689                         res = SCST_CMD_STATE_RES_CONT_SAME;
690                         goto out;
691                 }
692
693                 res = SCST_CMD_STATE_RES_CONT_NEXT;
694                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
695
696                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
697                 cmd->tgtt->preprocessing_done(cmd);
698                 TRACE_DBG("%s", "preprocessing_done() returned");
699                 goto out;
700
701         }
702
703         switch (cmd->data_direction) {
704         case SCST_DATA_WRITE:
705                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
706                 break;
707
708         default:
709                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
710                 break;
711         }
712
713 out:
714         TRACE_EXIT_HRES(res);
715         return res;
716
717 out_no_space:
718         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
719                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
720         scst_set_busy(cmd);
721         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
722         res = SCST_CMD_STATE_RES_CONT_SAME;
723         goto out;
724
725 out_error:
726         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
727         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
728         res = SCST_CMD_STATE_RES_CONT_SAME;
729         goto out;
730 }
731
732 void scst_restart_cmd(struct scst_cmd *cmd, int status, int pref_context)
733 {
734         TRACE_ENTRY();
735
736         TRACE_DBG("Preferred context: %d", pref_context);
737         TRACE_DBG("tag=%llu, status=%#x", scst_cmd_get_tag(cmd), status);
738
739 #ifdef EXTRACHECKS
740         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
741                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
742         {
743                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
744                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
745                         cmd->tgtt->name);
746                 pref_context = SCST_CONTEXT_TASKLET;
747         }
748 #endif
749
750         switch (status) {
751         case SCST_PREPROCESS_STATUS_SUCCESS:
752                 switch (cmd->data_direction) {
753                 case SCST_DATA_WRITE:
754                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
755                         break;
756                 default:
757                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
758                         break;
759                 }
760                 if (cmd->set_sn_on_restart_cmd)
761                         scst_cmd_set_sn(cmd);
762                 /* Small context optimization */
763                 if ((pref_context == SCST_CONTEXT_TASKLET) || 
764                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) {
765                         if (cmd->data_direction == SCST_DATA_WRITE) {
766                                 if ( !test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
767                                                 &cmd->tgt_dev->tgt_dev_flags))
768                                         pref_context = SCST_CONTEXT_THREAD;
769                         } else {
770                                 if ( !test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
771                                                 &cmd->tgt_dev->tgt_dev_flags))
772                                         pref_context = SCST_CONTEXT_THREAD;
773                         }
774                 }
775                 break;
776
777         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
778                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
779                 break;
780
781         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
782                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
783                 /* go through */
784         case SCST_PREPROCESS_STATUS_ERROR:
785                 scst_set_cmd_error(cmd,
786                            SCST_LOAD_SENSE(scst_sense_hardw_error));
787                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
788                 break;
789
790         default:
791                 PRINT_ERROR("%s() received unknown status %x", __func__,
792                         status);
793                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
794                 break;
795         }
796
797         scst_proccess_redirect_cmd(cmd, pref_context, 1);
798
799         TRACE_EXIT();
800         return;
801 }
802
803 /* No locks */
804 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
805 {
806         struct scst_tgt *tgt = cmd->sess->tgt;
807         int res = 0;
808         unsigned long flags;
809
810         TRACE_ENTRY();
811
812         spin_lock_irqsave(&tgt->tgt_lock, flags);
813         tgt->retry_cmds++;
814         smp_mb();
815         TRACE_RETRY("TGT QUEUE FULL: incrementing retry_cmds %d",
816               tgt->retry_cmds);
817         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
818                 /* At least one cmd finished, so try again */
819                 tgt->retry_cmds--;
820                 TRACE_RETRY("Some command(s) finished, direct retry "
821                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
822                       "retry_cmds=%d)", finished_cmds,
823                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
824                 res = -1;
825                 goto out_unlock_tgt;
826         }
827
828         TRACE_RETRY("Adding cmd %p to retry cmd list", cmd);
829         list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
830
831         if (!tgt->retry_timer_active) {
832                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
833                 add_timer(&tgt->retry_timer);
834                 tgt->retry_timer_active = 1;
835         }
836
837 out_unlock_tgt:
838         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
839
840         TRACE_EXIT_RES(res);
841         return res;
842 }
843
844 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
845 {
846         int res, rc;
847         int atomic = scst_cmd_atomic(cmd);
848
849         TRACE_ENTRY();
850
851         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
852                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
853                 goto out_dev_done;
854         }
855
856         if (cmd->tgtt->rdy_to_xfer == NULL) {
857                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
858                 res = SCST_CMD_STATE_RES_CONT_SAME;
859                 goto out;
860         }
861
862         if (atomic && !cmd->tgtt->rdy_to_xfer_atomic) {
863                 TRACE_DBG("%s", "rdy_to_xfer() can not be "
864                       "called in atomic context, rescheduling to the thread");
865                 res = SCST_CMD_STATE_RES_NEED_THREAD;
866                 goto out;
867         }
868
869         while (1) {
870                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
871
872                 res = SCST_CMD_STATE_RES_CONT_NEXT;
873                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
874
875                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
876 #ifdef DEBUG_RETRY
877                 if (((scst_random() % 100) == 75))
878                         rc = SCST_TGT_RES_QUEUE_FULL;
879                 else
880 #endif
881                         rc = cmd->tgtt->rdy_to_xfer(cmd);
882                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
883
884                 if (likely(rc == SCST_TGT_RES_SUCCESS))
885                         goto out;
886
887                 /* Restore the previous state */
888                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
889
890                 switch (rc) {
891                 case SCST_TGT_RES_QUEUE_FULL:
892                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
893                                 break;
894                         else
895                                 continue;
896
897                 case SCST_TGT_RES_NEED_THREAD_CTX:
898                         TRACE_DBG("Target driver %s "
899                               "rdy_to_xfer() requested thread "
900                               "context, rescheduling", cmd->tgtt->name);
901                         res = SCST_CMD_STATE_RES_NEED_THREAD;
902                         break;
903
904                 default:
905                         goto out_error_rc;
906                 }
907                 break;
908         }
909
910 out:
911         TRACE_EXIT_HRES(res);
912         return res;
913
914 out_error_rc:
915         if (rc == SCST_TGT_RES_FATAL_ERROR) {
916                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
917                      "fatal error", cmd->tgtt->name);
918         } else {
919                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
920                             "value %d", cmd->tgtt->name, rc);
921         }
922         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
923
924 out_dev_done:
925         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
926         res = SCST_CMD_STATE_RES_CONT_SAME;
927         goto out;
928 }
929
930 /* No locks, but might be in IRQ */
931 void scst_proccess_redirect_cmd(struct scst_cmd *cmd, int context,
932         int check_retries)
933 {
934         unsigned long flags;
935
936         TRACE_ENTRY();
937
938         TRACE_DBG("Context: %x", context);
939
940         switch(context & ~SCST_CONTEXT_PROCESSABLE) {
941         case SCST_CONTEXT_DIRECT_ATOMIC:
942                 context &= ~SCST_CONTEXT_PROCESSABLE;
943                 /* go through */
944         case SCST_CONTEXT_DIRECT:
945                 if (check_retries)
946                         scst_check_retries(cmd->tgt);
947                 scst_process_active_cmd(cmd, context);
948                 break;
949
950         default:
951                 PRINT_ERROR("Context %x is unknown, using the thread one",
952                             context);
953                 /* go through */
954         case SCST_CONTEXT_THREAD:
955                 if (check_retries)
956                         scst_check_retries(cmd->tgt);
957                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
958                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
959                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
960                         list_add(&cmd->cmd_list_entry,
961                                 &cmd->cmd_lists->active_cmd_list);
962                 else
963                         list_add_tail(&cmd->cmd_list_entry,
964                                 &cmd->cmd_lists->active_cmd_list);
965                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
966                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
967                 break;
968
969         case SCST_CONTEXT_TASKLET:
970                 if (check_retries)
971                         scst_check_retries(cmd->tgt);
972                 scst_schedule_tasklet(cmd);
973                 break;
974         }
975
976         TRACE_EXIT();
977         return;
978 }
979
980 void scst_rx_data(struct scst_cmd *cmd, int status, int pref_context)
981 {
982         TRACE_ENTRY();
983
984         TRACE_DBG("Preferred context: %d", pref_context);
985         TRACE(TRACE_SCSI, "tag=%llu status=%#x", scst_cmd_get_tag(cmd), status);
986
987 #ifdef EXTRACHECKS
988         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
989                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
990         {
991                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
992                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
993                         cmd->tgtt->name);
994                 pref_context = SCST_CONTEXT_TASKLET;
995         }
996 #endif
997
998         switch (status) {
999         case SCST_RX_STATUS_SUCCESS:
1000                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1001                 /* Small context optimization */
1002                 if ((pref_context == SCST_CONTEXT_TASKLET) || 
1003                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1004                         if ( !test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC, 
1005                                         &cmd->tgt_dev->tgt_dev_flags))
1006                                 pref_context = SCST_CONTEXT_THREAD;
1007                 }
1008                 break;
1009
1010         case SCST_RX_STATUS_ERROR_SENSE_SET:
1011                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1012                 break;
1013
1014         case SCST_RX_STATUS_ERROR_FATAL:
1015                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1016                 /* go through */
1017         case SCST_RX_STATUS_ERROR:
1018                 scst_set_cmd_error(cmd,
1019                            SCST_LOAD_SENSE(scst_sense_hardw_error));
1020                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1021                 break;
1022
1023         default:
1024                 PRINT_ERROR("scst_rx_data() received unknown status %x",
1025                         status);
1026                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1027                 break;
1028         }
1029
1030         scst_proccess_redirect_cmd(cmd, pref_context, 1);
1031
1032         TRACE_EXIT();
1033         return;
1034 }
1035
1036 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
1037 {
1038         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
1039
1040         TRACE_ENTRY();
1041
1042         cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
1043
1044         if (cmd->tgtt->pre_exec == NULL)
1045                 goto out;
1046
1047         TRACE_DBG("Calling pre_exec(%p)", cmd);
1048         rc = cmd->tgtt->pre_exec(cmd);
1049         TRACE_DBG("pre_exec() returned %d", rc);
1050
1051         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
1052                 switch(rc) {
1053                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
1054                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1055                         break;
1056                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
1057                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1058                         /* go through */
1059                 case SCST_PREPROCESS_STATUS_ERROR:
1060                         scst_set_cmd_error(cmd,
1061                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1062                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1063                         break;
1064                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1065                         TRACE_DBG("Target driver's %s pre_exec() requested "
1066                                 "thread context, rescheduling", cmd->tgtt->name);
1067                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1068                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1069                         break;
1070                 default:
1071                         sBUG();
1072                         break;
1073                 }
1074         }
1075
1076 out:
1077         TRACE_EXIT_RES(res);
1078         return res;
1079 }
1080
1081 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1082         const uint8_t *rq_sense, int rq_sense_len, int resid)
1083 {
1084         TRACE_ENTRY();
1085
1086 #ifdef MEASURE_LATENCY
1087         {
1088                 struct timespec ts;
1089                 getnstimeofday(&ts);
1090                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1091                 TRACE_DBG("cmd %p (sess %p): post_exec_start %Ld (tv_sec %ld, "
1092                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start, ts.tv_sec,
1093                         ts.tv_nsec);
1094         }
1095 #endif
1096
1097         cmd->status = result & 0xff;
1098         cmd->msg_status = msg_byte(result);
1099         cmd->host_status = host_byte(result);
1100         cmd->driver_status = driver_byte(result);
1101         if (unlikely(resid != 0)) {
1102 #ifdef EXTRACHECKS
1103                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1104                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1105                                 "op %x)", resid, cmd->resp_data_len,
1106                                 cmd->cdb[0]);
1107                 } else
1108 #endif
1109                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1110         }
1111
1112         if (cmd->status == SAM_STAT_CHECK_CONDITION)
1113                 scst_alloc_set_sense(cmd, in_irq(), rq_sense, rq_sense_len);
1114
1115         TRACE(TRACE_SCSI, "result=%x, cmd->status=%x, resid=%d, "
1116               "cmd->msg_status=%x, cmd->host_status=%x, "
1117               "cmd->driver_status=%x", result, cmd->status, resid,
1118               cmd->msg_status, cmd->host_status, cmd->driver_status);
1119
1120         cmd->completed = 1;
1121
1122         TRACE_EXIT();
1123         return;
1124 }
1125
1126 /* For small context optimization */
1127 static inline int scst_optimize_post_exec_context(struct scst_cmd *cmd,
1128         int context)
1129 {
1130         if ((context == SCST_CONTEXT_TASKLET) || 
1131             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1132                 if ( !test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC, 
1133                                 &cmd->tgt_dev->tgt_dev_flags))
1134                         context = SCST_CONTEXT_THREAD;
1135         }
1136         return context;
1137 }
1138
1139 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18)
1140 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1141                                             struct scsi_request **req)
1142 {
1143         struct scst_cmd *cmd = NULL;
1144
1145         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1146                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1147
1148         if (cmd == NULL) {
1149                 PRINT_ERROR("%s", "Request with NULL cmd");
1150                 if (*req)
1151                         scsi_release_request(*req);
1152         }
1153
1154         return cmd;
1155 }
1156
1157 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1158 {
1159         struct scsi_request *req = NULL;
1160         struct scst_cmd *cmd;
1161
1162         TRACE_ENTRY();
1163
1164         cmd = scst_get_cmd(scsi_cmd, &req);
1165         if (cmd == NULL)
1166                 goto out;
1167
1168         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1169                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1170
1171         /* Clear out request structure */
1172         req->sr_use_sg = 0;
1173         req->sr_sglist_len = 0;
1174         req->sr_bufflen = 0;
1175         req->sr_buffer = NULL;
1176         req->sr_underflow = 0;
1177         req->sr_request->rq_disk = NULL; /* disown request blk */
1178
1179         scst_release_request(cmd);
1180
1181         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1182
1183         scst_proccess_redirect_cmd(cmd,
1184                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1185
1186 out:
1187         TRACE_EXIT();
1188         return;
1189 }
1190 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) */
1191 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1192 {
1193         struct scst_cmd *cmd;
1194
1195         TRACE_ENTRY();
1196
1197         cmd = (struct scst_cmd *)data;
1198         if (cmd == NULL)
1199                 goto out;
1200
1201         scst_do_cmd_done(cmd, result, sense, SCST_SENSE_BUFFERSIZE, resid);
1202
1203         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1204
1205         scst_proccess_redirect_cmd(cmd,
1206                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1207
1208 out:
1209         TRACE_EXIT();
1210         return;
1211 }
1212 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) */
1213
1214 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state)
1215 {
1216         int context;
1217
1218         TRACE_ENTRY();
1219
1220 #ifdef MEASURE_LATENCY
1221         {
1222                 struct timespec ts;
1223                 getnstimeofday(&ts);
1224                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1225                 TRACE_DBG("cmd %p (sess %p): post_exec_start %Ld (tv_sec %ld, "
1226                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start, ts.tv_sec,
1227                         ts.tv_nsec);
1228         }
1229 #endif
1230
1231         if (next_state == SCST_CMD_STATE_DEFAULT)
1232                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1233
1234 #if defined(DEBUG)
1235         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1236                 if (cmd->sg) {
1237                         int i;
1238                         struct scatterlist *sg = cmd->sg;
1239                         TRACE_RECV_TOP("Exec'd %d S/G(s) at %p sg[0].page at "
1240                                 "%p", cmd->sg_cnt, sg, (void*)sg_page(&sg[0]));
1241                         for(i = 0; i < cmd->sg_cnt; ++i) {
1242                                 TRACE_BUFF_FLAG(TRACE_RCV_TOP, 
1243                                         "Exec'd sg", sg_virt(&sg[i]),
1244                                         sg[i].length);
1245                         }
1246                 }
1247         }
1248 #endif
1249
1250
1251 #ifdef EXTRACHECKS
1252         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1253             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1254             (next_state != SCST_CMD_STATE_FINISHED)) 
1255         {
1256                 PRINT_ERROR("scst_cmd_done_local() received invalid cmd "
1257                             "state %d (opcode %d)", next_state, cmd->cdb[0]);
1258                 scst_set_cmd_error(cmd,
1259                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1260                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1261         }
1262 #endif
1263         cmd->state = next_state;
1264
1265         context = scst_optimize_post_exec_context(cmd, scst_get_context());
1266         if (cmd->context_processable)
1267                 context |= SCST_CONTEXT_PROCESSABLE;
1268         scst_proccess_redirect_cmd(cmd, context, 0);
1269
1270         TRACE_EXIT();
1271         return;
1272 }
1273
1274 static int scst_report_luns_local(struct scst_cmd *cmd)
1275 {
1276         int rc;
1277         int dev_cnt = 0;
1278         int buffer_size;
1279         int i;
1280         struct scst_tgt_dev *tgt_dev = NULL;
1281         uint8_t *buffer;
1282         int offs, overflow = 0;
1283
1284         TRACE_ENTRY();
1285
1286         rc = scst_check_local_events(cmd);
1287         if (unlikely(rc != 0))
1288                 goto out_done;
1289
1290         cmd->status = 0;
1291         cmd->msg_status = 0;
1292         cmd->host_status = DID_OK;
1293         cmd->driver_status = 0;
1294
1295         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1296                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1297                         "LUNS command", cmd->cdb[2]);
1298                 goto out_err;
1299         }
1300
1301         buffer_size = scst_get_buf_first(cmd, &buffer);
1302         if (unlikely(buffer_size <= 0))
1303                 goto out_err;
1304
1305         if (buffer_size < 16)
1306                 goto out_put_err;
1307
1308         memset(buffer, 0, buffer_size);
1309         offs = 8;
1310
1311         /* sess->sess_tgt_dev_list_hash is protected by suspended activity */
1312         for(i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1313                 struct list_head *sess_tgt_dev_list_head =
1314                         &cmd->sess->sess_tgt_dev_list_hash[i];
1315                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1316                                 sess_tgt_dev_list_entry) {
1317                         if (!overflow) {
1318                                 if (offs >= buffer_size) {
1319                                         scst_put_buf(cmd, buffer);
1320                                         buffer_size = scst_get_buf_next(cmd, &buffer);
1321                                         if (buffer_size > 0) {
1322                                                 memset(buffer, 0, buffer_size);
1323                                                 offs = 0;
1324                                         } else {
1325                                                 overflow = 1;
1326                                                 goto inc_dev_cnt;
1327                                         }
1328                                 }
1329                                 if ((buffer_size - offs) < 8) {
1330                                         PRINT_ERROR("Buffer allocated for REPORT "
1331                                                 "LUNS command doesn't allow to fit 8 "
1332                                                 "byte entry (buffer_size=%d)",
1333                                                 buffer_size);
1334                                         goto out_put_hw_err;
1335                                 }
1336                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1337                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1338                                 offs += 8;
1339                         }
1340 inc_dev_cnt:
1341                         dev_cnt++;
1342                 }
1343         }
1344         if (!overflow)
1345                 scst_put_buf(cmd, buffer);
1346
1347         /* Set the response header */
1348         buffer_size = scst_get_buf_first(cmd, &buffer);
1349         if (unlikely(buffer_size <= 0))
1350                 goto out_err;
1351         dev_cnt *= 8;
1352         buffer[0] = (dev_cnt >> 24) & 0xff;
1353         buffer[1] = (dev_cnt >> 16) & 0xff;
1354         buffer[2] = (dev_cnt >> 8) & 0xff;
1355         buffer[3] = dev_cnt & 0xff;
1356         scst_put_buf(cmd, buffer);
1357
1358         dev_cnt += 8;
1359         if (dev_cnt < cmd->resp_data_len)
1360                 scst_set_resp_data_len(cmd, dev_cnt);
1361
1362 out_compl:
1363         cmd->completed = 1;
1364
1365 out_done:
1366         /* Report the result */
1367         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1368
1369         TRACE_EXIT();
1370         return SCST_EXEC_COMPLETED;
1371         
1372 out_put_err:
1373         scst_put_buf(cmd, buffer);
1374
1375 out_err:
1376         scst_set_cmd_error(cmd,
1377                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1378         goto out_compl;
1379
1380 out_put_hw_err:
1381         scst_put_buf(cmd, buffer);
1382         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1383         goto out_compl;
1384 }
1385
1386 static int scst_pre_select(struct scst_cmd *cmd)
1387 {
1388         int res = SCST_EXEC_NOT_COMPLETED;
1389
1390         TRACE_ENTRY();
1391
1392         if (scst_cmd_atomic(cmd)) {
1393                 res = SCST_EXEC_NEED_THREAD;
1394                 goto out;
1395         }
1396
1397         if (cmd->local_exec_done)
1398                 goto out;
1399
1400         cmd->local_exec_done = 1;
1401
1402         scst_block_dev_cmd(cmd, 1);
1403
1404         /* Check for local events will be done when cmd will be executed */
1405
1406 out:
1407         TRACE_EXIT_RES(res);
1408         return res;
1409 }
1410
1411 static int scst_reserve_local(struct scst_cmd *cmd)
1412 {
1413         int res = SCST_EXEC_NOT_COMPLETED, rc;
1414         struct scst_device *dev;
1415         struct scst_tgt_dev *tgt_dev_tmp;
1416
1417         TRACE_ENTRY();
1418
1419         if (scst_cmd_atomic(cmd)) {
1420                 res = SCST_EXEC_NEED_THREAD;
1421                 goto out;
1422         }
1423
1424         if (cmd->local_exec_done)
1425                 goto out;
1426
1427         cmd->local_exec_done = 1;
1428
1429         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1430                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1431                      "(lun=%Ld)", (uint64_t)cmd->lun);
1432                 scst_set_cmd_error(cmd,
1433                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1434                 goto out_done;
1435         }
1436
1437         dev = cmd->dev;
1438
1439         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1440                 scst_block_dev_cmd(cmd, 1);
1441
1442         rc = scst_check_local_events(cmd);
1443         if (unlikely(rc != 0))
1444                 goto out_done;
1445
1446         spin_lock_bh(&dev->dev_lock);
1447
1448         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1449                 spin_unlock_bh(&dev->dev_lock);
1450                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1451                 goto out_done;
1452         }
1453
1454         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1455                             dev_tgt_dev_list_entry) {
1456                 if (cmd->tgt_dev != tgt_dev_tmp)
1457                         set_bit(SCST_TGT_DEV_RESERVED, 
1458                                 &tgt_dev_tmp->tgt_dev_flags);
1459         }
1460         dev->dev_reserved = 1;
1461
1462         spin_unlock_bh(&dev->dev_lock);
1463         
1464 out:
1465         TRACE_EXIT_RES(res);
1466         return res;
1467
1468 out_done:
1469         /* Report the result */
1470         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1471         res = SCST_EXEC_COMPLETED;
1472         goto out;
1473 }
1474
1475 static int scst_release_local(struct scst_cmd *cmd)
1476 {
1477         int res = SCST_EXEC_NOT_COMPLETED, rc;
1478         struct scst_tgt_dev *tgt_dev_tmp;
1479         struct scst_device *dev;
1480
1481         TRACE_ENTRY();
1482
1483         if (scst_cmd_atomic(cmd)) {
1484                 res = SCST_EXEC_NEED_THREAD;
1485                 goto out;
1486         }
1487
1488         if (cmd->local_exec_done)
1489                 goto out;
1490
1491         cmd->local_exec_done = 1;
1492
1493         dev = cmd->dev;
1494
1495         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1496                 scst_block_dev_cmd(cmd, 1);
1497
1498         rc = scst_check_local_events(cmd);
1499         if (unlikely(rc != 0))
1500                 goto out_done;
1501
1502         spin_lock_bh(&dev->dev_lock);
1503
1504         /* 
1505          * The device could be RELEASED behind us, if RESERVING session 
1506          * is closed (see scst_free_tgt_dev()), but this actually doesn't 
1507          * matter, so use lock and no retest for DEV_RESERVED bits again
1508          */
1509         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1510                 res = SCST_EXEC_COMPLETED;
1511                 cmd->status = 0;
1512                 cmd->msg_status = 0;
1513                 cmd->host_status = DID_OK;
1514                 cmd->driver_status = 0;
1515                 cmd->completed = 1;
1516         } else {
1517                 list_for_each_entry(tgt_dev_tmp,
1518                                     &dev->dev_tgt_dev_list,
1519                                     dev_tgt_dev_list_entry) {
1520                         clear_bit(SCST_TGT_DEV_RESERVED, 
1521                                 &tgt_dev_tmp->tgt_dev_flags);
1522                 }
1523                 dev->dev_reserved = 0;
1524         }
1525
1526         spin_unlock_bh(&dev->dev_lock);
1527
1528         if (res == SCST_EXEC_COMPLETED)
1529                 goto out_done;
1530
1531 out:
1532         TRACE_EXIT_RES(res);
1533         return res;
1534
1535 out_done:
1536         res = SCST_EXEC_COMPLETED;
1537         /* Report the result */
1538         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1539         goto out;
1540 }
1541
1542 /* No locks, no IRQ or IRQ-safe context allowed */
1543 int scst_check_local_events(struct scst_cmd *cmd)
1544 {
1545         int res, rc;
1546         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1547
1548         TRACE_ENTRY();
1549
1550         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1551                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1552                 goto out_uncomplete;
1553         }
1554
1555         /* Reserve check before Unit Attention */
1556         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED, &tgt_dev->tgt_dev_flags))) {
1557                 if ((cmd->cdb[0] != INQUIRY) && (cmd->cdb[0] != REPORT_LUNS) &&
1558                     (cmd->cdb[0] != RELEASE) && (cmd->cdb[0] != RELEASE_10) &&
1559                     (cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER) &&
1560                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL || (cmd->cdb[4] & 3)) &&
1561                     (cmd->cdb[0] != LOG_SENSE) && (cmd->cdb[0] != REQUEST_SENSE)) {
1562                         scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1563                         goto out_complete;
1564                 }
1565         }
1566
1567         /* If we had internal bus reset, set the command error unit attention */
1568         if ((cmd->dev->scsi_dev != NULL) &&
1569             unlikely(cmd->dev->scsi_dev->was_reset)) {
1570                 if (scst_is_ua_command(cmd)) {
1571                         struct scst_device *dev = cmd->dev;
1572                         int done = 0;
1573                         /* Prevent more than 1 cmd to be triggered by was_reset */
1574                         spin_lock_bh(&dev->dev_lock);
1575                         barrier(); /* to reread was_reset */
1576                         if (dev->scsi_dev->was_reset) {
1577                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1578                                 scst_set_cmd_error(cmd,
1579                                            SCST_LOAD_SENSE(scst_sense_reset_UA));
1580                                 /* It looks like it is safe to clear was_reset here */
1581                                 dev->scsi_dev->was_reset = 0;
1582                                 done = 1;
1583                         }
1584                         spin_unlock_bh(&dev->dev_lock);
1585
1586                         if (done)
1587                                 goto out_complete;
1588                 }
1589         }
1590
1591         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING, 
1592                         &cmd->tgt_dev->tgt_dev_flags))) {
1593                 if (scst_is_ua_command(cmd)) {
1594                         rc = scst_set_pending_UA(cmd);
1595                         if (rc == 0)
1596                                 goto out_complete;
1597                 }
1598         }
1599
1600         res = 0;
1601
1602 out:
1603         TRACE_EXIT_RES(res);
1604         return res;
1605
1606 out_complete:
1607         res = 1;
1608         sBUG_ON(!cmd->completed);
1609         goto out;
1610
1611 out_uncomplete:
1612         res = -1;
1613         goto out;
1614 }
1615
1616 /* 
1617  * The result of cmd execution, if any, should be reported 
1618  * via scst_cmd_done_local() 
1619  */
1620 static int scst_pre_exec(struct scst_cmd *cmd)
1621 {
1622         int res = SCST_EXEC_NOT_COMPLETED;
1623         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1624
1625         TRACE_ENTRY();
1626
1627         /*
1628          * This function can be called several times for the same cmd, so it
1629          * can't change any state in a non-reentrable way or use something
1630          * like local_exec_done!!
1631          */
1632
1633         /* Check READ_ONLY device status */
1634         if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
1635             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
1636              cmd->cdb[0] == WRITE_10 ||
1637              cmd->cdb[0] == WRITE_12 ||
1638              cmd->cdb[0] == WRITE_16 ||
1639              cmd->cdb[0] == WRITE_VERIFY ||
1640              cmd->cdb[0] == WRITE_VERIFY_12 ||
1641              cmd->cdb[0] == WRITE_VERIFY_16 ||
1642              (cmd->dev->handler->type == TYPE_TAPE &&
1643               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
1644                 scst_set_cmd_error(cmd,
1645                            SCST_LOAD_SENSE(scst_sense_data_protect));
1646                 goto out_done;
1647         }
1648
1649 out:
1650         TRACE_EXIT_RES(res);
1651         return res;
1652
1653 out_done:
1654         res = SCST_EXEC_COMPLETED;
1655         /* Report the result */
1656         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1657         goto out;
1658 }
1659
1660 /* 
1661  * The result of cmd execution, if any, should be reported 
1662  * via scst_cmd_done_local() 
1663  */
1664 static inline int scst_local_exec(struct scst_cmd *cmd)
1665 {
1666         int res = SCST_EXEC_NOT_COMPLETED;
1667
1668         TRACE_ENTRY();
1669
1670         /*
1671          * Adding new commands here don't forget to update
1672          * scst_is_cmd_local() in scsi_tgt.h, if necessary
1673          */
1674
1675         switch (cmd->cdb[0]) {
1676         case MODE_SELECT:
1677         case MODE_SELECT_10:
1678         case LOG_SELECT:
1679                 res = scst_pre_select(cmd);
1680                 break;
1681         case RESERVE:
1682         case RESERVE_10:
1683                 res = scst_reserve_local(cmd);
1684                 break;
1685         case RELEASE:
1686         case RELEASE_10:
1687                 res = scst_release_local(cmd);
1688                 break;
1689         case REPORT_LUNS:
1690                 res = scst_report_luns_local(cmd);
1691                 break;
1692         }
1693
1694         TRACE_EXIT_RES(res);
1695         return res;
1696 }
1697
1698 /* cmd must be additionally referenced to not die inside */
1699 static int scst_do_send_to_midlev(struct scst_cmd *cmd)
1700 {
1701         int rc = SCST_EXEC_NOT_COMPLETED;
1702         struct scst_device *dev = cmd->dev;
1703         struct scst_dev_type *handler = dev->handler;
1704
1705         TRACE_ENTRY();
1706
1707         /* Check here to let an out of SN cmd be queued w/o context switch */
1708         if (scst_cmd_atomic(cmd) && !handler->exec_atomic) {
1709                 TRACE_DBG("Dev handler %s exec() can not be "
1710                       "called in atomic context, rescheduling to the thread",
1711                       handler->name);
1712                 rc = SCST_EXEC_NEED_THREAD;
1713                 goto out;
1714         }
1715
1716         cmd->sent_to_midlev = 1;
1717         cmd->state = SCST_CMD_STATE_EXECUTING;
1718         cmd->scst_cmd_done = scst_cmd_done_local;
1719
1720         rc = scst_pre_exec(cmd);
1721         if (rc != SCST_EXEC_NOT_COMPLETED) {
1722                 if (rc == SCST_EXEC_COMPLETED)
1723                         goto out;
1724                 else if (rc == SCST_EXEC_NEED_THREAD)
1725                         goto out_clear;
1726                 else
1727                         goto out_rc_error;
1728         }
1729
1730         rc = scst_local_exec(cmd);
1731         if (rc != SCST_EXEC_NOT_COMPLETED) {
1732                 if (rc == SCST_EXEC_COMPLETED)
1733                         goto out;
1734                 else if (rc == SCST_EXEC_NEED_THREAD)
1735                         goto out_clear;
1736                 else
1737                         goto out_rc_error;
1738         }
1739
1740         if (!handler->exec_sync)
1741                 cmd->context_processable = 0;
1742
1743         if (handler->exec) {
1744                 TRACE_DBG("Calling dev handler %s exec(%p)",
1745                       handler->name, cmd);
1746                 TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb, cmd->cdb_len);
1747                 cmd->scst_cmd_done = scst_cmd_done_local;
1748                 rc = handler->exec(cmd);
1749                 TRACE_DBG("Dev handler %s exec() returned %d",
1750                       handler->name, rc);
1751                 if (rc == SCST_EXEC_COMPLETED)
1752                         goto out;
1753                 else if (rc == SCST_EXEC_NEED_THREAD)
1754                         goto out_clear;
1755                 else if (rc != SCST_EXEC_NOT_COMPLETED)
1756                         goto out_rc_error;
1757         }
1758
1759         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1760         
1761         if (unlikely(dev->scsi_dev == NULL)) {
1762                 PRINT_ERROR("Command for virtual device must be "
1763                         "processed by device handler (lun %Ld)!",
1764                         (uint64_t)cmd->lun);
1765                 goto out_error;
1766         }
1767
1768         rc = scst_check_local_events(cmd);
1769         if (unlikely(rc != 0))
1770                 goto out_done;
1771
1772 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) 
1773         if (unlikely(scst_alloc_request(cmd) != 0)) {
1774                 if (scst_cmd_atomic(cmd)) {
1775                         rc = SCST_EXEC_NEED_THREAD;
1776                         goto out_clear;
1777                 } else {
1778                         PRINT_INFO("%s", "Unable to allocate request, "
1779                                 "sending BUSY status");
1780                         goto out_busy;
1781                 }
1782         }
1783         
1784         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1785                     (void *)cmd->scsi_req->sr_buffer,
1786                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1787                     cmd->retries);
1788 #else
1789         rc = scst_exec_req(dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1790                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1791                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1792                         scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
1793         if (unlikely(rc != 0)) {
1794                 if (scst_cmd_atomic(cmd)) {
1795                         rc = SCST_EXEC_NEED_THREAD;
1796                         goto out_clear;
1797                 } else {
1798                         PRINT_INFO("scst_exec_req() failed: %d", rc);
1799                         goto out_error;
1800                 }
1801         }
1802 #endif
1803
1804         rc = SCST_EXEC_COMPLETED;
1805
1806 out:
1807         TRACE_EXIT();
1808         return rc;
1809
1810 out_clear:
1811         /* Restore the state */
1812         cmd->sent_to_midlev = 0;
1813         cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
1814         goto out;
1815
1816 out_rc_error:
1817         PRINT_ERROR("Dev handler %s exec() or scst_local_exec() returned "
1818                     "invalid code %d", handler->name, rc);
1819         /* go through */
1820
1821 out_error:
1822         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1823
1824 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) 
1825 out_busy:
1826         scst_set_busy(cmd);
1827         /* go through */
1828 #endif
1829
1830 out_done:
1831         rc = SCST_EXEC_COMPLETED;
1832         /* Report the result */
1833         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1834         goto out;
1835 }
1836
1837 /* No locks */
1838 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1839 {
1840         if (slot == NULL)
1841                 goto inc;
1842
1843         /* Optimized for lockless fast path */
1844
1845         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1846                 atomic_read(slot));
1847
1848         if (!atomic_dec_and_test(slot))
1849                 goto out;
1850
1851         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1852                 tgt_dev->num_free_sn_slots);
1853         if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
1854                 spin_lock_irq(&tgt_dev->sn_lock);
1855                 if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
1856                         if (tgt_dev->num_free_sn_slots < 0)
1857                                 tgt_dev->cur_sn_slot = slot;
1858                         smp_mb(); /* to be in-sync with SIMPLE case in scst_cmd_set_sn() */
1859                         tgt_dev->num_free_sn_slots++;
1860                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1861                                 tgt_dev->num_free_sn_slots);
1862                         
1863                 }
1864                 spin_unlock_irq(&tgt_dev->sn_lock);
1865         }
1866
1867 inc:
1868         /*
1869          * No locks is needed, because only one thread at time can 
1870          * be here (serialized by sn). Also it is supposed that there
1871          * could not be half-incremented halves.
1872          */
1873         tgt_dev->expected_sn++;
1874         smp_mb(); /* write must be before def_cmd_count read */
1875         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1876
1877 out:
1878         return;
1879 }
1880
1881 static int scst_process_internal_cmd(struct scst_cmd *cmd)
1882 {
1883         int res = SCST_CMD_STATE_RES_CONT_NEXT, rc;
1884
1885         TRACE_ENTRY();
1886
1887         __scst_cmd_get(cmd);
1888
1889         rc = scst_do_send_to_midlev(cmd);
1890         if (rc == SCST_EXEC_NEED_THREAD) {
1891                 TRACE_DBG("%s", "scst_do_send_to_midlev() requested "
1892                       "thread context, rescheduling");
1893                 res = SCST_CMD_STATE_RES_NEED_THREAD;
1894         } else {
1895                 struct scst_device *dev = cmd->dev;
1896                 sBUG_ON(rc != SCST_EXEC_COMPLETED);
1897                 if (dev->scsi_dev != NULL)
1898                         generic_unplug_device(dev->scsi_dev->request_queue);
1899         }
1900
1901         __scst_cmd_put(cmd);
1902
1903         TRACE_EXIT_RES(res);
1904         return res;
1905 }
1906
1907 static int scst_send_to_midlev(struct scst_cmd **active_cmd)
1908 {
1909         int res, rc;
1910         struct scst_cmd *cmd = *active_cmd;
1911         struct scst_cmd *ref_cmd;
1912         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1913         struct scst_device *dev = cmd->dev;
1914         typeof(tgt_dev->expected_sn) expected_sn;
1915         int count;
1916
1917         TRACE_ENTRY();
1918
1919         res = SCST_CMD_STATE_RES_CONT_NEXT;
1920
1921         if (unlikely(cmd->internal || cmd->retry)) {
1922                 res = scst_process_internal_cmd(cmd);
1923                 goto out;
1924         }
1925
1926 #ifdef MEASURE_LATENCY
1927         if (cmd->pre_exec_finish == 0) {
1928                 struct timespec ts;
1929                 getnstimeofday(&ts);
1930                 cmd->pre_exec_finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1931                 TRACE_DBG("cmd %p (sess %p): pre_exec_finish %Ld (tv_sec %ld, "
1932                         "tv_nsec %ld)", cmd, sess, cmd->pre_exec_finish, ts.tv_sec,
1933                         ts.tv_nsec);
1934         }
1935 #endif
1936
1937         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
1938                 goto out;
1939
1940         ref_cmd = cmd;
1941         __scst_cmd_get(ref_cmd);
1942
1943         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
1944                 goto exec;
1945
1946         sBUG_ON(!cmd->sn_set);
1947
1948         expected_sn = tgt_dev->expected_sn;
1949         /* Optimized for lockless fast path */
1950         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
1951                 spin_lock_irq(&tgt_dev->sn_lock);
1952
1953                 tgt_dev->def_cmd_count++;
1954                 smp_mb();
1955
1956                 expected_sn = tgt_dev->expected_sn;
1957                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
1958                         /* We are under IRQ lock, but dev->dev_lock is BH one */
1959                         int cmd_blocking = scst_pre_dec_on_dev_cmd(cmd);
1960                         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1961                                 /* Necessary to allow aborting out of sn cmds */
1962                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p (tag %llu)",
1963                                         cmd, cmd->tag);
1964                                 tgt_dev->def_cmd_count--;
1965                                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1966                                 res = SCST_CMD_STATE_RES_CONT_SAME;
1967                         } else {
1968                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
1969                                         "expected_sn=%ld)", cmd, cmd->sn,
1970                                         cmd->sn_set, expected_sn);
1971                                 list_add_tail(&cmd->sn_cmd_list_entry,
1972                                               &tgt_dev->deferred_cmd_list);
1973                         }
1974                         spin_unlock_irq(&tgt_dev->sn_lock);
1975
1976                         __scst_dec_on_dev_cmd(dev, cmd_blocking);
1977
1978                         goto out_put;
1979                 } else {
1980                         TRACE_SN("Somebody incremented expected_sn %ld, "
1981                                 "continuing", expected_sn);
1982                         tgt_dev->def_cmd_count--;
1983                         spin_unlock_irq(&tgt_dev->sn_lock);
1984                 }
1985         }
1986
1987 exec:
1988         count = 0;
1989         while(1) {
1990                 atomic_t *slot = cmd->sn_slot;
1991                 /* For HQ commands SN is not set */
1992                 int inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1993                                       cmd->sn_set;
1994
1995                 rc = scst_do_send_to_midlev(cmd);
1996                 if (rc == SCST_EXEC_NEED_THREAD) {
1997                         TRACE_DBG("%s", "scst_do_send_to_midlev() requested "
1998                               "thread context, rescheduling");
1999                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2000                         scst_dec_on_dev_cmd(cmd);
2001                         *active_cmd = cmd;
2002                         if (count != 0)
2003                                 goto out_unplug;
2004                         else
2005                                 goto out_put;
2006                 }
2007                 sBUG_ON(rc != SCST_EXEC_COMPLETED);
2008
2009                 count++;
2010
2011                 if (inc_expected_sn)
2012                         scst_inc_expected_sn(tgt_dev, slot);
2013
2014                 cmd = scst_check_deferred_commands(tgt_dev);
2015                 if (cmd == NULL)
2016                         break;
2017
2018                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2019                         break;
2020
2021                 __scst_cmd_put(ref_cmd);
2022                 ref_cmd = cmd;
2023                 __scst_cmd_get(ref_cmd);
2024         }
2025
2026 out_unplug:
2027         if (dev->scsi_dev != NULL)
2028                 generic_unplug_device(dev->scsi_dev->request_queue);
2029
2030 out_put:
2031         __scst_cmd_put(ref_cmd);
2032         /* !! At this point sess, dev and tgt_dev can be already freed !! */
2033
2034 out:
2035         TRACE_EXIT_HRES(res);
2036         return res;
2037 }
2038
2039 /* No locks supposed to be held */
2040 static int scst_check_sense(struct scst_cmd *cmd)
2041 {
2042         int res = 0;
2043         struct scst_device *dev = cmd->dev;
2044         int dbl_ua_possible, ua_sent = 0;
2045
2046         TRACE_ENTRY();
2047
2048         /* If we had internal bus reset behind us, set the command error UA */
2049         if ((dev->scsi_dev != NULL) &&
2050             unlikely(cmd->host_status == DID_RESET) &&
2051             scst_is_ua_command(cmd)) {
2052                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
2053                       dev->scsi_dev->was_reset, cmd->host_status);
2054                 scst_set_cmd_error(cmd,
2055                    SCST_LOAD_SENSE(scst_sense_reset_UA));
2056                 /* just in case */
2057                 cmd->ua_ignore = 0;
2058                 /* It looks like it is safe to clear was_reset here */
2059                 dev->scsi_dev->was_reset = 0;
2060         }
2061
2062         dbl_ua_possible = dev->dev_double_ua_possible;
2063         TRACE_DBG("cmd %p dbl_ua_possible %d", cmd, dbl_ua_possible);
2064         if (unlikely(dbl_ua_possible)) {
2065                 spin_lock_bh(&dev->dev_lock);
2066                 barrier(); /* to reread dev_double_ua_possible */
2067                 dbl_ua_possible = dev->dev_double_ua_possible;
2068                 if (dbl_ua_possible)
2069                         ua_sent = dev->dev_reset_ua_sent;
2070                 else
2071                         spin_unlock_bh(&dev->dev_lock);
2072         }
2073
2074         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) && 
2075             SCST_SENSE_VALID(cmd->sense)) {
2076                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2077                         SCST_SENSE_BUFFERSIZE);
2078                 /* Check Unit Attention Sense Key */
2079                 if (scst_is_ua_sense(cmd->sense)) {
2080                         if (cmd->sense[12] == SCST_SENSE_ASC_UA_RESET) {
2081                                 if (dbl_ua_possible) {
2082                                         if (ua_sent) {
2083                                                 TRACE(TRACE_MGMT_MINOR, "%s", 
2084                                                         "Double UA detected");
2085                                                 /* Do retry */
2086                                                 TRACE(TRACE_MGMT_MINOR, "Retrying cmd %p "
2087                                                         "(tag %llu)", cmd, cmd->tag);
2088                                                 cmd->status = 0;
2089                                                 cmd->msg_status = 0;
2090                                                 cmd->host_status = DID_OK;
2091                                                 cmd->driver_status = 0;
2092                                                 mempool_free(cmd->sense, scst_sense_mempool);
2093                                                 cmd->sense = NULL;
2094                                                 cmd->retry = 1;
2095                                                 cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
2096                                                 res = 1;
2097                                                 /* 
2098                                                  * Dev is still blocked by this cmd, so
2099                                                  * it's OK to clear SCST_DEV_SERIALIZED
2100                                                  * here.
2101                                                  */
2102                                                 dev->dev_double_ua_possible = 0;
2103                                                 dev->dev_serialized = 0;
2104                                                 dev->dev_reset_ua_sent = 0;
2105                                                 goto out_unlock;
2106                                         } else
2107                                                 dev->dev_reset_ua_sent = 1;
2108                                 }
2109                         }
2110                         if (cmd->ua_ignore == 0) {
2111                                 if (unlikely(dbl_ua_possible)) {
2112                                         __scst_dev_check_set_UA(dev, cmd,
2113                                                 cmd->sense,
2114                                                 SCST_SENSE_BUFFERSIZE);
2115                                 } else {
2116                                         scst_dev_check_set_UA(dev, cmd,
2117                                                 cmd->sense,
2118                                                 SCST_SENSE_BUFFERSIZE);
2119                                 }
2120                         }
2121                 }
2122         }
2123
2124         if (unlikely(dbl_ua_possible)) {
2125                 if (ua_sent && scst_is_ua_command(cmd)) {
2126                         TRACE_MGMT_DBG("%s", "Clearing dbl_ua_possible flag");
2127                         dev->dev_double_ua_possible = 0;
2128                         dev->dev_serialized = 0;
2129                         dev->dev_reset_ua_sent = 0;
2130                 }
2131                 spin_unlock_bh(&dev->dev_lock);
2132         }
2133
2134 out:
2135         TRACE_EXIT_RES(res);
2136         return res;
2137
2138 out_unlock:
2139         spin_unlock_bh(&dev->dev_lock);
2140         goto out;
2141 }
2142
2143 static int scst_check_auto_sense(struct scst_cmd *cmd)
2144 {
2145         int res = 0;
2146
2147         TRACE_ENTRY();
2148
2149         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2150             (!SCST_SENSE_VALID(cmd->sense) ||
2151              SCST_NO_SENSE(cmd->sense))) {
2152                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2153                       "cmd->status=%x, cmd->msg_status=%x, "
2154                       "cmd->host_status=%x, cmd->driver_status=%x", cmd->status,
2155                       cmd->msg_status, cmd->host_status, cmd->driver_status);
2156                 res = 1;
2157         } else if (unlikely(cmd->host_status)) {
2158                 if ((cmd->host_status == DID_REQUEUE) ||
2159                     (cmd->host_status == DID_IMM_RETRY) ||
2160                     (cmd->host_status == DID_SOFT_ERROR) ||
2161                     (cmd->host_status == DID_ABORT)) {
2162                         scst_set_busy(cmd);
2163                 } else {
2164                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2165                                 "received, returning HARDWARE ERROR instead",
2166                                 cmd->host_status);
2167                         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2168                 }
2169         }
2170
2171         TRACE_EXIT_RES(res);
2172         return res;
2173 }
2174
2175 static int scst_done_cmd_check(struct scst_cmd *cmd, int *pres)
2176 {
2177         int res = 0, rc;
2178
2179         TRACE_ENTRY();
2180
2181         if (unlikely(cmd->cdb[0] == REQUEST_SENSE)) {
2182                 if (cmd->internal)
2183                         cmd = scst_complete_request_sense(cmd);
2184         } else if (unlikely(scst_check_auto_sense(cmd))) {
2185                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2186                             "without sense data (opcode 0x%x), issuing "
2187                             "REQUEST SENSE", cmd->cdb[0]);
2188                 rc = scst_prepare_request_sense(cmd);
2189                 if (rc > 0) {
2190                         *pres = rc;
2191                         res = 1;
2192                         goto out;
2193                 } else {
2194                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2195                                     "returning HARDWARE ERROR");
2196                         scst_set_cmd_error(cmd,
2197                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2198                 }
2199         } else if (unlikely(scst_check_sense(cmd))) {
2200                 *pres = SCST_CMD_STATE_RES_CONT_SAME;
2201                 res = 1;
2202                 goto out;
2203         }
2204
2205         if (likely(scsi_status_is_good(cmd->status))) {
2206                 unsigned char type = cmd->dev->handler->type;
2207                 if ((cmd->cdb[0] == MODE_SENSE || cmd->cdb[0] == MODE_SENSE_10) &&
2208                     cmd->tgt_dev->acg_dev->rd_only_flag &&
2209                     (type == TYPE_DISK || type == TYPE_WORM || type == TYPE_MOD ||
2210                      type == TYPE_TAPE)) {
2211                         int32_t length;
2212                         uint8_t *address;
2213
2214                         length = scst_get_buf_first(cmd, &address);
2215                         if (length <= 0)
2216                                 goto out;
2217                         if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2218                                 address[2] |= 0x80;   /* Write Protect*/
2219                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2220                                 address[3] |= 0x80;   /* Write Protect*/
2221                         scst_put_buf(cmd, address);
2222                 }
2223
2224                 /* 
2225                  * Check and clear NormACA option for the device, if necessary,
2226                  * since we don't support ACA
2227                  */
2228                 if ((cmd->cdb[0] == INQUIRY) &&
2229                     !(cmd->cdb[1] & SCST_INQ_EVPD/* Std INQUIRY data (no EVPD) */) &&
2230                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2231                         uint8_t *buffer;
2232                         int buflen;
2233
2234                         /* ToDo: all pages ?? */
2235                         buflen = scst_get_buf_first(cmd, &buffer);
2236                         if (buflen > SCST_INQ_BYTE3) {
2237 #ifdef EXTRACHECKS
2238                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2239                                         PRINT_INFO("NormACA set for device: "
2240                                             "lun=%Ld, type 0x%02x", 
2241                                             (uint64_t)cmd->lun, buffer[0]);
2242                                 }
2243 #endif
2244                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2245                         } else {
2246                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2247                                     "buffer");
2248                                 scst_set_cmd_error(cmd,
2249                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
2250                         }
2251                         if (buflen > 0)
2252                                 scst_put_buf(cmd, buffer);
2253                 }
2254
2255                 if (unlikely((cmd->cdb[0] == MODE_SELECT) || 
2256                     (cmd->cdb[0] == MODE_SELECT_10) ||
2257                     (cmd->cdb[0] == LOG_SELECT))) {
2258                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded (LUN %Ld)",
2259                                 (uint64_t)cmd->lun);
2260                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2261                         *pres = SCST_CMD_STATE_RES_CONT_SAME;
2262                         res = 1;
2263                         goto out;
2264                 }
2265         } else {
2266                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2267                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2268                                         &cmd->tgt_dev->tgt_dev_flags)) {
2269                                 struct scst_tgt_dev *tgt_dev_tmp;
2270                                 struct scst_device *dev = cmd->dev;
2271
2272                                 TRACE(TRACE_SCSI, "Real RESERVE failed lun=%Ld, "
2273                                         "status=%x", (uint64_t)cmd->lun, cmd->status);
2274                                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2275                                         SCST_SENSE_BUFFERSIZE);
2276
2277                                 /* Clearing the reservation */
2278                                 spin_lock_bh(&dev->dev_lock);
2279                                 list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
2280                                                     dev_tgt_dev_list_entry) {
2281                                         clear_bit(SCST_TGT_DEV_RESERVED, 
2282                                                 &tgt_dev_tmp->tgt_dev_flags);
2283                                 }
2284                                 dev->dev_reserved = 0;
2285                                 spin_unlock_bh(&dev->dev_lock);
2286                         }
2287                 }
2288
2289                 /* Check for MODE PARAMETERS CHANGED UA */
2290                 if ((cmd->dev->scsi_dev != NULL) &&
2291                     (cmd->status == SAM_STAT_CHECK_CONDITION) && 
2292                     SCST_SENSE_VALID(cmd->sense) &&
2293                     scst_is_ua_sense(cmd->sense) &&
2294                     (cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) {
2295                         TRACE(TRACE_SCSI, "MODE PARAMETERS CHANGED UA (lun %Ld)",
2296                                 (uint64_t)cmd->lun);
2297                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2298                         *pres = SCST_CMD_STATE_RES_CONT_SAME;
2299                         res = 1;
2300                         goto out;
2301                 }
2302         }
2303
2304 out:
2305         TRACE_EXIT_RES(res);
2306         return res;
2307 }
2308
2309 static int scst_pre_dev_done(struct scst_cmd *cmd)
2310 {
2311         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2312
2313         TRACE_ENTRY();
2314
2315         rc = scst_done_cmd_check(cmd, &res);
2316         if (rc)
2317                 goto out;
2318
2319         cmd->state = SCST_CMD_STATE_DEV_DONE;
2320
2321 out:
2322         TRACE_EXIT_HRES(res);
2323         return res;
2324 }
2325
2326 static int scst_mode_select_checks(struct scst_cmd *cmd)
2327 {
2328         int res = SCST_CMD_STATE_RES_CONT_SAME;
2329         int atomic = scst_cmd_atomic(cmd);
2330
2331         TRACE_ENTRY();
2332
2333         if (likely(scsi_status_is_good(cmd->status))) {
2334                 if (unlikely((cmd->cdb[0] == MODE_SELECT) || 
2335                     (cmd->cdb[0] == MODE_SELECT_10) ||
2336                     (cmd->cdb[0] == LOG_SELECT))) {
2337                         struct scst_device *dev = cmd->dev;
2338                         if (atomic && (dev->scsi_dev != NULL)) {
2339                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2340                                         "context required");
2341                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2342                                 goto out;
2343                         }
2344
2345                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2346                                 "setting the SELECT UA (lun=%Ld)", 
2347                                 (uint64_t)cmd->lun);
2348
2349                         spin_lock_bh(&dev->dev_lock);
2350                         spin_lock(&scst_temp_UA_lock);
2351                         if (cmd->cdb[0] == LOG_SELECT) {
2352                                 scst_set_sense(scst_temp_UA,
2353                                         sizeof(scst_temp_UA),
2354                                         UNIT_ATTENTION, 0x2a, 0x02);
2355                         } else {
2356                                 scst_set_sense(scst_temp_UA,
2357                                         sizeof(scst_temp_UA),
2358                                         UNIT_ATTENTION, 0x2a, 0x01);
2359                         }
2360                         scst_dev_check_set_local_UA(dev, cmd, scst_temp_UA,
2361                                 sizeof(scst_temp_UA));
2362                         spin_unlock(&scst_temp_UA_lock);
2363                         spin_unlock_bh(&dev->dev_lock);
2364
2365                         if (dev->scsi_dev != NULL)
2366                                 scst_obtain_device_parameters(dev);
2367                 }
2368         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) && 
2369                     SCST_SENSE_VALID(cmd->sense) &&
2370                     scst_is_ua_sense(cmd->sense) &&
2371                     (cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) {
2372                 if (atomic) {
2373                         TRACE_DBG("%s", "MODE PARAMETERS CHANGED UA: thread "
2374                                 "context required");
2375                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2376                         goto out;
2377                 }
2378
2379                 TRACE(TRACE_SCSI, "MODE PARAMETERS CHANGED UA (lun %Ld): "
2380                         "getting new parameters", (uint64_t)cmd->lun);
2381
2382                 scst_obtain_device_parameters(cmd->dev);
2383         } else
2384                 sBUG();
2385
2386         cmd->state = SCST_CMD_STATE_DEV_DONE;
2387
2388 out:
2389         TRACE_EXIT_HRES(res);
2390         return res;
2391 }
2392
2393 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2394 {
2395         if (likely(cmd->sn_set))
2396                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2397
2398         scst_make_deferred_commands_active(cmd->tgt_dev, cmd);
2399 }
2400
2401 static int scst_dev_done(struct scst_cmd *cmd)
2402 {
2403         int res = SCST_CMD_STATE_RES_CONT_SAME;
2404         int state;
2405         int atomic = scst_cmd_atomic(cmd);
2406
2407         TRACE_ENTRY();
2408
2409         if (atomic && !cmd->dev->handler->dev_done_atomic) {
2410                 TRACE_DBG("Dev handler %s dev_done() can not be "
2411                       "called in atomic context, rescheduling to the thread",
2412                       cmd->dev->handler->name);
2413                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2414                 goto out;
2415         }
2416
2417         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2418         if (likely(!scst_is_cmd_local(cmd)) && 
2419             likely(cmd->dev->handler->dev_done != NULL))
2420         {
2421                 int rc;
2422                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2423                       cmd->dev->handler->name, cmd);
2424                 rc = cmd->dev->handler->dev_done(cmd);
2425                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2426                       cmd->dev->handler->name, rc);
2427                 if (rc != SCST_CMD_STATE_DEFAULT)
2428                         state = rc;
2429         }
2430
2431         switch (state) {
2432         case SCST_CMD_STATE_PRE_XMIT_RESP:
2433         case SCST_CMD_STATE_DEV_PARSE:
2434         case SCST_CMD_STATE_PRE_PARSE:
2435         case SCST_CMD_STATE_PREPARE_SPACE:
2436         case SCST_CMD_STATE_RDY_TO_XFER:
2437         case SCST_CMD_STATE_TGT_PRE_EXEC:
2438         case SCST_CMD_STATE_SEND_TO_MIDLEV:
2439         case SCST_CMD_STATE_PRE_DEV_DONE:
2440         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2441         case SCST_CMD_STATE_DEV_DONE:
2442         case SCST_CMD_STATE_XMIT_RESP:
2443         case SCST_CMD_STATE_FINISHED:
2444                 cmd->state = state;
2445                 res = SCST_CMD_STATE_RES_CONT_SAME;
2446                 break;
2447
2448         case SCST_CMD_STATE_NEED_THREAD_CTX:
2449                 TRACE_DBG("Dev handler %s dev_done() requested "
2450                       "thread context, rescheduling",
2451                       cmd->dev->handler->name);
2452                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2453                 break;
2454
2455         default:
2456                 if (state >= 0) {
2457                         PRINT_ERROR("Dev handler %s dev_done() returned "
2458                                 "invalid cmd state %d", 
2459                                 cmd->dev->handler->name, state);
2460                 } else {
2461                         PRINT_ERROR("Dev handler %s dev_done() returned "
2462                                 "error %d", cmd->dev->handler->name, 
2463                                 state);
2464                 }
2465                 scst_set_cmd_error(cmd,
2466                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2467                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2468                 res = SCST_CMD_STATE_RES_CONT_SAME;
2469                 break;
2470         }
2471
2472         if (cmd->needs_unblocking)
2473                 scst_unblock_dev_cmd(cmd);
2474
2475         if (likely(cmd->dec_on_dev_needed))
2476                 scst_dec_on_dev_cmd(cmd);
2477
2478         if (cmd->inc_expected_sn_on_done && cmd->sent_to_midlev)
2479                 scst_inc_check_expected_sn(cmd);
2480
2481 out:
2482         TRACE_EXIT_HRES(res);
2483         return res;
2484 }
2485
2486 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2487 {
2488         int res;
2489
2490         TRACE_ENTRY();
2491
2492 #ifdef DEBUG_TM
2493         if (cmd->tm_dbg_delayed && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2494                 if (scst_cmd_atomic(cmd)) {
2495                         TRACE_MGMT_DBG("%s", "DEBUG_TM delayed cmd needs a thread");
2496                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2497                         return res;
2498                 }
2499                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2500                         cmd, cmd->tag);
2501                 schedule_timeout_uninterruptible(HZ);
2502         }
2503 #endif
2504
2505         if (likely(cmd->tgt_dev != NULL)) {
2506                 atomic_dec(&cmd->tgt_dev->tgt_dev_cmd_count);
2507                 atomic_dec(&cmd->dev->dev_cmd_count);
2508
2509                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2510                         scst_on_hq_cmd_response(cmd);
2511
2512                 if (unlikely(!cmd->sent_to_midlev)) {
2513                         TRACE_SN("cmd %p was not sent to mid-lev (sn %ld, set %d)",
2514                                 cmd, cmd->sn, cmd->sn_set);
2515                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2516                         cmd->sent_to_midlev = 1;
2517                 }
2518         }
2519
2520         /*
2521          * If we don't remove cmd from the search list here, before
2522          * submitting it for transmittion, we will have a race, when for
2523          * some reason cmd's release is delayed after transmittion and
2524          * initiator sends cmd with the same tag => it is possible that
2525          * a wrong cmd will be found by find() functions.
2526          */
2527         spin_lock_irq(&cmd->sess->sess_list_lock);
2528         list_del(&cmd->search_cmd_list_entry);
2529         spin_unlock_irq(&cmd->sess->sess_list_lock);
2530
2531         cmd->done = 1;
2532         smp_mb(); /* to sync with scst_abort_cmd() */
2533
2534         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2535                 scst_xmit_process_aborted_cmd(cmd);
2536
2537         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2538                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu), skipping",
2539                         cmd, cmd->tag);
2540                 cmd->state = SCST_CMD_STATE_FINISHED;
2541                 res = SCST_CMD_STATE_RES_CONT_SAME;
2542                 goto out;
2543         }
2544
2545         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2546         res = SCST_CMD_STATE_RES_CONT_SAME;
2547
2548 out:
2549 #ifdef MEASURE_LATENCY
2550         {
2551                 struct timespec ts;
2552                 uint64_t finish, scst_time, proc_time;
2553                 struct scst_session *sess = cmd->sess;
2554
2555                 getnstimeofday(&ts);
2556                 finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2557
2558                 spin_lock_bh(&sess->meas_lock);
2559
2560                 scst_time = cmd->pre_exec_finish - cmd->start;
2561                 scst_time += finish - cmd->post_exec_start;
2562                 proc_time = finish - cmd->start;
2563
2564                 sess->scst_time += scst_time;
2565                 sess->processing_time += proc_time;
2566                 sess->processed_cmds++;
2567
2568                 spin_unlock_bh(&sess->meas_lock);
2569
2570                 TRACE_DBG("cmd %p (sess %p): finish %Ld (tv_sec %ld, "
2571                         "tv_nsec %ld), scst_time %Ld, proc_time %Ld", cmd, sess,
2572                         finish, ts.tv_sec, ts.tv_nsec, scst_time, proc_time);
2573         }
2574 #endif
2575         TRACE_EXIT_HRES(res);
2576         return res;
2577 }
2578
2579 static int scst_xmit_response(struct scst_cmd *cmd)
2580 {
2581         int res, rc;
2582         int atomic = scst_cmd_atomic(cmd);
2583
2584         TRACE_ENTRY();
2585
2586         if (atomic && !cmd->tgtt->xmit_response_atomic) {
2587                 TRACE_DBG("%s", "xmit_response() can not be "
2588                       "called in atomic context, rescheduling to the thread");
2589                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2590                 goto out;
2591         }
2592
2593         while (1) {
2594                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2595
2596                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2597                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2598
2599                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2600
2601 #if defined(DEBUG)
2602                 if (cmd->sg) {
2603                         int i;
2604                         struct scatterlist *sg = cmd->sg;
2605                         TRACE_SEND_BOT("Xmitting %d S/G(s) at %p sg[0].page at %p",
2606                               cmd->sg_cnt, sg, (void*)sg_page(&sg[0]));
2607                         for(i = 0; i < cmd->sg_cnt; ++i) {
2608                                 TRACE_BUFF_FLAG(TRACE_SND_BOT,
2609                                     "Xmitting sg", sg_virt(&sg[i]),
2610                                     sg[i].length);
2611                         }
2612                 }
2613 #endif
2614
2615 #ifdef DEBUG_RETRY
2616                 if (((scst_random() % 100) == 77))
2617                         rc = SCST_TGT_RES_QUEUE_FULL;
2618                 else
2619 #endif
2620                         rc = cmd->tgtt->xmit_response(cmd);
2621                 TRACE_DBG("xmit_response() returned %d", rc);
2622
2623                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2624                         goto out;
2625
2626                 /* Restore the previous state */
2627                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2628
2629                 switch (rc) {
2630                 case SCST_TGT_RES_QUEUE_FULL:
2631                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2632                                 break;
2633                         else
2634                                 continue;
2635
2636                 case SCST_TGT_RES_NEED_THREAD_CTX:
2637                         TRACE_DBG("Target driver %s xmit_response() "
2638                               "requested thread context, rescheduling",
2639                               cmd->tgtt->name);
2640                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2641                         break;
2642
2643                 default:
2644                         goto out_error;
2645                 }
2646                 break;
2647         }
2648
2649 out:
2650         /* Caution: cmd can be already dead here */
2651         TRACE_EXIT_HRES(res);
2652         return res;
2653
2654 out_error:
2655         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2656                 PRINT_ERROR("Target driver %s xmit_response() returned "
2657                         "fatal error", cmd->tgtt->name);
2658         } else {
2659                 PRINT_ERROR("Target driver %s xmit_response() returned "
2660                         "invalid value %d", cmd->tgtt->name, rc);
2661         }
2662         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2663         cmd->state = SCST_CMD_STATE_FINISHED;
2664         res = SCST_CMD_STATE_RES_CONT_SAME;
2665         goto out;
2666 }
2667
2668 void scst_tgt_cmd_done(struct scst_cmd *cmd)
2669 {
2670         TRACE_ENTRY();
2671
2672         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2673
2674         cmd->state = SCST_CMD_STATE_FINISHED;
2675         scst_proccess_redirect_cmd(cmd, scst_get_context(), 1);
2676
2677         TRACE_EXIT();
2678         return;
2679 }
2680
2681 static int scst_finish_cmd(struct scst_cmd *cmd)
2682 {
2683         int res;
2684
2685         TRACE_ENTRY();
2686
2687         atomic_dec(&cmd->sess->sess_cmd_count);
2688
2689         cmd->finished = 1;
2690         smp_mb(); /* to sync with scst_abort_cmd() */
2691
2692         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2693                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
2694                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
2695                         atomic_read(&scst_cmd_count));
2696
2697                 scst_finish_cmd_mgmt(cmd);
2698         }
2699
2700         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
2701                 if ((cmd->tgt_dev != NULL) &&
2702                     scst_is_ua_sense(cmd->sense)) {
2703                         /* This UA delivery failed, so requeue it */
2704                         TRACE_MGMT_DBG("Requeuing UA for delivery failed cmd "
2705                                 "%p", cmd);
2706                         scst_check_set_UA(cmd->tgt_dev, cmd->sense,
2707                                         SCST_SENSE_BUFFERSIZE, 1);
2708                 }
2709         }
2710
2711         __scst_cmd_put(cmd);
2712
2713         res = SCST_CMD_STATE_RES_CONT_NEXT;
2714
2715         TRACE_EXIT_HRES(res);
2716         return res;
2717 }
2718
2719 /*
2720  * No locks, but it must be externally serialized (see comment for
2721  * scst_cmd_init_done() in scsi_tgt.h)
2722  */
2723 static void scst_cmd_set_sn(struct scst_cmd *cmd)
2724 {
2725         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2726         unsigned long flags;
2727
2728         if (scst_is_implicit_hq(cmd)) {
2729                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "Implicit HQ cmd %p", cmd);
2730                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
2731         }
2732
2733         /* Optimized for lockless fast path */
2734
2735         scst_check_debug_sn(cmd);
2736
2737         if (cmd->dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) {
2738                 /*
2739                  * Not the best way, but well enough until there will be a
2740                  * possibility to specify queue type during pass-through
2741                  * commands submission.
2742                  */
2743                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
2744         }
2745
2746         switch(cmd->queue_type) {
2747         case SCST_CMD_QUEUE_SIMPLE:
2748         case SCST_CMD_QUEUE_UNTAGGED:
2749 #if 1 /* temporary, ToDo */
2750                 if (scst_cmd_is_expected_set(cmd)) {
2751                         if (cmd->expected_data_direction == SCST_DATA_READ)
2752                                 goto ordered;
2753                 } else
2754                         goto ordered;
2755 #endif
2756                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
2757                         /*
2758                          * atomic_inc_return() implies memory barrier to sync
2759                          * with scst_inc_expected_sn()
2760                          */
2761                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
2762                                 tgt_dev->curr_sn++;
2763                                 TRACE_SN("Incremented curr_sn %ld",
2764                                         tgt_dev->curr_sn);
2765                         }
2766                         cmd->sn_slot = tgt_dev->cur_sn_slot;
2767                         cmd->sn = tgt_dev->curr_sn;
2768                         
2769                         tgt_dev->prev_cmd_ordered = 0;
2770                 } else {
2771                         TRACE(TRACE_MINOR, "***WARNING*** Not enough SN slots "
2772                                 "%d", ARRAY_SIZE(tgt_dev->sn_slots));
2773                         goto ordered;
2774                 }
2775                 break;
2776
2777         case SCST_CMD_QUEUE_ORDERED:
2778                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "ORDERED cmd %p "
2779                         "(op %x)", cmd, cmd->cdb[0]);
2780 ordered:
2781                 if (!tgt_dev->prev_cmd_ordered) {
2782                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2783                         if (tgt_dev->num_free_sn_slots >= 0) {
2784                                 tgt_dev->num_free_sn_slots--;
2785                                 if (tgt_dev->num_free_sn_slots >= 0) {
2786                                         int i = 0;
2787                                         /*
2788                                          * Commands can finish in any order, so we don't
2789                                          * know, which slot is empty.
2790                                          */
2791                                         while(1) {
2792                                                 tgt_dev->cur_sn_slot++;
2793                                                 if (tgt_dev->cur_sn_slot == tgt_dev->sn_slots +
2794                                                                 ARRAY_SIZE(tgt_dev->sn_slots))
2795                                                         tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
2796
2797                                                 if (atomic_read(tgt_dev->cur_sn_slot) == 0)
2798                                                         break;
2799
2800                                                 i++;
2801                                                 sBUG_ON(i == ARRAY_SIZE(tgt_dev->sn_slots));
2802                                         }
2803                                         TRACE_SN("New cur SN slot %zd",
2804                                                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
2805                                 }
2806                         }
2807                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2808                 }
2809                 tgt_dev->prev_cmd_ordered = 1;
2810                 tgt_dev->curr_sn++;
2811                 cmd->sn = tgt_dev->curr_sn;
2812                 break;
2813
2814         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
2815                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "HQ cmd %p "
2816                         "(op %x)", cmd, cmd->cdb[0]);
2817                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2818                 tgt_dev->hq_cmd_count++;
2819                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2820                 goto out;
2821
2822         default:
2823                 sBUG();
2824         }
2825
2826         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
2827                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
2828                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
2829                 atomic_read(tgt_dev->cur_sn_slot), 
2830                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
2831                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
2832
2833         cmd->sn_set = 1;
2834
2835 out:
2836         return;
2837 }
2838
2839 /*
2840  * Returns 0 on success, > 0 when we need to wait for unblock,
2841  * < 0 if there is no device (lun) or device type handler.
2842  *
2843  * No locks, but might be on IRQ, protection is done by the
2844  * suspended activity.
2845  */
2846 static int scst_translate_lun(struct scst_cmd *cmd)
2847 {
2848         struct scst_tgt_dev *tgt_dev = NULL;
2849         int res;
2850
2851         TRACE_ENTRY();
2852
2853         __scst_get(1);
2854
2855         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
2856                 struct list_head *sess_tgt_dev_list_head =
2857                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
2858                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %Ld)", cmd,
2859                         (uint64_t)cmd->lun);
2860                 res = -1;
2861                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
2862                                 sess_tgt_dev_list_entry) {
2863                         if (tgt_dev->lun == cmd->lun) {
2864                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
2865
2866                                 if (unlikely(tgt_dev->dev->handler == &scst_null_devtype)) {
2867                                         PRINT_INFO("Dev handler for device "
2868                                           "%Ld is NULL, the device will not be "
2869                                           "visible remotely", (uint64_t)cmd->lun);
2870                                         break;
2871                                 }
2872                                 
2873                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
2874                                 cmd->tgt_dev = tgt_dev;
2875                                 cmd->dev = tgt_dev->dev;
2876
2877                                 res = 0;
2878                                 break;
2879                         }
2880                 }
2881                 if (res != 0) {
2882                         TRACE(TRACE_MINOR, "tgt_dev for lun %Ld not found, command to "
2883                                 "unexisting LU?", (uint64_t)cmd->lun);
2884                         __scst_put();
2885                 }
2886         } else {
2887                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
2888                 __scst_put();
2889                 res = 1;
2890         }
2891
2892         TRACE_EXIT_RES(res);
2893         return res;
2894 }
2895
2896 /*
2897  * No locks, but might be on IRQ
2898  *
2899  * Returns 0 on success, > 0 when we need to wait for unblock,
2900  * < 0 if there is no device (lun) or device type handler.
2901  */
2902 static int __scst_init_cmd(struct scst_cmd *cmd)
2903 {
2904         int res = 0;
2905
2906         TRACE_ENTRY();
2907
2908         res = scst_translate_lun(cmd);
2909         if (likely(res == 0)) {
2910                 int cnt;
2911                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
2912                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
2913                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
2914                         TRACE(TRACE_MGMT_MINOR, "Too many pending commands (%d) in "
2915                                 "session, returning BUSY to initiator \"%s\"",
2916                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
2917                                   "Anonymous" : cmd->sess->initiator_name);
2918                         goto out_busy;
2919                 }
2920                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
2921                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
2922                         TRACE(TRACE_MGMT_MINOR, "Too many pending device commands "
2923                                 "(%d), returning BUSY to initiator \"%s\"",
2924                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
2925                                   "Anonymous" : cmd->sess->initiator_name);
2926                         goto out_busy;
2927                 }
2928                 if (!cmd->set_sn_on_restart_cmd)
2929                         scst_cmd_set_sn(cmd);
2930         } else if (res < 0) {
2931                 TRACE_DBG("Finishing cmd %p", cmd);
2932                 scst_set_cmd_error(cmd,
2933                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
2934                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2935         } else
2936                 goto out;
2937
2938 out:
2939         TRACE_EXIT_RES(res);
2940         return res;
2941
2942 out_busy:
2943         scst_set_busy(cmd);
2944         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2945         goto out;
2946 }
2947
2948 /* Called under scst_init_lock and IRQs disabled */
2949 static void scst_do_job_init(void)
2950 {
2951         struct scst_cmd *cmd;
2952         int susp;
2953
2954         TRACE_ENTRY();
2955
2956 restart:
2957         /*
2958          * There is no need for read barrier here, because we don't care where
2959          * this check will be done.
2960          */
2961         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
2962         if (scst_init_poll_cnt > 0)
2963                 scst_init_poll_cnt--;
2964
2965         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
2966                 int rc;
2967                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
2968                         continue;
2969                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2970                         spin_unlock_irq(&scst_init_lock);
2971                         rc = __scst_init_cmd(cmd);
2972                         spin_lock_irq(&scst_init_lock);
2973                         if (rc > 0) {
2974                                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, restarting");
2975                                 goto restart;
2976                         }
2977                 } else {
2978                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
2979                                 cmd, cmd->tag);
2980                         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2981                 }
2982
2983                 /*
2984                  * Deleting cmd from init cmd list after __scst_init_cmd()
2985                  * is necessary to keep the check in scst_init_cmd() correct
2986                  * to preserve the commands order.
2987                  *
2988                  * We don't care about the race, when init cmd list is empty
2989                  * and one command detected that it just was not empty, so
2990                  * it's inserting to it, but another command at the same time
2991                  * seeing init cmd list empty and goes directly, because it
2992                  * could affect only commands from the same initiator to the
2993                  * same tgt_dev, but init_cmd_done() doesn't guarantee the order
2994                  * in case of simultaneous such calls anyway.
2995                  */
2996                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
2997                 smp_wmb();
2998                 list_del(&cmd->cmd_list_entry);
2999                 spin_unlock(&scst_init_lock);
3000
3001                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
3002                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
3003                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
3004                         list_add(&cmd->cmd_list_entry,
3005                                 &cmd->cmd_lists->active_cmd_list);
3006                 else
3007                         list_add_tail(&cmd->cmd_list_entry,
3008                                 &cmd->cmd_lists->active_cmd_list);
3009                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3010                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
3011
3012                 spin_lock(&scst_init_lock);
3013                 goto restart;
3014         }
3015
3016         /* It isn't really needed, but let's keep it */
3017         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
3018                 goto restart;
3019
3020         TRACE_EXIT();
3021         return;
3022 }
3023
3024 static inline int test_init_cmd_list(void)
3025 {
3026         int res = (!list_empty(&scst_init_cmd_list) &&
3027                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
3028                   unlikely(kthread_should_stop()) ||
3029                   (scst_init_poll_cnt > 0);
3030         return res;
3031 }
3032
3033 int scst_init_cmd_thread(void *arg)
3034 {
3035         TRACE_ENTRY();
3036
3037         PRINT_INFO("Init thread started, PID %d", current->pid);
3038
3039         current->flags |= PF_NOFREEZE;
3040
3041         set_user_nice(current, -10);
3042
3043         spin_lock_irq(&scst_init_lock);
3044         while(!kthread_should_stop()) {
3045                 wait_queue_t wait;
3046                 init_waitqueue_entry(&wait, current);
3047
3048                 if (!test_init_cmd_list()) {
3049                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
3050                                                  &wait);
3051                         for (;;) {
3052                                 set_current_state(TASK_INTERRUPTIBLE);
3053                                 if (test_init_cmd_list())
3054                                         break;
3055                                 spin_unlock_irq(&scst_init_lock);
3056                                 schedule();
3057                                 spin_lock_irq(&scst_init_lock);
3058                         }
3059                         set_current_state(TASK_RUNNING);
3060                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
3061                 }
3062                 scst_do_job_init();
3063         }
3064         spin_unlock_irq(&scst_init_lock);
3065
3066         /*
3067          * If kthread_should_stop() is true, we are guaranteed to be
3068          * on the module unload, so scst_init_cmd_list must be empty.
3069          */
3070         sBUG_ON(!list_empty(&scst_init_cmd_list));
3071
3072         PRINT_INFO("Init thread PID %d finished", current->pid);
3073
3074         TRACE_EXIT();
3075         return 0;
3076 }
3077
3078 /* Called with no locks held */
3079 void scst_process_active_cmd(struct scst_cmd *cmd, int context)
3080 {
3081         int res;
3082
3083         TRACE_ENTRY();
3084
3085         EXTRACHECKS_BUG_ON(in_irq());
3086
3087         cmd->context_processable = context | SCST_CONTEXT_PROCESSABLE;
3088         context &= ~SCST_CONTEXT_PROCESSABLE;
3089         cmd->atomic = (context == SCST_CONTEXT_DIRECT_ATOMIC);
3090
3091         TRACE_DBG("cmd %p, context_processable %d, atomic %d", cmd,
3092                 cmd->context_processable, cmd->atomic);
3093
3094         do {
3095                 switch (cmd->state) {
3096                 case SCST_CMD_STATE_PRE_PARSE:
3097                         res = scst_pre_parse(cmd);
3098                         EXTRACHECKS_BUG_ON(res ==
3099                                 SCST_CMD_STATE_RES_NEED_THREAD);
3100                         break;
3101
3102                 case SCST_CMD_STATE_DEV_PARSE:
3103                         res = scst_parse_cmd(cmd);
3104                         break;
3105
3106                 case SCST_CMD_STATE_PREPARE_SPACE:
3107                         res = scst_prepare_space(cmd);
3108                         break;
3109
3110                 case SCST_CMD_STATE_RDY_TO_XFER:
3111                         res = scst_rdy_to_xfer(cmd);
3112                         break;
3113
3114                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3115                         res = scst_tgt_pre_exec(cmd);
3116                         break;
3117
3118                 case SCST_CMD_STATE_SEND_TO_MIDLEV:
3119                         if (tm_dbg_check_cmd(cmd) != 0) {
3120                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
3121                                 TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
3122                                         "because of TM DBG delay", cmd,
3123                                         cmd->tag);
3124                                 break;
3125                         }
3126                         res = scst_send_to_midlev(&cmd);
3127                         /* !! At this point cmd, sess & tgt_dev can be already freed !! */
3128                         break;
3129
3130                 case SCST_CMD_STATE_PRE_DEV_DONE:
3131                         res = scst_pre_dev_done(cmd);
3132                         EXTRACHECKS_BUG_ON(res ==
3133                                 SCST_CMD_STATE_RES_NEED_THREAD);
3134                         break;
3135
3136                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3137                         res = scst_mode_select_checks(cmd);
3138                         break;
3139
3140                 case SCST_CMD_STATE_DEV_DONE:
3141                         res = scst_dev_done(cmd);
3142                         break;
3143
3144                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3145                         res = scst_pre_xmit_response(cmd);
3146                         EXTRACHECKS_BUG_ON(res ==
3147                                 SCST_CMD_STATE_RES_NEED_THREAD);
3148                         break;
3149
3150                 case SCST_CMD_STATE_XMIT_RESP:
3151                         res = scst_xmit_response(cmd);
3152                         break;
3153
3154                 case SCST_CMD_STATE_FINISHED:
3155                         res = scst_finish_cmd(cmd);
3156                         EXTRACHECKS_BUG_ON(res ==
3157                                 SCST_CMD_STATE_RES_NEED_THREAD);
3158                         break;
3159
3160                 default:
3161                         PRINT_CRIT_ERROR("cmd (%p) in state %d, but shouldn't "
3162                                 "be", cmd, cmd->state);
3163                         sBUG();
3164                         res = SCST_CMD_STATE_RES_CONT_NEXT;
3165                         break;
3166                 }
3167         } while(res == SCST_CMD_STATE_RES_CONT_SAME);
3168
3169         if (res == SCST_CMD_STATE_RES_CONT_NEXT) {
3170                 /* None */
3171         } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
3172                 spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3173                 switch (cmd->state) {
3174                 case SCST_CMD_STATE_PRE_PARSE:
3175                 case SCST_CMD_STATE_DEV_PARSE:
3176                 case SCST_CMD_STATE_PREPARE_SPACE:
3177                 case SCST_CMD_STATE_RDY_TO_XFER:
3178                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3179                 case SCST_CMD_STATE_SEND_TO_MIDLEV:
3180                 case SCST_CMD_STATE_PRE_DEV_DONE:
3181                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3182                 case SCST_CMD_STATE_DEV_DONE:
3183                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3184                 case SCST_CMD_STATE_XMIT_RESP:
3185                 case SCST_CMD_STATE_FINISHED:
3186                         TRACE_DBG("Adding cmd %p to head of active cmd list", cmd);
3187                         list_add(&cmd->cmd_list_entry,
3188                                 &cmd->cmd_lists->active_cmd_list);
3189                         break;
3190 #ifdef EXTRACHECKS
3191                 /* not very valid commands */
3192                 case SCST_CMD_STATE_DEFAULT:
3193                 case SCST_CMD_STATE_NEED_THREAD_CTX:
3194                         PRINT_CRIT_ERROR("cmd %p is in state %d, not putting on "
3195                                 "useful list (left on scst cmd list)", cmd, 
3196                                 cmd->state);
3197                         spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3198                         sBUG();
3199                         spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3200                         break;
3201 #endif
3202                 default:
3203                         break;
3204                 }
3205                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3206                 spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3207         } else
3208                 sBUG();
3209
3210         TRACE_EXIT();
3211         return;
3212 }
3213
3214 /* Called under cmd_list_lock and IRQs disabled */
3215 static void scst_do_job_active(struct list_head *cmd_list,
3216         spinlock_t *cmd_list_lock, int context)
3217 {
3218         TRACE_ENTRY();
3219
3220 #ifdef EXTRACHECKS
3221         {
3222                 int c = context & ~SCST_CONTEXT_PROCESSABLE;
3223                 sBUG_ON((c != SCST_CONTEXT_DIRECT_ATOMIC) && 
3224                         (c != SCST_CONTEXT_DIRECT));
3225         }
3226 #endif
3227
3228         while (!list_empty(cmd_list)) {
3229                 struct scst_cmd *cmd = list_entry(cmd_list->next, typeof(*cmd),
3230                                         cmd_list_entry);
3231                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
3232                 list_del(&cmd->cmd_list_entry);
3233                 spin_unlock_irq(cmd_list_lock);
3234                 scst_process_active_cmd(cmd, context);
3235                 spin_lock_irq(cmd_list_lock);
3236         }
3237
3238         TRACE_EXIT();
3239         return;
3240 }
3241
3242 static inline int test_cmd_lists(struct scst_cmd_lists *p_cmd_lists)
3243 {
3244         int res = !list_empty(&p_cmd_lists->active_cmd_list) ||
3245             unlikely(kthread_should_stop()) ||
3246             tm_dbg_is_release();
3247         return res;
3248 }
3249
3250 int scst_cmd_thread(void *arg)
3251 {
3252         struct scst_cmd_lists *p_cmd_lists = (struct scst_cmd_lists*)arg;
3253
3254         TRACE_ENTRY();
3255
3256         PRINT_INFO("Processing thread started, PID %d", current->pid);
3257
3258 #if 0
3259         set_user_nice(current, 10);
3260 #endif
3261         current->flags |= PF_NOFREEZE;
3262
3263         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3264         while (!kthread_should_stop()) {
3265                 wait_queue_t wait;
3266                 init_waitqueue_entry(&wait, current);
3267
3268                 if (!test_cmd_lists(p_cmd_lists)) {
3269                         add_wait_queue_exclusive(&p_cmd_lists->cmd_list_waitQ,
3270                                 &wait);
3271                         for (;;) {
3272                                 set_current_state(TASK_INTERRUPTIBLE);
3273                                 if (test_cmd_lists(p_cmd_lists))
3274                                         break;
3275                                 spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3276                                 schedule();
3277                                 spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3278                         }
3279                         set_current_state(TASK_RUNNING);
3280                         remove_wait_queue(&p_cmd_lists->cmd_list_waitQ, &wait);
3281                 }
3282
3283                 if (tm_dbg_is_release()) {
3284                         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3285                         tm_dbg_check_released_cmds();
3286                         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3287                 }
3288
3289                 scst_do_job_active(&p_cmd_lists->active_cmd_list,
3290                         &p_cmd_lists->cmd_list_lock, SCST_CONTEXT_DIRECT |
3291                                                 SCST_CONTEXT_PROCESSABLE);
3292         }
3293         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3294
3295 #ifdef EXTRACHECKS
3296         /*
3297          * If kthread_should_stop() is true, we are guaranteed to be either
3298          * on the module unload, or there must be at least one other thread to
3299          * process the commands lists.
3300          */
3301         if (p_cmd_lists == &scst_main_cmd_lists) {
3302                 sBUG_ON((scst_threads_info.nr_cmd_threads == 1) &&
3303                          !list_empty(&scst_main_cmd_lists.active_cmd_list));
3304         }
3305 #endif
3306
3307         PRINT_INFO("Processing thread PID %d finished", current->pid);
3308
3309         TRACE_EXIT();
3310         return 0;
3311 }
3312
3313 void scst_cmd_tasklet(long p)
3314 {
3315         struct scst_tasklet *t = (struct scst_tasklet*)p;
3316
3317         TRACE_ENTRY();
3318
3319         spin_lock_irq(&t->tasklet_lock);
3320         scst_do_job_active(&t->tasklet_cmd_list, &t->tasklet_lock,
3321                 SCST_CONTEXT_DIRECT_ATOMIC);
3322         spin_unlock_irq(&t->tasklet_lock);
3323
3324         TRACE_EXIT();
3325         return;
3326 }
3327
3328 /*
3329  * Returns 0 on success, < 0 if there is no device handler or
3330  * > 0 if SCST_FLAG_SUSPENDED set and SCST_FLAG_SUSPENDING - not.
3331  * No locks, protection is done by the suspended activity.
3332  */
3333 static int scst_mgmt_translate_lun(struct scst_mgmt_cmd *mcmd)
3334 {
3335         struct scst_tgt_dev *tgt_dev = NULL;
3336         struct list_head *sess_tgt_dev_list_head;
3337         int res = -1;
3338
3339         TRACE_ENTRY();
3340
3341         TRACE_DBG("Finding tgt_dev for mgmt cmd %p (lun %Ld)", mcmd,
3342               (uint64_t)mcmd->lun);
3343
3344         __scst_get(1);
3345
3346         if (unlikely(test_bit(SCST_FLAG_SUSPENDED, &scst_flags) &&
3347                      !test_bit(SCST_FLAG_SUSPENDING, &scst_flags))) {
3348                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3349                 __scst_put();
3350                 res = 1;
3351                 goto out;
3352         }
3353
3354         sess_tgt_dev_list_head =
3355                 &mcmd->sess->sess_tgt_dev_list_hash[HASH_VAL(mcmd->lun)];
3356         list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3357                         sess_tgt_dev_list_entry) {
3358                 if (tgt_dev->lun == mcmd->lun) {
3359                         TRACE_DBG("tgt_dev %p found", tgt_dev);
3360                         mcmd->mcmd_tgt_dev = tgt_dev;
3361                         res = 0;
3362                         break;
3363                 }
3364         }
3365         if (mcmd->mcmd_tgt_dev == NULL)
3366                 __scst_put();
3367
3368 out:
3369         TRACE_EXIT_HRES(res);
3370         return res;
3371 }
3372
3373 /* No locks */
3374 void scst_done_cmd_mgmt(struct scst_cmd *cmd)
3375 {
3376         struct scst_mgmt_cmd_stub *mstb;
3377         bool wake = 0;
3378         unsigned long flags;
3379
3380         TRACE_ENTRY();
3381
3382         TRACE_MGMT_DBG("cmd %p done (tag %llu)", cmd, cmd->tag);
3383
3384         spin_lock_irqsave(&scst_mcmd_lock, flags);
3385
3386         list_for_each_entry(mstb, &cmd->mgmt_cmd_list,
3387                         cmd_mgmt_cmd_list_entry) {
3388                 struct scst_mgmt_cmd *mcmd = mstb->mcmd;
3389
3390                 TRACE_MGMT_DBG("mcmd %p, mcmd->cmd_done_wait_count %d", 
3391                         mcmd, mcmd->cmd_done_wait_count);
3392
3393                 mcmd->cmd_done_wait_count--;
3394                 if (mcmd->cmd_done_wait_count > 0) {
3395                         TRACE_MGMT_DBG("cmd_done_wait_count(%d) not 0, "
3396                                 "skipping", mcmd->cmd_done_wait_count);
3397                         continue;
3398                 }
3399
3400                 if (mcmd->completed) {
3401                         sBUG_ON(mcmd->nexus_loss_check_done);
3402                         mcmd->completed = 0;
3403                         mcmd->state = SCST_MGMT_CMD_STATE_CHECK_NEXUS_LOSS;
3404                         TRACE_MGMT_DBG("Adding mgmt cmd %p to active mgmt cmd "
3405                                 "list", mcmd);
3406                         list_add_tail(&mcmd->mgmt_cmd_list_entry,
3407                                 &scst_active_mgmt_cmd_list);
3408                         wake = 1;
3409                 }
3410         }
3411
3412         spin_unlock_irqrestore(&scst_mcmd_lock, flags);
3413
3414         if (wake)
3415                 wake_up(&scst_mgmt_cmd_list_waitQ);
3416
3417         TRACE_EXIT();
3418         return;
3419 }
3420
3421 /* No locks */
3422 static void scst_finish_cmd_mgmt(struct scst_cmd *cmd)
3423 {
3424         struct scst_mgmt_cmd_stub *mstb, *t;
3425         bool wake = 0;
3426         unsigned long flags;
3427
3428         TRACE_ENTRY();
3429
3430         TRACE_MGMT_DBG("cmd %p finished (tag %llu)", cmd, cmd->tag);
3431
3432         spin_lock_irqsave(&scst_mcmd_lock, flags);
3433
3434         list_for_each_entry_safe(mstb, t, &cmd->mgmt_cmd_list,
3435                         cmd_mgmt_cmd_list_entry) {
3436                 struct scst_mgmt_cmd *mcmd = mstb->mcmd;
3437
3438                 TRACE_MGMT_DBG("mcmd %p, mcmd->cmd_finish_wait_count %d", 
3439                         mcmd, mcmd->cmd_finish_wait_count);
3440
3441                 list_del(&mstb->cmd_mgmt_cmd_list_entry);
3442                 mempool_free(mstb, scst_mgmt_stub_mempool);
3443
3444                 if (cmd->completed)
3445                         mcmd->completed_cmd_count++;
3446
3447                 mcmd->cmd_finish_wait_count--;
3448                 if (mcmd->cmd_finish_wait_count > 0) {
3449                         TRACE_MGMT_DBG("cmd_finish_wait_count(%d) not 0, "
3450                                 "skipping", mcmd->cmd_finish_wait_count);
3451                         continue;
3452                 }
3453
3454                 if (mcmd->completed) {
3455                         mcmd->state = SCST_MGMT_CMD_STATE_DONE;
3456                         TRACE_MGMT_DBG("Adding mgmt cmd %p to active mgmt cmd "
3457                                 "list", mcmd);
3458                         list_add_tail(&mcmd->mgmt_cmd_list_entry,
3459                                 &scst_active_mgmt_cmd_list);
3460                         wake = 1;
3461                 }
3462         }
3463
3464         spin_unlock_irqrestore(&scst_mcmd_lock, flags);
3465
3466         if (wake)
3467                 wake_up(&scst_mgmt_cmd_list_waitQ);
3468
3469         TRACE_EXIT();
3470         return;
3471 }
3472
3473 static int scst_call_dev_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
3474         struct scst_tgt_dev *tgt_dev, int set_status)
3475 {
3476         int res = SCST_DEV_TM_NOT_COMPLETED;
3477         struct scst_dev_type *h = tgt_dev->dev->handler;
3478
3479         if (h->task_mgmt_fn) {
3480                 TRACE_MGMT_DBG("Calling dev handler %s task_mgmt_fn(fn=%d)",
3481                         h->name, mcmd->fn);
3482                 EXTRACHECKS_BUG_ON(in_irq());
3483                 res = h->task_mgmt_fn(mcmd, tgt_dev);
3484                 TRACE_MGMT_DBG("Dev handler %s task_mgmt_fn() returned %d",
3485                       h->name, res);
3486                 if (set_status && (res != SCST_DEV_TM_NOT_COMPLETED))
3487                         mcmd->status = res;
3488         }
3489         return res;
3490 }
3491
3492 static inline int scst_is_strict_mgmt_fn(int mgmt_fn)
3493 {
3494         switch(mgmt_fn) {
3495 #ifdef ABORT_CONSIDER_FINISHED_TASKS_AS_NOT_EXISTING
3496                 case SCST_ABORT_TASK:
3497 #endif
3498 #if 0
3499                 case SCST_ABORT_TASK_SET:
3500                 case SCST_CLEAR_TASK_SET:
3501 #endif
3502                         return 1;
3503                 default:
3504                         return 0;
3505         }
3506 }
3507
3508 /* 
3509  * Might be called under sess_list_lock and IRQ off + BHs also off
3510  * Returns -1 if command is being executed (ABORT failed), 0 otherwise
3511  */
3512 void scst_abort_cmd(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd,
3513         int other_ini, int call_dev_task_mgmt_fn)
3514 {
3515         unsigned long flags;
3516
3517         TRACE_ENTRY();
3518
3519         TRACE(((mcmd != NULL) && (mcmd->fn == SCST_ABORT_TASK)) ? TRACE_MGMT_MINOR : TRACE_MGMT,
3520                 "Aborting cmd %p (tag %llu, op %x)", cmd, cmd->tag, cmd->cdb[0]);
3521
3522         if (other_ini) {
3523                 set_bit(SCST_CMD_ABORTED_OTHER, &cmd->cmd_flags);
3524                 smp_mb__after_set_bit();
3525         } else {
3526                 /* Might be necessary if command aborted several times */
3527                 clear_bit(SCST_CMD_ABORTED_OTHER, &cmd->cmd_flags);
3528