A bunch of pending changes:
[mirror/scst/.git] / scst / src / scst_targ.c
1 /*
2  *  scst_targ.c
3  *  
4  *  Copyright (C) 2004-2007 Vladislav Bolkhovitin <vst@vlnb.net>
5  *                 and Leonid Stoljar
6  *  
7  *  This program is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU General Public License
9  *  as published by the Free Software Foundation, version 2
10  *  of the License.
11  * 
12  *  This program is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  *  GNU General Public License for more details.
16  */
17
18 #include <linux/init.h>
19 #include <linux/kernel.h>
20 #include <linux/errno.h>
21 #include <linux/list.h>
22 #include <linux/spinlock.h>
23 #include <linux/slab.h>
24 #include <linux/sched.h>
25 #include <linux/smp_lock.h>
26 #include <asm/unistd.h>
27 #include <asm/string.h>
28 #include <linux/kthread.h>
29 #include <linux/delay.h>
30
31 #include "scsi_tgt.h"
32 #include "scst_priv.h"
33
34 static void scst_cmd_set_sn(struct scst_cmd *cmd);
35 static int __scst_init_cmd(struct scst_cmd *cmd);
36 static void scst_finish_cmd_mgmt(struct scst_cmd *cmd);
37
38 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
39 {
40         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
41         unsigned long flags;
42
43         spin_lock_irqsave(&t->tasklet_lock, flags);
44         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
45                 smp_processor_id());
46         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
47         spin_unlock_irqrestore(&t->tasklet_lock, flags);
48
49         tasklet_schedule(&t->tasklet);
50 }
51
52 /* 
53  * Must not be called in parallel with scst_unregister_session_ex() for the 
54  * same sess
55  */
56 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
57                              const uint8_t *lun, int lun_len,
58                              const uint8_t *cdb, int cdb_len, int atomic)
59 {
60         struct scst_cmd *cmd;
61
62         TRACE_ENTRY();
63
64 #ifdef EXTRACHECKS
65         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
66                 PRINT_ERROR("%s", "New cmd while shutting down the session");
67                 sBUG();
68         }
69 #endif
70
71         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
72         if (cmd == NULL)
73                 goto out;
74
75         cmd->sess = sess;
76         cmd->tgt = sess->tgt;
77         cmd->tgtt = sess->tgt->tgtt;
78
79         /* 
80          * For both wrong lun and CDB defer the error reporting for
81          * scst_cmd_init_done()
82          */
83
84         cmd->lun = scst_unpack_lun(lun, lun_len);
85
86         if (cdb_len <= SCST_MAX_CDB_SIZE) {
87                 memcpy(cmd->cdb, cdb, cdb_len);
88                 cmd->cdb_len = cdb_len;
89         }
90
91         TRACE_DBG("cmd %p, sess %p", cmd, sess);
92         scst_sess_get(sess);
93
94 out:
95         TRACE_EXIT();
96         return cmd;
97 }
98
99 static int scst_init_cmd(struct scst_cmd *cmd, int context)
100 {
101         int rc;
102
103         TRACE_ENTRY();
104
105         /* See the comment in scst_do_job_init() */
106         if (unlikely(!list_empty(&scst_init_cmd_list))) {
107                 TRACE_MGMT_DBG("%s", "init cmd list busy");
108                 goto out_redirect;
109         }
110         /*
111          * Memory barrier isn't necessary here, because CPU appears to
112          * be self-consistent
113          */
114
115         rc = __scst_init_cmd(cmd);
116         if (unlikely(rc > 0))
117                 goto out_redirect;
118         else if (unlikely(rc != 0))
119                 goto out;
120
121         /* Small context optimization */
122         if (((context == SCST_CONTEXT_TASKLET) ||
123              (context == SCST_CONTEXT_DIRECT_ATOMIC)) && 
124             scst_cmd_is_expected_set(cmd)) {
125                 if (cmd->expected_data_direction == SCST_DATA_WRITE) {
126                         if ( !test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
127                                         &cmd->tgt_dev->tgt_dev_flags))
128                                 context = SCST_CONTEXT_THREAD;
129                 } else {
130                         if ( !test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
131                                         &cmd->tgt_dev->tgt_dev_flags))
132                                 context = SCST_CONTEXT_THREAD;
133                 }
134         }
135
136 out:
137         TRACE_EXIT_RES(context);
138         return context;
139
140 out_redirect:
141         if (cmd->preprocessing_only) {
142                 /*
143                  * Poor man solution for single threaded targets, where 
144                  * blocking receiver at least sometimes means blocking all.
145                  */
146                 sBUG_ON(context != SCST_CONTEXT_DIRECT);
147                 scst_set_busy(cmd);
148                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
149                 /* Keep initiator away from too many BUSY commands */
150                 if (!in_interrupt() && !in_atomic())
151                         msleep(50);
152                 else
153                         WARN_ON_ONCE(1);
154         } else {
155                 unsigned long flags;
156                 spin_lock_irqsave(&scst_init_lock, flags);
157                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
158                         "%d)", cmd, atomic_read(&scst_cmd_count));
159                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
160                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
161                         scst_init_poll_cnt++;
162                 spin_unlock_irqrestore(&scst_init_lock, flags);
163                 wake_up(&scst_init_cmd_list_waitQ);
164                 context = -1;
165         }
166         goto out;
167 }
168
169 #ifdef MEASURE_LATENCY
170 static inline uint64_t scst_sec_to_nsec(time_t sec)
171 {
172         return (uint64_t)sec * 1000000000;
173 }
174 #endif
175
176 void scst_cmd_init_done(struct scst_cmd *cmd, int pref_context)
177 {
178         unsigned long flags;
179         struct scst_session *sess = cmd->sess;
180
181         TRACE_ENTRY();
182
183 #ifdef MEASURE_LATENCY
184         {
185                 struct timespec ts;
186                 getnstimeofday(&ts);
187                 cmd->start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
188                 TRACE_DBG("cmd %p (sess %p): start %Ld (tv_sec %ld, "
189                         "tv_nsec %ld)", cmd, sess, cmd->start, ts.tv_sec,
190                         ts.tv_nsec);
191         }
192 #endif
193
194         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
195         TRACE(TRACE_SCSI, "tag=%llu, lun=%Ld, CDB len=%d", cmd->tag, 
196                 (uint64_t)cmd->lun, cmd->cdb_len);
197         PRINT_BUFF_FLAG(TRACE_SCSI|TRACE_RCV_BOT, "Recieving CDB",
198                 cmd->cdb, cmd->cdb_len);
199
200 #ifdef EXTRACHECKS
201         if (unlikely(in_irq()) && ((pref_context == SCST_CONTEXT_DIRECT) ||
202                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
203         {
204                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
205                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
206                         cmd->tgtt->name);
207                 pref_context = SCST_CONTEXT_TASKLET;
208         }
209 #endif
210
211         atomic_inc(&sess->sess_cmd_count);
212
213         spin_lock_irqsave(&sess->sess_list_lock, flags);
214
215         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
216                 /*
217                  * We have to always keep command in the search list from the
218                  * very beginning, because otherwise it can be missed during
219                  * TM processing. This check is needed because there might be
220                  * old, i.e. deferred, commands and new, i.e. just coming, ones.
221                  */
222                 if (cmd->search_cmd_list_entry.next == NULL)
223                         list_add_tail(&cmd->search_cmd_list_entry,
224                                 &sess->search_cmd_list);
225                 switch(sess->init_phase) {
226                 case SCST_SESS_IPH_SUCCESS:
227                         break;
228                 case SCST_SESS_IPH_INITING:
229                         TRACE_DBG("Adding cmd %p to init deferred cmd list", cmd);
230                         list_add_tail(&cmd->cmd_list_entry, 
231                                 &sess->init_deferred_cmd_list);
232                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
233                         goto out;
234                 case SCST_SESS_IPH_FAILED:
235                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
236                         scst_set_busy(cmd);
237                         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
238                         goto active;
239                 default:
240                         sBUG();
241                 }
242         } else
243                 list_add_tail(&cmd->search_cmd_list_entry, &sess->search_cmd_list);
244
245         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
246
247         if (unlikely(cmd->lun == (lun_t)-1)) {
248                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
249                 scst_set_cmd_error(cmd,
250                         SCST_LOAD_SENSE(scst_sense_lun_not_supported));
251                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
252                 goto active;
253         }
254
255         if (unlikely(cmd->cdb_len == 0)) {
256                 PRINT_ERROR("Wrong CDB len %d, finishing cmd", 0);
257                 scst_set_cmd_error(cmd,
258                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
259                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
260                 goto active;
261         }
262
263         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
264                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
265                 scst_set_cmd_error(cmd,
266                         SCST_LOAD_SENSE(scst_sense_invalid_message));
267                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
268                 goto active;
269         }
270
271         cmd->state = SCST_CMD_STATE_INIT;
272         /* cmd must be inited here to keep the order */
273         pref_context = scst_init_cmd(cmd, pref_context);
274         if (unlikely(pref_context < 0))
275                 goto out;
276
277 active:
278         /* Here cmd must not be in any cmd list, no locks */
279         switch (pref_context) {
280         case SCST_CONTEXT_TASKLET:
281                 scst_schedule_tasklet(cmd);
282                 break;
283
284         case SCST_CONTEXT_DIRECT:
285         case SCST_CONTEXT_DIRECT_ATOMIC:
286                 scst_process_active_cmd(cmd, pref_context);
287                 /* For *NEED_THREAD wake_up() is already done */
288                 break;
289
290         default:
291                 PRINT_ERROR("Context %x is undefined, using the thread one",
292                         pref_context);
293                 /* go through */
294         case SCST_CONTEXT_THREAD:
295                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
296                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
297                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
298                         list_add(&cmd->cmd_list_entry,
299                                 &cmd->cmd_lists->active_cmd_list);
300                 else
301                         list_add_tail(&cmd->cmd_list_entry,
302                                 &cmd->cmd_lists->active_cmd_list);
303                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
304                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
305                 break;
306         }
307
308 out:
309         TRACE_EXIT();
310         return;
311 }
312
313 static int scst_pre_parse(struct scst_cmd *cmd)
314 {
315         int res = SCST_CMD_STATE_RES_CONT_SAME;
316         struct scst_device *dev = cmd->dev;
317
318         TRACE_ENTRY();
319
320         cmd->inc_expected_sn_on_done = dev->handler->exec_sync ||
321                 (!dev->has_own_order_mgmt &&
322                  ((dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) ||
323                   (cmd->queue_type == SCST_CMD_QUEUE_ORDERED)));
324
325         sBUG_ON(cmd->internal);
326
327         /*
328          * Expected transfer data supplied by the SCSI transport via the
329          * target driver are untrusted, so we prefer to fetch them from CDB.
330          * Additionally, not all transports support supplying the expected
331          * transfer data.
332          */
333
334         if (unlikely(scst_get_cdb_info(cmd->cdb, dev->handler->type, 
335                         &cmd->op_flags, &cmd->data_direction,
336                         &cmd->bufflen, &cmd->cdb_len, &cmd->op_name) != 0)) {
337                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
338                         "Should you update scst_scsi_op_table?",
339                         cmd->cdb[0], dev->handler->name);
340 #ifdef USE_EXPECTED_VALUES
341                 if (scst_cmd_is_expected_set(cmd)) {
342                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
343                                 "direction %d, transfer_len %d",
344                                 cmd->expected_data_direction,
345                                 cmd->expected_transfer_len);
346                         cmd->data_direction = cmd->expected_data_direction;
347                         
348                         cmd->bufflen = cmd->expected_transfer_len;
349                         /* Restore (likely) lost CDB length */
350                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
351                         if (cmd->cdb_len == -1) {
352                                 PRINT_ERROR("Unable to get CDB length for "
353                                         "opcode 0x%02x. Returning INVALID "
354                                         "OPCODE", cmd->cdb[0]);
355                                 scst_set_cmd_error(cmd,
356                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
357                                 goto out_xmit;
358                         }
359                 } else {
360                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
361                              "target %s not supplied expected values",
362                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
363                         scst_set_cmd_error(cmd,
364                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
365                         goto out_xmit;
366                 }
367 #else
368                 scst_set_cmd_error(cmd,
369                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
370                 goto out_xmit;
371 #endif
372         } else {
373                 TRACE(TRACE_SCSI, "op_name <%s>, direction=%d (expected %d, "
374                         "set %s), transfer_len=%d (expected len %d), flags=%d",
375                         cmd->op_name, cmd->data_direction,
376                         cmd->expected_data_direction,
377                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
378                         cmd->bufflen, cmd->expected_transfer_len, cmd->op_flags);
379
380                 if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
381                         if (scst_cmd_is_expected_set(cmd)) {
382                                 /*
383                                  * Command data length can't be easily
384                                  * determined from the CDB. ToDo, that should 
385                                  * be fixed. Until it's fixed, get it from
386                                  * the supplied expected value, but
387                                  * limit it to some reasonable value (15MB).
388                                  */
389                                 cmd->bufflen = min(cmd->expected_transfer_len,
390                                                         15*1024*1024);
391                         } else
392                                 cmd->bufflen = 0;
393                 }
394         }
395
396         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
397                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
398                             "(opcode 0x%02x)", cmd->cdb[0]);
399                 scst_set_cmd_error(cmd,
400                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
401                 goto out_xmit;
402         }
403
404         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
405                 PRINT_ERROR("Linked commands are not supported "
406                             "(opcode 0x%02x)", cmd->cdb[0]);
407                 scst_set_cmd_error(cmd,
408                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
409                 goto out_xmit;
410         }
411
412         cmd->state = SCST_CMD_STATE_DEV_PARSE;
413
414 out:
415         TRACE_EXIT_RES(res);
416         return res;
417
418 out_xmit:
419         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
420         res = SCST_CMD_STATE_RES_CONT_SAME;
421         goto out;
422 }
423
424 static int scst_parse_cmd(struct scst_cmd *cmd)
425 {
426         int res = SCST_CMD_STATE_RES_CONT_SAME;
427         int state;
428         struct scst_device *dev = cmd->dev;
429         int atomic = scst_cmd_atomic(cmd);
430         int orig_bufflen = cmd->bufflen;
431
432         TRACE_ENTRY();
433
434         if (atomic && !dev->handler->parse_atomic) {
435                 TRACE_DBG("Dev handler %s parse() can not be "
436                       "called in atomic context, rescheduling to the thread",
437                       dev->handler->name);
438                 res = SCST_CMD_STATE_RES_NEED_THREAD;
439                 goto out;
440         }
441
442         if (likely(!scst_is_cmd_local(cmd))) {
443                 TRACE_DBG("Calling dev handler %s parse(%p)",
444                       dev->handler->name, cmd);
445                 TRACE_BUFF_FLAG(TRACE_SND_BOT, "Parsing: ", cmd->cdb, cmd->cdb_len);
446                 state = dev->handler->parse(cmd);
447                 /* Caution: cmd can be already dead here */
448                 TRACE_DBG("Dev handler %s parse() returned %d",
449                         dev->handler->name, state);
450
451                 switch (state) {
452                 case SCST_CMD_STATE_NEED_THREAD_CTX:
453                         TRACE_DBG("Dev handler %s parse() requested thread "
454                               "context, rescheduling", dev->handler->name);
455                         res = SCST_CMD_STATE_RES_NEED_THREAD;
456                         goto out;
457
458                 case SCST_CMD_STATE_STOP:
459                         TRACE_DBG("Dev handler %s parse() requested stop "
460                                 "processing", dev->handler->name);
461                         res = SCST_CMD_STATE_RES_CONT_NEXT;
462                         goto out;
463                 }
464
465                 if (state == SCST_CMD_STATE_DEFAULT)
466                         state = SCST_CMD_STATE_PREPARE_SPACE;
467         } else
468                 state = SCST_CMD_STATE_PREPARE_SPACE;
469
470         if (cmd->data_len == -1)
471                 cmd->data_len = cmd->bufflen;
472
473         if (cmd->data_buf_alloced && unlikely((orig_bufflen > cmd->bufflen))) {
474                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
475                         "is less, than required (size %d)", cmd->bufflen,
476                         orig_bufflen);
477                 goto out_error;
478         }
479
480         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
481                 goto set_res;
482
483         if (unlikely((cmd->bufflen == 0) &&
484                      (cmd->op_flags & SCST_UNKNOWN_LENGTH))) {
485                 PRINT_ERROR("Unknown data transfer length for opcode 0x%x "
486                         "(handler %s, target %s)", cmd->cdb[0],
487                         dev->handler->name, cmd->tgtt->name);
488                 goto out_error;
489         }
490
491 #ifdef EXTRACHECKS
492         if ((cmd->bufflen != 0) &&
493             ((cmd->data_direction == SCST_DATA_NONE) ||
494              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
495                 PRINT_ERROR("Dev handler %s parse() returned "
496                         "invalid cmd data_direction %d, bufflen %d, state %d "
497                         "or sg %p (opcode 0x%x)", dev->handler->name, 
498                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
499                         cmd->cdb[0]);
500                 goto out_error;
501         }
502 #endif
503
504         if (scst_cmd_is_expected_set(cmd)) {
505 #ifdef USE_EXPECTED_VALUES
506 #       ifdef EXTRACHECKS
507                 if ((cmd->data_direction != cmd->expected_data_direction) ||
508                     (cmd->bufflen != cmd->expected_transfer_len)) {
509                         PRINT_ERROR("Expected values don't match decoded ones: "
510                                 "data_direction %d, expected_data_direction %d, "
511                                 "bufflen %d, expected_transfer_len %d",
512                                 cmd->data_direction, cmd->expected_data_direction,
513                                 cmd->bufflen, cmd->expected_transfer_len);
514                 }
515 #       endif
516                 cmd->data_direction = cmd->expected_data_direction;
517                 cmd->bufflen = cmd->expected_transfer_len;
518 #else
519                 if (unlikely(cmd->data_direction != cmd->expected_data_direction)) {
520                         PRINT_ERROR("Expected data direction %d for opcode "
521                                 "0x%02x (handler %s, target %s) doesn't match "
522                                 "decoded value %d", cmd->data_direction,
523                                 cmd->cdb[0], dev->handler->name,
524                                 cmd->tgtt->name, cmd->expected_data_direction);
525                         scst_set_cmd_error(cmd,
526                                 SCST_LOAD_SENSE(scst_sense_invalid_message));
527                         goto out_dev_done;
528                 }
529                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
530                         TRACE(TRACE_MINOR, "Warning: expected transfer length "
531                                 "%d for opcode 0x%02x (handler %s, target %s) "
532                                 "doesn't match decoded value %d. Faulty "
533                                 "initiator (e.g. VMware is known to be such) or "
534                                 "scst_scsi_op_table should be updated?",
535                                 cmd->expected_transfer_len, cmd->cdb[0],
536                                 dev->handler->name, cmd->tgtt->name,
537                                 cmd->bufflen);
538                         PRINT_BUFF_FLAG(TRACE_MINOR, "Suspicious CDB", cmd->cdb,
539                                 cmd->cdb_len);
540                 }
541 #endif
542         }
543
544         if (unlikely(cmd->data_direction == SCST_DATA_UNKNOWN)) {
545                 PRINT_ERROR("Unknown data direction. Opcode 0x%x, handler %s, "
546                         "target %s", cmd->cdb[0], dev->handler->name,
547                         cmd->tgtt->name);
548                 goto out_error;
549         }
550
551 set_res:
552         switch (state) {
553         case SCST_CMD_STATE_PREPARE_SPACE:
554         case SCST_CMD_STATE_PRE_PARSE:
555         case SCST_CMD_STATE_DEV_PARSE:
556         case SCST_CMD_STATE_RDY_TO_XFER:
557         case SCST_CMD_STATE_TGT_PRE_EXEC:
558         case SCST_CMD_STATE_SEND_TO_MIDLEV:
559         case SCST_CMD_STATE_PRE_DEV_DONE:
560         case SCST_CMD_STATE_DEV_DONE:
561         case SCST_CMD_STATE_PRE_XMIT_RESP:
562         case SCST_CMD_STATE_XMIT_RESP:
563         case SCST_CMD_STATE_FINISHED:
564                 cmd->state = state;
565                 res = SCST_CMD_STATE_RES_CONT_SAME;
566                 break;
567
568         default:
569                 if (state >= 0) {
570                         PRINT_ERROR("Dev handler %s parse() returned "
571                              "invalid cmd state %d (opcode %d)", 
572                              dev->handler->name, state, cmd->cdb[0]);
573                 } else {
574                         PRINT_ERROR("Dev handler %s parse() returned "
575                                 "error %d (opcode %d)", dev->handler->name, 
576                                 state, cmd->cdb[0]);
577                 }
578                 goto out_error;
579         }
580
581         if (cmd->resp_data_len == -1) {
582                 if (cmd->data_direction == SCST_DATA_READ)
583                         cmd->resp_data_len = cmd->bufflen;
584                 else
585                          cmd->resp_data_len = 0;
586         }
587         
588 out:
589         TRACE_EXIT_HRES(res);
590         return res;
591
592 out_error:
593         /* dev_done() will be called as part of the regular cmd's finish */
594         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
595
596 #ifndef USE_EXPECTED_VALUES
597 out_dev_done:
598 #endif
599         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
600         res = SCST_CMD_STATE_RES_CONT_SAME;
601         goto out;
602 }
603
604 static int scst_prepare_space(struct scst_cmd *cmd)
605 {
606         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
607
608         TRACE_ENTRY();
609
610         if (cmd->data_direction == SCST_DATA_NONE)
611                 goto prep_done;
612
613         if (cmd->data_buf_tgt_alloc) {
614                 int orig_bufflen = cmd->bufflen;
615
616                 TRACE_MEM("%s", "Custom tgt data buf allocation requested");
617
618                 r = cmd->tgtt->alloc_data_buf(cmd);
619                 if (r > 0)
620                         goto alloc;
621                 else if (r == 0) {
622                         if (unlikely(cmd->bufflen == 0)) {
623                                 /* See comment in scst_alloc_space() */
624                                 if (cmd->sg == NULL)
625                                         goto alloc;
626                         }
627                         cmd->data_buf_alloced = 1;
628                         if (unlikely(orig_bufflen < cmd->bufflen)) {
629                                 PRINT_ERROR("Target driver allocated data "
630                                         "buffer (size %d), is less, than "
631                                         "required (size %d)", orig_bufflen,
632                                         cmd->bufflen);
633                                 goto out_error;
634                         }
635                         TRACE_MEM("%s", "data_buf_alloced, returning");
636                 }
637                 goto check;
638         }
639
640 alloc:
641         r = scst_alloc_space(cmd);
642         
643 check:
644         if (r != 0) {
645                 if (scst_cmd_atomic(cmd)) {
646                         TRACE_MEM("%s", "Atomic memory allocation failed, "
647                               "rescheduling to the thread");
648                         res = SCST_CMD_STATE_RES_NEED_THREAD;
649                         goto out;
650                 } else
651                         goto out_no_space;
652         }
653
654 prep_done:
655         if (cmd->preprocessing_only) {
656                 if (scst_cmd_atomic(cmd) && 
657                     !cmd->tgtt->preprocessing_done_atomic) {
658                         TRACE_DBG("%s", "preprocessing_done() can not be "
659                               "called in atomic context, rescheduling to "
660                               "the thread");
661                         res = SCST_CMD_STATE_RES_NEED_THREAD;
662                         goto out;
663                 }
664
665                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
666                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
667                                 "cmd %p", cmd);
668                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
669                         res = SCST_CMD_STATE_RES_CONT_SAME;
670                         goto out;
671                 }
672
673                 res = SCST_CMD_STATE_RES_CONT_NEXT;
674                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
675
676                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
677                 cmd->tgtt->preprocessing_done(cmd);
678                 TRACE_DBG("%s", "preprocessing_done() returned");
679                 goto out;
680
681         }
682
683         switch (cmd->data_direction) {
684         case SCST_DATA_WRITE:
685                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
686                 break;
687
688         default:
689                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
690                 break;
691         }
692
693 out:
694         TRACE_EXIT_HRES(res);
695         return res;
696
697 out_no_space:
698         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
699                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
700         scst_set_busy(cmd);
701         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
702         res = SCST_CMD_STATE_RES_CONT_SAME;
703         goto out;
704
705 out_error:
706         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
707         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
708         res = SCST_CMD_STATE_RES_CONT_SAME;
709         goto out;
710 }
711
712 void scst_restart_cmd(struct scst_cmd *cmd, int status, int pref_context)
713 {
714         TRACE_ENTRY();
715
716         TRACE_DBG("Preferred context: %d", pref_context);
717         TRACE_DBG("tag=%llu, status=%#x", scst_cmd_get_tag(cmd), status);
718
719 #ifdef EXTRACHECKS
720         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
721                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
722         {
723                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
724                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
725                         cmd->tgtt->name);
726                 pref_context = SCST_CONTEXT_TASKLET;
727         }
728 #endif
729
730         switch (status) {
731         case SCST_PREPROCESS_STATUS_SUCCESS:
732                 switch (cmd->data_direction) {
733                 case SCST_DATA_WRITE:
734                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
735                         break;
736                 default:
737                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
738                         break;
739                 }
740                 if (cmd->set_sn_on_restart_cmd)
741                         scst_cmd_set_sn(cmd);
742                 /* Small context optimization */
743                 if ((pref_context == SCST_CONTEXT_TASKLET) || 
744                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) {
745                         if (cmd->data_direction == SCST_DATA_WRITE) {
746                                 if ( !test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
747                                                 &cmd->tgt_dev->tgt_dev_flags))
748                                         pref_context = SCST_CONTEXT_THREAD;
749                         } else {
750                                 if ( !test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
751                                                 &cmd->tgt_dev->tgt_dev_flags))
752                                         pref_context = SCST_CONTEXT_THREAD;
753                         }
754                 }
755                 break;
756
757         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
758                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
759                 break;
760
761         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
762                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
763                 /* go through */
764         case SCST_PREPROCESS_STATUS_ERROR:
765                 scst_set_cmd_error(cmd,
766                            SCST_LOAD_SENSE(scst_sense_hardw_error));
767                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
768                 break;
769
770         default:
771                 PRINT_ERROR("%s() received unknown status %x", __func__,
772                         status);
773                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
774                 break;
775         }
776
777         scst_proccess_redirect_cmd(cmd, pref_context, 1);
778
779         TRACE_EXIT();
780         return;
781 }
782
783 /* No locks */
784 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
785 {
786         struct scst_tgt *tgt = cmd->sess->tgt;
787         int res = 0;
788         unsigned long flags;
789
790         TRACE_ENTRY();
791
792         spin_lock_irqsave(&tgt->tgt_lock, flags);
793         tgt->retry_cmds++;
794         smp_mb();
795         TRACE_RETRY("TGT QUEUE FULL: incrementing retry_cmds %d",
796               tgt->retry_cmds);
797         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
798                 /* At least one cmd finished, so try again */
799                 tgt->retry_cmds--;
800                 TRACE_RETRY("Some command(s) finished, direct retry "
801                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
802                       "retry_cmds=%d)", finished_cmds,
803                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
804                 res = -1;
805                 goto out_unlock_tgt;
806         }
807
808         TRACE_RETRY("Adding cmd %p to retry cmd list", cmd);
809         list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
810
811         if (!tgt->retry_timer_active) {
812                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
813                 add_timer(&tgt->retry_timer);
814                 tgt->retry_timer_active = 1;
815         }
816
817 out_unlock_tgt:
818         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
819
820         TRACE_EXIT_RES(res);
821         return res;
822 }
823
824 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
825 {
826         int res, rc;
827         int atomic = scst_cmd_atomic(cmd);
828
829         TRACE_ENTRY();
830
831         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
832                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
833                 goto out_dev_done;
834         }
835
836         if (cmd->tgtt->rdy_to_xfer == NULL) {
837                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
838                 res = SCST_CMD_STATE_RES_CONT_SAME;
839                 goto out;
840         }
841
842         if (atomic && !cmd->tgtt->rdy_to_xfer_atomic) {
843                 TRACE_DBG("%s", "rdy_to_xfer() can not be "
844                       "called in atomic context, rescheduling to the thread");
845                 res = SCST_CMD_STATE_RES_NEED_THREAD;
846                 goto out;
847         }
848
849         while (1) {
850                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
851
852                 res = SCST_CMD_STATE_RES_CONT_NEXT;
853                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
854
855                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
856 #ifdef DEBUG_RETRY
857                 if (((scst_random() % 100) == 75))
858                         rc = SCST_TGT_RES_QUEUE_FULL;
859                 else
860 #endif
861                         rc = cmd->tgtt->rdy_to_xfer(cmd);
862                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
863
864                 if (likely(rc == SCST_TGT_RES_SUCCESS))
865                         goto out;
866
867                 /* Restore the previous state */
868                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
869
870                 switch (rc) {
871                 case SCST_TGT_RES_QUEUE_FULL:
872                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
873                                 break;
874                         else
875                                 continue;
876
877                 case SCST_TGT_RES_NEED_THREAD_CTX:
878                         TRACE_DBG("Target driver %s "
879                               "rdy_to_xfer() requested thread "
880                               "context, rescheduling", cmd->tgtt->name);
881                         res = SCST_CMD_STATE_RES_NEED_THREAD;
882                         break;
883
884                 default:
885                         goto out_error_rc;
886                 }
887                 break;
888         }
889
890 out:
891         TRACE_EXIT_HRES(res);
892         return res;
893
894 out_error_rc:
895         if (rc == SCST_TGT_RES_FATAL_ERROR) {
896                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
897                      "fatal error", cmd->tgtt->name);
898         } else {
899                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
900                             "value %d", cmd->tgtt->name, rc);
901         }
902         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
903
904 out_dev_done:
905         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
906         res = SCST_CMD_STATE_RES_CONT_SAME;
907         goto out;
908 }
909
910 /* No locks, but might be in IRQ */
911 void scst_proccess_redirect_cmd(struct scst_cmd *cmd, int context,
912         int check_retries)
913 {
914         unsigned long flags;
915
916         TRACE_ENTRY();
917
918         TRACE_DBG("Context: %x", context);
919
920         switch(context & ~SCST_CONTEXT_PROCESSABLE) {
921         case SCST_CONTEXT_DIRECT_ATOMIC:
922                 context &= ~SCST_CONTEXT_PROCESSABLE;
923                 /* go through */
924         case SCST_CONTEXT_DIRECT:
925                 if (check_retries)
926                         scst_check_retries(cmd->tgt);
927                 scst_process_active_cmd(cmd, context);
928                 break;
929
930         default:
931                 PRINT_ERROR("Context %x is unknown, using the thread one",
932                             context);
933                 /* go through */
934         case SCST_CONTEXT_THREAD:
935                 if (check_retries)
936                         scst_check_retries(cmd->tgt);
937                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
938                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
939                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
940                         list_add(&cmd->cmd_list_entry,
941                                 &cmd->cmd_lists->active_cmd_list);
942                 else
943                         list_add_tail(&cmd->cmd_list_entry,
944                                 &cmd->cmd_lists->active_cmd_list);
945                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
946                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
947                 break;
948
949         case SCST_CONTEXT_TASKLET:
950                 if (check_retries)
951                         scst_check_retries(cmd->tgt);
952                 scst_schedule_tasklet(cmd);
953                 break;
954         }
955
956         TRACE_EXIT();
957         return;
958 }
959
960 void scst_rx_data(struct scst_cmd *cmd, int status, int pref_context)
961 {
962         TRACE_ENTRY();
963
964         TRACE_DBG("Preferred context: %d", pref_context);
965         TRACE(TRACE_SCSI, "tag=%llu status=%#x", scst_cmd_get_tag(cmd), status);
966
967 #ifdef EXTRACHECKS
968         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
969                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
970         {
971                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
972                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
973                         cmd->tgtt->name);
974                 pref_context = SCST_CONTEXT_TASKLET;
975         }
976 #endif
977
978         switch (status) {
979         case SCST_RX_STATUS_SUCCESS:
980                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
981                 /* Small context optimization */
982                 if ((pref_context == SCST_CONTEXT_TASKLET) || 
983                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) {
984                         if ( !test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC, 
985                                         &cmd->tgt_dev->tgt_dev_flags))
986                                 pref_context = SCST_CONTEXT_THREAD;
987                 }
988                 break;
989
990         case SCST_RX_STATUS_ERROR_SENSE_SET:
991                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
992                 break;
993
994         case SCST_RX_STATUS_ERROR_FATAL:
995                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
996                 /* go through */
997         case SCST_RX_STATUS_ERROR:
998                 scst_set_cmd_error(cmd,
999                            SCST_LOAD_SENSE(scst_sense_hardw_error));
1000                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1001                 break;
1002
1003         default:
1004                 PRINT_ERROR("scst_rx_data() received unknown status %x",
1005                         status);
1006                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1007                 break;
1008         }
1009
1010         scst_proccess_redirect_cmd(cmd, pref_context, 1);
1011
1012         TRACE_EXIT();
1013         return;
1014 }
1015
1016 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
1017 {
1018         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
1019
1020         TRACE_ENTRY();
1021
1022         cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
1023
1024         if (cmd->tgtt->pre_exec == NULL)
1025                 goto out;
1026
1027         TRACE_DBG("Calling pre_exec(%p)", cmd);
1028         rc = cmd->tgtt->pre_exec(cmd);
1029         TRACE_DBG("pre_exec() returned %d", rc);
1030
1031         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
1032                 switch(rc) {
1033                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
1034                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1035                         break;
1036                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
1037                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1038                         /* go through */
1039                 case SCST_PREPROCESS_STATUS_ERROR:
1040                         scst_set_cmd_error(cmd,
1041                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1042                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1043                         break;
1044                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1045                         TRACE_DBG("Target driver's %s pre_exec() requested "
1046                                 "thread context, rescheduling", cmd->tgtt->name);
1047                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1048                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1049                         break;
1050                 default:
1051                         sBUG();
1052                         break;
1053                 }
1054         }
1055
1056 out:
1057         TRACE_EXIT_RES(res);
1058         return res;
1059 }
1060
1061 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1062         const uint8_t *rq_sense, int rq_sense_len, int resid)
1063 {
1064         TRACE_ENTRY();
1065
1066 #ifdef MEASURE_LATENCY
1067         {
1068                 struct timespec ts;
1069                 getnstimeofday(&ts);
1070                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1071                 TRACE_DBG("cmd %p (sess %p): post_exec_start %Ld (tv_sec %ld, "
1072                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start, ts.tv_sec,
1073                         ts.tv_nsec);
1074         }
1075 #endif
1076
1077         cmd->status = result & 0xff;
1078         cmd->msg_status = msg_byte(result);
1079         cmd->host_status = host_byte(result);
1080         cmd->driver_status = driver_byte(result);
1081         if (unlikely(resid != 0)) {
1082 #ifdef EXTRACHECKS
1083                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1084                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1085                                 "op %x)", resid, cmd->resp_data_len,
1086                                 cmd->cdb[0]);
1087                 } else
1088 #endif
1089                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1090         }
1091
1092         if (cmd->status == SAM_STAT_CHECK_CONDITION)
1093                 scst_alloc_set_sense(cmd, in_irq(), rq_sense, rq_sense_len);
1094
1095         TRACE(TRACE_SCSI, "result=%x, cmd->status=%x, resid=%d, "
1096               "cmd->msg_status=%x, cmd->host_status=%x, "
1097               "cmd->driver_status=%x", result, cmd->status, resid,
1098               cmd->msg_status, cmd->host_status, cmd->driver_status);
1099
1100         cmd->completed = 1;
1101
1102         TRACE_EXIT();
1103         return;
1104 }
1105
1106 /* For small context optimization */
1107 static inline int scst_optimize_post_exec_context(struct scst_cmd *cmd,
1108         int context)
1109 {
1110         if ((context == SCST_CONTEXT_TASKLET) || 
1111             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1112                 if ( !test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC, 
1113                                 &cmd->tgt_dev->tgt_dev_flags))
1114                         context = SCST_CONTEXT_THREAD;
1115         }
1116         return context;
1117 }
1118
1119 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18)
1120 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1121                                             struct scsi_request **req)
1122 {
1123         struct scst_cmd *cmd = NULL;
1124
1125         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1126                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1127
1128         if (cmd == NULL) {
1129                 PRINT_ERROR("%s", "Request with NULL cmd");
1130                 if (*req)
1131                         scsi_release_request(*req);
1132         }
1133
1134         return cmd;
1135 }
1136
1137 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1138 {
1139         struct scsi_request *req = NULL;
1140         struct scst_cmd *cmd;
1141
1142         TRACE_ENTRY();
1143
1144         cmd = scst_get_cmd(scsi_cmd, &req);
1145         if (cmd == NULL)
1146                 goto out;
1147
1148         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1149                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1150
1151         /* Clear out request structure */
1152         req->sr_use_sg = 0;
1153         req->sr_sglist_len = 0;
1154         req->sr_bufflen = 0;
1155         req->sr_buffer = NULL;
1156         req->sr_underflow = 0;
1157         req->sr_request->rq_disk = NULL; /* disown request blk */
1158
1159         scst_release_request(cmd);
1160
1161         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1162
1163         scst_proccess_redirect_cmd(cmd,
1164                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1165
1166 out:
1167         TRACE_EXIT();
1168         return;
1169 }
1170 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) */
1171 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1172 {
1173         struct scst_cmd *cmd;
1174
1175         TRACE_ENTRY();
1176
1177         cmd = (struct scst_cmd *)data;
1178         if (cmd == NULL)
1179                 goto out;
1180
1181         scst_do_cmd_done(cmd, result, sense, SCST_SENSE_BUFFERSIZE, resid);
1182
1183         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1184
1185         scst_proccess_redirect_cmd(cmd,
1186                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1187
1188 out:
1189         TRACE_EXIT();
1190         return;
1191 }
1192 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) */
1193
1194 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state)
1195 {
1196         int context;
1197
1198         TRACE_ENTRY();
1199
1200 #ifdef MEASURE_LATENCY
1201         {
1202                 struct timespec ts;
1203                 getnstimeofday(&ts);
1204                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1205                 TRACE_DBG("cmd %p (sess %p): post_exec_start %Ld (tv_sec %ld, "
1206                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start, ts.tv_sec,
1207                         ts.tv_nsec);
1208         }
1209 #endif
1210
1211         if (next_state == SCST_CMD_STATE_DEFAULT)
1212                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1213
1214 #if defined(DEBUG)
1215         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1216                 if (cmd->sg) {
1217                         int i;
1218                         struct scatterlist *sg = cmd->sg;
1219                         TRACE_RECV_TOP("Exec'd %d S/G(s) at %p sg[0].page at "
1220                                 "%p", cmd->sg_cnt, sg, (void*)sg_page(&sg[0]));
1221                         for(i = 0; i < cmd->sg_cnt; ++i) {
1222                                 TRACE_BUFF_FLAG(TRACE_RCV_TOP, 
1223                                         "Exec'd sg", sg_virt(&sg[i]),
1224                                         sg[i].length);
1225                         }
1226                 }
1227         }
1228 #endif
1229
1230
1231 #ifdef EXTRACHECKS
1232         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1233             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1234             (next_state != SCST_CMD_STATE_FINISHED)) 
1235         {
1236                 PRINT_ERROR("scst_cmd_done_local() received invalid cmd "
1237                             "state %d (opcode %d)", next_state, cmd->cdb[0]);
1238                 scst_set_cmd_error(cmd,
1239                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1240                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1241         }
1242 #endif
1243         cmd->state = next_state;
1244
1245         context = scst_optimize_post_exec_context(cmd, scst_get_context());
1246         if (cmd->context_processable)
1247                 context |= SCST_CONTEXT_PROCESSABLE;
1248         scst_proccess_redirect_cmd(cmd, context, 0);
1249
1250         TRACE_EXIT();
1251         return;
1252 }
1253
1254 static int scst_report_luns_local(struct scst_cmd *cmd)
1255 {
1256         int rc;
1257         int dev_cnt = 0;
1258         int buffer_size;
1259         int i;
1260         struct scst_tgt_dev *tgt_dev = NULL;
1261         uint8_t *buffer;
1262         int offs, overflow = 0;
1263
1264         TRACE_ENTRY();
1265
1266         rc = scst_check_local_events(cmd);
1267         if (unlikely(rc != 0))
1268                 goto out_done;
1269
1270         cmd->status = 0;
1271         cmd->msg_status = 0;
1272         cmd->host_status = DID_OK;
1273         cmd->driver_status = 0;
1274
1275         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1276                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1277                         "LUNS command", cmd->cdb[2]);
1278                 goto out_err;
1279         }
1280
1281         buffer_size = scst_get_buf_first(cmd, &buffer);
1282         if (unlikely(buffer_size <= 0))
1283                 goto out_err;
1284
1285         if (buffer_size < 16)
1286                 goto out_put_err;
1287
1288         memset(buffer, 0, buffer_size);
1289         offs = 8;
1290
1291         /* sess->sess_tgt_dev_list_hash is protected by suspended activity */
1292         for(i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1293                 struct list_head *sess_tgt_dev_list_head =
1294                         &cmd->sess->sess_tgt_dev_list_hash[i];
1295                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1296                                 sess_tgt_dev_list_entry) {
1297                         if (!overflow) {
1298                                 if (offs >= buffer_size) {
1299                                         scst_put_buf(cmd, buffer);
1300                                         buffer_size = scst_get_buf_next(cmd, &buffer);
1301                                         if (buffer_size > 0) {
1302                                                 memset(buffer, 0, buffer_size);
1303                                                 offs = 0;
1304                                         } else {
1305                                                 overflow = 1;
1306                                                 goto inc_dev_cnt;
1307                                         }
1308                                 }
1309                                 if ((buffer_size - offs) < 8) {
1310                                         PRINT_ERROR("Buffer allocated for REPORT "
1311                                                 "LUNS command doesn't allow to fit 8 "
1312                                                 "byte entry (buffer_size=%d)",
1313                                                 buffer_size);
1314                                         goto out_put_hw_err;
1315                                 }
1316                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1317                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1318                                 offs += 8;
1319                         }
1320 inc_dev_cnt:
1321                         dev_cnt++;
1322                 }
1323         }
1324         if (!overflow)
1325                 scst_put_buf(cmd, buffer);
1326
1327         /* Set the response header */
1328         buffer_size = scst_get_buf_first(cmd, &buffer);
1329         if (unlikely(buffer_size <= 0))
1330                 goto out_err;
1331         dev_cnt *= 8;
1332         buffer[0] = (dev_cnt >> 24) & 0xff;
1333         buffer[1] = (dev_cnt >> 16) & 0xff;
1334         buffer[2] = (dev_cnt >> 8) & 0xff;
1335         buffer[3] = dev_cnt & 0xff;
1336         scst_put_buf(cmd, buffer);
1337
1338         dev_cnt += 8;
1339         if (dev_cnt < cmd->resp_data_len)
1340                 scst_set_resp_data_len(cmd, dev_cnt);
1341
1342 out_compl:
1343         cmd->completed = 1;
1344
1345 out_done:
1346         /* Report the result */
1347         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1348
1349         TRACE_EXIT();
1350         return SCST_EXEC_COMPLETED;
1351         
1352 out_put_err:
1353         scst_put_buf(cmd, buffer);
1354
1355 out_err:
1356         scst_set_cmd_error(cmd,
1357                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1358         goto out_compl;
1359
1360 out_put_hw_err:
1361         scst_put_buf(cmd, buffer);
1362         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1363         goto out_compl;
1364 }
1365
1366 static int scst_pre_select(struct scst_cmd *cmd)
1367 {
1368         int res = SCST_EXEC_NOT_COMPLETED;
1369
1370         TRACE_ENTRY();
1371
1372         if (scst_cmd_atomic(cmd)) {
1373                 res = SCST_EXEC_NEED_THREAD;
1374                 goto out;
1375         }
1376
1377         if (cmd->local_exec_done)
1378                 goto out;
1379
1380         cmd->local_exec_done = 1;
1381
1382         scst_block_dev_cmd(cmd, 1);
1383
1384         /* Check for local events will be done when cmd will be executed */
1385
1386 out:
1387         TRACE_EXIT_RES(res);
1388         return res;
1389 }
1390
1391 static int scst_reserve_local(struct scst_cmd *cmd)
1392 {
1393         int res = SCST_EXEC_NOT_COMPLETED, rc;
1394         struct scst_device *dev;
1395         struct scst_tgt_dev *tgt_dev_tmp;
1396
1397         TRACE_ENTRY();
1398
1399         if (scst_cmd_atomic(cmd)) {
1400                 res = SCST_EXEC_NEED_THREAD;
1401                 goto out;
1402         }
1403
1404         if (cmd->local_exec_done)
1405                 goto out;
1406
1407         cmd->local_exec_done = 1;
1408
1409         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1410                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1411                      "(lun=%Ld)", (uint64_t)cmd->lun);
1412                 scst_set_cmd_error(cmd,
1413                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1414                 goto out_done;
1415         }
1416
1417         dev = cmd->dev;
1418
1419         scst_block_dev_cmd(cmd, 1);
1420
1421         rc = scst_check_local_events(cmd);
1422         if (unlikely(rc != 0))
1423                 goto out_done;
1424
1425         spin_lock_bh(&dev->dev_lock);
1426
1427         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1428                 spin_unlock_bh(&dev->dev_lock);
1429                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1430                 goto out_done;
1431         }
1432
1433         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1434                             dev_tgt_dev_list_entry) {
1435                 if (cmd->tgt_dev != tgt_dev_tmp)
1436                         set_bit(SCST_TGT_DEV_RESERVED, 
1437                                 &tgt_dev_tmp->tgt_dev_flags);
1438         }
1439         dev->dev_reserved = 1;
1440
1441         spin_unlock_bh(&dev->dev_lock);
1442         
1443 out:
1444         TRACE_EXIT_RES(res);
1445         return res;
1446
1447 out_done:
1448         /* Report the result */
1449         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1450         res = SCST_EXEC_COMPLETED;
1451         goto out;
1452 }
1453
1454 static int scst_release_local(struct scst_cmd *cmd)
1455 {
1456         int res = SCST_EXEC_NOT_COMPLETED, rc;
1457         struct scst_tgt_dev *tgt_dev_tmp;
1458         struct scst_device *dev;
1459
1460         TRACE_ENTRY();
1461
1462         if (scst_cmd_atomic(cmd)) {
1463                 res = SCST_EXEC_NEED_THREAD;
1464                 goto out;
1465         }
1466
1467         if (cmd->local_exec_done)
1468                 goto out;
1469
1470         cmd->local_exec_done = 1;
1471
1472         dev = cmd->dev;
1473
1474         scst_block_dev_cmd(cmd, 1);
1475
1476         rc = scst_check_local_events(cmd);
1477         if (unlikely(rc != 0))
1478                 goto out_done;
1479
1480         spin_lock_bh(&dev->dev_lock);
1481
1482         /* 
1483          * The device could be RELEASED behind us, if RESERVING session 
1484          * is closed (see scst_free_tgt_dev()), but this actually doesn't 
1485          * matter, so use lock and no retest for DEV_RESERVED bits again
1486          */
1487         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1488                 res = SCST_EXEC_COMPLETED;
1489                 cmd->status = 0;
1490                 cmd->msg_status = 0;
1491                 cmd->host_status = DID_OK;
1492                 cmd->driver_status = 0;
1493                 cmd->completed = 1;
1494         } else {
1495                 list_for_each_entry(tgt_dev_tmp,
1496                                     &dev->dev_tgt_dev_list,
1497                                     dev_tgt_dev_list_entry) {
1498                         clear_bit(SCST_TGT_DEV_RESERVED, 
1499                                 &tgt_dev_tmp->tgt_dev_flags);
1500                 }
1501                 dev->dev_reserved = 0;
1502         }
1503
1504         spin_unlock_bh(&dev->dev_lock);
1505
1506         if (res == SCST_EXEC_COMPLETED)
1507                 goto out_done;
1508
1509 out:
1510         TRACE_EXIT_RES(res);
1511         return res;
1512
1513 out_done:
1514         res = SCST_EXEC_COMPLETED;
1515         /* Report the result */
1516         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1517         goto out;
1518 }
1519
1520 /* No locks, no IRQ or IRQ-safe context allowed */
1521 int scst_check_local_events(struct scst_cmd *cmd)
1522 {
1523         int res, rc;
1524         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1525
1526         TRACE_ENTRY();
1527
1528         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1529                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1530                 goto out_uncomplete;
1531         }
1532
1533         /* Reserve check before Unit Attention */
1534         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED, &tgt_dev->tgt_dev_flags))) {
1535                 if ((cmd->cdb[0] != INQUIRY) && (cmd->cdb[0] != REPORT_LUNS) &&
1536                     (cmd->cdb[0] != RELEASE) && (cmd->cdb[0] != RELEASE_10) &&
1537                     (cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER) &&
1538                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL || (cmd->cdb[4] & 3)) &&
1539                     (cmd->cdb[0] != LOG_SENSE) && (cmd->cdb[0] != REQUEST_SENSE)) {
1540                         scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1541                         goto out_complete;
1542                 }
1543         }
1544
1545         /* If we had internal bus reset, set the command error unit attention */
1546         if ((cmd->dev->scsi_dev != NULL) &&
1547             unlikely(cmd->dev->scsi_dev->was_reset)) {
1548                 if (scst_is_ua_command(cmd)) {
1549                         struct scst_device *dev = cmd->dev;
1550                         int done = 0;
1551                         /* Prevent more than 1 cmd to be triggered by was_reset */
1552                         spin_lock_bh(&dev->dev_lock);
1553                         barrier(); /* to reread was_reset */
1554                         if (dev->scsi_dev->was_reset) {
1555                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1556                                 scst_set_cmd_error(cmd,
1557                                            SCST_LOAD_SENSE(scst_sense_reset_UA));
1558                                 /* It looks like it is safe to clear was_reset here */
1559                                 dev->scsi_dev->was_reset = 0;
1560                                 done = 1;
1561                         }
1562                         spin_unlock_bh(&dev->dev_lock);
1563
1564                         if (done)
1565                                 goto out_complete;
1566                 }
1567         }
1568
1569         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING, 
1570                         &cmd->tgt_dev->tgt_dev_flags))) {
1571                 if (scst_is_ua_command(cmd)) {
1572                         rc = scst_set_pending_UA(cmd);
1573                         if (rc == 0)
1574                                 goto out_complete;
1575                 }
1576         }
1577
1578         res = 0;
1579
1580 out:
1581         TRACE_EXIT_RES(res);
1582         return res;
1583
1584 out_complete:
1585         res = 1;
1586         sBUG_ON(!cmd->completed);
1587         goto out;
1588
1589 out_uncomplete:
1590         res = -1;
1591         goto out;
1592 }
1593
1594 /* 
1595  * The result of cmd execution, if any, should be reported 
1596  * via scst_cmd_done_local() 
1597  */
1598 static int scst_pre_exec(struct scst_cmd *cmd)
1599 {
1600         int res = SCST_EXEC_NOT_COMPLETED;
1601         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1602
1603         TRACE_ENTRY();
1604
1605         /*
1606          * This function can be called several times for the same cmd, so it
1607          * can't change any state in a non-reentrable way or use something
1608          * like local_exec_done!!
1609          */
1610
1611         /* Check READ_ONLY device status */
1612         if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
1613             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
1614              cmd->cdb[0] == WRITE_10 ||
1615              cmd->cdb[0] == WRITE_12 ||
1616              cmd->cdb[0] == WRITE_16 ||
1617              cmd->cdb[0] == WRITE_VERIFY ||
1618              cmd->cdb[0] == WRITE_VERIFY_12 ||
1619              cmd->cdb[0] == WRITE_VERIFY_16 ||
1620              (cmd->dev->handler->type == TYPE_TAPE &&
1621               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
1622                 scst_set_cmd_error(cmd,
1623                            SCST_LOAD_SENSE(scst_sense_data_protect));
1624                 goto out_done;
1625         }
1626
1627 out:
1628         TRACE_EXIT_RES(res);
1629         return res;
1630
1631 out_done:
1632         res = SCST_EXEC_COMPLETED;
1633         /* Report the result */
1634         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1635         goto out;
1636 }
1637
1638 /* 
1639  * The result of cmd execution, if any, should be reported 
1640  * via scst_cmd_done_local() 
1641  */
1642 static inline int scst_local_exec(struct scst_cmd *cmd)
1643 {
1644         int res = SCST_EXEC_NOT_COMPLETED;
1645
1646         TRACE_ENTRY();
1647
1648         /*
1649          * Adding new commands here don't forget to update
1650          * scst_is_cmd_local() in scsi_tgt.h, if necessary
1651          */
1652
1653         switch (cmd->cdb[0]) {
1654         case MODE_SELECT:
1655         case MODE_SELECT_10:
1656         case LOG_SELECT:
1657                 res = scst_pre_select(cmd);
1658                 break;
1659         case RESERVE:
1660         case RESERVE_10:
1661                 res = scst_reserve_local(cmd);
1662                 break;
1663         case RELEASE:
1664         case RELEASE_10:
1665                 res = scst_release_local(cmd);
1666                 break;
1667         case REPORT_LUNS:
1668                 res = scst_report_luns_local(cmd);
1669                 break;
1670         }
1671
1672         TRACE_EXIT_RES(res);
1673         return res;
1674 }
1675
1676 /* cmd must be additionally referenced to not die inside */
1677 static int scst_do_send_to_midlev(struct scst_cmd *cmd)
1678 {
1679         int rc = SCST_EXEC_NOT_COMPLETED;
1680         struct scst_device *dev = cmd->dev;
1681         struct scst_dev_type *handler = dev->handler;
1682
1683         TRACE_ENTRY();
1684
1685         /* Check here to let an out of SN cmd be queued w/o context switch */
1686         if (scst_cmd_atomic(cmd) && !handler->exec_atomic) {
1687                 TRACE_DBG("Dev handler %s exec() can not be "
1688                       "called in atomic context, rescheduling to the thread",
1689                       handler->name);
1690                 rc = SCST_EXEC_NEED_THREAD;
1691                 goto out;
1692         }
1693
1694         cmd->sent_to_midlev = 1;
1695         cmd->state = SCST_CMD_STATE_EXECUTING;
1696         cmd->scst_cmd_done = scst_cmd_done_local;
1697
1698         rc = scst_pre_exec(cmd);
1699         if (rc != SCST_EXEC_NOT_COMPLETED) {
1700                 if (rc == SCST_EXEC_COMPLETED)
1701                         goto out;
1702                 else if (rc == SCST_EXEC_NEED_THREAD)
1703                         goto out_clear;
1704                 else
1705                         goto out_rc_error;
1706         }
1707
1708         rc = scst_local_exec(cmd);
1709         if (rc != SCST_EXEC_NOT_COMPLETED) {
1710                 if (rc == SCST_EXEC_COMPLETED)
1711                         goto out;
1712                 else if (rc == SCST_EXEC_NEED_THREAD)
1713                         goto out_clear;
1714                 else
1715                         goto out_rc_error;
1716         }
1717
1718         if (!handler->exec_sync)
1719                 cmd->context_processable = 0;
1720
1721         if (handler->exec) {
1722                 TRACE_DBG("Calling dev handler %s exec(%p)",
1723                       handler->name, cmd);
1724                 TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb, cmd->cdb_len);
1725                 cmd->scst_cmd_done = scst_cmd_done_local;
1726                 rc = handler->exec(cmd);
1727                 TRACE_DBG("Dev handler %s exec() returned %d",
1728                       handler->name, rc);
1729                 if (rc == SCST_EXEC_COMPLETED)
1730                         goto out;
1731                 else if (rc == SCST_EXEC_NEED_THREAD)
1732                         goto out_clear;
1733                 else if (rc != SCST_EXEC_NOT_COMPLETED)
1734                         goto out_rc_error;
1735         }
1736
1737         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1738         
1739         if (unlikely(dev->scsi_dev == NULL)) {
1740                 PRINT_ERROR("Command for virtual device must be "
1741                         "processed by device handler (lun %Ld)!",
1742                         (uint64_t)cmd->lun);
1743                 goto out_error;
1744         }
1745
1746         rc = scst_check_local_events(cmd);
1747         if (unlikely(rc != 0))
1748                 goto out_done;
1749
1750 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) 
1751         if (unlikely(scst_alloc_request(cmd) != 0)) {
1752                 if (scst_cmd_atomic(cmd)) {
1753                         rc = SCST_EXEC_NEED_THREAD;
1754                         goto out_clear;
1755                 } else {
1756                         PRINT_INFO("%s", "Unable to allocate request, "
1757                                 "sending BUSY status");
1758                         goto out_busy;
1759                 }
1760         }
1761         
1762         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1763                     (void *)cmd->scsi_req->sr_buffer,
1764                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1765                     cmd->retries);
1766 #else
1767         rc = scst_exec_req(dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1768                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1769                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1770                         scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
1771         if (unlikely(rc != 0)) {
1772                 if (scst_cmd_atomic(cmd)) {
1773                         rc = SCST_EXEC_NEED_THREAD;
1774                         goto out_clear;
1775                 } else {
1776                         PRINT_INFO("scst_exec_req() failed: %d", rc);
1777                         goto out_error;
1778                 }
1779         }
1780 #endif
1781
1782         rc = SCST_EXEC_COMPLETED;
1783
1784 out:
1785         TRACE_EXIT();
1786         return rc;
1787
1788 out_clear:
1789         /* Restore the state */
1790         cmd->sent_to_midlev = 0;
1791         cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
1792         goto out;
1793
1794 out_rc_error:
1795         PRINT_ERROR("Dev handler %s exec() or scst_local_exec() returned "
1796                     "invalid code %d", handler->name, rc);
1797         /* go through */
1798
1799 out_error:
1800         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1801
1802 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) 
1803 out_busy:
1804         scst_set_busy(cmd);
1805         /* go through */
1806 #endif
1807
1808 out_done:
1809         rc = SCST_EXEC_COMPLETED;
1810         /* Report the result */
1811         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1812         goto out;
1813 }
1814
1815 /* No locks */
1816 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1817 {
1818         if (slot == NULL)
1819                 goto inc;
1820
1821         /* Optimized for lockless fast path */
1822
1823         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1824                 atomic_read(slot));
1825
1826         if (!atomic_dec_and_test(slot))
1827                 goto out;
1828
1829         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1830                 tgt_dev->num_free_sn_slots);
1831         if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
1832                 spin_lock_irq(&tgt_dev->sn_lock);
1833                 if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
1834                         if (tgt_dev->num_free_sn_slots < 0)
1835                                 tgt_dev->cur_sn_slot = slot;
1836                         smp_mb(); /* to be in-sync with SIMPLE case in scst_cmd_set_sn() */
1837                         tgt_dev->num_free_sn_slots++;
1838                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1839                                 tgt_dev->num_free_sn_slots);
1840                         
1841                 }
1842                 spin_unlock_irq(&tgt_dev->sn_lock);
1843         }
1844
1845 inc:
1846         /*
1847          * No locks is needed, because only one thread at time can 
1848          * be here (serialized by sn). Also it is supposed that there
1849          * could not be half-incremented halves.
1850          */
1851         tgt_dev->expected_sn++;
1852         smp_mb(); /* write must be before def_cmd_count read */
1853         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1854
1855 out:
1856         return;
1857 }
1858
1859 static int scst_process_internal_cmd(struct scst_cmd *cmd)
1860 {
1861         int res = SCST_CMD_STATE_RES_CONT_NEXT, rc;
1862
1863         TRACE_ENTRY();
1864
1865         __scst_cmd_get(cmd);
1866
1867         rc = scst_do_send_to_midlev(cmd);
1868         if (rc == SCST_EXEC_NEED_THREAD) {
1869                 TRACE_DBG("%s", "scst_do_send_to_midlev() requested "
1870                       "thread context, rescheduling");
1871                 res = SCST_CMD_STATE_RES_NEED_THREAD;
1872         } else {
1873                 struct scst_device *dev = cmd->dev;
1874                 sBUG_ON(rc != SCST_EXEC_COMPLETED);
1875                 if (dev->scsi_dev != NULL)
1876                         generic_unplug_device(dev->scsi_dev->request_queue);
1877         }
1878
1879         __scst_cmd_put(cmd);
1880
1881         TRACE_EXIT_RES(res);
1882         return res;
1883 }
1884
1885 static int scst_send_to_midlev(struct scst_cmd **active_cmd)
1886 {
1887         int res, rc;
1888         struct scst_cmd *cmd = *active_cmd;
1889         struct scst_cmd *ref_cmd;
1890         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1891         struct scst_device *dev = cmd->dev;
1892         typeof(tgt_dev->expected_sn) expected_sn;
1893         int count;
1894
1895         TRACE_ENTRY();
1896
1897         res = SCST_CMD_STATE_RES_CONT_NEXT;
1898
1899         if (unlikely(cmd->internal || cmd->retry)) {
1900                 res = scst_process_internal_cmd(cmd);
1901                 goto out;
1902         }
1903
1904 #ifdef MEASURE_LATENCY
1905         if (cmd->pre_exec_finish == 0) {
1906                 struct timespec ts;
1907                 getnstimeofday(&ts);
1908                 cmd->pre_exec_finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1909                 TRACE_DBG("cmd %p (sess %p): pre_exec_finish %Ld (tv_sec %ld, "
1910                         "tv_nsec %ld)", cmd, sess, cmd->pre_exec_finish, ts.tv_sec,
1911                         ts.tv_nsec);
1912         }
1913 #endif
1914
1915         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
1916                 goto out;
1917
1918         ref_cmd = cmd;
1919         __scst_cmd_get(ref_cmd);
1920
1921         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
1922                 goto exec;
1923
1924         sBUG_ON(!cmd->sn_set);
1925
1926         expected_sn = tgt_dev->expected_sn;
1927         /* Optimized for lockless fast path */
1928         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
1929                 spin_lock_irq(&tgt_dev->sn_lock);
1930
1931                 tgt_dev->def_cmd_count++;
1932                 smp_mb();
1933
1934                 expected_sn = tgt_dev->expected_sn;
1935                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
1936                         /* We are under IRQ lock, but dev->dev_lock is BH one */
1937                         int cmd_blocking = scst_pre_dec_on_dev_cmd(cmd);
1938                         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1939                                 /* Necessary to allow aborting out of sn cmds */
1940                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p (tag %llu)",
1941                                         cmd, cmd->tag);
1942                                 tgt_dev->def_cmd_count--;
1943                                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1944                                 res = SCST_CMD_STATE_RES_CONT_SAME;
1945                         } else {
1946                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
1947                                         "expected_sn=%ld)", cmd, cmd->sn,
1948                                         cmd->sn_set, expected_sn);
1949                                 list_add_tail(&cmd->sn_cmd_list_entry,
1950                                               &tgt_dev->deferred_cmd_list);
1951                         }
1952                         spin_unlock_irq(&tgt_dev->sn_lock);
1953
1954                         __scst_dec_on_dev_cmd(dev, cmd_blocking);
1955
1956                         goto out_put;
1957                 } else {
1958                         TRACE_SN("Somebody incremented expected_sn %ld, "
1959                                 "continuing", expected_sn);
1960                         tgt_dev->def_cmd_count--;
1961                         spin_unlock_irq(&tgt_dev->sn_lock);
1962                 }
1963         }
1964
1965 exec:
1966         count = 0;
1967         while(1) {
1968                 atomic_t *slot = cmd->sn_slot;
1969                 /* For HQ commands SN is not set */
1970                 int inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1971                                       cmd->sn_set;
1972
1973                 rc = scst_do_send_to_midlev(cmd);
1974                 if (rc == SCST_EXEC_NEED_THREAD) {
1975                         TRACE_DBG("%s", "scst_do_send_to_midlev() requested "
1976                               "thread context, rescheduling");
1977                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1978                         scst_dec_on_dev_cmd(cmd);
1979                         *active_cmd = cmd;
1980                         if (count != 0)
1981                                 goto out_unplug;
1982                         else
1983                                 goto out_put;
1984                 }
1985                 sBUG_ON(rc != SCST_EXEC_COMPLETED);
1986
1987                 count++;
1988
1989                 if (inc_expected_sn)
1990                         scst_inc_expected_sn(tgt_dev, slot);
1991
1992                 cmd = scst_check_deferred_commands(tgt_dev);
1993                 if (cmd == NULL)
1994                         break;
1995
1996                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
1997                         break;
1998
1999                 __scst_cmd_put(ref_cmd);
2000                 ref_cmd = cmd;
2001                 __scst_cmd_get(ref_cmd);
2002         }
2003
2004 out_unplug:
2005         if (dev->scsi_dev != NULL)
2006                 generic_unplug_device(dev->scsi_dev->request_queue);
2007
2008 out_put:
2009         __scst_cmd_put(ref_cmd);
2010         /* !! At this point sess, dev and tgt_dev can be already freed !! */
2011
2012 out:
2013         TRACE_EXIT_HRES(res);
2014         return res;
2015 }
2016
2017 /* No locks supposed to be held */
2018 static int scst_check_sense(struct scst_cmd *cmd)
2019 {
2020         int res = 0;
2021         struct scst_device *dev = cmd->dev;
2022         int dbl_ua_possible, ua_sent = 0;
2023
2024         TRACE_ENTRY();
2025
2026         /* If we had internal bus reset behind us, set the command error UA */
2027         if ((dev->scsi_dev != NULL) &&
2028             unlikely(cmd->host_status == DID_RESET) &&
2029             scst_is_ua_command(cmd)) {
2030                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
2031                       dev->scsi_dev->was_reset, cmd->host_status);
2032                 scst_set_cmd_error(cmd,
2033                    SCST_LOAD_SENSE(scst_sense_reset_UA));
2034                 /* just in case */
2035                 cmd->ua_ignore = 0;
2036                 /* It looks like it is safe to clear was_reset here */
2037                 dev->scsi_dev->was_reset = 0;
2038         }
2039
2040         dbl_ua_possible = dev->dev_double_ua_possible;
2041         TRACE_DBG("cmd %p dbl_ua_possible %d", cmd, dbl_ua_possible);
2042         if (unlikely(dbl_ua_possible)) {
2043                 spin_lock_bh(&dev->dev_lock);
2044                 barrier(); /* to reread dev_double_ua_possible */
2045                 dbl_ua_possible = dev->dev_double_ua_possible;
2046                 if (dbl_ua_possible)
2047                         ua_sent = dev->dev_reset_ua_sent;
2048                 else
2049                         spin_unlock_bh(&dev->dev_lock);
2050         }
2051
2052         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) && 
2053             SCST_SENSE_VALID(cmd->sense)) {
2054                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2055                         SCST_SENSE_BUFFERSIZE);
2056                 /* Check Unit Attention Sense Key */
2057                 if (scst_is_ua_sense(cmd->sense)) {
2058                         if (cmd->sense[12] == SCST_SENSE_ASC_UA_RESET) {
2059                                 if (dbl_ua_possible) {
2060                                         if (ua_sent) {
2061                                                 TRACE(TRACE_MGMT_MINOR, "%s", 
2062                                                         "Double UA detected");
2063                                                 /* Do retry */
2064                                                 TRACE(TRACE_MGMT_MINOR, "Retrying cmd %p "
2065                                                         "(tag %llu)", cmd, cmd->tag);
2066                                                 cmd->status = 0;
2067                                                 cmd->msg_status = 0;
2068                                                 cmd->host_status = DID_OK;
2069                                                 cmd->driver_status = 0;
2070                                                 mempool_free(cmd->sense, scst_sense_mempool);
2071                                                 cmd->sense = NULL;
2072                                                 cmd->retry = 1;
2073                                                 cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
2074                                                 res = 1;
2075                                                 /* 
2076                                                  * Dev is still blocked by this cmd, so
2077                                                  * it's OK to clear SCST_DEV_SERIALIZED
2078                                                  * here.
2079                                                  */
2080                                                 dev->dev_double_ua_possible = 0;
2081                                                 dev->dev_serialized = 0;
2082                                                 dev->dev_reset_ua_sent = 0;
2083                                                 goto out_unlock;
2084                                         } else
2085                                                 dev->dev_reset_ua_sent = 1;
2086                                 }
2087                         }
2088                         if (cmd->ua_ignore == 0) {
2089                                 if (unlikely(dbl_ua_possible)) {
2090                                         __scst_dev_check_set_UA(dev, cmd,
2091                                                 cmd->sense,
2092                                                 SCST_SENSE_BUFFERSIZE);
2093                                 } else {
2094                                         scst_dev_check_set_UA(dev, cmd,
2095                                                 cmd->sense,
2096                                                 SCST_SENSE_BUFFERSIZE);
2097                                 }
2098                         }
2099                 }
2100         }
2101
2102         if (unlikely(dbl_ua_possible)) {
2103                 if (ua_sent && scst_is_ua_command(cmd)) {
2104                         TRACE_MGMT_DBG("%s", "Clearing dbl_ua_possible flag");
2105                         dev->dev_double_ua_possible = 0;
2106                         dev->dev_serialized = 0;
2107                         dev->dev_reset_ua_sent = 0;
2108                 }
2109                 spin_unlock_bh(&dev->dev_lock);
2110         }
2111
2112 out:
2113         TRACE_EXIT_RES(res);
2114         return res;
2115
2116 out_unlock:
2117         spin_unlock_bh(&dev->dev_lock);
2118         goto out;
2119 }
2120
2121 static int scst_check_auto_sense(struct scst_cmd *cmd)
2122 {
2123         int res = 0;
2124
2125         TRACE_ENTRY();
2126
2127         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2128             (!SCST_SENSE_VALID(cmd->sense) ||
2129              SCST_NO_SENSE(cmd->sense))) {
2130                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2131                       "cmd->status=%x, cmd->msg_status=%x, "
2132                       "cmd->host_status=%x, cmd->driver_status=%x", cmd->status,
2133                       cmd->msg_status, cmd->host_status, cmd->driver_status);
2134                 res = 1;
2135         } else if (unlikely(cmd->host_status)) {
2136                 if ((cmd->host_status == DID_REQUEUE) ||
2137                     (cmd->host_status == DID_IMM_RETRY) ||
2138                     (cmd->host_status == DID_SOFT_ERROR) ||
2139                     (cmd->host_status == DID_ABORT)) {
2140                         scst_set_busy(cmd);
2141                 } else {
2142                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2143                                 "received, returning HARDWARE ERROR instead",
2144                                 cmd->host_status);
2145                         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2146                 }
2147         }
2148
2149         TRACE_EXIT_RES(res);
2150         return res;
2151 }
2152
2153 static int scst_done_cmd_check(struct scst_cmd *cmd, int *pres)
2154 {
2155         int res = 0, rc;
2156
2157         TRACE_ENTRY();
2158
2159         if (unlikely(cmd->cdb[0] == REQUEST_SENSE)) {
2160                 if (cmd->internal)
2161                         cmd = scst_complete_request_sense(cmd);
2162         } else if (unlikely(scst_check_auto_sense(cmd))) {
2163                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2164                             "without sense data (opcode 0x%x), issuing "
2165                             "REQUEST SENSE", cmd->cdb[0]);
2166                 rc = scst_prepare_request_sense(cmd);
2167                 if (rc > 0) {
2168                         *pres = rc;
2169                         res = 1;
2170                         goto out;
2171                 } else {
2172                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2173                                     "returning HARDWARE ERROR");
2174                         scst_set_cmd_error(cmd,
2175                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2176                 }
2177         } else if (unlikely(scst_check_sense(cmd))) {
2178                 *pres = SCST_CMD_STATE_RES_CONT_SAME;
2179                 res = 1;
2180                 goto out;
2181         }
2182
2183         if (likely(scsi_status_is_good(cmd->status))) {
2184                 unsigned char type = cmd->dev->handler->type;
2185                 if ((cmd->cdb[0] == MODE_SENSE || cmd->cdb[0] == MODE_SENSE_10) &&
2186                     cmd->tgt_dev->acg_dev->rd_only_flag &&
2187                     (type == TYPE_DISK || type == TYPE_WORM || type == TYPE_MOD ||
2188                      type == TYPE_TAPE)) {
2189                         int32_t length;
2190                         uint8_t *address;
2191
2192                         length = scst_get_buf_first(cmd, &address);
2193                         if (length <= 0)
2194                                 goto out;
2195                         if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2196                                 address[2] |= 0x80;   /* Write Protect*/
2197                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2198                                 address[3] |= 0x80;   /* Write Protect*/
2199                         scst_put_buf(cmd, address);
2200                 }
2201
2202                 /* 
2203                  * Check and clear NormACA option for the device, if necessary,
2204                  * since we don't support ACA
2205                  */
2206                 if ((cmd->cdb[0] == INQUIRY) &&
2207                     !(cmd->cdb[1] & SCST_INQ_EVPD/* Std INQUIRY data (no EVPD) */) &&
2208                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2209                         uint8_t *buffer;
2210                         int buflen;
2211
2212                         /* ToDo: all pages ?? */
2213                         buflen = scst_get_buf_first(cmd, &buffer);
2214                         if (buflen > SCST_INQ_BYTE3) {
2215 #ifdef EXTRACHECKS
2216                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2217                                         PRINT_INFO("NormACA set for device: "
2218                                             "lun=%Ld, type 0x%02x", 
2219                                             (uint64_t)cmd->lun, buffer[0]);
2220                                 }
2221 #endif
2222                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2223                         } else {
2224                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2225                                     "buffer");
2226                                 scst_set_cmd_error(cmd,
2227                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
2228                         }
2229                         if (buflen > 0)
2230                                 scst_put_buf(cmd, buffer);
2231                 }
2232
2233                 if (unlikely((cmd->cdb[0] == MODE_SELECT) || 
2234                     (cmd->cdb[0] == MODE_SELECT_10) ||
2235                     (cmd->cdb[0] == LOG_SELECT))) {
2236                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded (LUN %Ld)",
2237                                 (uint64_t)cmd->lun);
2238                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2239                         *pres = SCST_CMD_STATE_RES_CONT_SAME;
2240                         res = 1;
2241                         goto out;
2242                 }
2243         } else {
2244                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2245                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2246                                         &cmd->tgt_dev->tgt_dev_flags)) {
2247                                 struct scst_tgt_dev *tgt_dev_tmp;
2248                                 struct scst_device *dev = cmd->dev;
2249
2250                                 TRACE(TRACE_SCSI, "Real RESERVE failed lun=%Ld, "
2251                                         "status=%x", (uint64_t)cmd->lun, cmd->status);
2252                                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2253                                         SCST_SENSE_BUFFERSIZE);
2254
2255                                 /* Clearing the reservation */
2256                                 spin_lock_bh(&dev->dev_lock);
2257                                 list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
2258                                                     dev_tgt_dev_list_entry) {
2259                                         clear_bit(SCST_TGT_DEV_RESERVED, 
2260                                                 &tgt_dev_tmp->tgt_dev_flags);
2261                                 }
2262                                 dev->dev_reserved = 0;
2263                                 spin_unlock_bh(&dev->dev_lock);
2264                         }
2265                 }
2266
2267                 /* Check for MODE PARAMETERS CHANGED UA */
2268                 if ((cmd->dev->scsi_dev != NULL) &&
2269                     (cmd->status == SAM_STAT_CHECK_CONDITION) && 
2270                     SCST_SENSE_VALID(cmd->sense) &&
2271                     scst_is_ua_sense(cmd->sense) &&
2272                     (cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) {
2273                         TRACE(TRACE_SCSI, "MODE PARAMETERS CHANGED UA (lun %Ld)",
2274                                 (uint64_t)cmd->lun);
2275                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2276                         *pres = SCST_CMD_STATE_RES_CONT_SAME;
2277                         res = 1;
2278                         goto out;
2279                 }
2280         }
2281
2282 out:
2283         TRACE_EXIT_RES(res);
2284         return res;
2285 }
2286
2287 static int scst_pre_dev_done(struct scst_cmd *cmd)
2288 {
2289         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2290
2291         TRACE_ENTRY();
2292
2293         atomic_dec(&cmd->tgt_dev->tgt_dev_cmd_count);
2294         atomic_dec(&cmd->dev->dev_cmd_count);
2295
2296         rc = scst_done_cmd_check(cmd, &res);
2297         if (rc)
2298                 goto out;
2299
2300         cmd->state = SCST_CMD_STATE_DEV_DONE;
2301
2302 out:
2303         TRACE_EXIT_HRES(res);
2304         return res;
2305 }
2306
2307 static int scst_mode_select_checks(struct scst_cmd *cmd)
2308 {
2309         int res = SCST_CMD_STATE_RES_CONT_SAME;
2310         int atomic = scst_cmd_atomic(cmd);
2311
2312         TRACE_ENTRY();
2313
2314         if (likely(scsi_status_is_good(cmd->status))) {
2315                 if (unlikely((cmd->cdb[0] == MODE_SELECT) || 
2316                     (cmd->cdb[0] == MODE_SELECT_10) ||
2317                     (cmd->cdb[0] == LOG_SELECT))) {
2318                         struct scst_device *dev = cmd->dev;
2319                         if (atomic && (dev->scsi_dev != NULL)) {
2320                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2321                                         "context required");
2322                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2323                                 goto out;
2324                         }
2325
2326                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2327                                 "setting the SELECT UA (lun=%Ld)", 
2328                                 (uint64_t)cmd->lun);
2329
2330                         spin_lock_bh(&dev->dev_lock);
2331                         spin_lock(&scst_temp_UA_lock);
2332                         if (cmd->cdb[0] == LOG_SELECT) {
2333                                 scst_set_sense(scst_temp_UA,
2334                                         sizeof(scst_temp_UA),
2335                                         UNIT_ATTENTION, 0x2a, 0x02);
2336                         } else {
2337                                 scst_set_sense(scst_temp_UA,
2338                                         sizeof(scst_temp_UA),
2339                                         UNIT_ATTENTION, 0x2a, 0x01);
2340                         }
2341                         scst_dev_check_set_local_UA(dev, cmd, scst_temp_UA,
2342                                 sizeof(scst_temp_UA));
2343                         spin_unlock(&scst_temp_UA_lock);
2344                         spin_unlock_bh(&dev->dev_lock);
2345
2346                         if (dev->scsi_dev != NULL)
2347                                 scst_obtain_device_parameters(dev);
2348                 }
2349         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) && 
2350                     SCST_SENSE_VALID(cmd->sense) &&
2351                     scst_is_ua_sense(cmd->sense) &&
2352                     (cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) {
2353                 if (atomic) {
2354                         TRACE_DBG("%s", "MODE PARAMETERS CHANGED UA: thread "
2355                                 "context required");
2356                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2357                         goto out;
2358                 }
2359
2360                 TRACE(TRACE_SCSI, "MODE PARAMETERS CHANGED UA (lun %Ld): "
2361                         "getting new parameters", (uint64_t)cmd->lun);
2362
2363                 scst_obtain_device_parameters(cmd->dev);
2364         } else
2365                 sBUG();
2366
2367         cmd->state = SCST_CMD_STATE_DEV_DONE;
2368
2369 out:
2370         TRACE_EXIT_HRES(res);
2371         return res;
2372 }
2373
2374 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2375 {
2376         if (likely(cmd->sn_set))
2377                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2378
2379         scst_make_deferred_commands_active(cmd->tgt_dev, cmd);
2380 }
2381
2382 static int scst_dev_done(struct scst_cmd *cmd)
2383 {
2384         int res = SCST_CMD_STATE_RES_CONT_SAME;
2385         int state;
2386         int atomic = scst_cmd_atomic(cmd);
2387
2388         TRACE_ENTRY();
2389
2390         if (atomic && !cmd->dev->handler->dev_done_atomic) {
2391                 TRACE_DBG("Dev handler %s dev_done() can not be "
2392                       "called in atomic context, rescheduling to the thread",
2393                       cmd->dev->handler->name);
2394                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2395                 goto out;
2396         }
2397
2398         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2399         if (likely(!scst_is_cmd_local(cmd)) && 
2400             likely(cmd->dev->handler->dev_done != NULL))
2401         {
2402                 int rc;
2403                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2404                       cmd->dev->handler->name, cmd);
2405                 rc = cmd->dev->handler->dev_done(cmd);
2406                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2407                       cmd->dev->handler->name, rc);
2408                 if (rc != SCST_CMD_STATE_DEFAULT)
2409                         state = rc;
2410         }
2411
2412         switch (state) {
2413         case SCST_CMD_STATE_PRE_XMIT_RESP:
2414         case SCST_CMD_STATE_DEV_PARSE:
2415         case SCST_CMD_STATE_PRE_PARSE:
2416         case SCST_CMD_STATE_PREPARE_SPACE:
2417         case SCST_CMD_STATE_RDY_TO_XFER:
2418         case SCST_CMD_STATE_TGT_PRE_EXEC:
2419         case SCST_CMD_STATE_SEND_TO_MIDLEV:
2420         case SCST_CMD_STATE_PRE_DEV_DONE:
2421         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2422         case SCST_CMD_STATE_DEV_DONE:
2423         case SCST_CMD_STATE_XMIT_RESP:
2424         case SCST_CMD_STATE_FINISHED:
2425                 cmd->state = state;
2426                 res = SCST_CMD_STATE_RES_CONT_SAME;
2427                 break;
2428
2429         case SCST_CMD_STATE_NEED_THREAD_CTX:
2430                 TRACE_DBG("Dev handler %s dev_done() requested "
2431                       "thread context, rescheduling",
2432                       cmd->dev->handler->name);
2433                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2434                 break;
2435
2436         default:
2437                 if (state >= 0) {
2438                         PRINT_ERROR("Dev handler %s dev_done() returned "
2439                                 "invalid cmd state %d", 
2440                                 cmd->dev->handler->name, state);
2441                 } else {
2442                         PRINT_ERROR("Dev handler %s dev_done() returned "
2443                                 "error %d", cmd->dev->handler->name, 
2444                                 state);
2445                 }
2446                 scst_set_cmd_error(cmd,
2447                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2448                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2449                 res = SCST_CMD_STATE_RES_CONT_SAME;
2450                 break;
2451         }
2452
2453         if (cmd->needs_unblocking)
2454                 scst_unblock_dev_cmd(cmd);
2455
2456         if (likely(cmd->dec_on_dev_needed))
2457                 scst_dec_on_dev_cmd(cmd);
2458
2459         if (cmd->inc_expected_sn_on_done && cmd->sent_to_midlev)
2460                 scst_inc_check_expected_sn(cmd);
2461
2462 out:
2463         TRACE_EXIT_HRES(res);
2464         return res;
2465 }
2466
2467 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2468 {
2469         int res;
2470
2471         TRACE_ENTRY();
2472
2473 #ifdef DEBUG_TM
2474         if (cmd->tm_dbg_delayed && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2475                 if (scst_cmd_atomic(cmd)) {
2476                         TRACE_MGMT_DBG("%s", "DEBUG_TM delayed cmd needs a thread");
2477                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2478                         return res;
2479                 }
2480                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2481                         cmd, cmd->tag);
2482                 schedule_timeout_uninterruptible(HZ);
2483         }
2484 #endif
2485
2486         if (cmd->tgt_dev != NULL) {
2487                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2488                         scst_on_hq_cmd_response(cmd);
2489
2490                 if (unlikely(!cmd->sent_to_midlev)) {
2491                         TRACE_SN("cmd %p was not sent to mid-lev (sn %ld, set %d)",
2492                                 cmd, cmd->sn, cmd->sn_set);
2493                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2494                         cmd->sent_to_midlev = 1;
2495                 }
2496         }
2497
2498         /*
2499          * If we don't remove cmd from the search list here, before
2500          * submitting it for transmittion, we will have a race, when for
2501          * some reason cmd's release is delayed after transmittion and
2502          * initiator sends cmd with the same tag => it is possible that
2503          * a wrong cmd will be found by find() functions.
2504          */
2505         spin_lock_irq(&cmd->sess->sess_list_lock);
2506         list_del(&cmd->search_cmd_list_entry);
2507         spin_unlock_irq(&cmd->sess->sess_list_lock);
2508
2509         cmd->done = 1;
2510         smp_mb(); /* to sync with scst_abort_cmd() */
2511
2512         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2513                 scst_xmit_process_aborted_cmd(cmd);
2514
2515         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2516                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu), skipping",
2517                         cmd, cmd->tag);
2518                 cmd->state = SCST_CMD_STATE_FINISHED;
2519                 res = SCST_CMD_STATE_RES_CONT_SAME;
2520                 goto out;
2521         }
2522
2523         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2524         res = SCST_CMD_STATE_RES_CONT_SAME;
2525
2526 out:
2527 #ifdef MEASURE_LATENCY
2528         {
2529                 struct timespec ts;
2530                 uint64_t finish, scst_time, proc_time;
2531                 struct scst_session *sess = cmd->sess;
2532
2533                 getnstimeofday(&ts);
2534                 finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2535
2536                 spin_lock_bh(&sess->meas_lock);
2537
2538                 scst_time = cmd->pre_exec_finish - cmd->start;
2539                 scst_time += finish - cmd->post_exec_start;
2540                 proc_time = finish - cmd->start;
2541
2542                 sess->scst_time += scst_time;
2543                 sess->processing_time += proc_time;
2544                 sess->processed_cmds++;
2545
2546                 spin_unlock_bh(&sess->meas_lock);
2547
2548                 TRACE_DBG("cmd %p (sess %p): finish %Ld (tv_sec %ld, "
2549                         "tv_nsec %ld), scst_time %Ld, proc_time %Ld", cmd, sess,
2550                         finish, ts.tv_sec, ts.tv_nsec, scst_time, proc_time);
2551         }
2552 #endif
2553         TRACE_EXIT_HRES(res);
2554         return res;
2555 }
2556
2557 static int scst_xmit_response(struct scst_cmd *cmd)
2558 {
2559         int res, rc;
2560         int atomic = scst_cmd_atomic(cmd);
2561
2562         TRACE_ENTRY();
2563
2564         if (atomic && !cmd->tgtt->xmit_response_atomic) {
2565                 TRACE_DBG("%s", "xmit_response() can not be "
2566                       "called in atomic context, rescheduling to the thread");
2567                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2568                 goto out;
2569         }
2570
2571         while (1) {
2572                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2573
2574                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2575                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2576
2577                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2578
2579 #if defined(DEBUG)
2580                 if (cmd->sg) {
2581                         int i;
2582                         struct scatterlist *sg = cmd->sg;
2583                         TRACE_SEND_BOT("Xmitting %d S/G(s) at %p sg[0].page at %p",
2584                               cmd->sg_cnt, sg, (void*)sg_page(&sg[0]));
2585                         for(i = 0; i < cmd->sg_cnt; ++i) {
2586                                 TRACE_BUFF_FLAG(TRACE_SND_BOT,
2587                                     "Xmitting sg", sg_virt(&sg[i]),
2588                                     sg[i].length);
2589                         }
2590                 }
2591 #endif
2592
2593 #ifdef DEBUG_RETRY
2594                 if (((scst_random() % 100) == 77))
2595                         rc = SCST_TGT_RES_QUEUE_FULL;
2596                 else
2597 #endif
2598                         rc = cmd->tgtt->xmit_response(cmd);
2599                 TRACE_DBG("xmit_response() returned %d", rc);
2600
2601                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2602                         goto out;
2603
2604                 /* Restore the previous state */
2605                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2606
2607                 switch (rc) {
2608                 case SCST_TGT_RES_QUEUE_FULL:
2609                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2610                                 break;
2611                         else
2612                                 continue;
2613
2614                 case SCST_TGT_RES_NEED_THREAD_CTX:
2615                         TRACE_DBG("Target driver %s xmit_response() "
2616                               "requested thread context, rescheduling",
2617                               cmd->tgtt->name);
2618                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2619                         break;
2620
2621                 default:
2622                         goto out_error;
2623                 }
2624                 break;
2625         }
2626
2627 out:
2628         /* Caution: cmd can be already dead here */
2629         TRACE_EXIT_HRES(res);
2630         return res;
2631
2632 out_error:
2633         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2634                 PRINT_ERROR("Target driver %s xmit_response() returned "
2635                         "fatal error", cmd->tgtt->name);
2636         } else {
2637                 PRINT_ERROR("Target driver %s xmit_response() returned "
2638                         "invalid value %d", cmd->tgtt->name, rc);
2639         }
2640         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2641         cmd->state = SCST_CMD_STATE_FINISHED;
2642         res = SCST_CMD_STATE_RES_CONT_SAME;
2643         goto out;
2644 }
2645
2646 void scst_tgt_cmd_done(struct scst_cmd *cmd)
2647 {
2648         TRACE_ENTRY();
2649
2650         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2651
2652         cmd->state = SCST_CMD_STATE_FINISHED;
2653         scst_proccess_redirect_cmd(cmd, scst_get_context(), 1);
2654
2655         TRACE_EXIT();
2656         return;
2657 }
2658
2659 static int scst_finish_cmd(struct scst_cmd *cmd)
2660 {
2661         int res;
2662
2663         TRACE_ENTRY();
2664
2665         atomic_dec(&cmd->sess->sess_cmd_count);
2666
2667         cmd->finished = 1;
2668         smp_mb(); /* to sync with scst_abort_cmd() */
2669
2670         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2671                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
2672                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
2673                         atomic_read(&scst_cmd_count));
2674
2675                 scst_finish_cmd_mgmt(cmd);
2676         }
2677
2678         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
2679                 if ((cmd->tgt_dev != NULL) &&
2680                     scst_is_ua_sense(cmd->sense)) {
2681                         /* This UA delivery failed, so requeue it */
2682                         TRACE_MGMT_DBG("Requeuing UA for delivery failed cmd "
2683                                 "%p", cmd);
2684                         scst_check_set_UA(cmd->tgt_dev, cmd->sense,
2685                                         SCST_SENSE_BUFFERSIZE, 1);
2686                 }
2687         }
2688
2689         __scst_cmd_put(cmd);
2690
2691         res = SCST_CMD_STATE_RES_CONT_NEXT;
2692
2693         TRACE_EXIT_HRES(res);
2694         return res;
2695 }
2696
2697 /*
2698  * No locks, but it must be externally serialized (see comment for
2699  * scst_cmd_init_done() in scsi_tgt.h)
2700  */
2701 static void scst_cmd_set_sn(struct scst_cmd *cmd)
2702 {
2703         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2704         unsigned long flags;
2705
2706         if (scst_is_implicit_hq(cmd)) {
2707                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "Implicit HQ cmd %p", cmd);
2708                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
2709         }
2710
2711         /* Optimized for lockless fast path */
2712
2713         scst_check_debug_sn(cmd);
2714
2715         if (cmd->dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) {
2716                 /*
2717                  * Not the best way, but well enough until there will be a
2718                  * possibility to specify queue type during pass-through
2719                  * commands submission.
2720                  */
2721                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
2722         }
2723
2724         switch(cmd->queue_type) {
2725         case SCST_CMD_QUEUE_SIMPLE:
2726         case SCST_CMD_QUEUE_UNTAGGED:
2727 #if 1 /* temporary, ToDo */
2728                 if (scst_cmd_is_expected_set(cmd)) {
2729                         if (cmd->expected_data_direction == SCST_DATA_READ)
2730                                 goto ordered;
2731                 } else
2732                         goto ordered;
2733 #endif
2734                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
2735                         /*
2736                          * atomic_inc_return() implies memory barrier to sync
2737                          * with scst_inc_expected_sn()
2738                          */
2739                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
2740                                 tgt_dev->curr_sn++;
2741                                 TRACE_SN("Incremented curr_sn %ld",
2742                                         tgt_dev->curr_sn);
2743                         }
2744                         cmd->sn_slot = tgt_dev->cur_sn_slot;
2745                         cmd->sn = tgt_dev->curr_sn;
2746                         
2747                         tgt_dev->prev_cmd_ordered = 0;
2748                 } else {
2749                         TRACE(TRACE_MINOR, "***WARNING*** Not enough SN slots "
2750                                 "%d", ARRAY_SIZE(tgt_dev->sn_slots));
2751                         goto ordered;
2752                 }
2753                 break;
2754
2755         case SCST_CMD_QUEUE_ORDERED:
2756                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "ORDERED cmd %p "
2757                         "(op %x)", cmd, cmd->cdb[0]);
2758 ordered:
2759                 if (!tgt_dev->prev_cmd_ordered) {
2760                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2761                         if (tgt_dev->num_free_sn_slots >= 0) {
2762                                 tgt_dev->num_free_sn_slots--;
2763                                 if (tgt_dev->num_free_sn_slots >= 0) {
2764                                         int i = 0;
2765                                         /*
2766                                          * Commands can finish in any order, so we don't
2767                                          * know, which slot is empty.
2768                                          */
2769                                         while(1) {
2770                                                 tgt_dev->cur_sn_slot++;
2771                                                 if (tgt_dev->cur_sn_slot == tgt_dev->sn_slots +
2772                                                                 ARRAY_SIZE(tgt_dev->sn_slots))
2773                                                         tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
2774
2775                                                 if (atomic_read(tgt_dev->cur_sn_slot) == 0)
2776                                                         break;
2777
2778                                                 i++;
2779                                                 sBUG_ON(i == ARRAY_SIZE(tgt_dev->sn_slots));
2780                                         }
2781                                         TRACE_SN("New cur SN slot %zd",
2782                                                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
2783                                 }
2784                         }
2785                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2786                 }
2787                 tgt_dev->prev_cmd_ordered = 1;
2788                 tgt_dev->curr_sn++;
2789                 cmd->sn = tgt_dev->curr_sn;
2790                 break;
2791
2792         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
2793                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "HQ cmd %p "
2794                         "(op %x)", cmd, cmd->cdb[0]);
2795                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2796                 tgt_dev->hq_cmd_count++;
2797                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2798                 goto out;
2799
2800         default:
2801                 sBUG();
2802         }
2803
2804         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
2805                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
2806                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
2807                 atomic_read(tgt_dev->cur_sn_slot), 
2808                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
2809                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
2810
2811         cmd->sn_set = 1;
2812
2813 out:
2814         return;
2815 }
2816
2817 /*
2818  * Returns 0 on success, > 0 when we need to wait for unblock,
2819  * < 0 if there is no device (lun) or device type handler.
2820  *
2821  * No locks, but might be on IRQ, protection is done by the
2822  * suspended activity.
2823  */
2824 static int scst_translate_lun(struct scst_cmd *cmd)
2825 {
2826         struct scst_tgt_dev *tgt_dev = NULL;
2827         int res;
2828
2829         TRACE_ENTRY();
2830
2831         __scst_get(1);
2832
2833         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
2834                 struct list_head *sess_tgt_dev_list_head =
2835                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
2836                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %Ld)", cmd,
2837                         (uint64_t)cmd->lun);
2838                 res = -1;
2839                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
2840                                 sess_tgt_dev_list_entry) {
2841                         if (tgt_dev->lun == cmd->lun) {
2842                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
2843
2844                                 if (unlikely(tgt_dev->dev->handler == &scst_null_devtype)) {
2845                                         PRINT_INFO("Dev handler for device "
2846                                           "%Ld is NULL, the device will not be "
2847                                           "visible remotely", (uint64_t)cmd->lun);
2848                                         break;
2849                                 }
2850                                 
2851                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
2852                                 cmd->tgt_dev = tgt_dev;
2853                                 cmd->dev = tgt_dev->dev;
2854
2855                                 res = 0;
2856                                 break;
2857                         }
2858                 }
2859                 if (res != 0) {
2860                         TRACE(TRACE_MINOR, "tgt_dev for lun %Ld not found, command to "
2861                                 "unexisting LU?", (uint64_t)cmd->lun);
2862                         __scst_put();
2863                 }
2864         } else {
2865                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
2866                 __scst_put();
2867                 res = 1;
2868         }
2869
2870         TRACE_EXIT_RES(res);
2871         return res;
2872 }
2873
2874 /*
2875  * No locks, but might be on IRQ
2876  *
2877  * Returns 0 on success, > 0 when we need to wait for unblock,
2878  * < 0 if there is no device (lun) or device type handler.
2879  */
2880 static int __scst_init_cmd(struct scst_cmd *cmd)
2881 {
2882         int res = 0;
2883
2884         TRACE_ENTRY();
2885
2886         res = scst_translate_lun(cmd);
2887         if (likely(res == 0)) {
2888                 int cnt;
2889                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
2890                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
2891                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
2892                         TRACE(TRACE_MGMT_MINOR, "Too many pending commands (%d) in "
2893                                 "session, returning BUSY to initiator \"%s\"",
2894                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
2895                                   "Anonymous" : cmd->sess->initiator_name);
2896                         goto out_busy;
2897                 }
2898                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
2899                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
2900                         TRACE(TRACE_MGMT_MINOR, "Too many pending device commands "
2901                                 "(%d), returning BUSY to initiator \"%s\"",
2902                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
2903                                   "Anonymous" : cmd->sess->initiator_name);
2904                         goto out_busy;
2905                 }
2906                 if (!cmd->set_sn_on_restart_cmd)
2907                         scst_cmd_set_sn(cmd);
2908         } else if (res < 0) {
2909                 TRACE_DBG("Finishing cmd %p", cmd);
2910                 scst_set_cmd_error(cmd,
2911                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
2912                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2913         } else
2914                 goto out;
2915
2916 out:
2917         TRACE_EXIT_RES(res);
2918         return res;
2919
2920 out_busy:
2921         scst_set_busy(cmd);
2922         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2923         goto out;
2924 }
2925
2926 /* Called under scst_init_lock and IRQs disabled */
2927 static void scst_do_job_init(void)
2928 {
2929         struct scst_cmd *cmd;
2930         int susp;
2931
2932         TRACE_ENTRY();
2933
2934 restart:
2935         /*
2936          * There is no need for read barrier here, because we don't care where
2937          * this check will be done.
2938          */
2939         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
2940         if (scst_init_poll_cnt > 0)
2941                 scst_init_poll_cnt--;
2942
2943         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
2944                 int rc;
2945                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
2946                         continue;
2947                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2948                         spin_unlock_irq(&scst_init_lock);
2949                         rc = __scst_init_cmd(cmd);
2950                         spin_lock_irq(&scst_init_lock);
2951                         if (rc > 0) {
2952                                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, restarting");
2953                                 goto restart;
2954                         }
2955                 } else {
2956                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
2957                                 cmd, cmd->tag);
2958                         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2959                 }
2960
2961                 /*
2962                  * Deleting cmd from init cmd list after __scst_init_cmd()
2963                  * is necessary to keep the check in scst_init_cmd() correct
2964                  * to preserve the commands order.
2965                  *
2966                  * We don't care about the race, when init cmd list is empty
2967                  * and one command detected that it just was not empty, so
2968                  * it's inserting to it, but another command at the same time
2969                  * seeing init cmd list empty and goes directly, because it
2970                  * could affect only commands from the same initiator to the
2971                  * same tgt_dev, but init_cmd_done() doesn't guarantee the order
2972                  * in case of simultaneous such calls anyway.
2973                  */
2974                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
2975                 smp_wmb();
2976                 list_del(&cmd->cmd_list_entry);
2977                 spin_unlock(&scst_init_lock);
2978
2979                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
2980                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
2981                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2982                         list_add(&cmd->cmd_list_entry,
2983                                 &cmd->cmd_lists->active_cmd_list);
2984                 else
2985                         list_add_tail(&cmd->cmd_list_entry,
2986                                 &cmd->cmd_lists->active_cmd_list);
2987                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
2988                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
2989
2990                 spin_lock(&scst_init_lock);
2991                 goto restart;
2992         }
2993
2994         /* It isn't really needed, but let's keep it */
2995         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
2996                 goto restart;
2997
2998         TRACE_EXIT();
2999         return;
3000 }
3001
3002 static inline int test_init_cmd_list(void)
3003 {
3004         int res = (!list_empty(&scst_init_cmd_list) &&
3005                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
3006                   unlikely(kthread_should_stop()) ||
3007                   (scst_init_poll_cnt > 0);
3008         return res;
3009 }
3010
3011 int scst_init_cmd_thread(void *arg)
3012 {
3013         TRACE_ENTRY();
3014
3015         PRINT_INFO("Init thread started, PID %d", current->pid);
3016
3017         current->flags |= PF_NOFREEZE;
3018
3019         set_user_nice(current, -10);
3020
3021         spin_lock_irq(&scst_init_lock);
3022         while(!kthread_should_stop()) {
3023                 wait_queue_t wait;
3024                 init_waitqueue_entry(&wait, current);
3025
3026                 if (!test_init_cmd_list()) {
3027                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
3028                                                  &wait);
3029                         for (;;) {
3030                                 set_current_state(TASK_INTERRUPTIBLE);
3031                                 if (test_init_cmd_list())
3032                                         break;
3033                                 spin_unlock_irq(&scst_init_lock);
3034                                 schedule();
3035                                 spin_lock_irq(&scst_init_lock);
3036                         }
3037                         set_current_state(TASK_RUNNING);
3038                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
3039                 }
3040                 scst_do_job_init();
3041         }
3042         spin_unlock_irq(&scst_init_lock);
3043
3044         /*
3045          * If kthread_should_stop() is true, we are guaranteed to be
3046          * on the module unload, so scst_init_cmd_list must be empty.
3047          */
3048         sBUG_ON(!list_empty(&scst_init_cmd_list));
3049
3050         PRINT_INFO("Init thread PID %d finished", current->pid);
3051
3052         TRACE_EXIT();
3053         return 0;
3054 }
3055
3056 /* Called with no locks held */
3057 void scst_process_active_cmd(struct scst_cmd *cmd, int context)
3058 {
3059         int res;
3060
3061         TRACE_ENTRY();
3062
3063         EXTRACHECKS_BUG_ON(in_irq());
3064
3065         cmd->context_processable = context | SCST_CONTEXT_PROCESSABLE;
3066         context &= ~SCST_CONTEXT_PROCESSABLE;
3067         cmd->atomic = (context == SCST_CONTEXT_DIRECT_ATOMIC);
3068
3069         TRACE_DBG("cmd %p, context_processable %d, atomic %d", cmd,
3070                 cmd->context_processable, cmd->atomic);
3071
3072         do {
3073                 switch (cmd->state) {
3074                 case SCST_CMD_STATE_PRE_PARSE:
3075                         res = scst_pre_parse(cmd);
3076                         EXTRACHECKS_BUG_ON(res ==
3077                                 SCST_CMD_STATE_RES_NEED_THREAD);
3078                         break;
3079
3080                 case SCST_CMD_STATE_DEV_PARSE:
3081                         res = scst_parse_cmd(cmd);
3082                         break;
3083
3084                 case SCST_CMD_STATE_PREPARE_SPACE:
3085                         res = scst_prepare_space(cmd);
3086                         break;
3087
3088                 case SCST_CMD_STATE_RDY_TO_XFER:
3089                         res = scst_rdy_to_xfer(cmd);
3090                         break;
3091
3092                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3093                         res = scst_tgt_pre_exec(cmd);
3094                         break;
3095
3096                 case SCST_CMD_STATE_SEND_TO_MIDLEV:
3097                         if (tm_dbg_check_cmd(cmd) != 0) {
3098                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
3099                                 TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
3100                                         "because of TM DBG delay", cmd,
3101                                         cmd->tag);
3102                                 break;
3103                         }
3104                         res = scst_send_to_midlev(&cmd);
3105                         /* !! At this point cmd, sess & tgt_dev can be already freed !! */
3106                         break;
3107
3108                 case SCST_CMD_STATE_PRE_DEV_DONE:
3109                         res = scst_pre_dev_done(cmd);
3110                         EXTRACHECKS_BUG_ON(res ==
3111                                 SCST_CMD_STATE_RES_NEED_THREAD);
3112                         break;
3113
3114                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3115                         res = scst_mode_select_checks(cmd);
3116                         break;
3117
3118                 case SCST_CMD_STATE_DEV_DONE:
3119                         res = scst_dev_done(cmd);
3120                         break;
3121
3122                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3123                         res = scst_pre_xmit_response(cmd);
3124                         EXTRACHECKS_BUG_ON(res ==
3125                                 SCST_CMD_STATE_RES_NEED_THREAD);
3126                         break;
3127
3128                 case SCST_CMD_STATE_XMIT_RESP:
3129                         res = scst_xmit_response(cmd);
3130                         break;
3131
3132                 case SCST_CMD_STATE_FINISHED:
3133                         res = scst_finish_cmd(cmd);
3134                         EXTRACHECKS_BUG_ON(res ==
3135                                 SCST_CMD_STATE_RES_NEED_THREAD);
3136                         break;
3137
3138                 default:
3139                         PRINT_ERROR("cmd (%p) in state %d, but shouldn't be",
3140                                cmd, cmd->state);
3141                         sBUG();
3142                         res = SCST_CMD_STATE_RES_CONT_NEXT;
3143                         break;
3144                 }
3145         } while(res == SCST_CMD_STATE_RES_CONT_SAME);
3146
3147         if (res == SCST_CMD_STATE_RES_CONT_NEXT) {
3148                 /* None */
3149         } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
3150                 spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3151                 switch (cmd->state) {
3152                 case SCST_CMD_STATE_PRE_PARSE:
3153                 case SCST_CMD_STATE_DEV_PARSE:
3154                 case SCST_CMD_STATE_PREPARE_SPACE:
3155                 case SCST_CMD_STATE_RDY_TO_XFER:
3156                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3157                 case SCST_CMD_STATE_SEND_TO_MIDLEV:
3158                 case SCST_CMD_STATE_PRE_DEV_DONE:
3159                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3160                 case SCST_CMD_STATE_DEV_DONE:
3161                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3162                 case SCST_CMD_STATE_XMIT_RESP:
3163                 case SCST_CMD_STATE_FINISHED:
3164                         TRACE_DBG("Adding cmd %p to head of active cmd list", cmd);
3165                         list_add(&cmd->cmd_list_entry,
3166                                 &cmd->cmd_lists->active_cmd_list);
3167                         break;
3168 #ifdef EXTRACHECKS
3169                 /* not very valid commands */
3170                 case SCST_CMD_STATE_DEFAULT:
3171                 case SCST_CMD_STATE_NEED_THREAD_CTX:
3172                         PRINT_ERROR("cmd %p is in state %d, not putting on "
3173                                 "useful list (left on scst cmd list)", cmd, 
3174                                 cmd->state);
3175                         spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3176                         sBUG();
3177                         spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3178                         break;
3179 #endif
3180                 default:
3181                         break;
3182                 }
3183                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3184                 spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3185         } else
3186                 sBUG();
3187
3188         TRACE_EXIT();
3189         return;
3190 }
3191
3192 /* Called under cmd_list_lock and IRQs disabled */
3193 static void scst_do_job_active(struct list_head *cmd_list,
3194         spinlock_t *cmd_list_lock, int context)
3195 {
3196         TRACE_ENTRY();
3197
3198 #ifdef EXTRACHECKS
3199         {
3200                 int c = context & ~SCST_CONTEXT_PROCESSABLE;
3201                 sBUG_ON((c != SCST_CONTEXT_DIRECT_ATOMIC) && 
3202                         (c != SCST_CONTEXT_DIRECT));
3203         }
3204 #endif
3205
3206         while (!list_empty(cmd_list)) {
3207                 struct scst_cmd *cmd = list_entry(cmd_list->next, typeof(*cmd),
3208                                         cmd_list_entry);
3209                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
3210                 list_del(&cmd->cmd_list_entry);
3211                 spin_unlock_irq(cmd_list_lock);
3212                 scst_process_active_cmd(cmd, context);
3213                 spin_lock_irq(cmd_list_lock);
3214         }
3215
3216         TRACE_EXIT();
3217         return;
3218 }
3219
3220 static inline int test_cmd_lists(struct scst_cmd_lists *p_cmd_lists)
3221 {
3222         int res = !list_empty(&p_cmd_lists->active_cmd_list) ||
3223             unlikely(kthread_should_stop()) ||
3224             tm_dbg_is_release();
3225         return res;
3226 }
3227
3228 int scst_cmd_thread(void *arg)
3229 {
3230         struct scst_cmd_lists *p_cmd_lists = (struct scst_cmd_lists*)arg;
3231
3232         TRACE_ENTRY();
3233
3234         PRINT_INFO("Processing thread started, PID %d", current->pid);
3235
3236 #if 0
3237         set_user_nice(current, 10);
3238 #endif
3239         current->flags |= PF_NOFREEZE;
3240
3241         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3242         while (!kthread_should_stop()) {
3243                 wait_queue_t wait;
3244                 init_waitqueue_entry(&wait, current);
3245
3246                 if (!test_cmd_lists(p_cmd_lists)) {
3247                         add_wait_queue_exclusive(&p_cmd_lists->cmd_list_waitQ,
3248                                 &wait);
3249                         for (;;) {
3250                                 set_current_state(TASK_INTERRUPTIBLE);
3251                                 if (test_cmd_lists(p_cmd_lists))
3252                                         break;
3253                                 spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3254                                 schedule();
3255                                 spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3256                         }
3257                         set_current_state(TASK_RUNNING);
3258                         remove_wait_queue(&p_cmd_lists->cmd_list_waitQ, &wait);
3259                 }
3260
3261                 if (tm_dbg_is_release()) {
3262                         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3263                         tm_dbg_check_released_cmds();
3264                         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3265                 }
3266
3267                 scst_do_job_active(&p_cmd_lists->active_cmd_list,
3268                         &p_cmd_lists->cmd_list_lock, SCST_CONTEXT_DIRECT |
3269                                                 SCST_CONTEXT_PROCESSABLE);
3270         }
3271         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3272
3273 #ifdef EXTRACHECKS
3274         /*
3275          * If kthread_should_stop() is true, we are guaranteed to be either
3276          * on the module unload, or there must be at least one other thread to
3277          * process the commands lists.
3278          */
3279         if (p_cmd_lists == &scst_main_cmd_lists) {
3280                 sBUG_ON((scst_threads_info.nr_cmd_threads == 1) &&
3281                          !list_empty(&scst_main_cmd_lists.active_cmd_list));
3282         }
3283 #endif
3284
3285         PRINT_INFO("Processing thread PID %d finished", current->pid);
3286
3287         TRACE_EXIT();
3288         return 0;
3289 }
3290
3291 void scst_cmd_tasklet(long p)
3292 {
3293         struct scst_tasklet *t = (struct scst_tasklet*)p;
3294
3295         TRACE_ENTRY();
3296
3297         spin_lock_irq(&t->tasklet_lock);
3298         scst_do_job_active(&t->tasklet_cmd_list, &t->tasklet_lock,
3299                 SCST_CONTEXT_DIRECT_ATOMIC);
3300         spin_unlock_irq(&t->tasklet_lock);
3301
3302         TRACE_EXIT();
3303         return;
3304 }
3305
3306 /*
3307  * Returns 0 on success, < 0 if there is no device handler or
3308  * > 0 if SCST_FLAG_SUSPENDED set and SCST_FLAG_SUSPENDING - not.
3309  * No locks, protection is done by the suspended activity.
3310  */
3311 static int scst_mgmt_translate_lun(struct scst_mgmt_cmd *mcmd)
3312 {
3313         struct scst_tgt_dev *tgt_dev = NULL;
3314         struct list_head *sess_tgt_dev_list_head;
3315         int res = -1;
3316
3317         TRACE_ENTRY();
3318
3319         TRACE_DBG("Finding tgt_dev for mgmt cmd %p (lun %Ld)", mcmd,
3320               (uint64_t)mcmd->lun);
3321
3322         __scst_get(1);
3323
3324         if (unlikely(test_bit(SCST_FLAG_SUSPENDED, &scst_flags) &&
3325                      !test_bit(SCST_FLAG_SUSPENDING, &scst_flags))) {
3326                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3327                 __scst_put();
3328                 res = 1;
3329                 goto out;
3330         }
3331
3332         sess_tgt_dev_list_head =
3333                 &mcmd->sess->sess_tgt_dev_list_hash[HASH_VAL(mcmd->lun)];
3334         list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3335                         sess_tgt_dev_list_entry) {
3336                 if (tgt_dev->lun == mcmd->lun) {
3337                         TRACE_DBG("tgt_dev %p found", tgt_dev);
3338                         mcmd->mcmd_tgt_dev = tgt_dev;
3339                         res = 0;
3340                         break;
3341                 }
3342         }
3343         if (mcmd->mcmd_tgt_dev == NULL)
3344                 __scst_put();
3345
3346 out:
3347         TRACE_EXIT_HRES(res);
3348         return res;
3349 }
3350
3351 /* No locks */
3352 void scst_done_cmd_mgmt(struct scst_cmd *cmd)
3353 {
3354         struct scst_mgmt_cmd_stub *mstb;
3355         bool wake = 0;
3356         unsigned long flags;
3357
3358         TRACE_ENTRY();
3359
3360         TRACE_MGMT_DBG("cmd %p done (tag %llu)", cmd, cmd->tag);
3361
3362         spin_lock_irqsave(&scst_mcmd_lock, flags);
3363
3364         list_for_each_entry(mstb, &cmd->mgmt_cmd_list,
3365                         cmd_mgmt_cmd_list_entry) {
3366                 struct scst_mgmt_cmd *mcmd = mstb->mcmd;
3367
3368                 TRACE_MGMT_DBG("mcmd %p, mcmd->cmd_done_wait_count %d", 
3369                         mcmd, mcmd->cmd_done_wait_count);
3370
3371                 mcmd->cmd_done_wait_count--;
3372                 if (mcmd->cmd_done_wait_count > 0) {
3373                         TRACE_MGMT_DBG("cmd_done_wait_count(%d) not 0, "
3374                                 "skipping", mcmd->cmd_done_wait_count);
3375                         continue;
3376                 }
3377
3378                 if (mcmd->completed) {
3379                         sBUG_ON(mcmd->nexus_loss_check_done);
3380                         mcmd->nexus_loss_check_active = 1;
3381                         mcmd->state = SCST_MGMT_CMD_STATE_CHECK_NEXUS_LOSS;
3382                         TRACE_MGMT_DBG("Adding mgmt cmd %p to active mgmt cmd "
3383                                 "list", mcmd);
3384                         list_add_tail(&mcmd->mgmt_cmd_list_entry,
3385                                 &scst_active_mgmt_cmd_list);
3386                         wake = 1;
3387                 }
3388         }
3389
3390         spin_unlock_irqrestore(&scst_mcmd_lock, flags);
3391
3392         if (wake)
3393                 wake_up(&scst_mgmt_cmd_list_waitQ);
3394
3395         TRACE_EXIT();
3396         return;
3397 }
3398
3399 /* No locks */
3400 static void scst_finish_cmd_mgmt(struct scst_cmd *cmd)
3401 {
3402         struct scst_mgmt_cmd_stub *mstb, *t;
3403         bool wake = 0;
3404         unsigned long flags;
3405
3406         TRACE_ENTRY();
3407
3408         TRACE_MGMT_DBG("cmd %p finished (tag %llu)", cmd, cmd->tag);
3409
3410         spin_lock_irqsave(&scst_mcmd_lock, flags);
3411
3412         list_for_each_entry_safe(mstb, t, &cmd->mgmt_cmd_list,
3413                         cmd_mgmt_cmd_list_entry) {
3414                 struct scst_mgmt_cmd *mcmd = mstb->mcmd;
3415
3416                 TRACE_MGMT_DBG("mcmd %p, mcmd->cmd_finish_wait_count %d", 
3417                         mcmd, mcmd->cmd_finish_wait_count);
3418
3419                 list_del(&mstb->cmd_mgmt_cmd_list_entry);
3420                 mempool_free(mstb, scst_mgmt_stub_mempool);
3421
3422                 if (cmd->completed)
3423                         mcmd->completed_cmd_count++;
3424
3425                 mcmd->cmd_finish_wait_count--;
3426                 if (mcmd->cmd_finish_wait_count > 0) {
3427                         TRACE_MGMT_DBG("cmd_finish_wait_count(%d) not 0, "
3428                                 "skipping", mcmd->cmd_finish_wait_count);
3429                         continue;
3430                 }
3431
3432                 if (mcmd->completed && !mcmd->nexus_loss_check_active) {
3433                         mcmd->state = SCST_MGMT_CMD_STATE_DONE;
3434                         TRACE_MGMT_DBG("Adding mgmt cmd %p to active mgmt cmd "
3435                                 "list", mcmd);
3436                         list_add_tail(&mcmd->mgmt_cmd_list_entry,
3437                                 &scst_active_mgmt_cmd_list);
3438                         wake = 1;
3439                 }
3440         }
3441
3442         spin_unlock_irqrestore(&scst_mcmd_lock, flags);
3443
3444         if (wake)
3445                 wake_up(&scst_mgmt_cmd_list_waitQ);
3446
3447         TRACE_EXIT();
3448         return;
3449 }
3450
3451 static int scst_call_dev_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
3452         struct scst_tgt_dev *tgt_dev, int set_status)
3453 {
3454         int res = SCST_DEV_TM_NOT_COMPLETED;
3455         struct scst_dev_type *h = tgt_dev->dev->handler;
3456
3457         if (h->task_mgmt_fn) {
3458                 TRACE_MGMT_DBG("Calling dev handler %s task_mgmt_fn(fn=%d)",
3459                         h->name, mcmd->fn);
3460                 EXTRACHECKS_BUG_ON(in_irq());
3461                 res = h->task_mgmt_fn(mcmd, tgt_dev);
3462                 TRACE_MGMT_DBG("Dev handler %s task_mgmt_fn() returned %d",
3463                       h->name, res);
3464                 if (set_status && (res != SCST_DEV_TM_NOT_COMPLETED))
3465                         mcmd->status = res;
3466         }
3467         return res;
3468 }
3469
3470 static inline int scst_is_strict_mgmt_fn(int mgmt_fn)
3471 {
3472         switch(mgmt_fn) {
3473 #ifdef ABORT_CONSIDER_FINISHED_TASKS_AS_NOT_EXISTING
3474                 case SCST_ABORT_TASK:
3475 #endif
3476 #if 0
3477                 case SCST_ABORT_TASK_SET:
3478                 case SCST_CLEAR_TASK_SET:
3479 #endif
3480                         return 1;
3481                 default:
3482                         return 0;
3483         }
3484 }
3485
3486 /* 
3487  * Might be called under sess_list_lock and IRQ off + BHs also off
3488  * Returns -1 if command is being executed (ABORT failed), 0 otherwise
3489  */
3490 void scst_abort_cmd(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd,
3491         int other_ini, int call_dev_task_mgmt_fn)
3492 {
3493         unsigned long flags;
3494
3495         TRACE_ENTRY();
3496
3497         TRACE(((mcmd != NULL) && (mcmd->fn == SCST_ABORT_TASK)) ? TRACE_MGMT_MINOR : TRACE_MGMT,
3498                 "Aborting cmd %p (tag %llu)", cmd, cmd->tag);
3499
3500         if (other_ini) {
3501                 set_bit(SCST_CMD_ABORTED_OTHER, &cmd->cmd_flags);
3502                 smp_mb__after_set_bit();
3503         } else {
3504                 /* Might be necessary if command aborted several times */
3505                 clear_bit(SCST_CMD_ABORTED_OTHER, &cmd->cmd_flags);
3506         }
3507         set_bit(SCST_CMD_ABORTED, &cmd->cmd_flags);
3508         /* 
3509          * To sync with cmd->finished/done set in
3510          * scst_finish_cmd()/scst_pre_xmit_response()
3511          */
3512         smp_mb__after_set_bit(); 
3513
3514         if (cmd->tgt_dev == NULL) {
3515                 unsigned long flags;
3516                 spin_lock_irqsave(&scst_init_lock, flags);
3517                 scst_init_poll_cnt++;
3518                 spin_unlock_irqrestore(&scst_init_lock, flags);
3519                 wake_up(&scst_init_cmd_list_waitQ);
3520         }
3521
3522         if (call_dev_task_mgmt_fn && (cmd->tgt_dev != NULL)) {
3523                 EXTRACHECKS_BUG_ON(irqs_disabled());
3524                 scst_call_dev_task_mgmt_fn(mcmd, cmd->tgt_dev, 1);
3525         }
3526
3527         spin_lock_irqsave(&scst_mcmd_lock, flags);
3528         if ((mcmd != NULL) && !cmd->finished) {
3529                 struct scst_mgmt_cmd_stub *mstb;
3530
3531                 mstb = mempool_alloc(scst_mgmt_stub_mempool, GFP_ATOMIC);
3532                 if (mstb == NULL) {
3533                         PRINT_CRIT_ERROR("Allocation of management command "
3534                                 "stub failed (mcmd %p, cmd %p)", mcmd, cmd);
3535                         goto unlock;
3536                 }
3537                 mstb->mcmd = mcmd;
3538
3539                 /*
3540                  * cmd can't die here or sess_list_lock already taken and
3541                  * cmd is in the search list
3542 &