Post r227 fixes and cleanups. Particulary, fixes brocken by it scst_user functionalit...
[mirror/scst/.git] / scst / src / scst_targ.c
1 /*
2  *  scst_targ.c
3  *  
4  *  Copyright (C) 2004-2007 Vladislav Bolkhovitin <vst@vlnb.net>
5  *                 and Leonid Stoljar
6  *  
7  *  This program is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU General Public License
9  *  as published by the Free Software Foundation, version 2
10  *  of the License.
11  * 
12  *  This program is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  *  GNU General Public License for more details.
16  */
17
18 #include <linux/init.h>
19 #include <linux/kernel.h>
20 #include <linux/errno.h>
21 #include <linux/list.h>
22 #include <linux/spinlock.h>
23 #include <linux/slab.h>
24 #include <linux/sched.h>
25 #include <linux/smp_lock.h>
26 #include <asm/unistd.h>
27 #include <asm/string.h>
28 #include <linux/kthread.h>
29 #include <linux/delay.h>
30
31 #include "scsi_tgt.h"
32 #include "scst_priv.h"
33
34 static void scst_cmd_set_sn(struct scst_cmd *cmd);
35 static int __scst_init_cmd(struct scst_cmd *cmd);
36
37 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
38 {
39         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
40         unsigned long flags;
41
42         spin_lock_irqsave(&t->tasklet_lock, flags);
43         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
44                 smp_processor_id());
45         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
46         spin_unlock_irqrestore(&t->tasklet_lock, flags);
47
48         tasklet_schedule(&t->tasklet);
49 }
50
51 /* 
52  * Must not be called in parallel with scst_unregister_session() for the 
53  * same sess
54  */
55 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
56                              const uint8_t *lun, int lun_len,
57                              const uint8_t *cdb, int cdb_len, int atomic)
58 {
59         struct scst_cmd *cmd;
60
61         TRACE_ENTRY();
62
63 #ifdef EXTRACHECKS
64         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
65                 PRINT_ERROR("%s", "New cmd while shutting down the session");
66                 sBUG();
67         }
68 #endif
69
70         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
71         if (cmd == NULL)
72                 goto out;
73
74         cmd->sess = sess;
75         cmd->tgt = sess->tgt;
76         cmd->tgtt = sess->tgt->tgtt;
77
78         /* 
79          * For both wrong lun and CDB defer the error reporting for
80          * scst_cmd_init_done()
81          */
82
83         cmd->lun = scst_unpack_lun(lun, lun_len);
84
85         if (cdb_len <= SCST_MAX_CDB_SIZE) {
86                 memcpy(cmd->cdb, cdb, cdb_len);
87                 cmd->cdb_len = cdb_len;
88         }
89
90         TRACE_DBG("cmd %p, sess %p", cmd, sess);
91         scst_sess_get(sess);
92
93 out:
94         TRACE_EXIT();
95         return cmd;
96 }
97
98 static int scst_init_cmd(struct scst_cmd *cmd, int context)
99 {
100         int rc;
101
102         TRACE_ENTRY();
103
104         /* See the comment in scst_do_job_init() */
105         if (unlikely(!list_empty(&scst_init_cmd_list))) {
106                 TRACE_MGMT_DBG("%s", "init cmd list busy");
107                 goto out_redirect;
108         }
109         smp_rmb();
110
111         rc = __scst_init_cmd(cmd);
112         if (unlikely(rc > 0))
113                 goto out_redirect;
114         else if (unlikely(rc != 0))
115                 goto out;
116
117         /* Small context optimization */
118         if (((context == SCST_CONTEXT_TASKLET) ||
119              (context == SCST_CONTEXT_DIRECT_ATOMIC)) && 
120             scst_cmd_is_expected_set(cmd)) {
121                 if (cmd->expected_data_direction == SCST_DATA_WRITE) {
122                         if ( !test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
123                                         &cmd->tgt_dev->tgt_dev_flags))
124                                 context = SCST_CONTEXT_THREAD;
125                 } else {
126                         if ( !test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
127                                         &cmd->tgt_dev->tgt_dev_flags))
128                                 context = SCST_CONTEXT_THREAD;
129                 }
130         }
131
132 out:
133         TRACE_EXIT_RES(context);
134         return context;
135
136 out_redirect:
137         if (cmd->preprocessing_only) {
138                 /*
139                  * Poor man solution for single threaded targets, where 
140                  * blocking receiver at least sometimes means blocking all.
141                  */
142                 sBUG_ON(context != SCST_CONTEXT_DIRECT);
143                 scst_set_busy(cmd);
144                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
145                 /* Keep initiator away from too many BUSY commands */
146                 if (!in_interrupt() && !in_atomic())
147                         msleep(50);
148                 else
149                         WARN_ON_ONCE(1);
150         } else {
151                 unsigned long flags;
152                 spin_lock_irqsave(&scst_init_lock, flags);
153                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
154                         "%d)", cmd, atomic_read(&scst_cmd_count));
155                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
156                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
157                         scst_init_poll_cnt++;
158                 spin_unlock_irqrestore(&scst_init_lock, flags);
159                 wake_up(&scst_init_cmd_list_waitQ);
160                 context = -1;
161         }
162         goto out;
163 }
164
165 void scst_cmd_init_done(struct scst_cmd *cmd, int pref_context)
166 {
167         unsigned long flags;
168         struct scst_session *sess = cmd->sess;
169
170         TRACE_ENTRY();
171
172         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
173         TRACE(TRACE_SCSI, "tag=%llu, lun=%Ld, CDB len=%d", cmd->tag, 
174                 (uint64_t)cmd->lun, cmd->cdb_len);
175         TRACE_BUFF_FLAG(TRACE_SCSI|TRACE_RECV_BOT, "Recieving CDB",
176                 cmd->cdb, cmd->cdb_len);
177
178 #ifdef EXTRACHECKS
179         if (unlikely(in_irq()) && ((pref_context == SCST_CONTEXT_DIRECT) ||
180                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
181         {
182                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
183                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
184                         cmd->tgtt->name);
185                 pref_context = SCST_CONTEXT_TASKLET;
186         }
187 #endif
188
189         atomic_inc(&sess->sess_cmd_count);
190
191         spin_lock_irqsave(&sess->sess_list_lock, flags);
192
193         list_add_tail(&cmd->search_cmd_list_entry, &sess->search_cmd_list);
194
195         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
196                 switch(sess->init_phase) {
197                 case SCST_SESS_IPH_SUCCESS:
198                         break;
199                 case SCST_SESS_IPH_INITING:
200                         TRACE_DBG("Adding cmd %p to init deferred cmd list", cmd);
201                         list_add_tail(&cmd->cmd_list_entry, 
202                                 &sess->init_deferred_cmd_list);
203                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
204                         goto out;
205                 case SCST_SESS_IPH_FAILED:
206                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
207                         scst_set_busy(cmd);
208                         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
209                         goto active;
210                 default:
211                         sBUG();
212                 }
213         }
214
215         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
216
217         if (unlikely(cmd->lun == (lun_t)-1)) {
218                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
219                 scst_set_cmd_error(cmd,
220                         SCST_LOAD_SENSE(scst_sense_lun_not_supported));
221                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
222                 goto active;
223         }
224
225         if (unlikely(cmd->cdb_len == 0)) {
226                 PRINT_ERROR("Wrong CDB len %d, finishing cmd", 0);
227                 scst_set_cmd_error(cmd,
228                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
229                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
230                 goto active;
231         }
232
233         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
234                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
235                 scst_set_cmd_error(cmd,
236                         SCST_LOAD_SENSE(scst_sense_invalid_message));
237                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
238                 goto active;
239         }
240
241         cmd->state = SCST_CMD_STATE_INIT;
242         /* cmd must be inited here to keep the order */
243         pref_context = scst_init_cmd(cmd, pref_context);
244         if (unlikely(pref_context < 0))
245                 goto out;
246
247 active:
248         /* Here cmd must not be in any cmd list, no locks */
249         switch (pref_context) {
250         case SCST_CONTEXT_TASKLET:
251                 scst_schedule_tasklet(cmd);
252                 break;
253
254         case SCST_CONTEXT_DIRECT:
255         case SCST_CONTEXT_DIRECT_ATOMIC:
256                 scst_process_active_cmd(cmd, pref_context);
257                 /* For *NEED_THREAD wake_up() is already done */
258                 break;
259
260         default:
261                 PRINT_ERROR("Context %x is undefined, using the thread one",
262                         pref_context);
263                 /* go through */
264         case SCST_CONTEXT_THREAD:
265                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
266                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
267                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
268                         list_add(&cmd->cmd_list_entry,
269                                 &cmd->cmd_lists->active_cmd_list);
270                 else
271                         list_add_tail(&cmd->cmd_list_entry,
272                                 &cmd->cmd_lists->active_cmd_list);
273                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
274                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
275                 break;
276         }
277
278 out:
279         TRACE_EXIT();
280         return;
281 }
282
283 static int scst_pre_parse(struct scst_cmd *cmd)
284 {
285         int res = SCST_CMD_STATE_RES_CONT_SAME;
286         struct scst_device *dev = cmd->dev;
287
288         TRACE_ENTRY();
289
290         cmd->inc_expected_sn_on_done = !dev->has_own_order_mgmt;
291
292         sBUG_ON(cmd->internal);
293
294         /*
295          * Expected transfer data supplied by the SCSI transport via the
296          * target driver are untrusted, so we prefer to fetch them from CDB.
297          * Additionally, not all transports support supplying the expected
298          * transfer data.
299          */
300
301         if (unlikely(scst_get_cdb_info(cmd->cdb, dev->handler->type, 
302                         &cmd->op_flags, &cmd->data_direction,
303                         &cmd->bufflen, &cmd->cdb_len, &cmd->op_name) != 0)) {
304                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
305                         "Should you update scst_scsi_op_table?",
306                         cmd->cdb[0], dev->handler->name);
307 #ifdef USE_EXPECTED_VALUES
308                 if (scst_cmd_is_expected_set(cmd)) {
309                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
310                                 "direction %d, transfer_len %d",
311                                 cmd->expected_data_direction,
312                                 cmd->expected_transfer_len);
313                         cmd->data_direction = cmd->expected_data_direction;
314                         
315                         cmd->bufflen = cmd->expected_transfer_len;
316                         /* Restore (likely) lost CDB length */
317                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
318                         if (cmd->cdb_len == -1) {
319                                 PRINT_ERROR("Unable to get CDB length for "
320                                         "opcode 0x%02x. Returning INVALID "
321                                         "OPCODE", cmd->cdb[0]);
322                                 scst_set_cmd_error(cmd,
323                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
324                                 goto out_xmit;
325                         }
326                 } else {
327                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
328                              "target %s not supplied expected values",
329                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
330                         scst_set_cmd_error(cmd,
331                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
332                         goto out_xmit;
333                 }
334 #else
335                 scst_set_cmd_error(cmd,
336                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
337                 goto out_xmit;
338 #endif
339         } else {
340                 TRACE(TRACE_SCSI, "op_name <%s>, direction=%d (expected %d, "
341                         "set %s), transfer_len=%d (expected len %d), flags=%d",
342                         cmd->op_name, cmd->data_direction,
343                         cmd->expected_data_direction,
344                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
345                         cmd->bufflen, cmd->expected_transfer_len, cmd->op_flags);
346
347                 if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
348                         if (scst_cmd_is_expected_set(cmd)) {
349                                 /*
350                                  * Command data length can't be easily
351                                  * determined from the CDB. ToDo, that should 
352                                  * be fixed. Until it's fixed, get it from
353                                  * the supplied expected value, but
354                                  * limit it to some reasonable value (15MB).
355                                  */
356                                 cmd->bufflen = min(cmd->expected_transfer_len,
357                                                         15*1024*1024);
358                         } else
359                                 cmd->bufflen = 0;
360                 }
361         }
362
363         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
364                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
365                             "(opcode 0x%02x)", cmd->cdb[0]);
366                 scst_set_cmd_error(cmd,
367                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
368                 goto out_xmit;
369         }
370
371         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
372                 PRINT_ERROR("Linked commands are not supported "
373                             "(opcode 0x%02x)", cmd->cdb[0]);
374                 scst_set_cmd_error(cmd,
375                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
376                 goto out_xmit;
377         }
378
379         cmd->state = SCST_CMD_STATE_DEV_PARSE;
380
381 out:
382         TRACE_EXIT_RES(res);
383         return res;
384
385 out_xmit:
386         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
387         res = SCST_CMD_STATE_RES_CONT_SAME;
388         goto out;
389 }
390
391 static int scst_parse_cmd(struct scst_cmd *cmd)
392 {
393         int res = SCST_CMD_STATE_RES_CONT_SAME;
394         int state;
395         struct scst_device *dev = cmd->dev;
396         int atomic = scst_cmd_atomic(cmd);
397         int orig_bufflen = cmd->bufflen;
398         scst_data_direction orig_data_direction = cmd->data_direction;
399
400         TRACE_ENTRY();
401
402         if (atomic && !dev->handler->parse_atomic) {
403                 TRACE_DBG("Dev handler %s parse() can not be "
404                       "called in atomic context, rescheduling to the thread",
405                       dev->handler->name);
406                 res = SCST_CMD_STATE_RES_NEED_THREAD;
407                 goto out;
408         }
409
410         if (likely(!scst_is_cmd_local(cmd))) {
411                 TRACE_DBG("Calling dev handler %s parse(%p)",
412                       dev->handler->name, cmd);
413                 TRACE_BUFF_FLAG(TRACE_SEND_BOT, "Parsing: ", cmd->cdb, cmd->cdb_len);
414                 state = dev->handler->parse(cmd);
415                 /* Caution: cmd can be already dead here */
416                 TRACE_DBG("Dev handler %s parse() returned %d",
417                         dev->handler->name, state);
418
419                 switch (state) {
420                 case SCST_CMD_STATE_NEED_THREAD_CTX:
421                         TRACE_DBG("Dev handler %s parse() requested thread "
422                               "context, rescheduling", dev->handler->name);
423                         res = SCST_CMD_STATE_RES_NEED_THREAD;
424                         goto out;
425
426                 case SCST_CMD_STATE_STOP:
427                         TRACE_DBG("Dev handler %s parse() requested stop "
428                                 "processing", dev->handler->name);
429                         res = SCST_CMD_STATE_RES_CONT_NEXT;
430                         goto out;
431                 }
432
433                 if (state == SCST_CMD_STATE_DEFAULT)
434                         state = SCST_CMD_STATE_PREPARE_SPACE;
435         } else
436                 state = SCST_CMD_STATE_PREPARE_SPACE;
437
438         if (cmd->data_len == -1)
439                 cmd->data_len = cmd->bufflen;
440
441         if (cmd->data_buf_alloced && unlikely((orig_bufflen > cmd->bufflen))) {
442                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
443                         "is less, than required (size %d)", cmd->bufflen,
444                         orig_bufflen);
445                 goto out_error;
446         }
447
448         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
449                 goto set_res;
450
451 #ifdef EXTRACHECKS
452         if ((cmd->bufflen != 0) &&
453             ((cmd->data_direction == SCST_DATA_NONE) ||
454              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
455                 PRINT_ERROR("Dev handler %s parse() returned "
456                         "invalid cmd data_direction %d, bufflen %d, state %d "
457                         "or sg %p (opcode 0x%x)", dev->handler->name, 
458                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
459                         cmd->cdb[0]);
460                 goto out_error;
461         }
462 #endif
463
464         if (scst_cmd_is_expected_set(cmd)) {
465 #ifdef USE_EXPECTED_VALUES
466 #       ifdef EXTRACHECKS
467                 if ((cmd->data_direction != cmd->expected_data_direction) ||
468                     (cmd->bufflen != cmd->expected_transfer_len)) {
469                         PRINT_ERROR("Expected values don't match decoded ones: "
470                                 "data_direction %d, expected_data_direction %d, "
471                                 "bufflen %d, expected_transfer_len %d",
472                                 cmd->data_direction, cmd->expected_data_direction,
473                                 cmd->bufflen, cmd->expected_transfer_len);
474                 }
475 #       endif
476                 cmd->data_direction = cmd->expected_data_direction;
477                 cmd->bufflen = cmd->expected_transfer_len;
478 #else
479                 if (unlikely(cmd->data_direction != orig_data_direction)) {
480                         PRINT_ERROR("Expected data direction %d for opcode "
481                                 "0x%02x (handler %s, target %s) doesn't match "
482                                 "decoded value %d", cmd->data_direction,
483                                 cmd->cdb[0], dev->handler->name,
484                                 cmd->tgtt->name, orig_data_direction);
485                         scst_set_cmd_error(cmd,
486                                 SCST_LOAD_SENSE(scst_sense_invalid_message));
487                         goto out_dev_done;
488                 }
489                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
490                         PRINT_INFO("Warning: expected transfer length %d for "
491                                 "opcode 0x%02x (handler %s, target %s) doesn't "
492                                 "match decoded value %d. Faulty initiator "
493                                 "(e.g. VMware is known to be such) or "
494                                 "scst_scsi_op_table should be updated?",
495                                 cmd->expected_transfer_len, cmd->cdb[0],
496                                 dev->handler->name, cmd->tgtt->name,
497                                 cmd->bufflen);
498                         PRINT_BUFFER("Suspicious CDB", cmd->cdb, cmd->cdb_len);
499                 }
500 #endif
501         }
502
503         if ((cmd->data_direction == SCST_DATA_UNKNOWN) ||
504             ((cmd->bufflen == 0) && (cmd->data_direction != SCST_DATA_NONE))) {
505                 PRINT_ERROR("Wrong data direction (%d) or/and buffer "
506                         "length (%d). Opcode 0x%x, handler %s, target %s",
507                         cmd->data_direction, cmd->bufflen, cmd->cdb[0],
508                         dev->handler->name, cmd->tgtt->name);
509                 goto out_error;
510         }
511
512 set_res:
513         switch (state) {
514         case SCST_CMD_STATE_PREPARE_SPACE:
515         case SCST_CMD_STATE_PRE_PARSE:
516         case SCST_CMD_STATE_DEV_PARSE:
517         case SCST_CMD_STATE_RDY_TO_XFER:
518         case SCST_CMD_STATE_TGT_PRE_EXEC:
519         case SCST_CMD_STATE_SEND_TO_MIDLEV:
520         case SCST_CMD_STATE_PRE_DEV_DONE:
521         case SCST_CMD_STATE_DEV_DONE:
522         case SCST_CMD_STATE_PRE_XMIT_RESP:
523         case SCST_CMD_STATE_XMIT_RESP:
524         case SCST_CMD_STATE_FINISHED:
525                 cmd->state = state;
526                 res = SCST_CMD_STATE_RES_CONT_SAME;
527                 break;
528
529         default:
530                 if (state >= 0) {
531                         PRINT_ERROR("Dev handler %s parse() returned "
532                              "invalid cmd state %d (opcode %d)", 
533                              dev->handler->name, state, cmd->cdb[0]);
534                 } else {
535                         PRINT_ERROR("Dev handler %s parse() returned "
536                                 "error %d (opcode %d)", dev->handler->name, 
537                                 state, cmd->cdb[0]);
538                 }
539                 goto out_error;
540         }
541
542         if (cmd->resp_data_len == -1) {
543                 if (cmd->data_direction == SCST_DATA_READ)
544                         cmd->resp_data_len = cmd->bufflen;
545                 else
546                          cmd->resp_data_len = 0;
547         }
548         
549 out:
550         TRACE_EXIT_HRES(res);
551         return res;
552
553 out_error:
554         /* dev_done() will be called as part of the regular cmd's finish */
555         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
556
557 #ifndef USE_EXPECTED_VALUES
558 out_dev_done:
559 #endif
560         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
561         res = SCST_CMD_STATE_RES_CONT_SAME;
562         goto out;
563 }
564
565 static int scst_prepare_space(struct scst_cmd *cmd)
566 {
567         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
568
569         TRACE_ENTRY();
570
571         if (cmd->data_direction == SCST_DATA_NONE)
572                 goto prep_done;
573
574         if (cmd->data_buf_tgt_alloc) {
575                 int orig_bufflen = cmd->bufflen;
576
577                 TRACE_MEM("%s", "Custom tgt data buf allocation requested");
578
579                 r = cmd->tgtt->alloc_data_buf(cmd);
580                 if (r > 0)
581                         goto alloc;
582                 else if (r == 0) {
583                         cmd->data_buf_alloced = 1;
584                         if (unlikely(orig_bufflen < cmd->bufflen)) {
585                                 PRINT_ERROR("Target driver allocated data "
586                                         "buffer (size %d), is less, than "
587                                         "required (size %d)", orig_bufflen,
588                                         cmd->bufflen);
589                                 goto out_error;
590                         }
591                 } else
592                         goto check;
593         }
594
595 alloc:
596         if (!cmd->data_buf_alloced) {
597                 r = scst_alloc_space(cmd);
598         } else {
599                 TRACE_MEM("%s", "data_buf_alloced set, returning");
600         }
601         
602 check:
603         if (r != 0) {
604                 if (scst_cmd_atomic(cmd)) {
605                         TRACE_MEM("%s", "Atomic memory allocation failed, "
606                               "rescheduling to the thread");
607                         res = SCST_CMD_STATE_RES_NEED_THREAD;
608                         goto out;
609                 } else
610                         goto out_no_space;
611         }
612
613 prep_done:
614         if (cmd->preprocessing_only) {
615                 if (scst_cmd_atomic(cmd) && 
616                     !cmd->tgtt->preprocessing_done_atomic) {
617                         TRACE_DBG("%s", "preprocessing_done() can not be "
618                               "called in atomic context, rescheduling to "
619                               "the thread");
620                         res = SCST_CMD_STATE_RES_NEED_THREAD;
621                         goto out;
622                 }
623
624                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
625                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
626                                 "cmd %p", cmd);
627                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
628                         res = SCST_CMD_STATE_RES_CONT_SAME;
629                         goto out;
630                 }
631
632                 res = SCST_CMD_STATE_RES_CONT_NEXT;
633                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
634
635                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
636                 cmd->tgtt->preprocessing_done(cmd);
637                 TRACE_DBG("%s", "preprocessing_done() returned");
638                 goto out;
639
640         }
641
642         switch (cmd->data_direction) {
643         case SCST_DATA_WRITE:
644                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
645                 break;
646
647         default:
648                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
649                 break;
650         }
651
652 out:
653         TRACE_EXIT_HRES(res);
654         return res;
655
656 out_no_space:
657         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
658                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
659         scst_set_busy(cmd);
660         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
661         res = SCST_CMD_STATE_RES_CONT_SAME;
662         goto out;
663
664 out_error:
665         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
666         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
667         res = SCST_CMD_STATE_RES_CONT_SAME;
668         goto out;
669 }
670
671 void scst_restart_cmd(struct scst_cmd *cmd, int status, int pref_context)
672 {
673         TRACE_ENTRY();
674
675         TRACE_DBG("Preferred context: %d", pref_context);
676         TRACE_DBG("tag=%llu, status=%#x", scst_cmd_get_tag(cmd), status);
677
678 #ifdef EXTRACHECKS
679         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
680                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
681         {
682                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
683                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
684                         cmd->tgtt->name);
685                 pref_context = SCST_CONTEXT_TASKLET;
686         }
687 #endif
688
689         switch (status) {
690         case SCST_PREPROCESS_STATUS_SUCCESS:
691                 switch (cmd->data_direction) {
692                 case SCST_DATA_WRITE:
693                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
694                         break;
695                 default:
696                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
697                         break;
698                 }
699                 if (cmd->set_sn_on_restart_cmd)
700                         scst_cmd_set_sn(cmd);
701                 /* Small context optimization */
702                 if ((pref_context == SCST_CONTEXT_TASKLET) || 
703                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) {
704                         if (cmd->data_direction == SCST_DATA_WRITE) {
705                                 if ( !test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
706                                                 &cmd->tgt_dev->tgt_dev_flags))
707                                         pref_context = SCST_CONTEXT_THREAD;
708                         } else {
709                                 if ( !test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
710                                                 &cmd->tgt_dev->tgt_dev_flags))
711                                         pref_context = SCST_CONTEXT_THREAD;
712                         }
713                 }
714                 break;
715
716         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
717                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
718                 break;
719
720         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
721                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
722                 /* go through */
723         case SCST_PREPROCESS_STATUS_ERROR:
724                 scst_set_cmd_error(cmd,
725                            SCST_LOAD_SENSE(scst_sense_hardw_error));
726                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
727                 break;
728
729         default:
730                 PRINT_ERROR("%s() received unknown status %x", __func__,
731                         status);
732                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
733                 break;
734         }
735
736         scst_proccess_redirect_cmd(cmd, pref_context, 1);
737
738         TRACE_EXIT();
739         return;
740 }
741
742 /* No locks */
743 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
744 {
745         struct scst_tgt *tgt = cmd->sess->tgt;
746         int res = 0;
747         unsigned long flags;
748
749         TRACE_ENTRY();
750
751         spin_lock_irqsave(&tgt->tgt_lock, flags);
752         tgt->retry_cmds++;
753         smp_mb();
754         TRACE(TRACE_RETRY, "TGT QUEUE FULL: incrementing retry_cmds %d",
755               tgt->retry_cmds);
756         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
757                 /* At least one cmd finished, so try again */
758                 tgt->retry_cmds--;
759                 TRACE(TRACE_RETRY, "Some command(s) finished, direct retry "
760                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
761                       "retry_cmds=%d)", finished_cmds,
762                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
763                 res = -1;
764                 goto out_unlock_tgt;
765         }
766
767         TRACE(TRACE_RETRY, "Adding cmd %p to retry cmd list", cmd);
768         list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
769
770         if (!tgt->retry_timer_active) {
771                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
772                 add_timer(&tgt->retry_timer);
773                 tgt->retry_timer_active = 1;
774         }
775
776 out_unlock_tgt:
777         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
778
779         TRACE_EXIT_RES(res);
780         return res;
781 }
782
783 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
784 {
785         int res, rc;
786         int atomic = scst_cmd_atomic(cmd);
787
788         TRACE_ENTRY();
789
790         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
791                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
792                 goto out_dev_done;
793         }
794
795         if (cmd->tgtt->rdy_to_xfer == NULL) {
796                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
797                 res = SCST_CMD_STATE_RES_CONT_SAME;
798                 goto out;
799         }
800
801         if (atomic && !cmd->tgtt->rdy_to_xfer_atomic) {
802                 TRACE_DBG("%s", "rdy_to_xfer() can not be "
803                       "called in atomic context, rescheduling to the thread");
804                 res = SCST_CMD_STATE_RES_NEED_THREAD;
805                 goto out;
806         }
807
808         while (1) {
809                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
810
811                 res = SCST_CMD_STATE_RES_CONT_NEXT;
812                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
813
814                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
815 #ifdef DEBUG_RETRY
816                 if (((scst_random() % 100) == 75))
817                         rc = SCST_TGT_RES_QUEUE_FULL;
818                 else
819 #endif
820                         rc = cmd->tgtt->rdy_to_xfer(cmd);
821                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
822
823                 if (likely(rc == SCST_TGT_RES_SUCCESS))
824                         goto out;
825
826                 /* Restore the previous state */
827                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
828
829                 switch (rc) {
830                 case SCST_TGT_RES_QUEUE_FULL:
831                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
832                                 break;
833                         else
834                                 continue;
835
836                 case SCST_TGT_RES_NEED_THREAD_CTX:
837                         TRACE_DBG("Target driver %s "
838                               "rdy_to_xfer() requested thread "
839                               "context, rescheduling", cmd->tgtt->name);
840                         res = SCST_CMD_STATE_RES_NEED_THREAD;
841                         break;
842
843                 default:
844                         goto out_error_rc;
845                 }
846                 break;
847         }
848
849 out:
850         TRACE_EXIT_HRES(res);
851         return res;
852
853 out_error_rc:
854         if (rc == SCST_TGT_RES_FATAL_ERROR) {
855                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
856                      "fatal error", cmd->tgtt->name);
857         } else {
858                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
859                             "value %d", cmd->tgtt->name, rc);
860         }
861         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
862
863 out_dev_done:
864         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
865         res = SCST_CMD_STATE_RES_CONT_SAME;
866         goto out;
867 }
868
869 /* No locks, but might be in IRQ */
870 void scst_proccess_redirect_cmd(struct scst_cmd *cmd, int context,
871         int check_retries)
872 {
873         unsigned long flags;
874
875         TRACE_ENTRY();
876
877         TRACE_DBG("Context: %d", context);
878
879         switch(context) {
880         case SCST_CONTEXT_DIRECT:
881         case SCST_CONTEXT_DIRECT_ATOMIC:
882                 if (check_retries)
883                         scst_check_retries(cmd->tgt);
884                 scst_process_active_cmd(cmd, context);
885                 break;
886
887         default:
888                 PRINT_ERROR("Context %x is unknown, using the thread one",
889                             context);
890                 /* go through */
891         case SCST_CONTEXT_THREAD:
892                 if (check_retries)
893                         scst_check_retries(cmd->tgt);
894                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
895                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
896                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
897                         list_add(&cmd->cmd_list_entry,
898                                 &cmd->cmd_lists->active_cmd_list);
899                 else
900                         list_add_tail(&cmd->cmd_list_entry,
901                                 &cmd->cmd_lists->active_cmd_list);
902                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
903                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
904                 break;
905
906         case SCST_CONTEXT_TASKLET:
907                 if (check_retries)
908                         scst_check_retries(cmd->tgt);
909                 scst_schedule_tasklet(cmd);
910                 break;
911         }
912
913         TRACE_EXIT();
914         return;
915 }
916
917 void scst_rx_data(struct scst_cmd *cmd, int status, int pref_context)
918 {
919         TRACE_ENTRY();
920
921         TRACE_DBG("Preferred context: %d", pref_context);
922         TRACE(TRACE_SCSI, "tag=%llu status=%#x", scst_cmd_get_tag(cmd), status);
923
924 #ifdef EXTRACHECKS
925         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
926                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
927         {
928                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
929                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
930                         cmd->tgtt->name);
931                 pref_context = SCST_CONTEXT_TASKLET;
932         }
933 #endif
934
935         switch (status) {
936         case SCST_RX_STATUS_SUCCESS:
937                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
938                 /* Small context optimization */
939                 if ((pref_context == SCST_CONTEXT_TASKLET) || 
940                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) {
941                         if ( !test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC, 
942                                         &cmd->tgt_dev->tgt_dev_flags))
943                                 pref_context = SCST_CONTEXT_THREAD;
944                 }
945                 break;
946
947         case SCST_RX_STATUS_ERROR_SENSE_SET:
948                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
949                 break;
950
951         case SCST_RX_STATUS_ERROR_FATAL:
952                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
953                 /* go through */
954         case SCST_RX_STATUS_ERROR:
955                 scst_set_cmd_error(cmd,
956                            SCST_LOAD_SENSE(scst_sense_hardw_error));
957                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
958                 break;
959
960         default:
961                 PRINT_ERROR("scst_rx_data() received unknown status %x",
962                         status);
963                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
964                 break;
965         }
966
967         scst_proccess_redirect_cmd(cmd, pref_context, 1);
968
969         TRACE_EXIT();
970         return;
971 }
972
973 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
974 {
975         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
976
977         TRACE_ENTRY();
978
979         cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
980
981         if (cmd->tgtt->pre_exec == NULL)
982                 goto out;
983
984         TRACE_DBG("Calling pre_exec(%p)", cmd);
985         rc = cmd->tgtt->pre_exec(cmd);
986         TRACE_DBG("pre_exec() returned %d", rc);
987
988         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
989                 switch(rc) {
990                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
991                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
992                         break;
993                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
994                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
995                         /* go through */
996                 case SCST_PREPROCESS_STATUS_ERROR:
997                         scst_set_cmd_error(cmd,
998                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
999                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1000                         break;
1001                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1002                         TRACE_DBG("Target driver's %s pre_exec() requested "
1003                                 "thread context, rescheduling", cmd->tgtt->name);
1004                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1005                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1006                         break;
1007                 default:
1008                         sBUG();
1009                         break;
1010                 }
1011         }
1012
1013 out:
1014         TRACE_EXIT_RES(res);
1015         return res;
1016 }
1017
1018 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1019         const uint8_t *rq_sense, int rq_sense_len, int resid)
1020 {
1021         TRACE_ENTRY();
1022
1023         cmd->status = result & 0xff;
1024         cmd->msg_status = msg_byte(result);
1025         cmd->host_status = host_byte(result);
1026         cmd->driver_status = driver_byte(result);
1027         if (unlikely(resid != 0)) {
1028 #ifdef EXTRACHECKS
1029                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1030                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1031                                 "op %x)", resid, cmd->resp_data_len,
1032                                 cmd->cdb[0]);
1033                 } else
1034 #endif
1035                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1036         }
1037
1038         /* 
1039          * We checked that rq_sense_len < sizeof(cmd->sense_buffer)
1040          * in init_scst()
1041          */
1042         memcpy(cmd->sense_buffer, rq_sense, rq_sense_len);
1043         memset(&cmd->sense_buffer[rq_sense_len], 0,
1044                 sizeof(cmd->sense_buffer) - rq_sense_len);
1045
1046         TRACE(TRACE_SCSI, "result=%x, cmd->status=%x, resid=%d, "
1047               "cmd->msg_status=%x, cmd->host_status=%x, "
1048               "cmd->driver_status=%x", result, cmd->status, resid,
1049               cmd->msg_status, cmd->host_status, cmd->driver_status);
1050
1051         cmd->completed = 1;
1052
1053         TRACE_EXIT();
1054         return;
1055 }
1056
1057 /* For small context optimization */
1058 static inline int scst_optimize_post_exec_context(struct scst_cmd *cmd,
1059         int context)
1060 {
1061         if ((context == SCST_CONTEXT_TASKLET) || 
1062             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1063                 if ( !test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC, 
1064                                 &cmd->tgt_dev->tgt_dev_flags))
1065                         context = SCST_CONTEXT_THREAD;
1066         }
1067         return context;
1068 }
1069
1070 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18)
1071 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1072                                             struct scsi_request **req)
1073 {
1074         struct scst_cmd *cmd = NULL;
1075
1076         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1077                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1078
1079         if (cmd == NULL) {
1080                 PRINT_ERROR("%s", "Request with NULL cmd");
1081                 if (*req)
1082                         scsi_release_request(*req);
1083         }
1084
1085         return cmd;
1086 }
1087
1088 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1089 {
1090         struct scsi_request *req = NULL;
1091         struct scst_cmd *cmd;
1092
1093         TRACE_ENTRY();
1094
1095         cmd = scst_get_cmd(scsi_cmd, &req);
1096         if (cmd == NULL)
1097                 goto out;
1098
1099         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1100                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1101
1102         /* Clear out request structure */
1103         req->sr_use_sg = 0;
1104         req->sr_sglist_len = 0;
1105         req->sr_bufflen = 0;
1106         req->sr_buffer = NULL;
1107         req->sr_underflow = 0;
1108         req->sr_request->rq_disk = NULL; /* disown request blk */
1109
1110         scst_release_request(cmd);
1111
1112         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1113
1114         scst_proccess_redirect_cmd(cmd,
1115                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1116
1117 out:
1118         TRACE_EXIT();
1119         return;
1120 }
1121 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) */
1122 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1123 {
1124         struct scst_cmd *cmd;
1125
1126         TRACE_ENTRY();
1127
1128         cmd = (struct scst_cmd *)data;
1129         if (cmd == NULL)
1130                 goto out;
1131
1132         scst_do_cmd_done(cmd, result, sense, SCST_SENSE_BUFFERSIZE, resid);
1133
1134         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1135
1136         scst_proccess_redirect_cmd(cmd,
1137                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1138
1139 out:
1140         TRACE_EXIT();
1141         return;
1142 }
1143 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) */
1144
1145 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state)
1146 {
1147         TRACE_ENTRY();
1148
1149         if (next_state == SCST_CMD_STATE_DEFAULT)
1150                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1151
1152 #if defined(DEBUG) || defined(TRACING)
1153         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1154                 if (cmd->sg) {
1155                         int i;
1156                         struct scatterlist *sg = cmd->sg;
1157                         TRACE(TRACE_RECV_TOP, 
1158                               "Exec'd %d S/G(s) at %p sg[0].page at %p",
1159                               cmd->sg_cnt, sg, (void*)sg[0].page);
1160                         for(i = 0; i < cmd->sg_cnt; ++i) {
1161                                 TRACE_BUFF_FLAG(TRACE_RECV_TOP, 
1162                                         "Exec'd sg", page_address(sg[i].page),
1163                                         sg[i].length);
1164                         }
1165                 }
1166         }
1167 #endif
1168
1169
1170 #ifdef EXTRACHECKS
1171         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1172             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1173             (next_state != SCST_CMD_STATE_FINISHED)) 
1174         {
1175                 PRINT_ERROR("scst_cmd_done_local() received invalid cmd "
1176                             "state %d (opcode %d)", next_state, cmd->cdb[0]);
1177                 scst_set_cmd_error(cmd,
1178                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1179                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1180         }
1181 #endif
1182         cmd->state = next_state;
1183
1184         scst_proccess_redirect_cmd(cmd,
1185                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1186
1187         TRACE_EXIT();
1188         return;
1189 }
1190
1191 static int scst_report_luns_local(struct scst_cmd *cmd)
1192 {
1193         int rc;
1194         int dev_cnt = 0;
1195         int buffer_size;
1196         int i;
1197         struct scst_tgt_dev *tgt_dev = NULL;
1198         uint8_t *buffer;
1199         int offs, overflow = 0;
1200
1201         TRACE_ENTRY();
1202
1203         rc = scst_check_local_events(cmd);
1204         if (unlikely(rc != 0))
1205                 goto out_done;
1206
1207         cmd->status = 0;
1208         cmd->msg_status = 0;
1209         cmd->host_status = DID_OK;
1210         cmd->driver_status = 0;
1211
1212         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1213                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1214                         "LUNS command", cmd->cdb[2]);
1215                 goto out_err;
1216         }
1217
1218         buffer_size = scst_get_buf_first(cmd, &buffer);
1219         if (unlikely(buffer_size <= 0))
1220                 goto out_err;
1221
1222         if (buffer_size < 16)
1223                 goto out_put_err;
1224
1225         memset(buffer, 0, buffer_size);
1226         offs = 8;
1227
1228         /* sess->sess_tgt_dev_list_hash is protected by suspended activity */
1229         for(i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1230                 struct list_head *sess_tgt_dev_list_head =
1231                         &cmd->sess->sess_tgt_dev_list_hash[i];
1232                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1233                                 sess_tgt_dev_list_entry) {
1234                         if (!overflow) {
1235                                 if (offs >= buffer_size) {
1236                                         scst_put_buf(cmd, buffer);
1237                                         buffer_size = scst_get_buf_next(cmd, &buffer);
1238                                         if (buffer_size > 0) {
1239                                                 memset(buffer, 0, buffer_size);
1240                                                 offs = 0;
1241                                         } else {
1242                                                 overflow = 1;
1243                                                 goto inc_dev_cnt;
1244                                         }
1245                                 }
1246                                 if ((buffer_size - offs) < 8) {
1247                                         PRINT_ERROR("Buffer allocated for REPORT "
1248                                                 "LUNS command doesn't allow to fit 8 "
1249                                                 "byte entry (buffer_size=%d)",
1250                                                 buffer_size);
1251                                         goto out_put_hw_err;
1252                                 }
1253                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1254                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1255                                 offs += 8;
1256                         }
1257 inc_dev_cnt:
1258                         dev_cnt++;
1259                 }
1260         }
1261         if (!overflow)
1262                 scst_put_buf(cmd, buffer);
1263
1264         /* Set the response header */
1265         buffer_size = scst_get_buf_first(cmd, &buffer);
1266         if (unlikely(buffer_size <= 0))
1267                 goto out_err;
1268         dev_cnt *= 8;
1269         buffer[0] = (dev_cnt >> 24) & 0xff;
1270         buffer[1] = (dev_cnt >> 16) & 0xff;
1271         buffer[2] = (dev_cnt >> 8) & 0xff;
1272         buffer[3] = dev_cnt & 0xff;
1273         scst_put_buf(cmd, buffer);
1274
1275         dev_cnt += 8;
1276         if (dev_cnt < cmd->resp_data_len)
1277                 scst_set_resp_data_len(cmd, dev_cnt);
1278
1279 out_compl:
1280         cmd->completed = 1;
1281
1282 out_done:
1283         /* Report the result */
1284         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1285
1286         TRACE_EXIT();
1287         return SCST_EXEC_COMPLETED;
1288         
1289 out_put_err:
1290         scst_put_buf(cmd, buffer);
1291
1292 out_err:
1293         scst_set_cmd_error(cmd,
1294                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1295         goto out_compl;
1296
1297 out_put_hw_err:
1298         scst_put_buf(cmd, buffer);
1299         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1300         goto out_compl;
1301 }
1302
1303 static int scst_pre_select(struct scst_cmd *cmd)
1304 {
1305         int res = SCST_EXEC_NOT_COMPLETED;
1306
1307         TRACE_ENTRY();
1308
1309         if (scst_cmd_atomic(cmd)) {
1310                 res = SCST_EXEC_NEED_THREAD;
1311                 goto out;
1312         }
1313
1314         if (cmd->local_exec_done)
1315                 goto out;
1316
1317         cmd->local_exec_done = 1;
1318
1319         scst_block_dev_cmd(cmd, 1);
1320
1321         /* Check for local events will be done when cmd will be executed */
1322
1323 out:
1324         TRACE_EXIT_RES(res);
1325         return res;
1326 }
1327
1328 static int scst_reserve_local(struct scst_cmd *cmd)
1329 {
1330         int res = SCST_EXEC_NOT_COMPLETED, rc;
1331         struct scst_device *dev;
1332         struct scst_tgt_dev *tgt_dev_tmp;
1333
1334         TRACE_ENTRY();
1335
1336         if (scst_cmd_atomic(cmd)) {
1337                 res = SCST_EXEC_NEED_THREAD;
1338                 goto out;
1339         }
1340
1341         if (cmd->local_exec_done)
1342                 goto out;
1343
1344         cmd->local_exec_done = 1;
1345
1346         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1347                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1348                      "(lun=%Ld)", (uint64_t)cmd->lun);
1349                 scst_set_cmd_error(cmd,
1350                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1351                 goto out_done;
1352         }
1353
1354         dev = cmd->dev;
1355
1356         scst_block_dev_cmd(cmd, 1);
1357
1358         rc = scst_check_local_events(cmd);
1359         if (unlikely(rc != 0))
1360                 goto out_done;
1361
1362         spin_lock_bh(&dev->dev_lock);
1363
1364         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1365                 spin_unlock_bh(&dev->dev_lock);
1366                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1367                 goto out_done;
1368         }
1369
1370         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1371                             dev_tgt_dev_list_entry) 
1372         {
1373                 if (cmd->tgt_dev != tgt_dev_tmp)
1374                         set_bit(SCST_TGT_DEV_RESERVED, 
1375                                 &tgt_dev_tmp->tgt_dev_flags);
1376         }
1377         dev->dev_reserved = 1;
1378
1379         spin_unlock_bh(&dev->dev_lock);
1380         
1381 out:
1382         TRACE_EXIT_RES(res);
1383         return res;
1384
1385 out_done:
1386         /* Report the result */
1387         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1388         res = SCST_EXEC_COMPLETED;
1389         goto out;
1390 }
1391
1392 static int scst_release_local(struct scst_cmd *cmd)
1393 {
1394         int res = SCST_EXEC_NOT_COMPLETED, rc;
1395         struct scst_tgt_dev *tgt_dev_tmp;
1396         struct scst_device *dev;
1397
1398         TRACE_ENTRY();
1399
1400         if (scst_cmd_atomic(cmd)) {
1401                 res = SCST_EXEC_NEED_THREAD;
1402                 goto out;
1403         }
1404
1405         if (cmd->local_exec_done)
1406                 goto out;
1407
1408         cmd->local_exec_done = 1;
1409
1410         dev = cmd->dev;
1411
1412         scst_block_dev_cmd(cmd, 1);
1413
1414         rc = scst_check_local_events(cmd);
1415         if (unlikely(rc != 0))
1416                 goto out_done;
1417
1418         spin_lock_bh(&dev->dev_lock);
1419
1420         /* 
1421          * The device could be RELEASED behind us, if RESERVING session 
1422          * is closed (see scst_free_tgt_dev()), but this actually doesn't 
1423          * matter, so use lock and no retest for DEV_RESERVED bits again
1424          */
1425         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1426                 res = SCST_EXEC_COMPLETED;
1427                 cmd->status = 0;
1428                 cmd->msg_status = 0;
1429                 cmd->host_status = DID_OK;
1430                 cmd->driver_status = 0;
1431                 cmd->completed = 1;
1432         } else {
1433                 list_for_each_entry(tgt_dev_tmp,
1434                                     &dev->dev_tgt_dev_list,
1435                                     dev_tgt_dev_list_entry) {
1436                         clear_bit(SCST_TGT_DEV_RESERVED, 
1437                                 &tgt_dev_tmp->tgt_dev_flags);
1438                 }
1439                 dev->dev_reserved = 0;
1440         }
1441
1442         spin_unlock_bh(&dev->dev_lock);
1443
1444         if (res == SCST_EXEC_COMPLETED)
1445                 goto out_done;
1446
1447 out:
1448         TRACE_EXIT_RES(res);
1449         return res;
1450
1451 out_done:
1452         res = SCST_EXEC_COMPLETED;
1453         /* Report the result */
1454         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1455         goto out;
1456 }
1457
1458 /* No locks, no IRQ or IRQ-safe context allowed */
1459 int scst_check_local_events(struct scst_cmd *cmd)
1460 {
1461         int res, rc;
1462         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1463
1464         TRACE_ENTRY();
1465
1466         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1467                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1468                 goto out_uncomplete;
1469         }
1470
1471         /* Reserve check before Unit Attention */
1472         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED, &tgt_dev->tgt_dev_flags))) {
1473                 if ((cmd->cdb[0] != INQUIRY) && (cmd->cdb[0] != REPORT_LUNS) &&
1474                     (cmd->cdb[0] != RELEASE) && (cmd->cdb[0] != RELEASE_10) &&
1475                     (cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER) &&
1476                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL || (cmd->cdb[4] & 3)) &&
1477                     (cmd->cdb[0] != LOG_SENSE) && (cmd->cdb[0] != REQUEST_SENSE)) {
1478                         scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1479                         goto out_complete;
1480                 }
1481         }
1482
1483         /* If we had internal bus reset, set the command error unit attention */
1484         if ((cmd->dev->scsi_dev != NULL) &&
1485             unlikely(cmd->dev->scsi_dev->was_reset)) {
1486                 if (scst_is_ua_command(cmd)) {
1487                         struct scst_device *dev = cmd->dev;
1488                         int done = 0;
1489                         /* Prevent more than 1 cmd to be triggered by was_reset */
1490                         spin_lock_bh(&dev->dev_lock);
1491                         barrier(); /* to reread was_reset */
1492                         if (dev->scsi_dev->was_reset) {
1493                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1494                                 scst_set_cmd_error(cmd,
1495                                            SCST_LOAD_SENSE(scst_sense_reset_UA));
1496                                 /* It looks like it is safe to clear was_reset here */
1497                                 dev->scsi_dev->was_reset = 0;
1498                                 smp_mb();
1499                                 done = 1;
1500                         }
1501                         spin_unlock_bh(&dev->dev_lock);
1502
1503                         if (done)
1504                                 goto out_complete;
1505                 }
1506         }
1507
1508         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING, 
1509                         &cmd->tgt_dev->tgt_dev_flags))) {
1510                 if (scst_is_ua_command(cmd)) {
1511                         rc = scst_set_pending_UA(cmd);
1512                         if (rc == 0)
1513                                 goto out_complete;
1514                 }
1515         }
1516
1517         res = 0;
1518
1519 out:
1520         TRACE_EXIT_RES(res);
1521         return res;
1522
1523 out_complete:
1524         res = 1;
1525         sBUG_ON(!cmd->completed);
1526         goto out;
1527
1528 out_uncomplete:
1529         res = -1;
1530         goto out;
1531 }
1532
1533 /* 
1534  * The result of cmd execution, if any, should be reported 
1535  * via scst_cmd_done_local() 
1536  */
1537 static int scst_pre_exec(struct scst_cmd *cmd)
1538 {
1539         int res = SCST_EXEC_NOT_COMPLETED;
1540         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1541
1542         TRACE_ENTRY();
1543
1544         /*
1545          * This function can be called several times for the same cmd, so it
1546          * can't change any state in a non-reentrable way or use something
1547          * like local_exec_done!!
1548          */
1549
1550         /* Check READ_ONLY device status */
1551         if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
1552             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
1553              cmd->cdb[0] == WRITE_10 ||
1554              cmd->cdb[0] == WRITE_12 ||
1555              cmd->cdb[0] == WRITE_16 ||
1556              cmd->cdb[0] == WRITE_VERIFY ||
1557              cmd->cdb[0] == WRITE_VERIFY_12 ||
1558              cmd->cdb[0] == WRITE_VERIFY_16 ||
1559              (cmd->dev->handler->type == TYPE_TAPE &&
1560               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
1561                 scst_set_cmd_error(cmd,
1562                            SCST_LOAD_SENSE(scst_sense_data_protect));
1563                 goto out_done;
1564         }
1565
1566 out:
1567         TRACE_EXIT_RES(res);
1568         return res;
1569
1570 out_done:
1571         res = SCST_EXEC_COMPLETED;
1572         /* Report the result */
1573         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1574         goto out;
1575 }
1576
1577 /* 
1578  * The result of cmd execution, if any, should be reported 
1579  * via scst_cmd_done_local() 
1580  */
1581 static inline int scst_local_exec(struct scst_cmd *cmd)
1582 {
1583         int res = SCST_EXEC_NOT_COMPLETED;
1584
1585         TRACE_ENTRY();
1586
1587         /*
1588          * Adding new commands here don't forget to update
1589          * scst_is_cmd_local() in scsi_tgt.h, if necessary
1590          */
1591
1592         switch (cmd->cdb[0]) {
1593         case MODE_SELECT:
1594         case MODE_SELECT_10:
1595         case LOG_SELECT:
1596                 res = scst_pre_select(cmd);
1597                 break;
1598         case RESERVE:
1599         case RESERVE_10:
1600                 res = scst_reserve_local(cmd);
1601                 break;
1602         case RELEASE:
1603         case RELEASE_10:
1604                 res = scst_release_local(cmd);
1605                 break;
1606         case REPORT_LUNS:
1607                 res = scst_report_luns_local(cmd);
1608                 break;
1609         }
1610
1611         TRACE_EXIT_RES(res);
1612         return res;
1613 }
1614
1615 static int scst_do_send_to_midlev(struct scst_cmd *cmd)
1616 {
1617         int rc = SCST_EXEC_NOT_COMPLETED;
1618
1619         TRACE_ENTRY();
1620
1621         /* Check here to let an out of SN cmd be queued w/o context switch */
1622         if (scst_cmd_atomic(cmd) && !cmd->dev->handler->exec_atomic) {
1623                 TRACE_DBG("Dev handler %s exec() can not be "
1624                       "called in atomic context, rescheduling to the thread",
1625                       cmd->dev->handler->name);
1626                 rc = SCST_EXEC_NEED_THREAD;
1627                 goto out;
1628         }
1629
1630         cmd->sent_to_midlev = 1;
1631         cmd->state = SCST_CMD_STATE_EXECUTING;
1632         cmd->scst_cmd_done = scst_cmd_done_local;
1633
1634         rc = scst_pre_exec(cmd);
1635         /* !! At this point cmd, sess & tgt_dev can be already freed !! */
1636         if (rc != SCST_EXEC_NOT_COMPLETED) {
1637                 if (rc == SCST_EXEC_COMPLETED)
1638                         goto out;
1639                 else if (rc == SCST_EXEC_NEED_THREAD)
1640                         goto out_clear;
1641                 else
1642                         goto out_rc_error;
1643         }
1644
1645         rc = scst_local_exec(cmd);
1646         /* !! At this point cmd, sess & tgt_dev can be already freed !! */
1647         if (rc != SCST_EXEC_NOT_COMPLETED) {
1648                 if (rc == SCST_EXEC_COMPLETED)
1649                         goto out;
1650                 else if (rc == SCST_EXEC_NEED_THREAD)
1651                         goto out_clear;
1652                 else
1653                         goto out_rc_error;
1654         }
1655
1656         if (cmd->dev->handler->exec) {
1657                 struct scst_device *dev = cmd->dev;
1658                 TRACE_DBG("Calling dev handler %s exec(%p)",
1659                       dev->handler->name, cmd);
1660                 TRACE_BUFF_FLAG(TRACE_SEND_TOP, "Execing: ", cmd->cdb, cmd->cdb_len);
1661                 cmd->scst_cmd_done = scst_cmd_done_local;
1662                 rc = dev->handler->exec(cmd);
1663                 /* !! At this point cmd, sess & tgt_dev can be already freed !! */
1664                 TRACE_DBG("Dev handler %s exec() returned %d",
1665                       dev->handler->name, rc);
1666                 if (rc == SCST_EXEC_COMPLETED)
1667                         goto out;
1668                 else if (rc == SCST_EXEC_NEED_THREAD)
1669                         goto out_clear;
1670                 else if (rc != SCST_EXEC_NOT_COMPLETED)
1671                         goto out_rc_error;
1672         }
1673
1674         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1675         
1676         if (unlikely(cmd->dev->scsi_dev == NULL)) {
1677                 PRINT_ERROR("Command for virtual device must be "
1678                         "processed by device handler (lun %Ld)!",
1679                         (uint64_t)cmd->lun);
1680                 goto out_error;
1681         }
1682
1683         rc = scst_check_local_events(cmd);
1684         if (unlikely(rc != 0))
1685                 goto out_done;
1686
1687 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) 
1688         if (unlikely(scst_alloc_request(cmd) != 0)) {
1689                 if (scst_cmd_atomic(cmd)) {
1690                         rc = SCST_EXEC_NEED_THREAD;
1691                         goto out_clear;
1692                 } else {
1693                         PRINT_INFO("%s", "Unable to allocate request, "
1694                                 "sending BUSY status");
1695                         goto out_busy;
1696                 }
1697         }
1698         
1699         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1700                     (void *)cmd->scsi_req->sr_buffer,
1701                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1702                     cmd->retries);
1703 #else
1704         rc = scst_exec_req(cmd->dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1705                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1706                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1707                         scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
1708         if (unlikely(rc != 0)) {
1709                 if (scst_cmd_atomic(cmd)) {
1710                         rc = SCST_EXEC_NEED_THREAD;
1711                         goto out_clear;
1712                 } else {
1713                         PRINT_INFO("scst_exec_req() failed: %d", rc);
1714                         goto out_error;
1715                 }
1716         }
1717 #endif
1718
1719         rc = SCST_EXEC_COMPLETED;
1720
1721 out:
1722         TRACE_EXIT();
1723         return rc;
1724
1725 out_clear:
1726         /* Restore the state */
1727         cmd->sent_to_midlev = 0;
1728         cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
1729         goto out;
1730
1731 out_rc_error:
1732         PRINT_ERROR("Dev handler %s exec() or scst_local_exec() returned "
1733                     "invalid code %d", cmd->dev->handler->name, rc);
1734         /* go through */
1735
1736 out_error:
1737         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1738
1739 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) 
1740 out_busy:
1741         scst_set_busy(cmd);
1742         /* go through */
1743 #endif
1744
1745 out_done:
1746         rc = SCST_EXEC_COMPLETED;
1747         /* Report the result */
1748         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1749         goto out;
1750 }
1751
1752 /* No locks */
1753 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1754 {
1755         if (slot == NULL)
1756                 goto inc;
1757
1758         /* Optimized for lockless fast path */
1759
1760         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1761                 atomic_read(slot));
1762
1763         if (!atomic_dec_and_test(slot))
1764                 goto out;
1765
1766         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1767                 tgt_dev->num_free_sn_slots);
1768         if (tgt_dev->num_free_sn_slots != ARRAY_SIZE(tgt_dev->sn_slots)) {
1769                 spin_lock_irq(&tgt_dev->sn_lock);
1770                 if (tgt_dev->num_free_sn_slots != ARRAY_SIZE(tgt_dev->sn_slots)) {
1771                         tgt_dev->num_free_sn_slots++;
1772                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1773                                 tgt_dev->num_free_sn_slots);
1774                         if (tgt_dev->num_free_sn_slots == 0)
1775                                 tgt_dev->cur_sn_slot = slot;
1776                 }
1777                 spin_unlock_irq(&tgt_dev->sn_lock);
1778         }
1779
1780 inc:
1781         /*
1782          * No locks is needed, because only one thread at time can 
1783          * be here (serialized by sn). Also it is supposed that there
1784          * could not be half-incremented halves.
1785          */
1786         tgt_dev->expected_sn++;
1787         smp_mb(); /* write must be before def_cmd_count read */
1788         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1789
1790 out:
1791         return;
1792 }
1793
1794 static int scst_send_to_midlev(struct scst_cmd **active_cmd)
1795 {
1796         int res, rc;
1797         struct scst_cmd *cmd = *active_cmd;
1798         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1799         struct scst_device *dev = cmd->dev;
1800         typeof(tgt_dev->expected_sn) expected_sn;
1801         int count;
1802
1803         TRACE_ENTRY();
1804
1805         res = SCST_CMD_STATE_RES_CONT_NEXT;
1806
1807         __scst_get(0); /* protect dev & tgt_dev */
1808
1809         if (unlikely(cmd->internal || cmd->retry)) {
1810                 rc = scst_do_send_to_midlev(cmd);
1811                 /* !! At this point cmd, sess & tgt_dev can be already freed !! */
1812                 if (rc == SCST_EXEC_NEED_THREAD) {
1813                         TRACE_DBG("%s", "scst_do_send_to_midlev() requested "
1814                               "thread context, rescheduling");
1815                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1816                         scst_dec_on_dev_cmd(cmd);
1817                         goto out_put;
1818                 } else {
1819                         sBUG_ON(rc != SCST_EXEC_COMPLETED);
1820                         goto out_unplug;
1821                 }
1822         }
1823
1824         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
1825                 goto out_put;
1826
1827         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
1828                 goto exec;
1829
1830         sBUG_ON(!cmd->sn_set);
1831
1832         expected_sn = tgt_dev->expected_sn;
1833         /* Optimized for lockless fast path */
1834         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
1835                 spin_lock_irq(&tgt_dev->sn_lock);
1836                 tgt_dev->def_cmd_count++;
1837                 smp_mb();
1838                 barrier(); /* to reread expected_sn & hq_cmd_count */
1839                 expected_sn = tgt_dev->expected_sn;
1840                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
1841                         /* We are under IRQ lock, but dev->dev_lock is BH one */
1842                         int cmd_blocking = scst_pre_dec_on_dev_cmd(cmd);
1843                         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1844                                 /* Necessary to allow aborting out of sn cmds */
1845                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p (tag %llu)",
1846                                         cmd, cmd->tag);
1847                                 tgt_dev->def_cmd_count--;
1848                                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1849                                 res = SCST_CMD_STATE_RES_CONT_SAME;
1850                         } else {
1851                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
1852                                         "expected_sn=%ld)", cmd, cmd->sn,
1853                                         cmd->sn_set, expected_sn);
1854                                 list_add_tail(&cmd->sn_cmd_list_entry,
1855                                               &tgt_dev->deferred_cmd_list);
1856                         }
1857                         spin_unlock_irq(&tgt_dev->sn_lock);
1858                         /* !! At this point cmd can be already freed !! */
1859                         __scst_dec_on_dev_cmd(dev, cmd_blocking);
1860                         goto out_put;
1861                 } else {
1862                         TRACE_SN("Somebody incremented expected_sn %ld, "
1863                                 "continuing", expected_sn);
1864                         tgt_dev->def_cmd_count--;
1865                         spin_unlock_irq(&tgt_dev->sn_lock);
1866                 }
1867         }
1868
1869 exec:
1870         count = 0;
1871         while(1) {
1872                 atomic_t *slot = cmd->sn_slot;
1873                 int inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1874                                       cmd->sn_set;
1875                 rc = scst_do_send_to_midlev(cmd);
1876                 if (rc == SCST_EXEC_NEED_THREAD) {
1877                         TRACE_DBG("%s", "scst_do_send_to_midlev() requested "
1878                               "thread context, rescheduling");
1879                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1880                         scst_dec_on_dev_cmd(cmd);
1881                         *active_cmd = cmd;
1882                         if (count != 0)
1883                                 goto out_unplug;
1884                         else
1885                                 goto out_put;
1886                 }
1887                 sBUG_ON(rc != SCST_EXEC_COMPLETED);
1888                 /* !! At this point cmd can be already freed !! */
1889                 count++;
1890                 if (inc_expected_sn)
1891                         scst_inc_expected_sn(tgt_dev, slot);
1892                 cmd = scst_check_deferred_commands(tgt_dev);
1893                 if (cmd == NULL)
1894                         break;
1895                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
1896                         break;
1897         }
1898
1899 out_unplug:
1900         if (dev->scsi_dev != NULL)
1901                 generic_unplug_device(dev->scsi_dev->request_queue);
1902
1903 out_put:
1904         __scst_put();
1905         /* !! At this point sess, dev and tgt_dev can be already freed !! */
1906
1907         TRACE_EXIT_HRES(res);
1908         return res;
1909 }
1910
1911 /* No locks supposed to be held */
1912 static int scst_check_sense(struct scst_cmd *cmd)
1913 {
1914         int res = 0;
1915         struct scst_device *dev = cmd->dev;
1916         int dbl_ua_possible, ua_sent = 0;
1917
1918         TRACE_ENTRY();
1919
1920         /* If we had internal bus reset behind us, set the command error UA */
1921         if ((dev->scsi_dev != NULL) &&
1922             unlikely(cmd->host_status == DID_RESET) &&
1923             scst_is_ua_command(cmd)) {
1924                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
1925                       dev->scsi_dev->was_reset, cmd->host_status);
1926                 scst_set_cmd_error(cmd,
1927                    SCST_LOAD_SENSE(scst_sense_reset_UA));
1928                 /* just in case */
1929                 cmd->ua_ignore = 0;
1930                 /* It looks like it is safe to clear was_reset here */
1931                 dev->scsi_dev->was_reset = 0;
1932                 smp_mb();
1933         }
1934
1935         dbl_ua_possible = dev->dev_double_ua_possible;
1936         TRACE_DBG("cmd %p dbl_ua_possible %d", cmd, dbl_ua_possible);
1937         if (unlikely(dbl_ua_possible)) {
1938                 spin_lock_bh(&dev->dev_lock);
1939                 barrier(); /* to reread dev_double_ua_possible */
1940                 dbl_ua_possible = dev->dev_double_ua_possible;
1941                 if (dbl_ua_possible)
1942                         ua_sent = dev->dev_reset_ua_sent;
1943                 else
1944                         spin_unlock_bh(&dev->dev_lock);
1945         }
1946
1947         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) && 
1948             SCST_SENSE_VALID(cmd->sense_buffer)) {
1949                 TRACE_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense_buffer,
1950                         sizeof(cmd->sense_buffer));
1951                 /* Check Unit Attention Sense Key */
1952                 if (scst_is_ua_sense(cmd->sense_buffer)) {
1953                         if (cmd->sense_buffer[12] == SCST_SENSE_ASC_UA_RESET) {
1954                                 if (dbl_ua_possible) {
1955                                         if (ua_sent) {
1956                                                 TRACE(TRACE_MGMT_MINOR, "%s", 
1957                                                         "Double UA detected");
1958                                                 /* Do retry */
1959                                                 TRACE(TRACE_MGMT_MINOR, "Retrying cmd %p "
1960                                                         "(tag %llu)", cmd, cmd->tag);
1961                                                 cmd->status = 0;
1962                                                 cmd->msg_status = 0;
1963                                                 cmd->host_status = DID_OK;
1964                                                 cmd->driver_status = 0;
1965                                                 memset(cmd->sense_buffer, 0,
1966                                                         sizeof(cmd->sense_buffer));
1967                                                 cmd->retry = 1;
1968                                                 cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
1969                                                 res = 1;
1970                                                 /* 
1971                                                  * Dev is still blocked by this cmd, so
1972                                                  * it's OK to clear SCST_DEV_SERIALIZED
1973                                                  * here.
1974                                                  */
1975                                                 dev->dev_double_ua_possible = 0;
1976                                                 dev->dev_serialized = 0;
1977                                                 dev->dev_reset_ua_sent = 0;
1978                                                 goto out_unlock;
1979                                         } else
1980                                                 dev->dev_reset_ua_sent = 1;
1981                                 }
1982                         }
1983                         if (cmd->ua_ignore == 0) {
1984                                 if (unlikely(dbl_ua_possible)) {
1985                                         __scst_dev_check_set_UA(dev, cmd,
1986                                                 cmd->sense_buffer,
1987                                                 sizeof(cmd->sense_buffer));
1988                                 } else {
1989                                         scst_dev_check_set_UA(dev, cmd,
1990                                                 cmd->sense_buffer,
1991                                                 sizeof(cmd->sense_buffer));
1992                                 }
1993                         }
1994                 }
1995         }
1996
1997         if (unlikely(dbl_ua_possible)) {
1998                 if (ua_sent && scst_is_ua_command(cmd)) {
1999                         TRACE_MGMT_DBG("%s", "Clearing dbl_ua_possible flag");
2000                         dev->dev_double_ua_possible = 0;
2001                         dev->dev_serialized = 0;
2002                         dev->dev_reset_ua_sent = 0;
2003                 }
2004                 spin_unlock_bh(&dev->dev_lock);
2005         }
2006
2007 out:
2008         TRACE_EXIT_RES(res);
2009         return res;
2010
2011 out_unlock:
2012         spin_unlock_bh(&dev->dev_lock);
2013         goto out;
2014 }
2015
2016 static int scst_check_auto_sense(struct scst_cmd *cmd)
2017 {
2018         int res = 0;
2019
2020         TRACE_ENTRY();
2021
2022         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2023             (!SCST_SENSE_VALID(cmd->sense_buffer) ||
2024              SCST_NO_SENSE(cmd->sense_buffer))) {
2025                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2026                       "cmd->status=%x, cmd->msg_status=%x, "
2027                       "cmd->host_status=%x, cmd->driver_status=%x", cmd->status,
2028                       cmd->msg_status, cmd->host_status, cmd->driver_status);
2029                 res = 1;
2030         } else if (unlikely(cmd->host_status)) {
2031                 if ((cmd->host_status == DID_REQUEUE) ||
2032                     (cmd->host_status == DID_IMM_RETRY) ||
2033                     (cmd->host_status == DID_SOFT_ERROR) ||
2034                     (cmd->host_status == DID_ABORT)) {
2035                         scst_set_busy(cmd);
2036                 } else {
2037                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2038                                 "received, returning HARDWARE ERROR instead",
2039                                 cmd->host_status);
2040                         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2041                 }
2042         }
2043
2044         TRACE_EXIT_RES(res);
2045         return res;
2046 }
2047
2048 static int scst_done_cmd_check(struct scst_cmd *cmd, int *pres)
2049 {
2050         int res = 0, rc;
2051
2052         TRACE_ENTRY();
2053
2054         if (unlikely(cmd->cdb[0] == REQUEST_SENSE)) {
2055                 if (cmd->internal)
2056                         cmd = scst_complete_request_sense(cmd);
2057         } else if (unlikely(scst_check_auto_sense(cmd))) {
2058                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2059                             "without sense data (opcode 0x%x), issuing "
2060                             "REQUEST SENSE", cmd->cdb[0]);
2061                 rc = scst_prepare_request_sense(cmd);
2062                 if (rc > 0) {
2063                         *pres = rc;
2064                         res = 1;
2065                         goto out;
2066                 } else {
2067                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2068                                     "returning HARDWARE ERROR");
2069                         scst_set_cmd_error(cmd,
2070                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2071                 }
2072         } else if (unlikely(scst_check_sense(cmd))) {
2073                 *pres = SCST_CMD_STATE_RES_CONT_SAME;
2074                 res = 1;
2075                 goto out;
2076         }
2077
2078         if (likely(scsi_status_is_good(cmd->status))) {
2079                 unsigned char type = cmd->dev->handler->type;
2080                 if ((cmd->cdb[0] == MODE_SENSE || cmd->cdb[0] == MODE_SENSE_10) &&
2081                     cmd->tgt_dev->acg_dev->rd_only_flag &&
2082                     (type == TYPE_DISK || type == TYPE_WORM || type == TYPE_MOD ||
2083                      type == TYPE_TAPE)) {
2084                         int32_t length;
2085                         uint8_t *address;
2086
2087                         length = scst_get_buf_first(cmd, &address);
2088                         if (length <= 0)
2089                                 goto out;
2090                         if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2091                                 address[2] |= 0x80;   /* Write Protect*/
2092                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2093                                 address[3] |= 0x80;   /* Write Protect*/
2094                         scst_put_buf(cmd, address);
2095                 }
2096
2097                 /* 
2098                  * Check and clear NormACA option for the device, if necessary,
2099                  * since we don't support ACA
2100                  */
2101                 if ((cmd->cdb[0] == INQUIRY) &&
2102                     !(cmd->cdb[1] & SCST_INQ_EVPD/* Std INQUIRY data (no EVPD) */) &&
2103                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2104                         uint8_t *buffer;
2105                         int buflen;
2106
2107                         /* ToDo: all pages ?? */
2108                         buflen = scst_get_buf_first(cmd, &buffer);
2109                         if (buflen > SCST_INQ_BYTE3) {
2110 #ifdef EXTRACHECKS
2111                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2112                                         PRINT_INFO("NormACA set for device: "
2113                                             "lun=%Ld, type 0x%02x", 
2114                                             (uint64_t)cmd->lun, buffer[0]);
2115                                 }
2116 #endif
2117                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2118                         } else {
2119                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2120                                     "buffer");
2121                                 scst_set_cmd_error(cmd,
2122                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
2123                         }
2124                         if (buflen > 0)
2125                                 scst_put_buf(cmd, buffer);
2126                 }
2127
2128                 if (unlikely((cmd->cdb[0] == MODE_SELECT) || 
2129                     (cmd->cdb[0] == MODE_SELECT_10) ||
2130                     (cmd->cdb[0] == LOG_SELECT))) {
2131                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded (LUN %Ld)",
2132                                 (uint64_t)cmd->lun);
2133                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2134                         *pres = SCST_CMD_STATE_RES_CONT_SAME;
2135                         res = 1;
2136                         goto out;
2137                 }
2138         } else {
2139                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2140                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2141                                         &cmd->tgt_dev->tgt_dev_flags)) {
2142                                 struct scst_tgt_dev *tgt_dev_tmp;
2143                                 TRACE(TRACE_SCSI, "Real RESERVE failed lun=%Ld, status=%x",
2144                                       (uint64_t)cmd->lun, cmd->status);
2145                                 TRACE_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense_buffer,
2146                                              sizeof(cmd->sense_buffer));
2147                                 /* Clearing the reservation */
2148                                 list_for_each_entry(tgt_dev_tmp, &cmd->dev->dev_tgt_dev_list,
2149                                                     dev_tgt_dev_list_entry) {
2150                                         clear_bit(SCST_TGT_DEV_RESERVED, 
2151                                                 &tgt_dev_tmp->tgt_dev_flags);
2152                                 }
2153                                 cmd->dev->dev_reserved = 0;
2154                         }
2155                 }
2156
2157                 /* Check for MODE PARAMETERS CHANGED UA */
2158                 if ((cmd->dev->scsi_dev != NULL) &&
2159                     (cmd->status == SAM_STAT_CHECK_CONDITION) && 
2160                     SCST_SENSE_VALID(cmd->sense_buffer) &&
2161                     scst_is_ua_sense(cmd->sense_buffer) &&
2162                     (cmd->sense_buffer[12] == 0x2a) &&
2163                     (cmd->sense_buffer[13] == 0x01)) {
2164                         TRACE(TRACE_SCSI, "MODE PARAMETERS CHANGED UA (lun %Ld)",
2165                                 (uint64_t)cmd->lun);
2166                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2167                         *pres = SCST_CMD_STATE_RES_CONT_SAME;
2168                         res = 1;
2169                         goto out;
2170                 }
2171         }
2172
2173 out:
2174         TRACE_EXIT_RES(res);
2175         return res;
2176 }
2177
2178 static int scst_pre_dev_done(struct scst_cmd *cmd)
2179 {
2180         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2181
2182         TRACE_ENTRY();
2183
2184         rc = scst_done_cmd_check(cmd, &res);
2185         if (rc)
2186                 goto out;
2187
2188         cmd->state = SCST_CMD_STATE_DEV_DONE;
2189
2190 out:
2191         TRACE_EXIT_HRES(res);
2192         return res;
2193 }
2194
2195 static int scst_mode_select_checks(struct scst_cmd *cmd)
2196 {
2197         int res = SCST_CMD_STATE_RES_CONT_SAME;
2198         int atomic = scst_cmd_atomic(cmd);
2199
2200         TRACE_ENTRY();
2201
2202         if (likely(scsi_status_is_good(cmd->status))) {
2203                 if (unlikely((cmd->cdb[0] == MODE_SELECT) || 
2204                     (cmd->cdb[0] == MODE_SELECT_10) ||
2205                     (cmd->cdb[0] == LOG_SELECT))) {
2206                         if (atomic && (cmd->dev->scsi_dev != NULL)) {
2207                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2208                                         "context required");
2209                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2210                                 goto out;
2211                         }
2212
2213                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2214                                 "setting the SELECT UA (lun=%Ld)", 
2215                                 (uint64_t)cmd->lun);
2216
2217                         spin_lock_bh(&scst_temp_UA_lock);
2218                         if (cmd->cdb[0] == LOG_SELECT) {
2219                                 scst_set_sense(scst_temp_UA,
2220                                         sizeof(scst_temp_UA),
2221                                         UNIT_ATTENTION, 0x2a, 0x02);
2222                         } else {
2223                                 scst_set_sense(scst_temp_UA,
2224                                         sizeof(scst_temp_UA),
2225                                         UNIT_ATTENTION, 0x2a, 0x01);
2226                         }
2227                         scst_dev_check_set_local_UA(cmd->dev, cmd, scst_temp_UA,
2228                                 sizeof(scst_temp_UA));
2229                         spin_unlock_bh(&scst_temp_UA_lock);
2230
2231                         if (cmd->dev->scsi_dev != NULL)
2232                                 scst_obtain_device_parameters(cmd->dev);
2233                 }
2234         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) && 
2235                     SCST_SENSE_VALID(cmd->sense_buffer) &&
2236                     scst_is_ua_sense(cmd->sense_buffer) &&
2237                     (cmd->sense_buffer[12] == 0x2a) &&
2238                     (cmd->sense_buffer[13] == 0x01)) {
2239                 if (atomic) {
2240                         TRACE_DBG("%s", "MODE PARAMETERS CHANGED UA: thread "
2241                                 "context required");
2242                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2243                         goto out;
2244                 }
2245
2246                 TRACE(TRACE_SCSI, "MODE PARAMETERS CHANGED UA (lun %Ld): "
2247                         "getting new parameters", (uint64_t)cmd->lun);
2248
2249                 scst_obtain_device_parameters(cmd->dev);
2250         } else
2251                 sBUG();
2252
2253         cmd->state = SCST_CMD_STATE_DEV_DONE;
2254
2255 out:
2256         TRACE_EXIT_HRES(res);
2257         return res;
2258 }
2259
2260 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2261 {
2262         struct scst_cmd *c;
2263
2264         if (likely(cmd->sn_set))
2265                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2266
2267         c = scst_check_deferred_commands(cmd->tgt_dev);
2268         if (c != NULL) {
2269                 unsigned long flags;
2270                 spin_lock_irqsave(&c->cmd_lists->cmd_list_lock, flags);
2271                 TRACE_SN("Adding cmd %p to active cmd list", c);
2272                 list_add_tail(&c->cmd_list_entry,
2273                         &c->cmd_lists->active_cmd_list);
2274                 wake_up(&c->cmd_lists->cmd_list_waitQ);
2275                 spin_unlock_irqrestore(&c->cmd_lists->cmd_list_lock, flags);
2276         }
2277 }
2278
2279 static int scst_dev_done(struct scst_cmd *cmd)
2280 {
2281         int res = SCST_CMD_STATE_RES_CONT_SAME;
2282         int state;
2283         int atomic = scst_cmd_atomic(cmd);
2284
2285         TRACE_ENTRY();
2286
2287         if (atomic && !cmd->dev->handler->dev_done_atomic) {
2288                 TRACE_DBG("Dev handler %s dev_done() can not be "
2289                       "called in atomic context, rescheduling to the thread",
2290                       cmd->dev->handler->name);
2291                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2292                 goto out;
2293         }
2294
2295         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2296         if (likely(!scst_is_cmd_local(cmd)) && 
2297             likely(cmd->dev->handler->dev_done != NULL))
2298         {
2299                 int rc;
2300                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2301                       cmd->dev->handler->name, cmd);
2302                 rc = cmd->dev->handler->dev_done(cmd);
2303                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2304                       cmd->dev->handler->name, rc);
2305                 if (rc != SCST_CMD_STATE_DEFAULT)
2306                         state = rc;
2307         }
2308
2309         switch (state) {
2310         case SCST_CMD_STATE_PRE_XMIT_RESP:
2311         case SCST_CMD_STATE_DEV_PARSE:
2312         case SCST_CMD_STATE_PRE_PARSE:
2313         case SCST_CMD_STATE_PREPARE_SPACE:
2314         case SCST_CMD_STATE_RDY_TO_XFER:
2315         case SCST_CMD_STATE_TGT_PRE_EXEC:
2316         case SCST_CMD_STATE_SEND_TO_MIDLEV:
2317         case SCST_CMD_STATE_PRE_DEV_DONE:
2318         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2319         case SCST_CMD_STATE_DEV_DONE:
2320         case SCST_CMD_STATE_XMIT_RESP:
2321         case SCST_CMD_STATE_FINISHED:
2322                 cmd->state = state;
2323                 res = SCST_CMD_STATE_RES_CONT_SAME;
2324                 break;
2325
2326         case SCST_CMD_STATE_NEED_THREAD_CTX:
2327                 TRACE_DBG("Dev handler %s dev_done() requested "
2328                       "thread context, rescheduling",
2329                       cmd->dev->handler->name);
2330                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2331                 break;
2332
2333         default:
2334                 if (state >= 0) {
2335                         PRINT_ERROR("Dev handler %s dev_done() returned "
2336                                 "invalid cmd state %d", 
2337                                 cmd->dev->handler->name, state);
2338                 } else {
2339                         PRINT_ERROR("Dev handler %s dev_done() returned "
2340                                 "error %d", cmd->dev->handler->name, 
2341                                 state);
2342                 }
2343                 scst_set_cmd_error(cmd,
2344                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2345                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2346                 res = SCST_CMD_STATE_RES_CONT_SAME;
2347                 break;
2348         }
2349
2350         if (cmd->needs_unblocking)
2351                 scst_unblock_dev_cmd(cmd);
2352
2353         if (likely(cmd->dec_on_dev_needed))
2354                 scst_dec_on_dev_cmd(cmd);
2355
2356         if (cmd->inc_expected_sn_on_done && cmd->sent_to_midlev)
2357                 scst_inc_check_expected_sn(cmd);
2358
2359 out:
2360         TRACE_EXIT_HRES(res);
2361         return res;
2362 }
2363
2364 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2365 {
2366         int res;
2367
2368         TRACE_ENTRY();
2369
2370         if (cmd->tgt_dev != NULL) {
2371                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2372                         scst_on_hq_cmd_response(cmd);
2373
2374                 if (unlikely(!cmd->sent_to_midlev)) {
2375                         TRACE_SN("cmd %p was not sent to mid-lev (sn %ld, set %d)",
2376                                 cmd, cmd->sn, cmd->sn_set);
2377                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2378                         cmd->sent_to_midlev = 1;
2379                 }
2380         }
2381
2382         /*
2383          * If we don't remove cmd from the search list here, before
2384          * submitting it for transmittion, we will have a race, when for
2385          * some reason cmd's release is delayed after transmittion and
2386          * initiator sends cmd with the same tag => it is possible that
2387          * a wrong cmd will be found by find() functions.
2388          */
2389         spin_lock_irq(&cmd->sess->sess_list_lock);
2390         list_del(&cmd->search_cmd_list_entry);
2391         spin_unlock_irq(&cmd->sess->sess_list_lock);
2392
2393         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2394                 scst_xmit_process_aborted_cmd(cmd);
2395
2396         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2397                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu), skipping",
2398                         cmd, cmd->tag);
2399                 cmd->state = SCST_CMD_STATE_FINISHED;
2400                 res = SCST_CMD_STATE_RES_CONT_SAME;
2401                 goto out;
2402         }
2403
2404         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2405         res = SCST_CMD_STATE_RES_CONT_SAME;
2406
2407 out:
2408         TRACE_EXIT_HRES(res);
2409         return res;
2410 }
2411
2412 static int scst_xmit_response(struct scst_cmd *cmd)
2413 {
2414         int res, rc;
2415         int atomic = scst_cmd_atomic(cmd);
2416
2417         TRACE_ENTRY();
2418
2419         if (atomic && !cmd->tgtt->xmit_response_atomic) {
2420                 TRACE_DBG("%s", "xmit_response() can not be "
2421                       "called in atomic context, rescheduling to the thread");
2422                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2423                 goto out;
2424         }
2425
2426 #ifdef DEBUG_TM
2427         if (cmd->tm_dbg_delayed && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2428                 if (atomic && !cmd->tgtt->xmit_response_atomic) {
2429                         TRACE_MGMT_DBG("%s", "DEBUG_TM delayed cmd needs a thread");
2430                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2431                         goto out;
2432                 }
2433                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2434                         cmd, cmd->tag);
2435                 schedule_timeout_uninterruptible(HZ);
2436         }
2437 #endif
2438
2439         while (1) {
2440                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2441
2442                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2443                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2444
2445                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2446
2447 #if defined(DEBUG) || defined(TRACING)
2448                 if (cmd->sg) {
2449                         int i;
2450                         struct scatterlist *sg = cmd->sg;
2451                         TRACE(TRACE_SEND_BOT,
2452                               "Xmitting %d S/G(s) at %p sg[0].page at %p",
2453                               cmd->sg_cnt, sg, (void*)sg[0].page);
2454                         for(i = 0; i < cmd->sg_cnt; ++i) {
2455                                 TRACE_BUFF_FLAG(TRACE_SEND_BOT,
2456                                     "Xmitting sg", page_address(sg[i].page),
2457                                     sg[i].length);
2458                         }
2459                 }
2460 #endif
2461
2462 #ifdef DEBUG_RETRY
2463                 if (((scst_random() % 100) == 77))
2464                         rc = SCST_TGT_RES_QUEUE_FULL;
2465                 else
2466 #endif
2467                         rc = cmd->tgtt->xmit_response(cmd);
2468                 TRACE_DBG("xmit_response() returned %d", rc);
2469
2470                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2471                         goto out;
2472
2473                 /* Restore the previous state */
2474                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2475
2476                 switch (rc) {
2477                 case SCST_TGT_RES_QUEUE_FULL:
2478                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2479                                 break;
2480                         else
2481                                 continue;
2482
2483                 case SCST_TGT_RES_NEED_THREAD_CTX:
2484                         TRACE_DBG("Target driver %s xmit_response() "
2485                               "requested thread context, rescheduling",
2486                               cmd->tgtt->name);
2487                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2488                         break;
2489
2490                 default:
2491                         goto out_error;
2492                 }
2493                 break;
2494         }
2495
2496 out:
2497         /* Caution: cmd can be already dead here */
2498         TRACE_EXIT_HRES(res);
2499         return res;
2500
2501 out_error:
2502         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2503                 PRINT_ERROR("Target driver %s xmit_response() returned "
2504                         "fatal error", cmd->tgtt->name);
2505         } else {
2506                 PRINT_ERROR("Target driver %s xmit_response() returned "
2507                         "invalid value %d", cmd->tgtt->name, rc);
2508         }
2509         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2510         cmd->state = SCST_CMD_STATE_FINISHED;
2511         res = SCST_CMD_STATE_RES_CONT_SAME;
2512         goto out;
2513 }
2514
2515 void scst_tgt_cmd_done(struct scst_cmd *cmd)
2516 {
2517         TRACE_ENTRY();
2518
2519         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2520
2521         cmd->state = SCST_CMD_STATE_FINISHED;
2522         scst_proccess_redirect_cmd(cmd, scst_get_context(), 1);
2523
2524         TRACE_EXIT();
2525         return;
2526 }
2527
2528 static int scst_finish_cmd(struct scst_cmd *cmd)
2529 {
2530         int res;
2531
2532         TRACE_ENTRY();
2533
2534         atomic_dec(&cmd->sess->sess_cmd_count);
2535
2536         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2537                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
2538                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
2539                         atomic_read(&scst_cmd_count));
2540         }
2541
2542         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
2543                 if ((cmd->tgt_dev != NULL) &&
2544                     scst_is_ua_sense(cmd->sense_buffer)) {
2545                         /* This UA delivery failed, so requeue it */
2546                         scst_check_set_UA(cmd->tgt_dev, cmd->sense_buffer,
2547                                         sizeof(cmd->sense_buffer), 1);
2548                 }
2549         }
2550
2551         scst_cmd_put(cmd);
2552
2553         res = SCST_CMD_STATE_RES_CONT_NEXT;
2554
2555         TRACE_EXIT_HRES(res);
2556         return res;
2557 }
2558
2559 /*
2560  * No locks, but it must be externally serialized (see comment for
2561  * scst_cmd_init_done() in scsi_tgt.h)
2562  */
2563 static void scst_cmd_set_sn(struct scst_cmd *cmd)
2564 {
2565         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2566         unsigned long flags;
2567
2568         if (scst_is_implicit_hq(cmd)) {
2569                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "Implicit HQ cmd %p", cmd);
2570                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
2571         }
2572
2573         /* Optimized for lockless fast path */
2574
2575         scst_check_debug_sn(cmd);
2576
2577         if (cmd->dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER)
2578                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
2579
2580         switch(cmd->queue_type) {
2581         case SCST_CMD_QUEUE_SIMPLE:
2582         case SCST_CMD_QUEUE_UNTAGGED:
2583 #if 1 /* temporary, ToDo */
2584                 if (scst_cmd_is_expected_set(cmd)) {
2585                         if (cmd->expected_data_direction == SCST_DATA_READ)
2586                                 goto ordered;
2587                 } else
2588                         goto ordered;
2589 #endif
2590                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
2591                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
2592                                 tgt_dev->curr_sn++;
2593                                 TRACE_SN("Incremented curr_sn %ld",
2594                                         tgt_dev->curr_sn);
2595                         }
2596                         cmd->sn_slot = tgt_dev->cur_sn_slot;
2597                         cmd->sn = tgt_dev->curr_sn;
2598                         
2599                         tgt_dev->prev_cmd_ordered = 0;
2600                 } else {
2601                         TRACE(TRACE_MINOR, "%s", "Not enough SN slots");
2602                         goto ordered;
2603                 }
2604                 break;
2605
2606         case SCST_CMD_QUEUE_ORDERED:
2607                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "ORDERED cmd %p "
2608                         "(op %x)", cmd, cmd->cdb[0]);
2609 ordered:
2610                 if (!tgt_dev->prev_cmd_ordered) {
2611                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2612                         tgt_dev->num_free_sn_slots--;
2613                         smp_mb();
2614                         if ((tgt_dev->num_free_sn_slots >= 0) &&
2615                             (atomic_read(tgt_dev->cur_sn_slot) > 0)) {
2616                                 do {
2617                                         tgt_dev->cur_sn_slot++;
2618                                         if (tgt_dev->cur_sn_slot == 
2619                                                 tgt_dev->sn_slots +
2620                                                 ARRAY_SIZE(tgt_dev->sn_slots))
2621                                             tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
2622                                 } while(atomic_read(tgt_dev->cur_sn_slot) != 0);
2623                                 TRACE_SN("New cur SN slot %zd",
2624                                         tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
2625                         } else
2626                                 tgt_dev->num_free_sn_slots++;
2627                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2628                 }
2629                 tgt_dev->prev_cmd_ordered = 1;
2630                 tgt_dev->curr_sn++;
2631                 cmd->sn = tgt_dev->curr_sn;
2632                 break;
2633
2634         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
2635                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "HQ cmd %p "
2636                         "(op %x)", cmd, cmd->cdb[0]);
2637                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2638                 tgt_dev->hq_cmd_count++;
2639                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2640                 goto out;
2641
2642         default:
2643                 sBUG();
2644         }
2645
2646         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
2647                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
2648                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
2649                 atomic_read(tgt_dev->cur_sn_slot), 
2650                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
2651                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
2652
2653         cmd->sn_set = 1;
2654 out:
2655         return;
2656 }
2657
2658 /*
2659  * Returns 0 on success, > 0 when we need to wait for unblock,
2660  * < 0 if there is no device (lun) or device type handler.
2661  *
2662  * No locks, but might be on IRQ, protection is done by the
2663  * suspended activity.
2664  */
2665 static int scst_translate_lun(struct scst_cmd *cmd)
2666 {
2667         struct scst_tgt_dev *tgt_dev = NULL;
2668         int res;
2669
2670         TRACE_ENTRY();
2671
2672         __scst_get(1);
2673
2674         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
2675                 struct list_head *sess_tgt_dev_list_head =
2676                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
2677                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %Ld)", cmd,
2678                         (uint64_t)cmd->lun);
2679                 res = -1;
2680                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
2681                                 sess_tgt_dev_list_entry) {
2682                         if (tgt_dev->lun == cmd->lun) {
2683                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
2684
2685                                 if (unlikely(tgt_dev->dev->handler == &scst_null_devtype)) {
2686                                         PRINT_INFO("Dev handler for device "
2687                                           "%Ld is NULL, the device will not be "
2688                                           "visible remotely", (uint64_t)cmd->lun);
2689                                         break;
2690                                 }
2691                                 
2692                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
2693                                 cmd->tgt_dev = tgt_dev;
2694                                 cmd->dev = tgt_dev->dev;
2695
2696                                 res = 0;
2697                                 break;
2698                         }
2699                 }
2700                 if (res != 0) {
2701                         TRACE(TRACE_MINOR, "tgt_dev for lun %Ld not found, command to "
2702                                 "unexisting LU?", (uint64_t)cmd->lun);
2703                         __scst_put();
2704                 }
2705         } else {
2706                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
2707                 __scst_put();
2708                 res = 1;
2709         }
2710
2711         TRACE_EXIT_RES(res);
2712         return res;
2713 }
2714
2715 /*
2716  * No locks, but might be on IRQ
2717  *
2718  * Returns 0 on success, > 0 when we need to wait for unblock,
2719  * < 0 if there is no device (lun) or device type handler.
2720  */
2721 static int __scst_init_cmd(struct scst_cmd *cmd)
2722 {
2723         int res = 0;
2724
2725         TRACE_ENTRY();
2726
2727         res = scst_translate_lun(cmd);
2728         if (likely(res == 0)) {
2729                 int cnt;
2730                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
2731                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
2732                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
2733                         TRACE(TRACE_MGMT_MINOR, "Too many pending commands (%d) in "
2734                                 "session, returning BUSY to initiator \"%s\"",
2735                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
2736                                   "Anonymous" : cmd->sess->initiator_name);
2737                         goto out_busy;
2738                 }
2739                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
2740                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
2741                         TRACE(TRACE_MGMT_MINOR, "Too many pending device commands "
2742                                 "(%d), returning BUSY to initiator \"%s\"",
2743                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
2744                                   "Anonymous" : cmd->sess->initiator_name);
2745                         goto out_busy;
2746                 }
2747                 if (!cmd->set_sn_on_restart_cmd)
2748                         scst_cmd_set_sn(cmd);
2749         } else if (res < 0) {
2750                 TRACE_DBG("Finishing cmd %p", cmd);
2751                 scst_set_cmd_error(cmd,
2752                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
2753                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2754         } else
2755                 goto out;
2756
2757 out:
2758         TRACE_EXIT_RES(res);
2759         return res;
2760
2761 out_busy:
2762         scst_set_busy(cmd);
2763         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2764         goto out;
2765 }
2766
2767 /* Called under scst_init_lock and IRQs disabled */
2768 static void scst_do_job_init(void)
2769 {
2770         struct scst_cmd *cmd;
2771         int susp;
2772
2773         TRACE_ENTRY();
2774
2775 restart:
2776         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
2777         if (scst_init_poll_cnt > 0)
2778                 scst_init_poll_cnt--;
2779
2780         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
2781                 int rc;
2782                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
2783                         continue;
2784                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2785                         spin_unlock_irq(&scst_init_lock);
2786                         rc = __scst_init_cmd(cmd);
2787                         spin_lock_irq(&scst_init_lock);
2788                         if (rc > 0) {
2789                                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, restarting");
2790                                 goto restart;
2791                         }
2792                 } else {
2793                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
2794                                 cmd, cmd->tag);
2795                         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2796                 }
2797
2798                 /*
2799                  * Deleting cmd from init cmd list after __scst_init_cmd()
2800                  * is necessary to keep the check in scst_init_cmd() correct
2801                  * to preserve the commands order.
2802                  *
2803                  * We don't care about the race, when init cmd list is empty
2804                  * and one command detected that it just was not empty, so
2805                  * it's inserting to it, but another command at the same time
2806                  * seeing init cmd list empty and goes directly, because it
2807                  * could affect only commands from the same initiator to the
2808                  * same tgt_dev, but init_cmd_done() doesn't guarantee the order
2809                  * in case of simultaneous such calls anyway.
2810                  */
2811                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
2812                 list_del(&cmd->cmd_list_entry);
2813                 spin_unlock(&scst_init_lock);
2814
2815                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
2816                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
2817                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2818                         list_add(&cmd->cmd_list_entry,
2819                                 &cmd->cmd_lists->active_cmd_list);
2820                 else
2821                         list_add_tail(&cmd->cmd_list_entry,
2822                                 &cmd->cmd_lists->active_cmd_list);
2823                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
2824                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
2825
2826                 spin_lock(&scst_init_lock);
2827                 goto restart;
2828         }
2829
2830         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
2831                 goto restart;
2832
2833         TRACE_EXIT();
2834         return;
2835 }
2836
2837 static inline int test_init_cmd_list(void)
2838 {
2839         int res = (!list_empty(&scst_init_cmd_list) &&
2840                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
2841                   unlikely(kthread_should_stop()) ||
2842                   (scst_init_poll_cnt > 0);
2843         return res;
2844 }
2845
2846 int scst_init_cmd_thread(void *arg)
2847 {
2848         TRACE_ENTRY();
2849
2850         current->flags |= PF_NOFREEZE;
2851
2852         set_user_nice(current, -20);
2853
2854         spin_lock_irq(&scst_init_lock);
2855         while(!kthread_should_stop()) {
2856                 wait_queue_t wait;
2857                 init_waitqueue_entry(&wait, current);
2858
2859                 if (!test_init_cmd_list()) {
2860                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
2861                                                  &wait);
2862                         for (;;) {
2863                                 set_current_state(TASK_INTERRUPTIBLE);
2864                                 if (test_init_cmd_list())
2865                                         break;
2866                                 spin_unlock_irq(&scst_init_lock);
2867                                 schedule();
2868                                 spin_lock_irq(&scst_init_lock);
2869                         }
2870                         set_current_state(TASK_RUNNING);
2871                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
2872                 }
2873                 scst_do_job_init();
2874         }
2875         spin_unlock_irq(&scst_init_lock);
2876
2877         /*
2878          * If kthread_should_stop() is true, we are guaranteed to be
2879          * on the module unload, so scst_init_cmd_list must be empty.
2880          */
2881         sBUG_ON(!list_empty(&scst_init_cmd_list));
2882
2883         TRACE_EXIT();
2884         return 0;
2885 }
2886
2887 /* Called with no locks held */
2888 void scst_process_active_cmd(struct scst_cmd *cmd, int context)
2889 {
2890         int res;
2891
2892         TRACE_ENTRY();
2893
2894         EXTRACHECKS_BUG_ON(in_irq());
2895
2896         cmd->atomic = (context == SCST_CONTEXT_DIRECT_ATOMIC);
2897
2898         do {
2899                 switch (cmd->state) {
2900                 case SCST_CMD_STATE_PRE_PARSE:
2901                         res = scst_pre_parse(cmd);
2902                         break;
2903
2904                 case SCST_CMD_STATE_DEV_PARSE:
2905                         res = scst_parse_cmd(cmd);
2906                         break;
2907
2908                 case SCST_CMD_STATE_PREPARE_SPACE:
2909                         res = scst_prepare_space(cmd);
2910                         break;
2911
2912                 case SCST_CMD_STATE_RDY_TO_XFER:
2913                         res = scst_rdy_to_xfer(cmd);
2914                         break;
2915
2916                 case SCST_CMD_STATE_TGT_PRE_EXEC:
2917                         res = scst_tgt_pre_exec(cmd);
2918                         break;
2919
2920                 case SCST_CMD_STATE_SEND_TO_MIDLEV:
2921                         if (tm_dbg_check_cmd(cmd) != 0) {
2922                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2923                                 TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
2924                                         "because of TM DBG delay", cmd,
2925                                         cmd->tag);
2926                                 break;
2927                         }
2928                         res = scst_send_to_midlev(&cmd);
2929                         /* !! At this point cmd, sess & tgt_dev can be already freed !! */
2930                         break;
2931
2932                 case SCST_CMD_STATE_PRE_DEV_DONE:
2933                         res = scst_pre_dev_done(cmd);
2934                         break;
2935
2936                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2937                         res = scst_mode_select_checks(cmd);
2938                         break;
2939
2940                 case SCST_CMD_STATE_DEV_DONE:
2941                         res = scst_dev_done(cmd);
2942                         break;
2943
2944                 case SCST_CMD_STATE_PRE_XMIT_RESP:
2945                         res = scst_pre_xmit_response(cmd);
2946                         break;
2947
2948                 case SCST_CMD_STATE_XMIT_RESP:
2949                         res = scst_xmit_response(cmd);
2950                         break;
2951
2952                 case SCST_CMD_STATE_FINISHED:
2953                         res = scst_finish_cmd(cmd);
2954                         break;
2955
2956                 default:
2957                         PRINT_ERROR("cmd (%p) in state %d, but shouldn't be",
2958                                cmd, cmd->state);
2959                         sBUG();
2960                         res = SCST_CMD_STATE_RES_CONT_NEXT;
2961                         break;
2962                 }
2963         } while(res == SCST_CMD_STATE_RES_CONT_SAME);
2964
2965         if (res == SCST_CMD_STATE_RES_CONT_NEXT) {
2966                 /* None */
2967         } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
2968                 spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
2969                 switch (cmd->state) {
2970                 case SCST_CMD_STATE_PRE_PARSE:
2971                 case SCST_CMD_STATE_DEV_PARSE:
2972                 case SCST_CMD_STATE_PREPARE_SPACE:
2973                 case SCST_CMD_STATE_RDY_TO_XFER:
2974                 case SCST_CMD_STATE_TGT_PRE_EXEC:
2975                 case SCST_CMD_STATE_SEND_TO_MIDLEV:
2976                 case SCST_CMD_STATE_PRE_DEV_DONE:
2977                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2978                 case SCST_CMD_STATE_DEV_DONE:
2979                 case SCST_CMD_STATE_PRE_XMIT_RESP:
2980                 case SCST_CMD_STATE_XMIT_RESP:
2981                 case SCST_CMD_STATE_FINISHED:
2982                         TRACE_DBG("Adding cmd %p to head of active cmd list", cmd);
2983                         list_add(&cmd->cmd_list_entry,
2984                                 &cmd->cmd_lists->active_cmd_list);
2985                         break;
2986 #ifdef EXTRACHECKS
2987                 /* not very valid commands */
2988                 case SCST_CMD_STATE_DEFAULT:
2989                 case SCST_CMD_STATE_NEED_THREAD_CTX:
2990                         PRINT_ERROR("cmd %p is in state %d, not putting on "
2991                                 "useful list (left on scst cmd list)", cmd, 
2992                                 cmd->state);
2993                         spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
2994                         sBUG();
2995                         spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
2996                         break;
2997 #endif
2998                 default:
2999                         break;
3000                 }
3001                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3002                 spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3003         } else
3004                 sBUG();
3005
3006         TRACE_EXIT();
3007         return;
3008 }
3009
3010 /* Called under cmd_list_lock and IRQs disabled */
3011 static void scst_do_job_active(struct list_head *cmd_list,
3012         spinlock_t *cmd_list_lock, int context)
3013 {
3014         TRACE_ENTRY();
3015
3016 #ifdef EXTRACHECKS
3017         WARN_ON((context != SCST_CONTEXT_DIRECT_ATOMIC) && 
3018                 (context != SCST_CONTEXT_DIRECT));
3019 #endif
3020
3021         while (!list_empty(cmd_list)) {
3022                 struct scst_cmd *cmd = list_entry(cmd_list->next, typeof(*cmd),
3023                                         cmd_list_entry);
3024                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
3025                 list_del(&cmd->cmd_list_entry);
3026                 spin_unlock_irq(cmd_list_lock);
3027                 scst_process_active_cmd(cmd, context);
3028                 spin_lock_irq(cmd_list_lock);
3029         }
3030
3031         TRACE_EXIT();
3032         return;
3033 }
3034
3035 static inline int test_cmd_lists(struct scst_cmd_lists *p_cmd_lists)
3036 {
3037         int res = !list_empty(&p_cmd_lists->active_cmd_list) ||
3038             unlikely(kthread_should_stop()) ||
3039             tm_dbg_is_release();
3040         return res;
3041 }
3042
3043 int scst_cmd_thread(void *arg)
3044 {
3045         struct scst_cmd_lists *p_cmd_lists = (struct scst_cmd_lists*)arg;
3046
3047         TRACE_ENTRY();
3048
3049 #if 0
3050         set_user_nice(current, 10);
3051 #endif
3052         current->flags |= PF_NOFREEZE;
3053
3054         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3055         while (!kthread_should_stop()) {
3056                 wait_queue_t wait;
3057                 init_waitqueue_entry(&wait, current);
3058
3059                 if (!test_cmd_lists(p_cmd_lists)) {
3060                         add_wait_queue_exclusive(&p_cmd_lists->cmd_list_waitQ,
3061                                 &wait);
3062                         for (;;) {
3063                                 set_current_state(TASK_INTERRUPTIBLE);
3064                                 if (test_cmd_lists(p_cmd_lists))
3065                                         break;
3066                                 spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3067                                 schedule();
3068                                 spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3069                         }
3070                         set_current_state(TASK_RUNNING);
3071                         remove_wait_queue(&p_cmd_lists->cmd_list_waitQ, &wait);
3072                 }
3073
3074                 if (tm_dbg_is_release()) {
3075                         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3076                         tm_dbg_check_released_cmds();
3077                         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3078                 }
3079
3080                 scst_do_job_active(&p_cmd_lists->active_cmd_list,
3081                         &p_cmd_lists->cmd_list_lock, SCST_CONTEXT_DIRECT);
3082         }
3083         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3084
3085 #ifdef EXTRACHECKS
3086         /*
3087          * If kthread_should_stop() is true, we are guaranteed to be either
3088          * on the module unload, or there must be at least one other thread to
3089          * process the commands lists.
3090          */
3091         if (p_cmd_lists == &scst_main_cmd_lists) {
3092                 sBUG_ON((scst_threads_info.nr_cmd_threads == 1) &&
3093                          !list_empty(&scst_main_cmd_lists.active_cmd_list));
3094         }
3095 #endif
3096
3097         TRACE_EXIT();
3098         return 0;
3099 }
3100
3101 void scst_cmd_tasklet(long p)
3102 {
3103         struct scst_tasklet *t = (struct scst_tasklet*)p;
3104
3105         TRACE_ENTRY();
3106
3107         spin_lock_irq(&t->tasklet_lock);
3108         scst_do_job_active(&t->tasklet_cmd_list, &t->tasklet_lock,
3109                 SCST_CONTEXT_DIRECT_ATOMIC);
3110         spin_unlock_irq(&t->tasklet_lock);
3111
3112         TRACE_EXIT();
3113         return;
3114 }
3115
3116 /*
3117  * Returns 0 on success, < 0 if there is no device handler or
3118  * > 0 if SCST_FLAG_SUSPENDED set and SCST_FLAG_SUSPENDING - not.
3119  * No locks, protection is done by the suspended activity.
3120  */
3121 static int scst_mgmt_translate_lun(struct scst_mgmt_cmd *mcmd)
3122 {
3123         struct scst_tgt_dev *tgt_dev = NULL;
3124         struct list_head *sess_tgt_dev_list_head;
3125         int res = -1;
3126
3127         TRACE_ENTRY();
3128
3129         TRACE_DBG("Finding tgt_dev for mgmt cmd %p (lun %Ld)", mcmd,
3130               (uint64_t)mcmd->lun);
3131
3132         __scst_get(1);
3133
3134         if (unlikely(test_bit(SCST_FLAG_SUSPENDED, &scst_flags) &&
3135                      !test_bit(SCST_FLAG_SUSPENDING, &scst_flags))) {
3136                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3137                 __scst_put();
3138                 res = 1;
3139                 goto out;
3140         }
3141
3142         sess_tgt_dev_list_head =
3143                 &mcmd->sess->sess_tgt_dev_list_hash[HASH_VAL(mcmd->lun)];
3144         list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3145                         sess_tgt_dev_list_entry) {
3146                 if (tgt_dev->lun == mcmd->lun) {
3147                         TRACE_DBG("tgt_dev %p found", tgt_dev);
3148                         mcmd->mcmd_tgt_dev = tgt_dev;
3149                         res = 0;
3150                         break;
3151                 }
3152         }
3153         if (mcmd->mcmd_tgt_dev == NULL)
3154                 __scst_put();
3155
3156 out:
3157         TRACE_EXIT_HRES(res);
3158         return res;
3159 }
3160
3161 /* No locks */
3162 void scst_complete_cmd_mgmt(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd)
3163 {
3164         TRACE_ENTRY();
3165
3166         spin_lock_irq(&scst_mcmd_lock);
3167
3168         TRACE_MGMT_DBG("cmd %p completed (tag %llu, mcmd %p, "
3169                 "mcmd->cmd_wait_count %d)", cmd, cmd->tag, mcmd,
3170                 mcmd->cmd_wait_count);
3171
3172         cmd->mgmt_cmnd = NULL;
3173
3174         if (cmd->completed)
3175                 mcmd->completed_cmd_count++;
3176
3177         mcmd->cmd_wait_count--;
3178         if (mcmd->cmd_wait_count > 0) {
3179                 TRACE_MGMT_DBG("cmd_wait_count(%d) not 0, skipping",
3180                         mcmd->cmd_wait_count);
3181                 goto out_unlock;
3182         }
3183
3184         mcmd->state = SCST_MGMT_CMD_STATE_DONE;
3185
3186         if (mcmd->completed) {
3187                 TRACE_MGMT_DBG("Adding mgmt cmd %p to active mgmt cmd list",
3188                         mcmd);
3189                 list_add_tail(&mcmd->mgmt_cmd_list_entry,
3190                         &scst_active_mgmt_cmd_list);
3191         }
3192
3193         spin_unlock_irq(&scst_mcmd_lock);
3194
3195         wake_up(&scst_mgmt_cmd_list_waitQ);
3196
3197 out:
3198         TRACE_EXIT();
3199         return;
3200
3201 out_unlock:
3202         spin_unlock_irq(&scst_mcmd_lock);
3203         goto out;
3204 }
3205
3206 static int scst_call_dev_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
3207         struct scst_tgt_dev *tgt_dev, int set_status)
3208 {
3209         int res = SCST_DEV_TM_NOT_COMPLETED;
3210         struct scst_dev_type *h = tgt_dev->dev->handler;
3211
3212         if (h->task_mgmt_fn) {
3213                 TRACE_MGMT_DBG("Calling dev handler %s task_mgmt_fn(fn=%d)",
3214                         h->name, mcmd->fn);
3215                 EXTRACHECKS_BUG_ON(in_irq());
3216                 res = h->task_mgmt_fn(mcmd, tgt_dev);
3217                 TRACE_MGMT_DBG("Dev handler %s task_mgmt_fn() returned %d",
3218                       h->name, res);
3219                 if (set_status && (res != SCST_DEV_TM_NOT_COMPLETED))
3220                         mcmd->status = res;
3221         }
3222         return res;
3223 }
3224
3225 static inline int scst_is_strict_mgmt_fn(int mgmt_fn)
3226 {
3227         switch(mgmt_fn) {
3228                 case SCST_ABORT_TASK:
3229 #if 0
3230                 case SCST_ABORT_TASK_SET:
3231                 case SCST_CLEAR_TASK_SET:
3232 #endif
3233                         return 1;
3234                 default:
3235                         return 0;
3236         }
3237 }
3238
3239 /* 
3240  * Might be called under sess_list_lock and IRQ off + BHs also off
3241  * Returns -1 if command is being executed (ABORT failed), 0 otherwise
3242  */
3243 void scst_abort_cmd(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd,
3244         int other_ini, int call_dev_task_mgmt_fn)
3245 {
3246         TRACE_ENTRY();
3247
3248         TRACE(((mcmd != NULL) && (mcmd->fn == SCST_ABORT_TASK)) ? TRACE_MGMT_MINOR : TRACE_MGMT,
3249                 "Aborting cmd %p (tag %llu)", cmd, cmd->tag);
3250
3251         if (other_ini) {
3252                 set_bit(SCST_CMD_ABORTED_OTHER, &cmd->cmd_flags);
3253                 smp_mb__after_set_bit();
3254         } else {
3255                 /* Might be necessary if command aborted several times */
3256                 clear_bit(SCST_CMD_ABORTED_OTHER, &cmd->cmd_flags);
3257         }
3258         set_bit(SCST_CMD_ABORTED, &cmd->cmd_flags);
3259         smp_mb__after_set_bit();
3260
3261         if (cmd->tgt_dev == NULL) {
3262                 unsigned long flags;
3263                 spin_lock_irqsave(&scst_init_lock, flags);
3264                 scst_init_poll_cnt++;
3265                 spin_unlock_irqrestore(&scst_init_lock, flags);
3266                 wake_up(&scst_init_cmd_list_waitQ);
3267         }
3268
3269         if (call_dev_task_mgmt_fn && (cmd->tgt_dev != NULL)) {
3270                 EXTRACHECKS_BUG_ON(irqs_disabled());
3271                 scst_call_dev_task_mgmt_fn(mcmd, cmd->tgt_dev, 1);
3272         }
3273
3274         if (mcmd) {
3275                 unsigned long flags;
3276                 /*
3277                  * Delay the response until the command's finish in
3278                  * order to guarantee that "no further responses from
3279                  * the task are sent to the SCSI initiator port" after
3280                  * response from the TM function is sent (SAM). Plus,
3281                  * we must wait here to be sure that we won't receive
3282                  * double commands with the same tag.
3283                  */
3284                 TRACE((mcmd->fn == SCST_ABORT_TASK) ? TRACE_MGMT_MINOR : TRACE_MGMT,
3285                         "cmd %p (tag %llu) being executed/xmitted (state %d), "
3286                         "deferring ABORT...", cmd, cmd->tag, cmd->state);
3287 #ifdef EXTRACHECKS
3288                 if (cmd->mgmt_cmnd) {
3289                         printk(KERN_ALERT "cmd %p (tag %llu, state %d) "
3290                                 "has non-NULL mgmt_cmnd %p!!! Current "
3291                                 "mcmd %p\n", cmd, cmd->tag, cmd->state,
3292                                 cmd->mgmt_cmnd, mcmd);
3293                 }
3294 #endif
3295                 sBUG_ON(cmd->mgmt_cmnd);
3296                 spin_lock_irqsave(&scst_mcmd_lock, flags);
3297                 mcmd->cmd_wait_count++;
3298                 spin_unlock_irqrestore(&scst_mcmd_lock, flags);
3299                 /* cmd can't die here or sess_list_lock already taken */
3300                 cmd->mgmt_cmnd = mcmd;
3301         }
3302
3303         tm_dbg_release_cmd(cmd);
3304
3305         TRACE_EXIT();
3306         return;
3307 }
3308
3309 /* No locks */
3310 static int scst_set_mcmd_next_state(struct scst_mgmt_cmd *mcmd)
3311 {
3312         int res;
3313         spin_lock_irq(&scst_mcmd_lock);
3314         if (mcmd->cmd_wait_count != 0) {
3315                 TRACE_MGMT_DBG("cmd_wait_count(%d) not 0, preparing to "
3316                         "wait", mcmd->cmd_wait_count);
3317                 mcmd->state = SCST_MGMT_CMD_STATE_EXECUTING;
3318                 res = -1;
3319         } else {
3320                 mcmd->state = SCST_MGMT_CMD_STATE_DONE;
3321                 res = 0;
3322         }
3323         mcmd->completed = 1;
3324         spin_unlock_irq(&scst_mcmd_lock);
3325         return res;
3326 }
3327
3328 static int __scst_check_unblock_aborted_cmd(struct scst_cmd *cmd)
3329 {
3330         int res;
3331         if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
3332                 TRACE_MGMT_DBG("Adding aborted blocked cmd %p to active cmd "
3333                         "list", cmd);
3334                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
3335                 list_add_tail(&cmd->cmd_list_entry,
3336                         &cmd->cmd_lists->active_cmd_list);
3337                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3338                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
3339                 res = 1;
3340         } else
3341                 res = 0;
3342         return res;
3343 }
3344
3345 static void scst_unblock_aborted_cmds(int scst_mutex_held)
3346 {
3347         struct scst_device *dev;
3348
3349         TRACE_ENTRY();
3350
3351         if (!scst_mutex_held)
3352                 mutex_lock(&scst_mutex);
3353
3354         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
3355                 struct scst_cmd *cmd, *tcmd;
3356                 struct scst_tgt_dev *tgt_dev;
3357                 spin_lock_bh(&dev->dev_lock);
3358                 local_irq_disable();
3359                 list_for_each_entry_safe(cmd, tcmd, &dev->blocked_cmd_list,
3360                                         blocked_cmd_list_entry) {
3361                         if (__scst_check_unblock_aborted_cmd(cmd))
3362                                 list_del(&cmd->blocked_cmd_list_entry);
3363                 }
3364                 local_irq_enable();
3365                 spin_unlock_bh(&dev->dev_lock);
3366
3367                 local_irq_disable();
3368                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
3369                                          dev_tgt_dev_list_entry) {
3370                         spin_lock(&tgt_dev->sn_lock);
3371                         list_for_each_entry_safe(cmd, tcmd,
3372                                         &tgt_dev->deferred_cmd_list,
3373                                         sn_cmd_list_entry) {
3374                                 if (__scst_check_unblock_aborted_cmd(cmd)) {
3375                                         TRACE_MGMT_DBG("Deleting aborted SN "
3376                                                 "cmd %p from SN list", cmd);
3377                                         tgt_dev->def_cmd_count--;
3378                                         list_del(&cmd->sn_cmd_list_entry);
3379                                 }
3380                         }
3381                         spin_unlock(&tgt_dev->sn_lock);
3382                 }
3383                 local_irq_enable();
3384         }
3385
3386         if (!scst_mutex_held)
3387                 mutex_unlock(&scst_mutex);
3388
3389         TRACE_EXIT();
3390         return;
3391 }
3392
3393 static void __scst_abort_task_set(struct scst_mgmt_cmd *mcmd,
3394         struct scst_tgt_dev *tgt_dev, int other_ini, int scst_mutex_held)
3395 {
3396         struct scst_cmd *cmd;
3397         struct scst_session *sess = tgt_dev->sess;
3398
3399         TRACE_ENTRY();
3400
3401         spin_lock_irq(&sess->sess_list_lock);
3402
3403         TRACE_DBG("Searching in search cmd list (sess=%p)", sess);
3404         list_for_each_entry(cmd, &sess->search_cmd_list, 
3405                         search_cmd_list_entry) {
3406                 if ((cmd->tgt_dev == tgt_dev) ||
3407                     ((cmd->tgt_dev == NULL) && 
3408                      (cmd->lun == tgt_dev->lun))) {
3409                         if (mcmd->cmd_sn_set) {
3410                                 sBUG_ON(!cmd->tgt_sn_set);
3411                                 if (scst_sn_before(mcmd->cmd_sn, cmd->tgt_sn) ||
3412                                     (mcmd->cmd_sn == cmd->tgt_sn))
3413                                         continue;
3414                         }
3415                         scst_abort_cmd(cmd, mcmd, other_ini, 0);
3416                 }
3417         }
3418         spin_unlock_irq(&sess->sess_list_lock);
3419
3420         scst_unblock_aborted_cmds(scst_mutex_held);
3421
3422         TRACE_EXIT();
3423         return;
3424 }
3425
3426 /* Returns 0 if the command processing should be continued, <0 otherwise */
3427 static int scst_abort_task_set(struct scst_mgmt_cmd *mcmd)
3428 {
3429         int res;
3430         struct scst_tgt_dev *tgt_dev = mcmd->mcmd_tgt_dev;
3431         struct scst_device *dev = tgt_dev->dev;
3432
3433         TRACE(TRACE_MGMT, "Aborting task set (lun=%Ld, mcmd=%p)",
3434                 tgt_dev->lun, mcmd);
3435
3436         mcmd->needs_unblocking = 1;
3437
3438         spin_lock_bh(&dev->dev_lock);
3439         __scst_block_dev(dev);
3440         spin_unlock_bh(&dev->dev_lock);
3441
3442         __scst_abort_task_set(mcmd, tgt_dev, 0, 0);
3443         scst_call_dev_task_mgmt_fn(mcmd, tgt_dev, 0);
3444
3445         res = scst_set_mcmd_next_state(mcmd);
3446
3447         TRACE_EXIT_RES(res);
3448         return res;
3449 }
3450
3451 static int scst_is_cmd_belongs_to_dev(struct scst_cmd *cmd,
3452         struct scst_device *dev)
3453 {
3454         struct scst_tgt_dev *tgt_dev = NULL;
3455         struct list_head *sess_tgt_dev_list_head;
3456         int res = 0;
3457
3458         TRACE_ENTRY();
3459
3460         TRACE_DBG("Finding match for dev %p and cmd %p (lun %Ld)", dev, cmd,
3461               (uint64_t)cmd->lun);
3462
3463         sess_tgt_dev_list_head =
3464                 &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
3465         list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3466                         sess_tgt_dev_list_entry) {
3467                 if (tgt_dev->lun == cmd->lun) {
3468                         TRACE_DBG("dev %p found", tgt_dev->dev);
3469                         res = (tgt_dev->dev == dev);
3470                         goto out;
3471                 }
3472         }
3473
3474 out:
3475         TRACE_EXIT_HRES(res);
3476         return res;
3477 }
3478
3479 /* Returns 0 if the command processing should be continued, <0 otherwise */
3480 static int scst_clear_task_set(struct scst_mgmt_cmd *mcmd)
3481 {
3482         int res;
3483         struct scst_device *dev = mcmd->mcmd_tgt_dev->dev;
3484         struct scst_tgt_dev *tgt_dev;
3485         LIST_HEAD(UA_tgt_devs);
3486
3487         TRACE_ENTRY();
3488
3489         TRACE(TRACE_MGMT, "Clearing task set (lun=%Ld, mcmd=%p)",
3490                 (uint64_t)mcmd->lun, mcmd);
3491
3492         mcmd->needs_unblocking = 1;
3493
3494         spin_lock_bh(&dev->dev_lock);
3495         __scst_block_dev(dev);
3496         spin_unlock_bh(&dev->dev_lock);
3497
3498         __scst_abort_task_set(mcmd, mcmd->mcmd_tgt_dev, 0, 0);
3499
3500         list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list, 
3501                 dev_tgt_dev_list_entry) 
3502         {
3503                 struct scst_session *sess = tgt_dev->sess;
3504                 struct scst_cmd *cmd;
3505                 int aborted = 0;
3506
3507                 if (tgt_dev == mcmd->mcmd_tgt_dev)
3508                         continue;
3509
3510                 spin_lock_irq(&sess->sess_list_lock);
3511
3512                 TRACE_DBG("Searching in search cmd list (sess=%p)", sess);
3513                 list_for_each_entry(cmd, &sess->search_cmd_list, 
3514                                 search_cmd_list_entry) {
3515                         if ((cmd->dev == dev) ||
3516                             ((cmd->dev == NULL) && 
3517                              scst_is_cmd_belongs_to_dev(cmd, dev))) {
3518                                 scst_abort_cmd(cmd, mcmd, 1, 0);
3519                                 aborted = 1;
3520                         }
3521                 }
3522                 spin_unlock_irq(&sess->sess_list_lock);
3523
3524                 if (aborted)
3525                         list_add_tail(&tgt_dev->extra_tgt_dev_list_entry,
3526                                         &UA_tgt_devs);
3527         }
3528
3529         scst_unblock_aborted_cmds(0);
3530
3531         if (!dev->tas) {
3532                 list_for_each_entry(tgt_dev, &UA_tgt_devs, extra_tgt_dev_list_entry) {
3533                         spin_lock_bh(&scst_temp_UA_lock);
3534                         scst_set_sense(scst_temp_UA, sizeof(scst_temp_UA),
3535                                 SCST_LOAD_SENSE(scst_sense_cleared_by_another_ini_UA));
3536                         scst_check_set_UA(tgt_dev, scst_temp_UA, sizeof(scst_temp_UA), 0);
3537                         spin_unlock_bh(&scst_temp_UA_lock);
3538                 }
3539         }
3540
3541         scst_call_dev_task_mgmt_fn(mcmd, mcmd->mcmd_tgt_dev, 0);
3542
3543         res = scst_set_mcmd_next_state(mcmd);
3544
3545         TRACE_EXIT_RES(res);
3546         return res;
3547 }