- Update to the latest IET r145
[mirror/scst/.git] / scst / src / scst_targ.c
1 /*
2  *  scst_targ.c
3  *  
4  *  Copyright (C) 2004-2007 Vladislav Bolkhovitin <vst@vlnb.net>
5  *                 and Leonid Stoljar
6  *  
7  *  This program is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU General Public License
9  *  as published by the Free Software Foundation, version 2
10  *  of the License.
11  * 
12  *  This program is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  *  GNU General Public License for more details.
16  */
17
18 #include <linux/init.h>
19 #include <linux/kernel.h>
20 #include <linux/errno.h>
21 #include <linux/list.h>
22 #include <linux/spinlock.h>
23 #include <linux/slab.h>
24 #include <linux/sched.h>
25 #include <linux/smp_lock.h>
26 #include <asm/unistd.h>
27 #include <asm/string.h>
28 #include <linux/kthread.h>
29 #include <linux/delay.h>
30
31 #include "scsi_tgt.h"
32 #include "scst_priv.h"
33
34 static void scst_cmd_set_sn(struct scst_cmd *cmd);
35 static int __scst_init_cmd(struct scst_cmd *cmd);
36
37 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
38 {
39         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
40         unsigned long flags;
41
42         spin_lock_irqsave(&t->tasklet_lock, flags);
43         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
44                 smp_processor_id());
45         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
46         spin_unlock_irqrestore(&t->tasklet_lock, flags);
47
48         tasklet_schedule(&t->tasklet);
49 }
50
51 /* 
52  * Must not be called in parallel with scst_unregister_session() for the 
53  * same sess
54  */
55 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
56                              const uint8_t *lun, int lun_len,
57                              const uint8_t *cdb, int cdb_len, int atomic)
58 {
59         struct scst_cmd *cmd;
60
61         TRACE_ENTRY();
62
63 #ifdef EXTRACHECKS
64         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
65                 PRINT_ERROR("%s", "New cmd while shutting down the session");
66                 sBUG();
67         }
68 #endif
69
70         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
71         if (cmd == NULL)
72                 goto out;
73
74         cmd->sess = sess;
75         cmd->tgt = sess->tgt;
76         cmd->tgtt = sess->tgt->tgtt;
77
78         /* 
79          * For both wrong lun and CDB defer the error reporting for
80          * scst_cmd_init_done()
81          */
82
83         cmd->lun = scst_unpack_lun(lun, lun_len);
84
85         if (cdb_len <= SCST_MAX_CDB_SIZE) {
86                 memcpy(cmd->cdb, cdb, cdb_len);
87                 cmd->cdb_len = cdb_len;
88         }
89
90         TRACE_DBG("cmd %p, sess %p", cmd, sess);
91         scst_sess_get(sess);
92
93 out:
94         TRACE_EXIT();
95         return cmd;
96 }
97
98 static int scst_init_cmd(struct scst_cmd *cmd, int context)
99 {
100         int rc;
101
102         TRACE_ENTRY();
103
104         /* See the comment in scst_do_job_init() */
105         if (unlikely(!list_empty(&scst_init_cmd_list))) {
106                 TRACE_MGMT_DBG("%s", "init cmd list busy");
107                 goto out_redirect;
108         }
109         smp_rmb();
110
111         rc = __scst_init_cmd(cmd);
112         if (unlikely(rc > 0))
113                 goto out_redirect;
114         else if (unlikely(rc != 0))
115                 goto out;
116
117         /* Small context optimization */
118         if (((context == SCST_CONTEXT_TASKLET) ||
119              (context == SCST_CONTEXT_DIRECT_ATOMIC)) && 
120             scst_cmd_is_expected_set(cmd)) {
121                 if (cmd->expected_data_direction == SCST_DATA_WRITE) {
122                         if ( !test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
123                                         &cmd->tgt_dev->tgt_dev_flags))
124                                 context = SCST_CONTEXT_THREAD;
125                 } else {
126                         if ( !test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
127                                         &cmd->tgt_dev->tgt_dev_flags))
128                                 context = SCST_CONTEXT_THREAD;
129                 }
130         }
131
132 out:
133         TRACE_EXIT_RES(context);
134         return context;
135
136 out_redirect:
137         if (cmd->preprocessing_only) {
138                 /*
139                  * Poor man solution for single threaded targets, where 
140                  * blocking receiver at least sometimes means blocking all.
141                  */
142                 sBUG_ON(context != SCST_CONTEXT_DIRECT);
143                 scst_set_busy(cmd);
144                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
145                 /* Keep initiator away from too many BUSY commands */
146                 if (!in_interrupt() && !in_atomic())
147                         msleep(50);
148                 else
149                         WARN_ON_ONCE(1);
150         } else {
151                 unsigned long flags;
152                 spin_lock_irqsave(&scst_init_lock, flags);
153                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
154                         "%d)", cmd, atomic_read(&scst_cmd_count));
155                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
156                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
157                         scst_init_poll_cnt++;
158                 spin_unlock_irqrestore(&scst_init_lock, flags);
159                 wake_up(&scst_init_cmd_list_waitQ);
160                 context = -1;
161         }
162         goto out;
163 }
164
165 void scst_cmd_init_done(struct scst_cmd *cmd, int pref_context)
166 {
167         unsigned long flags;
168         struct scst_session *sess = cmd->sess;
169
170         TRACE_ENTRY();
171
172         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
173         TRACE(TRACE_SCSI, "tag=%llu, lun=%Ld, CDB len=%d", cmd->tag, 
174                 (uint64_t)cmd->lun, cmd->cdb_len);
175         TRACE_BUFF_FLAG(TRACE_SCSI|TRACE_RECV_BOT, "Recieving CDB",
176                 cmd->cdb, cmd->cdb_len);
177
178 #ifdef EXTRACHECKS
179         if (unlikely(in_irq()) && ((pref_context == SCST_CONTEXT_DIRECT) ||
180                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
181         {
182                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
183                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
184                         cmd->tgtt->name);
185                 pref_context = SCST_CONTEXT_TASKLET;
186         }
187 #endif
188
189         atomic_inc(&sess->sess_cmd_count);
190
191         spin_lock_irqsave(&sess->sess_list_lock, flags);
192
193         list_add_tail(&cmd->search_cmd_list_entry, &sess->search_cmd_list);
194
195         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
196                 switch(sess->init_phase) {
197                 case SCST_SESS_IPH_SUCCESS:
198                         break;
199                 case SCST_SESS_IPH_INITING:
200                         TRACE_DBG("Adding cmd %p to init deferred cmd list", cmd);
201                         list_add_tail(&cmd->cmd_list_entry, 
202                                 &sess->init_deferred_cmd_list);
203                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
204                         goto out;
205                 case SCST_SESS_IPH_FAILED:
206                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
207                         scst_set_busy(cmd);
208                         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
209                         goto active;
210                 default:
211                         sBUG();
212                 }
213         }
214
215         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
216
217         if (unlikely(cmd->lun == (lun_t)-1)) {
218                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
219                 scst_set_cmd_error(cmd,
220                         SCST_LOAD_SENSE(scst_sense_lun_not_supported));
221                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
222                 goto active;
223         }
224
225         if (unlikely(cmd->cdb_len == 0)) {
226                 PRINT_ERROR("Wrong CDB len %d, finishing cmd", 0);
227                 scst_set_cmd_error(cmd,
228                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
229                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
230                 goto active;
231         }
232
233         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
234                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
235                 scst_set_cmd_error(cmd,
236                         SCST_LOAD_SENSE(scst_sense_invalid_message));
237                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
238                 goto active;
239         }
240
241         cmd->state = SCST_CMD_STATE_INIT;
242         /* cmd must be inited here to keep the order */
243         pref_context = scst_init_cmd(cmd, pref_context);
244         if (unlikely(pref_context < 0))
245                 goto out;
246
247 active:
248         /* Here cmd must not be in any cmd list, no locks */
249         switch (pref_context) {
250         case SCST_CONTEXT_TASKLET:
251                 scst_schedule_tasklet(cmd);
252                 break;
253
254         case SCST_CONTEXT_DIRECT:
255         case SCST_CONTEXT_DIRECT_ATOMIC:
256                 scst_process_active_cmd(cmd, pref_context);
257                 /* For *NEED_THREAD wake_up() is already done */
258                 break;
259
260         default:
261                 PRINT_ERROR("Context %x is undefined, using the thread one",
262                         pref_context);
263                 /* go through */
264         case SCST_CONTEXT_THREAD:
265                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
266                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
267                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
268                         list_add(&cmd->cmd_list_entry,
269                                 &cmd->cmd_lists->active_cmd_list);
270                 else
271                         list_add_tail(&cmd->cmd_list_entry,
272                                 &cmd->cmd_lists->active_cmd_list);
273                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
274                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
275                 break;
276         }
277
278 out:
279         TRACE_EXIT();
280         return;
281 }
282
283 static int scst_pre_parse(struct scst_cmd *cmd)
284 {
285         int res = SCST_CMD_STATE_RES_CONT_SAME;
286         struct scst_device *dev = cmd->dev;
287
288         TRACE_ENTRY();
289
290         cmd->inc_expected_sn_on_done = !dev->has_own_order_mgmt;
291
292         sBUG_ON(cmd->internal);
293
294         /*
295          * Expected transfer data supplied by the SCSI transport via the
296          * target driver are untrusted, so we prefer to fetch them from CDB.
297          * Additionally, not all transports support supplying the expected
298          * transfer data.
299          */
300
301         if (unlikely(scst_get_cdb_info(cmd->cdb, dev->handler->type, 
302                         &cmd->op_flags, &cmd->data_direction,
303                         &cmd->bufflen, &cmd->cdb_len, &cmd->op_name) != 0)) {
304                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
305                         "Should you update scst_scsi_op_table?",
306                         cmd->cdb[0], dev->handler->name);
307 #ifdef USE_EXPECTED_VALUES
308                 if (scst_cmd_is_expected_set(cmd)) {
309                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
310                                 "direction %d, transfer_len %d",
311                                 cmd->expected_data_direction,
312                                 cmd->expected_transfer_len);
313                         cmd->data_direction = cmd->expected_data_direction;
314                         
315                         cmd->bufflen = cmd->expected_transfer_len;
316                         /* Restore (likely) lost CDB length */
317                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
318                         if (cmd->cdb_len == -1) {
319                                 PRINT_ERROR("Unable to get CDB length for "
320                                         "opcode 0x%02x. Returning INVALID "
321                                         "OPCODE", cmd->cdb[0]);
322                                 scst_set_cmd_error(cmd,
323                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
324                                 goto out_xmit;
325                         }
326                 } else {
327                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
328                              "target %s not supplied expected values",
329                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
330                         scst_set_cmd_error(cmd,
331                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
332                         goto out_xmit;
333                 }
334 #else
335                 scst_set_cmd_error(cmd,
336                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
337                 goto out_xmit;
338 #endif
339         } else {
340                 TRACE(TRACE_SCSI, "op_name <%s>, direction=%d (expected %d, "
341                         "set %s), transfer_len=%d (expected len %d), flags=%d",
342                         cmd->op_name, cmd->data_direction,
343                         cmd->expected_data_direction,
344                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
345                         cmd->bufflen, cmd->expected_transfer_len, cmd->op_flags);
346
347                 if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
348                         if (scst_cmd_is_expected_set(cmd)) {
349                                 /*
350                                  * Command data length can't be easily
351                                  * determined from the CDB. ToDo, that should 
352                                  * be fixed. Until it's fixed, get it from
353                                  * the supplied expected value, but
354                                  * limit it to some reasonable value (15MB).
355                                  */
356                                 cmd->bufflen = min(cmd->expected_transfer_len,
357                                                         15*1024*1024);
358                         } else
359                                 cmd->bufflen = 0;
360                 }
361         }
362
363         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
364                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
365                             "(opcode 0x%02x)", cmd->cdb[0]);
366                 scst_set_cmd_error(cmd,
367                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
368                 goto out_xmit;
369         }
370
371         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
372                 PRINT_ERROR("Linked commands are not supported "
373                             "(opcode 0x%02x)", cmd->cdb[0]);
374                 scst_set_cmd_error(cmd,
375                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
376                 goto out_xmit;
377         }
378
379         cmd->state = SCST_CMD_STATE_DEV_PARSE;
380
381 out:
382         TRACE_EXIT_RES(res);
383         return res;
384
385 out_xmit:
386         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
387         res = SCST_CMD_STATE_RES_CONT_SAME;
388         goto out;
389 }
390
391 static int scst_parse_cmd(struct scst_cmd *cmd)
392 {
393         int res = SCST_CMD_STATE_RES_CONT_SAME;
394         int state;
395         struct scst_device *dev = cmd->dev;
396         int atomic = scst_cmd_atomic(cmd);
397         int orig_bufflen = cmd->bufflen;
398         scst_data_direction orig_data_direction = cmd->data_direction;
399
400         TRACE_ENTRY();
401
402         if (atomic && !dev->handler->parse_atomic) {
403                 TRACE_DBG("Dev handler %s parse() can not be "
404                       "called in atomic context, rescheduling to the thread",
405                       dev->handler->name);
406                 res = SCST_CMD_STATE_RES_NEED_THREAD;
407                 goto out;
408         }
409
410         if (likely(!scst_is_cmd_local(cmd))) {
411                 TRACE_DBG("Calling dev handler %s parse(%p)",
412                       dev->handler->name, cmd);
413                 TRACE_BUFF_FLAG(TRACE_SEND_BOT, "Parsing: ", cmd->cdb, cmd->cdb_len);
414                 state = dev->handler->parse(cmd);
415                 /* Caution: cmd can be already dead here */
416                 TRACE_DBG("Dev handler %s parse() returned %d",
417                         dev->handler->name, state);
418
419                 switch (state) {
420                 case SCST_CMD_STATE_NEED_THREAD_CTX:
421                         TRACE_DBG("Dev handler %s parse() requested thread "
422                               "context, rescheduling", dev->handler->name);
423                         res = SCST_CMD_STATE_RES_NEED_THREAD;
424                         goto out;
425
426                 case SCST_CMD_STATE_STOP:
427                         TRACE_DBG("Dev handler %s parse() requested stop "
428                                 "processing", dev->handler->name);
429                         res = SCST_CMD_STATE_RES_CONT_NEXT;
430                         goto out;
431                 }
432
433                 if (state == SCST_CMD_STATE_DEFAULT)
434                         state = SCST_CMD_STATE_PREPARE_SPACE;
435         } else
436                 state = SCST_CMD_STATE_PREPARE_SPACE;
437
438         if (cmd->data_len == -1)
439                 cmd->data_len = cmd->bufflen;
440
441         if (cmd->data_buf_alloced && unlikely((orig_bufflen > cmd->bufflen))) {
442                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
443                         "is less, than required (size %d)", cmd->bufflen,
444                         orig_bufflen);
445                 goto out_error;
446         }
447
448         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
449                 goto set_res;
450
451         if (unlikely((cmd->bufflen == 0) &&
452                      (cmd->op_flags & SCST_UNKNOWN_LENGTH))) {
453                 PRINT_ERROR("Unknown data transfer length for opcode 0x%x "
454                         "(handler %s, target %s)", cmd->cdb[0],
455                         dev->handler->name, cmd->tgtt->name);
456                 goto out_error;
457         }
458
459 #ifdef EXTRACHECKS
460         if ((cmd->bufflen != 0) &&
461             ((cmd->data_direction == SCST_DATA_NONE) ||
462              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
463                 PRINT_ERROR("Dev handler %s parse() returned "
464                         "invalid cmd data_direction %d, bufflen %d, state %d "
465                         "or sg %p (opcode 0x%x)", dev->handler->name, 
466                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
467                         cmd->cdb[0]);
468                 goto out_error;
469         }
470 #endif
471
472         if (scst_cmd_is_expected_set(cmd)) {
473 #ifdef USE_EXPECTED_VALUES
474 #       ifdef EXTRACHECKS
475                 if ((cmd->data_direction != cmd->expected_data_direction) ||
476                     (cmd->bufflen != cmd->expected_transfer_len)) {
477                         PRINT_ERROR("Expected values don't match decoded ones: "
478                                 "data_direction %d, expected_data_direction %d, "
479                                 "bufflen %d, expected_transfer_len %d",
480                                 cmd->data_direction, cmd->expected_data_direction,
481                                 cmd->bufflen, cmd->expected_transfer_len);
482                 }
483 #       endif
484                 cmd->data_direction = cmd->expected_data_direction;
485                 cmd->bufflen = cmd->expected_transfer_len;
486 #else
487                 if (unlikely(cmd->data_direction != orig_data_direction)) {
488                         PRINT_ERROR("Expected data direction %d for opcode "
489                                 "0x%02x (handler %s, target %s) doesn't match "
490                                 "decoded value %d", cmd->data_direction,
491                                 cmd->cdb[0], dev->handler->name,
492                                 cmd->tgtt->name, orig_data_direction);
493                         scst_set_cmd_error(cmd,
494                                 SCST_LOAD_SENSE(scst_sense_invalid_message));
495                         goto out_dev_done;
496                 }
497                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
498                         PRINT_INFO("Warning: expected transfer length %d for "
499                                 "opcode 0x%02x (handler %s, target %s) doesn't "
500                                 "match decoded value %d. Faulty initiator "
501                                 "(e.g. VMware is known to be such) or "
502                                 "scst_scsi_op_table should be updated?",
503                                 cmd->expected_transfer_len, cmd->cdb[0],
504                                 dev->handler->name, cmd->tgtt->name,
505                                 cmd->bufflen);
506                         PRINT_BUFFER("Suspicious CDB", cmd->cdb, cmd->cdb_len);
507                 }
508 #endif
509         }
510
511         if ((cmd->data_direction == SCST_DATA_UNKNOWN) ||
512             ((cmd->bufflen == 0) && (cmd->data_direction != SCST_DATA_NONE))) {
513                 PRINT_ERROR("Wrong data direction (%d) or/and buffer "
514                         "length (%d). Opcode 0x%x, handler %s, target %s",
515                         cmd->data_direction, cmd->bufflen, cmd->cdb[0],
516                         dev->handler->name, cmd->tgtt->name);
517                 goto out_error;
518         }
519
520 set_res:
521         switch (state) {
522         case SCST_CMD_STATE_PREPARE_SPACE:
523         case SCST_CMD_STATE_PRE_PARSE:
524         case SCST_CMD_STATE_DEV_PARSE:
525         case SCST_CMD_STATE_RDY_TO_XFER:
526         case SCST_CMD_STATE_TGT_PRE_EXEC:
527         case SCST_CMD_STATE_SEND_TO_MIDLEV:
528         case SCST_CMD_STATE_PRE_DEV_DONE:
529         case SCST_CMD_STATE_DEV_DONE:
530         case SCST_CMD_STATE_PRE_XMIT_RESP:
531         case SCST_CMD_STATE_XMIT_RESP:
532         case SCST_CMD_STATE_FINISHED:
533                 cmd->state = state;
534                 res = SCST_CMD_STATE_RES_CONT_SAME;
535                 break;
536
537         default:
538                 if (state >= 0) {
539                         PRINT_ERROR("Dev handler %s parse() returned "
540                              "invalid cmd state %d (opcode %d)", 
541                              dev->handler->name, state, cmd->cdb[0]);
542                 } else {
543                         PRINT_ERROR("Dev handler %s parse() returned "
544                                 "error %d (opcode %d)", dev->handler->name, 
545                                 state, cmd->cdb[0]);
546                 }
547                 goto out_error;
548         }
549
550         if (cmd->resp_data_len == -1) {
551                 if (cmd->data_direction == SCST_DATA_READ)
552                         cmd->resp_data_len = cmd->bufflen;
553                 else
554                          cmd->resp_data_len = 0;
555         }
556         
557 out:
558         TRACE_EXIT_HRES(res);
559         return res;
560
561 out_error:
562         /* dev_done() will be called as part of the regular cmd's finish */
563         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
564
565 #ifndef USE_EXPECTED_VALUES
566 out_dev_done:
567 #endif
568         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
569         res = SCST_CMD_STATE_RES_CONT_SAME;
570         goto out;
571 }
572
573 static int scst_prepare_space(struct scst_cmd *cmd)
574 {
575         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
576
577         TRACE_ENTRY();
578
579         if (cmd->data_direction == SCST_DATA_NONE)
580                 goto prep_done;
581
582         if (cmd->data_buf_tgt_alloc) {
583                 int orig_bufflen = cmd->bufflen;
584
585                 TRACE_MEM("%s", "Custom tgt data buf allocation requested");
586
587                 r = cmd->tgtt->alloc_data_buf(cmd);
588                 if (r > 0)
589                         goto alloc;
590                 else if (r == 0) {
591                         cmd->data_buf_alloced = 1;
592                         if (unlikely(orig_bufflen < cmd->bufflen)) {
593                                 PRINT_ERROR("Target driver allocated data "
594                                         "buffer (size %d), is less, than "
595                                         "required (size %d)", orig_bufflen,
596                                         cmd->bufflen);
597                                 goto out_error;
598                         }
599                 } else
600                         goto check;
601         }
602
603 alloc:
604         if (!cmd->data_buf_alloced) {
605                 r = scst_alloc_space(cmd);
606         } else {
607                 TRACE_MEM("%s", "data_buf_alloced set, returning");
608         }
609         
610 check:
611         if (r != 0) {
612                 if (scst_cmd_atomic(cmd)) {
613                         TRACE_MEM("%s", "Atomic memory allocation failed, "
614                               "rescheduling to the thread");
615                         res = SCST_CMD_STATE_RES_NEED_THREAD;
616                         goto out;
617                 } else
618                         goto out_no_space;
619         }
620
621 prep_done:
622         if (cmd->preprocessing_only) {
623                 if (scst_cmd_atomic(cmd) && 
624                     !cmd->tgtt->preprocessing_done_atomic) {
625                         TRACE_DBG("%s", "preprocessing_done() can not be "
626                               "called in atomic context, rescheduling to "
627                               "the thread");
628                         res = SCST_CMD_STATE_RES_NEED_THREAD;
629                         goto out;
630                 }
631
632                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
633                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
634                                 "cmd %p", cmd);
635                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
636                         res = SCST_CMD_STATE_RES_CONT_SAME;
637                         goto out;
638                 }
639
640                 res = SCST_CMD_STATE_RES_CONT_NEXT;
641                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
642
643                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
644                 cmd->tgtt->preprocessing_done(cmd);
645                 TRACE_DBG("%s", "preprocessing_done() returned");
646                 goto out;
647
648         }
649
650         switch (cmd->data_direction) {
651         case SCST_DATA_WRITE:
652                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
653                 break;
654
655         default:
656                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
657                 break;
658         }
659
660 out:
661         TRACE_EXIT_HRES(res);
662         return res;
663
664 out_no_space:
665         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
666                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
667         scst_set_busy(cmd);
668         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
669         res = SCST_CMD_STATE_RES_CONT_SAME;
670         goto out;
671
672 out_error:
673         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
674         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
675         res = SCST_CMD_STATE_RES_CONT_SAME;
676         goto out;
677 }
678
679 void scst_restart_cmd(struct scst_cmd *cmd, int status, int pref_context)
680 {
681         TRACE_ENTRY();
682
683         TRACE_DBG("Preferred context: %d", pref_context);
684         TRACE_DBG("tag=%llu, status=%#x", scst_cmd_get_tag(cmd), status);
685
686 #ifdef EXTRACHECKS
687         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
688                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
689         {
690                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
691                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
692                         cmd->tgtt->name);
693                 pref_context = SCST_CONTEXT_TASKLET;
694         }
695 #endif
696
697         switch (status) {
698         case SCST_PREPROCESS_STATUS_SUCCESS:
699                 switch (cmd->data_direction) {
700                 case SCST_DATA_WRITE:
701                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
702                         break;
703                 default:
704                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
705                         break;
706                 }
707                 if (cmd->set_sn_on_restart_cmd)
708                         scst_cmd_set_sn(cmd);
709                 /* Small context optimization */
710                 if ((pref_context == SCST_CONTEXT_TASKLET) || 
711                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) {
712                         if (cmd->data_direction == SCST_DATA_WRITE) {
713                                 if ( !test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
714                                                 &cmd->tgt_dev->tgt_dev_flags))
715                                         pref_context = SCST_CONTEXT_THREAD;
716                         } else {
717                                 if ( !test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
718                                                 &cmd->tgt_dev->tgt_dev_flags))
719                                         pref_context = SCST_CONTEXT_THREAD;
720                         }
721                 }
722                 break;
723
724         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
725                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
726                 break;
727
728         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
729                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
730                 /* go through */
731         case SCST_PREPROCESS_STATUS_ERROR:
732                 scst_set_cmd_error(cmd,
733                            SCST_LOAD_SENSE(scst_sense_hardw_error));
734                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
735                 break;
736
737         default:
738                 PRINT_ERROR("%s() received unknown status %x", __func__,
739                         status);
740                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
741                 break;
742         }
743
744         scst_proccess_redirect_cmd(cmd, pref_context, 1);
745
746         TRACE_EXIT();
747         return;
748 }
749
750 /* No locks */
751 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
752 {
753         struct scst_tgt *tgt = cmd->sess->tgt;
754         int res = 0;
755         unsigned long flags;
756
757         TRACE_ENTRY();
758
759         spin_lock_irqsave(&tgt->tgt_lock, flags);
760         tgt->retry_cmds++;
761         smp_mb();
762         TRACE(TRACE_RETRY, "TGT QUEUE FULL: incrementing retry_cmds %d",
763               tgt->retry_cmds);
764         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
765                 /* At least one cmd finished, so try again */
766                 tgt->retry_cmds--;
767                 TRACE(TRACE_RETRY, "Some command(s) finished, direct retry "
768                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
769                       "retry_cmds=%d)", finished_cmds,
770                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
771                 res = -1;
772                 goto out_unlock_tgt;
773         }
774
775         TRACE(TRACE_RETRY, "Adding cmd %p to retry cmd list", cmd);
776         list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
777
778         if (!tgt->retry_timer_active) {
779                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
780                 add_timer(&tgt->retry_timer);
781                 tgt->retry_timer_active = 1;
782         }
783
784 out_unlock_tgt:
785         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
786
787         TRACE_EXIT_RES(res);
788         return res;
789 }
790
791 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
792 {
793         int res, rc;
794         int atomic = scst_cmd_atomic(cmd);
795
796         TRACE_ENTRY();
797
798         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
799                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
800                 goto out_dev_done;
801         }
802
803         if (cmd->tgtt->rdy_to_xfer == NULL) {
804                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
805                 res = SCST_CMD_STATE_RES_CONT_SAME;
806                 goto out;
807         }
808
809         if (atomic && !cmd->tgtt->rdy_to_xfer_atomic) {
810                 TRACE_DBG("%s", "rdy_to_xfer() can not be "
811                       "called in atomic context, rescheduling to the thread");
812                 res = SCST_CMD_STATE_RES_NEED_THREAD;
813                 goto out;
814         }
815
816         while (1) {
817                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
818
819                 res = SCST_CMD_STATE_RES_CONT_NEXT;
820                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
821
822                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
823 #ifdef DEBUG_RETRY
824                 if (((scst_random() % 100) == 75))
825                         rc = SCST_TGT_RES_QUEUE_FULL;
826                 else
827 #endif
828                         rc = cmd->tgtt->rdy_to_xfer(cmd);
829                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
830
831                 if (likely(rc == SCST_TGT_RES_SUCCESS))
832                         goto out;
833
834                 /* Restore the previous state */
835                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
836
837                 switch (rc) {
838                 case SCST_TGT_RES_QUEUE_FULL:
839                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
840                                 break;
841                         else
842                                 continue;
843
844                 case SCST_TGT_RES_NEED_THREAD_CTX:
845                         TRACE_DBG("Target driver %s "
846                               "rdy_to_xfer() requested thread "
847                               "context, rescheduling", cmd->tgtt->name);
848                         res = SCST_CMD_STATE_RES_NEED_THREAD;
849                         break;
850
851                 default:
852                         goto out_error_rc;
853                 }
854                 break;
855         }
856
857 out:
858         TRACE_EXIT_HRES(res);
859         return res;
860
861 out_error_rc:
862         if (rc == SCST_TGT_RES_FATAL_ERROR) {
863                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
864                      "fatal error", cmd->tgtt->name);
865         } else {
866                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
867                             "value %d", cmd->tgtt->name, rc);
868         }
869         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
870
871 out_dev_done:
872         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
873         res = SCST_CMD_STATE_RES_CONT_SAME;
874         goto out;
875 }
876
877 /* No locks, but might be in IRQ */
878 void scst_proccess_redirect_cmd(struct scst_cmd *cmd, int context,
879         int check_retries)
880 {
881         unsigned long flags;
882
883         TRACE_ENTRY();
884
885         TRACE_DBG("Context: %d", context);
886
887         switch(context) {
888         case SCST_CONTEXT_DIRECT:
889         case SCST_CONTEXT_DIRECT_ATOMIC:
890                 if (check_retries)
891                         scst_check_retries(cmd->tgt);
892                 scst_process_active_cmd(cmd, context);
893                 break;
894
895         default:
896                 PRINT_ERROR("Context %x is unknown, using the thread one",
897                             context);
898                 /* go through */
899         case SCST_CONTEXT_THREAD:
900                 if (check_retries)
901                         scst_check_retries(cmd->tgt);
902                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
903                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
904                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
905                         list_add(&cmd->cmd_list_entry,
906                                 &cmd->cmd_lists->active_cmd_list);
907                 else
908                         list_add_tail(&cmd->cmd_list_entry,
909                                 &cmd->cmd_lists->active_cmd_list);
910                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
911                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
912                 break;
913
914         case SCST_CONTEXT_TASKLET:
915                 if (check_retries)
916                         scst_check_retries(cmd->tgt);
917                 scst_schedule_tasklet(cmd);
918                 break;
919         }
920
921         TRACE_EXIT();
922         return;
923 }
924
925 void scst_rx_data(struct scst_cmd *cmd, int status, int pref_context)
926 {
927         TRACE_ENTRY();
928
929         TRACE_DBG("Preferred context: %d", pref_context);
930         TRACE(TRACE_SCSI, "tag=%llu status=%#x", scst_cmd_get_tag(cmd), status);
931
932 #ifdef EXTRACHECKS
933         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
934                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
935         {
936                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
937                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
938                         cmd->tgtt->name);
939                 pref_context = SCST_CONTEXT_TASKLET;
940         }
941 #endif
942
943         switch (status) {
944         case SCST_RX_STATUS_SUCCESS:
945                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
946                 /* Small context optimization */
947                 if ((pref_context == SCST_CONTEXT_TASKLET) || 
948                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) {
949                         if ( !test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC, 
950                                         &cmd->tgt_dev->tgt_dev_flags))
951                                 pref_context = SCST_CONTEXT_THREAD;
952                 }
953                 break;
954
955         case SCST_RX_STATUS_ERROR_SENSE_SET:
956                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
957                 break;
958
959         case SCST_RX_STATUS_ERROR_FATAL:
960                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
961                 /* go through */
962         case SCST_RX_STATUS_ERROR:
963                 scst_set_cmd_error(cmd,
964                            SCST_LOAD_SENSE(scst_sense_hardw_error));
965                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
966                 break;
967
968         default:
969                 PRINT_ERROR("scst_rx_data() received unknown status %x",
970                         status);
971                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
972                 break;
973         }
974
975         scst_proccess_redirect_cmd(cmd, pref_context, 1);
976
977         TRACE_EXIT();
978         return;
979 }
980
981 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
982 {
983         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
984
985         TRACE_ENTRY();
986
987         cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
988
989         if (cmd->tgtt->pre_exec == NULL)
990                 goto out;
991
992         TRACE_DBG("Calling pre_exec(%p)", cmd);
993         rc = cmd->tgtt->pre_exec(cmd);
994         TRACE_DBG("pre_exec() returned %d", rc);
995
996         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
997                 switch(rc) {
998                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
999                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1000                         break;
1001                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
1002                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1003                         /* go through */
1004                 case SCST_PREPROCESS_STATUS_ERROR:
1005                         scst_set_cmd_error(cmd,
1006                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1007                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1008                         break;
1009                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1010                         TRACE_DBG("Target driver's %s pre_exec() requested "
1011                                 "thread context, rescheduling", cmd->tgtt->name);
1012                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1013                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1014                         break;
1015                 default:
1016                         sBUG();
1017                         break;
1018                 }
1019         }
1020
1021 out:
1022         TRACE_EXIT_RES(res);
1023         return res;
1024 }
1025
1026 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1027         const uint8_t *rq_sense, int rq_sense_len, int resid)
1028 {
1029         TRACE_ENTRY();
1030
1031         cmd->status = result & 0xff;
1032         cmd->msg_status = msg_byte(result);
1033         cmd->host_status = host_byte(result);
1034         cmd->driver_status = driver_byte(result);
1035         if (unlikely(resid != 0)) {
1036 #ifdef EXTRACHECKS
1037                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1038                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1039                                 "op %x)", resid, cmd->resp_data_len,
1040                                 cmd->cdb[0]);
1041                 } else
1042 #endif
1043                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1044         }
1045
1046         /* 
1047          * We checked that rq_sense_len < sizeof(cmd->sense_buffer)
1048          * in init_scst()
1049          */
1050         memcpy(cmd->sense_buffer, rq_sense, rq_sense_len);
1051         memset(&cmd->sense_buffer[rq_sense_len], 0,
1052                 sizeof(cmd->sense_buffer) - rq_sense_len);
1053
1054         TRACE(TRACE_SCSI, "result=%x, cmd->status=%x, resid=%d, "
1055               "cmd->msg_status=%x, cmd->host_status=%x, "
1056               "cmd->driver_status=%x", result, cmd->status, resid,
1057               cmd->msg_status, cmd->host_status, cmd->driver_status);
1058
1059         cmd->completed = 1;
1060
1061         TRACE_EXIT();
1062         return;
1063 }
1064
1065 /* For small context optimization */
1066 static inline int scst_optimize_post_exec_context(struct scst_cmd *cmd,
1067         int context)
1068 {
1069         if ((context == SCST_CONTEXT_TASKLET) || 
1070             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1071                 if ( !test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC, 
1072                                 &cmd->tgt_dev->tgt_dev_flags))
1073                         context = SCST_CONTEXT_THREAD;
1074         }
1075         return context;
1076 }
1077
1078 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18)
1079 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1080                                             struct scsi_request **req)
1081 {
1082         struct scst_cmd *cmd = NULL;
1083
1084         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1085                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1086
1087         if (cmd == NULL) {
1088                 PRINT_ERROR("%s", "Request with NULL cmd");
1089                 if (*req)
1090                         scsi_release_request(*req);
1091         }
1092
1093         return cmd;
1094 }
1095
1096 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1097 {
1098         struct scsi_request *req = NULL;
1099         struct scst_cmd *cmd;
1100
1101         TRACE_ENTRY();
1102
1103         cmd = scst_get_cmd(scsi_cmd, &req);
1104         if (cmd == NULL)
1105                 goto out;
1106
1107         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1108                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1109
1110         /* Clear out request structure */
1111         req->sr_use_sg = 0;
1112         req->sr_sglist_len = 0;
1113         req->sr_bufflen = 0;
1114         req->sr_buffer = NULL;
1115         req->sr_underflow = 0;
1116         req->sr_request->rq_disk = NULL; /* disown request blk */
1117
1118         scst_release_request(cmd);
1119
1120         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1121
1122         scst_proccess_redirect_cmd(cmd,
1123                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1124
1125 out:
1126         TRACE_EXIT();
1127         return;
1128 }
1129 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) */
1130 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1131 {
1132         struct scst_cmd *cmd;
1133
1134         TRACE_ENTRY();
1135
1136         cmd = (struct scst_cmd *)data;
1137         if (cmd == NULL)
1138                 goto out;
1139
1140         scst_do_cmd_done(cmd, result, sense, SCST_SENSE_BUFFERSIZE, resid);
1141
1142         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1143
1144         scst_proccess_redirect_cmd(cmd,
1145                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1146
1147 out:
1148         TRACE_EXIT();
1149         return;
1150 }
1151 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) */
1152
1153 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state)
1154 {
1155         TRACE_ENTRY();
1156
1157         if (next_state == SCST_CMD_STATE_DEFAULT)
1158                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1159
1160 #if defined(DEBUG) || defined(TRACING)
1161         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1162                 if (cmd->sg) {
1163                         int i;
1164                         struct scatterlist *sg = cmd->sg;
1165                         TRACE(TRACE_RECV_TOP, 
1166                               "Exec'd %d S/G(s) at %p sg[0].page at %p",
1167                               cmd->sg_cnt, sg, (void*)sg[0].page);
1168                         for(i = 0; i < cmd->sg_cnt; ++i) {
1169                                 TRACE_BUFF_FLAG(TRACE_RECV_TOP, 
1170                                         "Exec'd sg", page_address(sg[i].page),
1171                                         sg[i].length);
1172                         }
1173                 }
1174         }
1175 #endif
1176
1177
1178 #ifdef EXTRACHECKS
1179         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1180             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1181             (next_state != SCST_CMD_STATE_FINISHED)) 
1182         {
1183                 PRINT_ERROR("scst_cmd_done_local() received invalid cmd "
1184                             "state %d (opcode %d)", next_state, cmd->cdb[0]);
1185                 scst_set_cmd_error(cmd,
1186                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1187                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1188         }
1189 #endif
1190         cmd->state = next_state;
1191
1192         scst_proccess_redirect_cmd(cmd,
1193                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1194
1195         TRACE_EXIT();
1196         return;
1197 }
1198
1199 static int scst_report_luns_local(struct scst_cmd *cmd)
1200 {
1201         int rc;
1202         int dev_cnt = 0;
1203         int buffer_size;
1204         int i;
1205         struct scst_tgt_dev *tgt_dev = NULL;
1206         uint8_t *buffer;
1207         int offs, overflow = 0;
1208
1209         TRACE_ENTRY();
1210
1211         rc = scst_check_local_events(cmd);
1212         if (unlikely(rc != 0))
1213                 goto out_done;
1214
1215         cmd->status = 0;
1216         cmd->msg_status = 0;
1217         cmd->host_status = DID_OK;
1218         cmd->driver_status = 0;
1219
1220         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1221                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1222                         "LUNS command", cmd->cdb[2]);
1223                 goto out_err;
1224         }
1225
1226         buffer_size = scst_get_buf_first(cmd, &buffer);
1227         if (unlikely(buffer_size <= 0))
1228                 goto out_err;
1229
1230         if (buffer_size < 16)
1231                 goto out_put_err;
1232
1233         memset(buffer, 0, buffer_size);
1234         offs = 8;
1235
1236         /* sess->sess_tgt_dev_list_hash is protected by suspended activity */
1237         for(i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1238                 struct list_head *sess_tgt_dev_list_head =
1239                         &cmd->sess->sess_tgt_dev_list_hash[i];
1240                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1241                                 sess_tgt_dev_list_entry) {
1242                         if (!overflow) {
1243                                 if (offs >= buffer_size) {
1244                                         scst_put_buf(cmd, buffer);
1245                                         buffer_size = scst_get_buf_next(cmd, &buffer);
1246                                         if (buffer_size > 0) {
1247                                                 memset(buffer, 0, buffer_size);
1248                                                 offs = 0;
1249                                         } else {
1250                                                 overflow = 1;
1251                                                 goto inc_dev_cnt;
1252                                         }
1253                                 }
1254                                 if ((buffer_size - offs) < 8) {
1255                                         PRINT_ERROR("Buffer allocated for REPORT "
1256                                                 "LUNS command doesn't allow to fit 8 "
1257                                                 "byte entry (buffer_size=%d)",
1258                                                 buffer_size);
1259                                         goto out_put_hw_err;
1260                                 }
1261                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1262                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1263                                 offs += 8;
1264                         }
1265 inc_dev_cnt:
1266                         dev_cnt++;
1267                 }
1268         }
1269         if (!overflow)
1270                 scst_put_buf(cmd, buffer);
1271
1272         /* Set the response header */
1273         buffer_size = scst_get_buf_first(cmd, &buffer);
1274         if (unlikely(buffer_size <= 0))
1275                 goto out_err;
1276         dev_cnt *= 8;
1277         buffer[0] = (dev_cnt >> 24) & 0xff;
1278         buffer[1] = (dev_cnt >> 16) & 0xff;
1279         buffer[2] = (dev_cnt >> 8) & 0xff;
1280         buffer[3] = dev_cnt & 0xff;
1281         scst_put_buf(cmd, buffer);
1282
1283         dev_cnt += 8;
1284         if (dev_cnt < cmd->resp_data_len)
1285                 scst_set_resp_data_len(cmd, dev_cnt);
1286
1287 out_compl:
1288         cmd->completed = 1;
1289
1290 out_done:
1291         /* Report the result */
1292         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1293
1294         TRACE_EXIT();
1295         return SCST_EXEC_COMPLETED;
1296         
1297 out_put_err:
1298         scst_put_buf(cmd, buffer);
1299
1300 out_err:
1301         scst_set_cmd_error(cmd,
1302                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1303         goto out_compl;
1304
1305 out_put_hw_err:
1306         scst_put_buf(cmd, buffer);
1307         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1308         goto out_compl;
1309 }
1310
1311 static int scst_pre_select(struct scst_cmd *cmd)
1312 {
1313         int res = SCST_EXEC_NOT_COMPLETED;
1314
1315         TRACE_ENTRY();
1316
1317         if (scst_cmd_atomic(cmd)) {
1318                 res = SCST_EXEC_NEED_THREAD;
1319                 goto out;
1320         }
1321
1322         if (cmd->local_exec_done)
1323                 goto out;
1324
1325         cmd->local_exec_done = 1;
1326
1327         scst_block_dev_cmd(cmd, 1);
1328
1329         /* Check for local events will be done when cmd will be executed */
1330
1331 out:
1332         TRACE_EXIT_RES(res);
1333         return res;
1334 }
1335
1336 static int scst_reserve_local(struct scst_cmd *cmd)
1337 {
1338         int res = SCST_EXEC_NOT_COMPLETED, rc;
1339         struct scst_device *dev;
1340         struct scst_tgt_dev *tgt_dev_tmp;
1341
1342         TRACE_ENTRY();
1343
1344         if (scst_cmd_atomic(cmd)) {
1345                 res = SCST_EXEC_NEED_THREAD;
1346                 goto out;
1347         }
1348
1349         if (cmd->local_exec_done)
1350                 goto out;
1351
1352         cmd->local_exec_done = 1;
1353
1354         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1355                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1356                      "(lun=%Ld)", (uint64_t)cmd->lun);
1357                 scst_set_cmd_error(cmd,
1358                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1359                 goto out_done;
1360         }
1361
1362         dev = cmd->dev;
1363
1364         scst_block_dev_cmd(cmd, 1);
1365
1366         rc = scst_check_local_events(cmd);
1367         if (unlikely(rc != 0))
1368                 goto out_done;
1369
1370         spin_lock_bh(&dev->dev_lock);
1371
1372         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1373                 spin_unlock_bh(&dev->dev_lock);
1374                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1375                 goto out_done;
1376         }
1377
1378         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1379                             dev_tgt_dev_list_entry) {
1380                 if (cmd->tgt_dev != tgt_dev_tmp)
1381                         set_bit(SCST_TGT_DEV_RESERVED, 
1382                                 &tgt_dev_tmp->tgt_dev_flags);
1383         }
1384         dev->dev_reserved = 1;
1385
1386         spin_unlock_bh(&dev->dev_lock);
1387         
1388 out:
1389         TRACE_EXIT_RES(res);
1390         return res;
1391
1392 out_done:
1393         /* Report the result */
1394         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1395         res = SCST_EXEC_COMPLETED;
1396         goto out;
1397 }
1398
1399 static int scst_release_local(struct scst_cmd *cmd)
1400 {
1401         int res = SCST_EXEC_NOT_COMPLETED, rc;
1402         struct scst_tgt_dev *tgt_dev_tmp;
1403         struct scst_device *dev;
1404
1405         TRACE_ENTRY();
1406
1407         if (scst_cmd_atomic(cmd)) {
1408                 res = SCST_EXEC_NEED_THREAD;
1409                 goto out;
1410         }
1411
1412         if (cmd->local_exec_done)
1413                 goto out;
1414
1415         cmd->local_exec_done = 1;
1416
1417         dev = cmd->dev;
1418
1419         scst_block_dev_cmd(cmd, 1);
1420
1421         rc = scst_check_local_events(cmd);
1422         if (unlikely(rc != 0))
1423                 goto out_done;
1424
1425         spin_lock_bh(&dev->dev_lock);
1426
1427         /* 
1428          * The device could be RELEASED behind us, if RESERVING session 
1429          * is closed (see scst_free_tgt_dev()), but this actually doesn't 
1430          * matter, so use lock and no retest for DEV_RESERVED bits again
1431          */
1432         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1433                 res = SCST_EXEC_COMPLETED;
1434                 cmd->status = 0;
1435                 cmd->msg_status = 0;
1436                 cmd->host_status = DID_OK;
1437                 cmd->driver_status = 0;
1438                 cmd->completed = 1;
1439         } else {
1440                 list_for_each_entry(tgt_dev_tmp,
1441                                     &dev->dev_tgt_dev_list,
1442                                     dev_tgt_dev_list_entry) {
1443                         clear_bit(SCST_TGT_DEV_RESERVED, 
1444                                 &tgt_dev_tmp->tgt_dev_flags);
1445                 }
1446                 dev->dev_reserved = 0;
1447         }
1448
1449         spin_unlock_bh(&dev->dev_lock);
1450
1451         if (res == SCST_EXEC_COMPLETED)
1452                 goto out_done;
1453
1454 out:
1455         TRACE_EXIT_RES(res);
1456         return res;
1457
1458 out_done:
1459         res = SCST_EXEC_COMPLETED;
1460         /* Report the result */
1461         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1462         goto out;
1463 }
1464
1465 /* No locks, no IRQ or IRQ-safe context allowed */
1466 int scst_check_local_events(struct scst_cmd *cmd)
1467 {
1468         int res, rc;
1469         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1470
1471         TRACE_ENTRY();
1472
1473         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1474                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1475                 goto out_uncomplete;
1476         }
1477
1478         /* Reserve check before Unit Attention */
1479         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED, &tgt_dev->tgt_dev_flags))) {
1480                 if ((cmd->cdb[0] != INQUIRY) && (cmd->cdb[0] != REPORT_LUNS) &&
1481                     (cmd->cdb[0] != RELEASE) && (cmd->cdb[0] != RELEASE_10) &&
1482                     (cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER) &&
1483                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL || (cmd->cdb[4] & 3)) &&
1484                     (cmd->cdb[0] != LOG_SENSE) && (cmd->cdb[0] != REQUEST_SENSE)) {
1485                         scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1486                         goto out_complete;
1487                 }
1488         }
1489
1490         /* If we had internal bus reset, set the command error unit attention */
1491         if ((cmd->dev->scsi_dev != NULL) &&
1492             unlikely(cmd->dev->scsi_dev->was_reset)) {
1493                 if (scst_is_ua_command(cmd)) {
1494                         struct scst_device *dev = cmd->dev;
1495                         int done = 0;
1496                         /* Prevent more than 1 cmd to be triggered by was_reset */
1497                         spin_lock_bh(&dev->dev_lock);
1498                         barrier(); /* to reread was_reset */
1499                         if (dev->scsi_dev->was_reset) {
1500                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1501                                 scst_set_cmd_error(cmd,
1502                                            SCST_LOAD_SENSE(scst_sense_reset_UA));
1503                                 /* It looks like it is safe to clear was_reset here */
1504                                 dev->scsi_dev->was_reset = 0;
1505                                 smp_mb();
1506                                 done = 1;
1507                         }
1508                         spin_unlock_bh(&dev->dev_lock);
1509
1510                         if (done)
1511                                 goto out_complete;
1512                 }
1513         }
1514
1515         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING, 
1516                         &cmd->tgt_dev->tgt_dev_flags))) {
1517                 if (scst_is_ua_command(cmd)) {
1518                         rc = scst_set_pending_UA(cmd);
1519                         if (rc == 0)
1520                                 goto out_complete;
1521                 }
1522         }
1523
1524         res = 0;
1525
1526 out:
1527         TRACE_EXIT_RES(res);
1528         return res;
1529
1530 out_complete:
1531         res = 1;
1532         sBUG_ON(!cmd->completed);
1533         goto out;
1534
1535 out_uncomplete:
1536         res = -1;
1537         goto out;
1538 }
1539
1540 /* 
1541  * The result of cmd execution, if any, should be reported 
1542  * via scst_cmd_done_local() 
1543  */
1544 static int scst_pre_exec(struct scst_cmd *cmd)
1545 {
1546         int res = SCST_EXEC_NOT_COMPLETED;
1547         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1548
1549         TRACE_ENTRY();
1550
1551         /*
1552          * This function can be called several times for the same cmd, so it
1553          * can't change any state in a non-reentrable way or use something
1554          * like local_exec_done!!
1555          */
1556
1557         /* Check READ_ONLY device status */
1558         if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
1559             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
1560              cmd->cdb[0] == WRITE_10 ||
1561              cmd->cdb[0] == WRITE_12 ||
1562              cmd->cdb[0] == WRITE_16 ||
1563              cmd->cdb[0] == WRITE_VERIFY ||
1564              cmd->cdb[0] == WRITE_VERIFY_12 ||
1565              cmd->cdb[0] == WRITE_VERIFY_16 ||
1566              (cmd->dev->handler->type == TYPE_TAPE &&
1567               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
1568                 scst_set_cmd_error(cmd,
1569                            SCST_LOAD_SENSE(scst_sense_data_protect));
1570                 goto out_done;
1571         }
1572
1573 out:
1574         TRACE_EXIT_RES(res);
1575         return res;
1576
1577 out_done:
1578         res = SCST_EXEC_COMPLETED;
1579         /* Report the result */
1580         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1581         goto out;
1582 }
1583
1584 /* 
1585  * The result of cmd execution, if any, should be reported 
1586  * via scst_cmd_done_local() 
1587  */
1588 static inline int scst_local_exec(struct scst_cmd *cmd)
1589 {
1590         int res = SCST_EXEC_NOT_COMPLETED;
1591
1592         TRACE_ENTRY();
1593
1594         /*
1595          * Adding new commands here don't forget to update
1596          * scst_is_cmd_local() in scsi_tgt.h, if necessary
1597          */
1598
1599         switch (cmd->cdb[0]) {
1600         case MODE_SELECT:
1601         case MODE_SELECT_10:
1602         case LOG_SELECT:
1603                 res = scst_pre_select(cmd);
1604                 break;
1605         case RESERVE:
1606         case RESERVE_10:
1607                 res = scst_reserve_local(cmd);
1608                 break;
1609         case RELEASE:
1610         case RELEASE_10:
1611                 res = scst_release_local(cmd);
1612                 break;
1613         case REPORT_LUNS:
1614                 res = scst_report_luns_local(cmd);
1615                 break;
1616         }
1617
1618         TRACE_EXIT_RES(res);
1619         return res;
1620 }
1621
1622 static int scst_do_send_to_midlev(struct scst_cmd *cmd)
1623 {
1624         int rc = SCST_EXEC_NOT_COMPLETED;
1625
1626         TRACE_ENTRY();
1627
1628         /* Check here to let an out of SN cmd be queued w/o context switch */
1629         if (scst_cmd_atomic(cmd) && !cmd->dev->handler->exec_atomic) {
1630                 TRACE_DBG("Dev handler %s exec() can not be "
1631                       "called in atomic context, rescheduling to the thread",
1632                       cmd->dev->handler->name);
1633                 rc = SCST_EXEC_NEED_THREAD;
1634                 goto out;
1635         }
1636
1637         cmd->sent_to_midlev = 1;
1638         cmd->state = SCST_CMD_STATE_EXECUTING;
1639         cmd->scst_cmd_done = scst_cmd_done_local;
1640
1641         rc = scst_pre_exec(cmd);
1642         /* !! At this point cmd, sess & tgt_dev can be already freed !! */
1643         if (rc != SCST_EXEC_NOT_COMPLETED) {
1644                 if (rc == SCST_EXEC_COMPLETED)
1645                         goto out;
1646                 else if (rc == SCST_EXEC_NEED_THREAD)
1647                         goto out_clear;
1648                 else
1649                         goto out_rc_error;
1650         }
1651
1652         rc = scst_local_exec(cmd);
1653         /* !! At this point cmd, sess & tgt_dev can be already freed !! */
1654         if (rc != SCST_EXEC_NOT_COMPLETED) {
1655                 if (rc == SCST_EXEC_COMPLETED)
1656                         goto out;
1657                 else if (rc == SCST_EXEC_NEED_THREAD)
1658                         goto out_clear;
1659                 else
1660                         goto out_rc_error;
1661         }
1662
1663         if (cmd->dev->handler->exec) {
1664                 struct scst_device *dev = cmd->dev;
1665                 TRACE_DBG("Calling dev handler %s exec(%p)",
1666                       dev->handler->name, cmd);
1667                 TRACE_BUFF_FLAG(TRACE_SEND_TOP, "Execing: ", cmd->cdb, cmd->cdb_len);
1668                 cmd->scst_cmd_done = scst_cmd_done_local;
1669                 rc = dev->handler->exec(cmd);
1670                 /* !! At this point cmd, sess & tgt_dev can be already freed !! */
1671                 TRACE_DBG("Dev handler %s exec() returned %d",
1672                       dev->handler->name, rc);
1673                 if (rc == SCST_EXEC_COMPLETED)
1674                         goto out;
1675                 else if (rc == SCST_EXEC_NEED_THREAD)
1676                         goto out_clear;
1677                 else if (rc != SCST_EXEC_NOT_COMPLETED)
1678                         goto out_rc_error;
1679         }
1680
1681         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1682         
1683         if (unlikely(cmd->dev->scsi_dev == NULL)) {
1684                 PRINT_ERROR("Command for virtual device must be "
1685                         "processed by device handler (lun %Ld)!",
1686                         (uint64_t)cmd->lun);
1687                 goto out_error;
1688         }
1689
1690         rc = scst_check_local_events(cmd);
1691         if (unlikely(rc != 0))
1692                 goto out_done;
1693
1694 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) 
1695         if (unlikely(scst_alloc_request(cmd) != 0)) {
1696                 if (scst_cmd_atomic(cmd)) {
1697                         rc = SCST_EXEC_NEED_THREAD;
1698                         goto out_clear;
1699                 } else {
1700                         PRINT_INFO("%s", "Unable to allocate request, "
1701                                 "sending BUSY status");
1702                         goto out_busy;
1703                 }
1704         }
1705         
1706         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1707                     (void *)cmd->scsi_req->sr_buffer,
1708                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1709                     cmd->retries);
1710 #else
1711         rc = scst_exec_req(cmd->dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1712                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1713                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1714                         scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
1715         if (unlikely(rc != 0)) {
1716                 if (scst_cmd_atomic(cmd)) {
1717                         rc = SCST_EXEC_NEED_THREAD;
1718                         goto out_clear;
1719                 } else {
1720                         PRINT_INFO("scst_exec_req() failed: %d", rc);
1721                         goto out_error;
1722                 }
1723         }
1724 #endif
1725
1726         rc = SCST_EXEC_COMPLETED;
1727
1728 out:
1729         TRACE_EXIT();
1730         return rc;
1731
1732 out_clear:
1733         /* Restore the state */
1734         cmd->sent_to_midlev = 0;
1735         cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
1736         goto out;
1737
1738 out_rc_error:
1739         PRINT_ERROR("Dev handler %s exec() or scst_local_exec() returned "
1740                     "invalid code %d", cmd->dev->handler->name, rc);
1741         /* go through */
1742
1743 out_error:
1744         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1745
1746 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) 
1747 out_busy:
1748         scst_set_busy(cmd);
1749         /* go through */
1750 #endif
1751
1752 out_done:
1753         rc = SCST_EXEC_COMPLETED;
1754         /* Report the result */
1755         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1756         goto out;
1757 }
1758
1759 /* No locks */
1760 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1761 {
1762         if (slot == NULL)
1763                 goto inc;
1764
1765         /* Optimized for lockless fast path */
1766
1767         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1768                 atomic_read(slot));
1769
1770         if (!atomic_dec_and_test(slot))
1771                 goto out;
1772
1773         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1774                 tgt_dev->num_free_sn_slots);
1775         if (tgt_dev->num_free_sn_slots != ARRAY_SIZE(tgt_dev->sn_slots)) {
1776                 spin_lock_irq(&tgt_dev->sn_lock);
1777                 if (tgt_dev->num_free_sn_slots != ARRAY_SIZE(tgt_dev->sn_slots)) {
1778                         tgt_dev->num_free_sn_slots++;
1779                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1780                                 tgt_dev->num_free_sn_slots);
1781                         if (tgt_dev->num_free_sn_slots == 0)
1782                                 tgt_dev->cur_sn_slot = slot;
1783                 }
1784                 spin_unlock_irq(&tgt_dev->sn_lock);
1785         }
1786
1787 inc:
1788         /*
1789          * No locks is needed, because only one thread at time can 
1790          * be here (serialized by sn). Also it is supposed that there
1791          * could not be half-incremented halves.
1792          */
1793         tgt_dev->expected_sn++;
1794         smp_mb(); /* write must be before def_cmd_count read */
1795         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1796
1797 out:
1798         return;
1799 }
1800
1801 static int scst_send_to_midlev(struct scst_cmd **active_cmd)
1802 {
1803         int res, rc;
1804         struct scst_cmd *cmd = *active_cmd;
1805         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1806         struct scst_device *dev = cmd->dev;
1807         typeof(tgt_dev->expected_sn) expected_sn;
1808         int count;
1809
1810         TRACE_ENTRY();
1811
1812         res = SCST_CMD_STATE_RES_CONT_NEXT;
1813
1814         __scst_get(0); /* protect dev & tgt_dev */
1815
1816         if (unlikely(cmd->internal || cmd->retry)) {
1817                 rc = scst_do_send_to_midlev(cmd);
1818                 /* !! At this point cmd, sess & tgt_dev can be already freed !! */
1819                 if (rc == SCST_EXEC_NEED_THREAD) {
1820                         TRACE_DBG("%s", "scst_do_send_to_midlev() requested "
1821                               "thread context, rescheduling");
1822                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1823                         scst_dec_on_dev_cmd(cmd);
1824                         goto out_put;
1825                 } else {
1826                         sBUG_ON(rc != SCST_EXEC_COMPLETED);
1827                         goto out_unplug;
1828                 }
1829         }
1830
1831         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
1832                 goto out_put;
1833
1834         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
1835                 goto exec;
1836
1837         sBUG_ON(!cmd->sn_set);
1838
1839         expected_sn = tgt_dev->expected_sn;
1840         /* Optimized for lockless fast path */
1841         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
1842                 spin_lock_irq(&tgt_dev->sn_lock);
1843                 tgt_dev->def_cmd_count++;
1844                 smp_mb();
1845                 barrier(); /* to reread expected_sn & hq_cmd_count */
1846                 expected_sn = tgt_dev->expected_sn;
1847                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
1848                         /* We are under IRQ lock, but dev->dev_lock is BH one */
1849                         int cmd_blocking = scst_pre_dec_on_dev_cmd(cmd);
1850                         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1851                                 /* Necessary to allow aborting out of sn cmds */
1852                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p (tag %llu)",
1853                                         cmd, cmd->tag);
1854                                 tgt_dev->def_cmd_count--;
1855                                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1856                                 res = SCST_CMD_STATE_RES_CONT_SAME;
1857                         } else {
1858                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
1859                                         "expected_sn=%ld)", cmd, cmd->sn,
1860                                         cmd->sn_set, expected_sn);
1861                                 list_add_tail(&cmd->sn_cmd_list_entry,
1862                                               &tgt_dev->deferred_cmd_list);
1863                         }
1864                         spin_unlock_irq(&tgt_dev->sn_lock);
1865                         /* !! At this point cmd can be already freed !! */
1866                         __scst_dec_on_dev_cmd(dev, cmd_blocking);
1867                         goto out_put;
1868                 } else {
1869                         TRACE_SN("Somebody incremented expected_sn %ld, "
1870                                 "continuing", expected_sn);
1871                         tgt_dev->def_cmd_count--;
1872                         spin_unlock_irq(&tgt_dev->sn_lock);
1873                 }
1874         }
1875
1876 exec:
1877         count = 0;
1878         while(1) {
1879                 atomic_t *slot = cmd->sn_slot;
1880                 int inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1881                                       cmd->sn_set;
1882                 rc = scst_do_send_to_midlev(cmd);
1883                 if (rc == SCST_EXEC_NEED_THREAD) {
1884                         TRACE_DBG("%s", "scst_do_send_to_midlev() requested "
1885                               "thread context, rescheduling");
1886                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1887                         scst_dec_on_dev_cmd(cmd);
1888                         *active_cmd = cmd;
1889                         if (count != 0)
1890                                 goto out_unplug;
1891                         else
1892                                 goto out_put;
1893                 }
1894                 sBUG_ON(rc != SCST_EXEC_COMPLETED);
1895                 /* !! At this point cmd can be already freed !! */
1896                 count++;
1897                 if (inc_expected_sn)
1898                         scst_inc_expected_sn(tgt_dev, slot);
1899                 cmd = scst_check_deferred_commands(tgt_dev);
1900                 if (cmd == NULL)
1901                         break;
1902                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
1903                         break;
1904         }
1905
1906 out_unplug:
1907         if (dev->scsi_dev != NULL)
1908                 generic_unplug_device(dev->scsi_dev->request_queue);
1909
1910 out_put:
1911         __scst_put();
1912         /* !! At this point sess, dev and tgt_dev can be already freed !! */
1913
1914         TRACE_EXIT_HRES(res);
1915         return res;
1916 }
1917
1918 /* No locks supposed to be held */
1919 static int scst_check_sense(struct scst_cmd *cmd)
1920 {
1921         int res = 0;
1922         struct scst_device *dev = cmd->dev;
1923         int dbl_ua_possible, ua_sent = 0;
1924
1925         TRACE_ENTRY();
1926
1927         /* If we had internal bus reset behind us, set the command error UA */
1928         if ((dev->scsi_dev != NULL) &&
1929             unlikely(cmd->host_status == DID_RESET) &&
1930             scst_is_ua_command(cmd)) {
1931                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
1932                       dev->scsi_dev->was_reset, cmd->host_status);
1933                 scst_set_cmd_error(cmd,
1934                    SCST_LOAD_SENSE(scst_sense_reset_UA));
1935                 /* just in case */
1936                 cmd->ua_ignore = 0;
1937                 /* It looks like it is safe to clear was_reset here */
1938                 dev->scsi_dev->was_reset = 0;
1939                 smp_mb();
1940         }
1941
1942         dbl_ua_possible = dev->dev_double_ua_possible;
1943         TRACE_DBG("cmd %p dbl_ua_possible %d", cmd, dbl_ua_possible);
1944         if (unlikely(dbl_ua_possible)) {
1945                 spin_lock_bh(&dev->dev_lock);
1946                 barrier(); /* to reread dev_double_ua_possible */
1947                 dbl_ua_possible = dev->dev_double_ua_possible;
1948                 if (dbl_ua_possible)
1949                         ua_sent = dev->dev_reset_ua_sent;
1950                 else
1951                         spin_unlock_bh(&dev->dev_lock);
1952         }
1953
1954         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) && 
1955             SCST_SENSE_VALID(cmd->sense_buffer)) {
1956                 TRACE_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense_buffer,
1957                         sizeof(cmd->sense_buffer));
1958                 /* Check Unit Attention Sense Key */
1959                 if (scst_is_ua_sense(cmd->sense_buffer)) {
1960                         if (cmd->sense_buffer[12] == SCST_SENSE_ASC_UA_RESET) {
1961                                 if (dbl_ua_possible) {
1962                                         if (ua_sent) {
1963                                                 TRACE(TRACE_MGMT_MINOR, "%s", 
1964                                                         "Double UA detected");
1965                                                 /* Do retry */
1966                                                 TRACE(TRACE_MGMT_MINOR, "Retrying cmd %p "
1967                                                         "(tag %llu)", cmd, cmd->tag);
1968                                                 cmd->status = 0;
1969                                                 cmd->msg_status = 0;
1970                                                 cmd->host_status = DID_OK;
1971                                                 cmd->driver_status = 0;
1972                                                 memset(cmd->sense_buffer, 0,
1973                                                         sizeof(cmd->sense_buffer));
1974                                                 cmd->retry = 1;
1975                                                 cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
1976                                                 res = 1;
1977                                                 /* 
1978                                                  * Dev is still blocked by this cmd, so
1979                                                  * it's OK to clear SCST_DEV_SERIALIZED
1980                                                  * here.
1981                                                  */
1982                                                 dev->dev_double_ua_possible = 0;
1983                                                 dev->dev_serialized = 0;
1984                                                 dev->dev_reset_ua_sent = 0;
1985                                                 goto out_unlock;
1986                                         } else
1987                                                 dev->dev_reset_ua_sent = 1;
1988                                 }
1989                         }
1990                         if (cmd->ua_ignore == 0) {
1991                                 if (unlikely(dbl_ua_possible)) {
1992                                         __scst_dev_check_set_UA(dev, cmd,
1993                                                 cmd->sense_buffer,
1994                                                 sizeof(cmd->sense_buffer));
1995                                 } else {
1996                                         scst_dev_check_set_UA(dev, cmd,
1997                                                 cmd->sense_buffer,
1998                                                 sizeof(cmd->sense_buffer));
1999                                 }
2000                         }
2001                 }
2002         }
2003
2004         if (unlikely(dbl_ua_possible)) {
2005                 if (ua_sent && scst_is_ua_command(cmd)) {
2006                         TRACE_MGMT_DBG("%s", "Clearing dbl_ua_possible flag");
2007                         dev->dev_double_ua_possible = 0;
2008                         dev->dev_serialized = 0;
2009                         dev->dev_reset_ua_sent = 0;
2010                 }
2011                 spin_unlock_bh(&dev->dev_lock);
2012         }
2013
2014 out:
2015         TRACE_EXIT_RES(res);
2016         return res;
2017
2018 out_unlock:
2019         spin_unlock_bh(&dev->dev_lock);
2020         goto out;
2021 }
2022
2023 static int scst_check_auto_sense(struct scst_cmd *cmd)
2024 {
2025         int res = 0;
2026
2027         TRACE_ENTRY();
2028
2029         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2030             (!SCST_SENSE_VALID(cmd->sense_buffer) ||
2031              SCST_NO_SENSE(cmd->sense_buffer))) {
2032                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2033                       "cmd->status=%x, cmd->msg_status=%x, "
2034                       "cmd->host_status=%x, cmd->driver_status=%x", cmd->status,
2035                       cmd->msg_status, cmd->host_status, cmd->driver_status);
2036                 res = 1;
2037         } else if (unlikely(cmd->host_status)) {
2038                 if ((cmd->host_status == DID_REQUEUE) ||
2039                     (cmd->host_status == DID_IMM_RETRY) ||
2040                     (cmd->host_status == DID_SOFT_ERROR) ||
2041                     (cmd->host_status == DID_ABORT)) {
2042                         scst_set_busy(cmd);
2043                 } else {
2044                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2045                                 "received, returning HARDWARE ERROR instead",
2046                                 cmd->host_status);
2047                         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2048                 }
2049         }
2050
2051         TRACE_EXIT_RES(res);
2052         return res;
2053 }
2054
2055 static int scst_done_cmd_check(struct scst_cmd *cmd, int *pres)
2056 {
2057         int res = 0, rc;
2058
2059         TRACE_ENTRY();
2060
2061         if (unlikely(cmd->cdb[0] == REQUEST_SENSE)) {
2062                 if (cmd->internal)
2063                         cmd = scst_complete_request_sense(cmd);
2064         } else if (unlikely(scst_check_auto_sense(cmd))) {
2065                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2066                             "without sense data (opcode 0x%x), issuing "
2067                             "REQUEST SENSE", cmd->cdb[0]);
2068                 rc = scst_prepare_request_sense(cmd);
2069                 if (rc > 0) {
2070                         *pres = rc;
2071                         res = 1;
2072                         goto out;
2073                 } else {
2074                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2075                                     "returning HARDWARE ERROR");
2076                         scst_set_cmd_error(cmd,
2077                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2078                 }
2079         } else if (unlikely(scst_check_sense(cmd))) {
2080                 *pres = SCST_CMD_STATE_RES_CONT_SAME;
2081                 res = 1;
2082                 goto out;
2083         }
2084
2085         if (likely(scsi_status_is_good(cmd->status))) {
2086                 unsigned char type = cmd->dev->handler->type;
2087                 if ((cmd->cdb[0] == MODE_SENSE || cmd->cdb[0] == MODE_SENSE_10) &&
2088                     cmd->tgt_dev->acg_dev->rd_only_flag &&
2089                     (type == TYPE_DISK || type == TYPE_WORM || type == TYPE_MOD ||
2090                      type == TYPE_TAPE)) {
2091                         int32_t length;
2092                         uint8_t *address;
2093
2094                         length = scst_get_buf_first(cmd, &address);
2095                         if (length <= 0)
2096                                 goto out;
2097                         if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2098                                 address[2] |= 0x80;   /* Write Protect*/
2099                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2100                                 address[3] |= 0x80;   /* Write Protect*/
2101                         scst_put_buf(cmd, address);
2102                 }
2103
2104                 /* 
2105                  * Check and clear NormACA option for the device, if necessary,
2106                  * since we don't support ACA
2107                  */
2108                 if ((cmd->cdb[0] == INQUIRY) &&
2109                     !(cmd->cdb[1] & SCST_INQ_EVPD/* Std INQUIRY data (no EVPD) */) &&
2110                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2111                         uint8_t *buffer;
2112                         int buflen;
2113
2114                         /* ToDo: all pages ?? */
2115                         buflen = scst_get_buf_first(cmd, &buffer);
2116                         if (buflen > SCST_INQ_BYTE3) {
2117 #ifdef EXTRACHECKS
2118                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2119                                         PRINT_INFO("NormACA set for device: "
2120                                             "lun=%Ld, type 0x%02x", 
2121                                             (uint64_t)cmd->lun, buffer[0]);
2122                                 }
2123 #endif
2124                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2125                         } else {
2126                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2127                                     "buffer");
2128                                 scst_set_cmd_error(cmd,
2129                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
2130                         }
2131                         if (buflen > 0)
2132                                 scst_put_buf(cmd, buffer);
2133                 }
2134
2135                 if (unlikely((cmd->cdb[0] == MODE_SELECT) || 
2136                     (cmd->cdb[0] == MODE_SELECT_10) ||
2137                     (cmd->cdb[0] == LOG_SELECT))) {
2138                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded (LUN %Ld)",
2139                                 (uint64_t)cmd->lun);
2140                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2141                         *pres = SCST_CMD_STATE_RES_CONT_SAME;
2142                         res = 1;
2143                         goto out;
2144                 }
2145         } else {
2146                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2147                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2148                                         &cmd->tgt_dev->tgt_dev_flags)) {
2149                                 struct scst_tgt_dev *tgt_dev_tmp;
2150                                 struct scst_device *dev = cmd->dev;
2151
2152                                 TRACE(TRACE_SCSI, "Real RESERVE failed lun=%Ld, status=%x",
2153                                       (uint64_t)cmd->lun, cmd->status);
2154                                 TRACE_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense_buffer,
2155                                              sizeof(cmd->sense_buffer));
2156
2157                                 /* Clearing the reservation */
2158                                 spin_lock_bh(&dev->dev_lock);
2159                                 list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
2160                                                     dev_tgt_dev_list_entry) {
2161                                         clear_bit(SCST_TGT_DEV_RESERVED, 
2162                                                 &tgt_dev_tmp->tgt_dev_flags);
2163                                 }
2164                                 dev->dev_reserved = 0;
2165                                 spin_unlock_bh(&dev->dev_lock);
2166                         }
2167                 }
2168
2169                 /* Check for MODE PARAMETERS CHANGED UA */
2170                 if ((cmd->dev->scsi_dev != NULL) &&
2171                     (cmd->status == SAM_STAT_CHECK_CONDITION) && 
2172                     SCST_SENSE_VALID(cmd->sense_buffer) &&
2173                     scst_is_ua_sense(cmd->sense_buffer) &&
2174                     (cmd->sense_buffer[12] == 0x2a) &&
2175                     (cmd->sense_buffer[13] == 0x01)) {
2176                         TRACE(TRACE_SCSI, "MODE PARAMETERS CHANGED UA (lun %Ld)",
2177                                 (uint64_t)cmd->lun);
2178                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2179                         *pres = SCST_CMD_STATE_RES_CONT_SAME;
2180                         res = 1;
2181                         goto out;
2182                 }
2183         }
2184
2185 out:
2186         TRACE_EXIT_RES(res);
2187         return res;
2188 }
2189
2190 static int scst_pre_dev_done(struct scst_cmd *cmd)
2191 {
2192         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2193
2194         TRACE_ENTRY();
2195
2196         rc = scst_done_cmd_check(cmd, &res);
2197         if (rc)
2198                 goto out;
2199
2200         cmd->state = SCST_CMD_STATE_DEV_DONE;
2201
2202 out:
2203         TRACE_EXIT_HRES(res);
2204         return res;
2205 }
2206
2207 static int scst_mode_select_checks(struct scst_cmd *cmd)
2208 {
2209         int res = SCST_CMD_STATE_RES_CONT_SAME;
2210         int atomic = scst_cmd_atomic(cmd);
2211
2212         TRACE_ENTRY();
2213
2214         if (likely(scsi_status_is_good(cmd->status))) {
2215                 if (unlikely((cmd->cdb[0] == MODE_SELECT) || 
2216                     (cmd->cdb[0] == MODE_SELECT_10) ||
2217                     (cmd->cdb[0] == LOG_SELECT))) {
2218                         struct scst_device *dev = cmd->dev;
2219                         if (atomic && (dev->scsi_dev != NULL)) {
2220                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2221                                         "context required");
2222                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2223                                 goto out;
2224                         }
2225
2226                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2227                                 "setting the SELECT UA (lun=%Ld)", 
2228                                 (uint64_t)cmd->lun);
2229
2230                         spin_lock_bh(&dev->dev_lock);
2231                         spin_lock(&scst_temp_UA_lock);
2232                         if (cmd->cdb[0] == LOG_SELECT) {
2233                                 scst_set_sense(scst_temp_UA,
2234                                         sizeof(scst_temp_UA),
2235                                         UNIT_ATTENTION, 0x2a, 0x02);
2236                         } else {
2237                                 scst_set_sense(scst_temp_UA,
2238                                         sizeof(scst_temp_UA),
2239                                         UNIT_ATTENTION, 0x2a, 0x01);
2240                         }
2241                         scst_dev_check_set_local_UA(dev, cmd, scst_temp_UA,
2242                                 sizeof(scst_temp_UA));
2243                         spin_unlock(&scst_temp_UA_lock);
2244                         spin_unlock_bh(&dev->dev_lock);
2245
2246                         if (dev->scsi_dev != NULL)
2247                                 scst_obtain_device_parameters(dev);
2248                 }
2249         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) && 
2250                     SCST_SENSE_VALID(cmd->sense_buffer) &&
2251                     scst_is_ua_sense(cmd->sense_buffer) &&
2252                     (cmd->sense_buffer[12] == 0x2a) &&
2253                     (cmd->sense_buffer[13] == 0x01)) {
2254                 if (atomic) {
2255                         TRACE_DBG("%s", "MODE PARAMETERS CHANGED UA: thread "
2256                                 "context required");
2257                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2258                         goto out;
2259                 }
2260
2261                 TRACE(TRACE_SCSI, "MODE PARAMETERS CHANGED UA (lun %Ld): "
2262                         "getting new parameters", (uint64_t)cmd->lun);
2263
2264                 scst_obtain_device_parameters(cmd->dev);
2265         } else
2266                 sBUG();
2267
2268         cmd->state = SCST_CMD_STATE_DEV_DONE;
2269
2270 out:
2271         TRACE_EXIT_HRES(res);
2272         return res;
2273 }
2274
2275 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2276 {
2277         struct scst_cmd *c;
2278
2279         if (likely(cmd->sn_set))
2280                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2281
2282         c = scst_check_deferred_commands(cmd->tgt_dev);
2283         if (c != NULL) {
2284                 unsigned long flags;
2285                 spin_lock_irqsave(&c->cmd_lists->cmd_list_lock, flags);
2286                 TRACE_SN("Adding cmd %p to active cmd list", c);
2287                 list_add_tail(&c->cmd_list_entry,
2288                         &c->cmd_lists->active_cmd_list);
2289                 wake_up(&c->cmd_lists->cmd_list_waitQ);
2290                 spin_unlock_irqrestore(&c->cmd_lists->cmd_list_lock, flags);
2291         }
2292 }
2293
2294 static int scst_dev_done(struct scst_cmd *cmd)
2295 {
2296         int res = SCST_CMD_STATE_RES_CONT_SAME;
2297         int state;
2298         int atomic = scst_cmd_atomic(cmd);
2299
2300         TRACE_ENTRY();
2301
2302         if (atomic && !cmd->dev->handler->dev_done_atomic) {
2303                 TRACE_DBG("Dev handler %s dev_done() can not be "
2304                       "called in atomic context, rescheduling to the thread",
2305                       cmd->dev->handler->name);
2306                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2307                 goto out;
2308         }
2309
2310         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2311         if (likely(!scst_is_cmd_local(cmd)) && 
2312             likely(cmd->dev->handler->dev_done != NULL))
2313         {
2314                 int rc;
2315                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2316                       cmd->dev->handler->name, cmd);
2317                 rc = cmd->dev->handler->dev_done(cmd);
2318                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2319                       cmd->dev->handler->name, rc);
2320                 if (rc != SCST_CMD_STATE_DEFAULT)
2321                         state = rc;
2322         }
2323
2324         switch (state) {
2325         case SCST_CMD_STATE_PRE_XMIT_RESP:
2326         case SCST_CMD_STATE_DEV_PARSE:
2327         case SCST_CMD_STATE_PRE_PARSE:
2328         case SCST_CMD_STATE_PREPARE_SPACE:
2329         case SCST_CMD_STATE_RDY_TO_XFER:
2330         case SCST_CMD_STATE_TGT_PRE_EXEC:
2331         case SCST_CMD_STATE_SEND_TO_MIDLEV:
2332         case SCST_CMD_STATE_PRE_DEV_DONE:
2333         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2334         case SCST_CMD_STATE_DEV_DONE:
2335         case SCST_CMD_STATE_XMIT_RESP:
2336         case SCST_CMD_STATE_FINISHED:
2337                 cmd->state = state;
2338                 res = SCST_CMD_STATE_RES_CONT_SAME;
2339                 break;
2340
2341         case SCST_CMD_STATE_NEED_THREAD_CTX:
2342                 TRACE_DBG("Dev handler %s dev_done() requested "
2343                       "thread context, rescheduling",
2344                       cmd->dev->handler->name);
2345                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2346                 break;
2347
2348         default:
2349                 if (state >= 0) {
2350                         PRINT_ERROR("Dev handler %s dev_done() returned "
2351                                 "invalid cmd state %d", 
2352                                 cmd->dev->handler->name, state);
2353                 } else {
2354                         PRINT_ERROR("Dev handler %s dev_done() returned "
2355                                 "error %d", cmd->dev->handler->name, 
2356                                 state);
2357                 }
2358                 scst_set_cmd_error(cmd,
2359                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2360                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2361                 res = SCST_CMD_STATE_RES_CONT_SAME;
2362                 break;
2363         }
2364
2365         if (cmd->needs_unblocking)
2366                 scst_unblock_dev_cmd(cmd);
2367
2368         if (likely(cmd->dec_on_dev_needed))
2369                 scst_dec_on_dev_cmd(cmd);
2370
2371         if (cmd->inc_expected_sn_on_done && cmd->sent_to_midlev)
2372                 scst_inc_check_expected_sn(cmd);
2373
2374 out:
2375         TRACE_EXIT_HRES(res);
2376         return res;
2377 }
2378
2379 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2380 {
2381         int res;
2382
2383         TRACE_ENTRY();
2384
2385         if (cmd->tgt_dev != NULL) {
2386                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2387                         scst_on_hq_cmd_response(cmd);
2388
2389                 if (unlikely(!cmd->sent_to_midlev)) {
2390                         TRACE_SN("cmd %p was not sent to mid-lev (sn %ld, set %d)",
2391                                 cmd, cmd->sn, cmd->sn_set);
2392                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2393                         cmd->sent_to_midlev = 1;
2394                 }
2395         }
2396
2397         /*
2398          * If we don't remove cmd from the search list here, before
2399          * submitting it for transmittion, we will have a race, when for
2400          * some reason cmd's release is delayed after transmittion and
2401          * initiator sends cmd with the same tag => it is possible that
2402          * a wrong cmd will be found by find() functions.
2403          */
2404         spin_lock_irq(&cmd->sess->sess_list_lock);
2405         list_del(&cmd->search_cmd_list_entry);
2406         spin_unlock_irq(&cmd->sess->sess_list_lock);
2407
2408         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2409                 scst_xmit_process_aborted_cmd(cmd);
2410
2411         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2412                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu), skipping",
2413                         cmd, cmd->tag);
2414                 cmd->state = SCST_CMD_STATE_FINISHED;
2415                 res = SCST_CMD_STATE_RES_CONT_SAME;
2416                 goto out;
2417         }
2418
2419         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2420         res = SCST_CMD_STATE_RES_CONT_SAME;
2421
2422 out:
2423         TRACE_EXIT_HRES(res);
2424         return res;
2425 }
2426
2427 static int scst_xmit_response(struct scst_cmd *cmd)
2428 {
2429         int res, rc;
2430         int atomic = scst_cmd_atomic(cmd);
2431
2432         TRACE_ENTRY();
2433
2434         if (atomic && !cmd->tgtt->xmit_response_atomic) {
2435                 TRACE_DBG("%s", "xmit_response() can not be "
2436                       "called in atomic context, rescheduling to the thread");
2437                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2438                 goto out;
2439         }
2440
2441 #ifdef DEBUG_TM
2442         if (cmd->tm_dbg_delayed && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2443                 if (atomic && !cmd->tgtt->xmit_response_atomic) {
2444                         TRACE_MGMT_DBG("%s", "DEBUG_TM delayed cmd needs a thread");
2445                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2446                         goto out;
2447                 }
2448                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2449                         cmd, cmd->tag);
2450                 schedule_timeout_uninterruptible(HZ);
2451         }
2452 #endif
2453
2454         while (1) {
2455                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2456
2457                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2458                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2459
2460                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2461
2462 #if defined(DEBUG) || defined(TRACING)
2463                 if (cmd->sg) {
2464                         int i;
2465                         struct scatterlist *sg = cmd->sg;
2466                         TRACE(TRACE_SEND_BOT,
2467                               "Xmitting %d S/G(s) at %p sg[0].page at %p",
2468                               cmd->sg_cnt, sg, (void*)sg[0].page);
2469                         for(i = 0; i < cmd->sg_cnt; ++i) {
2470                                 TRACE_BUFF_FLAG(TRACE_SEND_BOT,
2471                                     "Xmitting sg", page_address(sg[i].page),
2472                                     sg[i].length);
2473                         }
2474                 }
2475 #endif
2476
2477 #ifdef DEBUG_RETRY
2478                 if (((scst_random() % 100) == 77))
2479                         rc = SCST_TGT_RES_QUEUE_FULL;
2480                 else
2481 #endif
2482                         rc = cmd->tgtt->xmit_response(cmd);
2483                 TRACE_DBG("xmit_response() returned %d", rc);
2484
2485                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2486                         goto out;
2487
2488                 /* Restore the previous state */
2489                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2490
2491                 switch (rc) {
2492                 case SCST_TGT_RES_QUEUE_FULL:
2493                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2494                                 break;
2495                         else
2496                                 continue;
2497
2498                 case SCST_TGT_RES_NEED_THREAD_CTX:
2499                         TRACE_DBG("Target driver %s xmit_response() "
2500                               "requested thread context, rescheduling",
2501                               cmd->tgtt->name);
2502                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2503                         break;
2504
2505                 default:
2506                         goto out_error;
2507                 }
2508                 break;
2509         }
2510
2511 out:
2512         /* Caution: cmd can be already dead here */
2513         TRACE_EXIT_HRES(res);
2514         return res;
2515
2516 out_error:
2517         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2518                 PRINT_ERROR("Target driver %s xmit_response() returned "
2519                         "fatal error", cmd->tgtt->name);
2520         } else {
2521                 PRINT_ERROR("Target driver %s xmit_response() returned "
2522                         "invalid value %d", cmd->tgtt->name, rc);
2523         }
2524         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2525         cmd->state = SCST_CMD_STATE_FINISHED;
2526         res = SCST_CMD_STATE_RES_CONT_SAME;
2527         goto out;
2528 }
2529
2530 void scst_tgt_cmd_done(struct scst_cmd *cmd)
2531 {
2532         TRACE_ENTRY();
2533
2534         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2535
2536         cmd->state = SCST_CMD_STATE_FINISHED;
2537         scst_proccess_redirect_cmd(cmd, scst_get_context(), 1);
2538
2539         TRACE_EXIT();
2540         return;
2541 }
2542
2543 static int scst_finish_cmd(struct scst_cmd *cmd)
2544 {
2545         int res;
2546
2547         TRACE_ENTRY();
2548
2549         atomic_dec(&cmd->sess->sess_cmd_count);
2550
2551         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2552                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
2553                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
2554                         atomic_read(&scst_cmd_count));
2555         }
2556
2557         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
2558                 if ((cmd->tgt_dev != NULL) &&
2559                     scst_is_ua_sense(cmd->sense_buffer)) {
2560                         /* This UA delivery failed, so requeue it */
2561                         scst_check_set_UA(cmd->tgt_dev, cmd->sense_buffer,
2562                                         sizeof(cmd->sense_buffer), 1);
2563                 }
2564         }
2565
2566         scst_cmd_put(cmd);
2567
2568         res = SCST_CMD_STATE_RES_CONT_NEXT;
2569
2570         TRACE_EXIT_HRES(res);
2571         return res;
2572 }
2573
2574 /*
2575  * No locks, but it must be externally serialized (see comment for
2576  * scst_cmd_init_done() in scsi_tgt.h)
2577  */
2578 static void scst_cmd_set_sn(struct scst_cmd *cmd)
2579 {
2580         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2581         unsigned long flags;
2582
2583         if (scst_is_implicit_hq(cmd)) {
2584                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "Implicit HQ cmd %p", cmd);
2585                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
2586         }
2587
2588         /* Optimized for lockless fast path */
2589
2590         scst_check_debug_sn(cmd);
2591
2592         if (cmd->dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER)
2593                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
2594
2595         switch(cmd->queue_type) {
2596         case SCST_CMD_QUEUE_SIMPLE:
2597         case SCST_CMD_QUEUE_UNTAGGED:
2598 #if 1 /* temporary, ToDo */
2599                 if (scst_cmd_is_expected_set(cmd)) {
2600                         if (cmd->expected_data_direction == SCST_DATA_READ)
2601                                 goto ordered;
2602                 } else
2603                         goto ordered;
2604 #endif
2605                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
2606                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
2607                                 tgt_dev->curr_sn++;
2608                                 TRACE_SN("Incremented curr_sn %ld",
2609                                         tgt_dev->curr_sn);
2610                         }
2611                         cmd->sn_slot = tgt_dev->cur_sn_slot;
2612                         cmd->sn = tgt_dev->curr_sn;
2613                         
2614                         tgt_dev->prev_cmd_ordered = 0;
2615                 } else {
2616                         TRACE(TRACE_MINOR, "%s", "Not enough SN slots");
2617                         goto ordered;
2618                 }
2619                 break;
2620
2621         case SCST_CMD_QUEUE_ORDERED:
2622                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "ORDERED cmd %p "
2623                         "(op %x)", cmd, cmd->cdb[0]);
2624 ordered:
2625                 if (!tgt_dev->prev_cmd_ordered) {
2626                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2627                         tgt_dev->num_free_sn_slots--;
2628                         smp_mb();
2629                         if ((tgt_dev->num_free_sn_slots >= 0) &&
2630                             (atomic_read(tgt_dev->cur_sn_slot) > 0)) {
2631                                 do {
2632                                         tgt_dev->cur_sn_slot++;
2633                                         if (tgt_dev->cur_sn_slot == 
2634                                                 tgt_dev->sn_slots +
2635                                                 ARRAY_SIZE(tgt_dev->sn_slots))
2636                                             tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
2637                                 } while(atomic_read(tgt_dev->cur_sn_slot) != 0);
2638                                 TRACE_SN("New cur SN slot %zd",
2639                                         tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
2640                         } else
2641                                 tgt_dev->num_free_sn_slots++;
2642                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2643                 }
2644                 tgt_dev->prev_cmd_ordered = 1;
2645                 tgt_dev->curr_sn++;
2646                 cmd->sn = tgt_dev->curr_sn;
2647                 break;
2648
2649         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
2650                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "HQ cmd %p "
2651                         "(op %x)", cmd, cmd->cdb[0]);
2652                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2653                 tgt_dev->hq_cmd_count++;
2654                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2655                 goto out;
2656
2657         default:
2658                 sBUG();
2659         }
2660
2661         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
2662                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
2663                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
2664                 atomic_read(tgt_dev->cur_sn_slot), 
2665                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
2666                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
2667
2668         cmd->sn_set = 1;
2669 out:
2670         return;
2671 }
2672
2673 /*
2674  * Returns 0 on success, > 0 when we need to wait for unblock,
2675  * < 0 if there is no device (lun) or device type handler.
2676  *
2677  * No locks, but might be on IRQ, protection is done by the
2678  * suspended activity.
2679  */
2680 static int scst_translate_lun(struct scst_cmd *cmd)
2681 {
2682         struct scst_tgt_dev *tgt_dev = NULL;
2683         int res;
2684
2685         TRACE_ENTRY();
2686
2687         __scst_get(1);
2688
2689         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
2690                 struct list_head *sess_tgt_dev_list_head =
2691                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
2692                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %Ld)", cmd,
2693                         (uint64_t)cmd->lun);
2694                 res = -1;
2695                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
2696                                 sess_tgt_dev_list_entry) {
2697                         if (tgt_dev->lun == cmd->lun) {
2698                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
2699
2700                                 if (unlikely(tgt_dev->dev->handler == &scst_null_devtype)) {
2701                                         PRINT_INFO("Dev handler for device "
2702                                           "%Ld is NULL, the device will not be "
2703                                           "visible remotely", (uint64_t)cmd->lun);
2704                                         break;
2705                                 }
2706                                 
2707                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
2708                                 cmd->tgt_dev = tgt_dev;
2709                                 cmd->dev = tgt_dev->dev;
2710
2711                                 res = 0;
2712                                 break;
2713                         }
2714                 }
2715                 if (res != 0) {
2716                         TRACE(TRACE_MINOR, "tgt_dev for lun %Ld not found, command to "
2717                                 "unexisting LU?", (uint64_t)cmd->lun);
2718                         __scst_put();
2719                 }
2720         } else {
2721                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
2722                 __scst_put();
2723                 res = 1;
2724         }
2725
2726         TRACE_EXIT_RES(res);
2727         return res;
2728 }
2729
2730 /*
2731  * No locks, but might be on IRQ
2732  *
2733  * Returns 0 on success, > 0 when we need to wait for unblock,
2734  * < 0 if there is no device (lun) or device type handler.
2735  */
2736 static int __scst_init_cmd(struct scst_cmd *cmd)
2737 {
2738         int res = 0;
2739
2740         TRACE_ENTRY();
2741
2742         res = scst_translate_lun(cmd);
2743         if (likely(res == 0)) {
2744                 int cnt;
2745                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
2746                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
2747                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
2748                         TRACE(TRACE_MGMT_MINOR, "Too many pending commands (%d) in "
2749                                 "session, returning BUSY to initiator \"%s\"",
2750                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
2751                                   "Anonymous" : cmd->sess->initiator_name);
2752                         goto out_busy;
2753                 }
2754                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
2755                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
2756                         TRACE(TRACE_MGMT_MINOR, "Too many pending device commands "
2757                                 "(%d), returning BUSY to initiator \"%s\"",
2758                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
2759                                   "Anonymous" : cmd->sess->initiator_name);
2760                         goto out_busy;
2761                 }
2762                 if (!cmd->set_sn_on_restart_cmd)
2763                         scst_cmd_set_sn(cmd);
2764         } else if (res < 0) {
2765                 TRACE_DBG("Finishing cmd %p", cmd);
2766                 scst_set_cmd_error(cmd,
2767                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
2768                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2769         } else
2770                 goto out;
2771
2772 out:
2773         TRACE_EXIT_RES(res);
2774         return res;
2775
2776 out_busy:
2777         scst_set_busy(cmd);
2778         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2779         goto out;
2780 }
2781
2782 /* Called under scst_init_lock and IRQs disabled */
2783 static void scst_do_job_init(void)
2784 {
2785         struct scst_cmd *cmd;
2786         int susp;
2787
2788         TRACE_ENTRY();
2789
2790 restart:
2791         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
2792         if (scst_init_poll_cnt > 0)
2793                 scst_init_poll_cnt--;
2794
2795         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
2796                 int rc;
2797                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
2798                         continue;
2799                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2800                         spin_unlock_irq(&scst_init_lock);
2801                         rc = __scst_init_cmd(cmd);
2802                         spin_lock_irq(&scst_init_lock);
2803                         if (rc > 0) {
2804                                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, restarting");
2805                                 goto restart;
2806                         }
2807                 } else {
2808                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
2809                                 cmd, cmd->tag);
2810                         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2811                 }
2812
2813                 /*
2814                  * Deleting cmd from init cmd list after __scst_init_cmd()
2815                  * is necessary to keep the check in scst_init_cmd() correct
2816                  * to preserve the commands order.
2817                  *
2818                  * We don't care about the race, when init cmd list is empty
2819                  * and one command detected that it just was not empty, so
2820                  * it's inserting to it, but another command at the same time
2821                  * seeing init cmd list empty and goes directly, because it
2822                  * could affect only commands from the same initiator to the
2823                  * same tgt_dev, but init_cmd_done() doesn't guarantee the order
2824                  * in case of simultaneous such calls anyway.
2825                  */
2826                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
2827                 list_del(&cmd->cmd_list_entry);
2828                 spin_unlock(&scst_init_lock);
2829
2830                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
2831                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
2832                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2833                         list_add(&cmd->cmd_list_entry,
2834                                 &cmd->cmd_lists->active_cmd_list);
2835                 else
2836                         list_add_tail(&cmd->cmd_list_entry,
2837                                 &cmd->cmd_lists->active_cmd_list);
2838                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
2839                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
2840
2841                 spin_lock(&scst_init_lock);
2842                 goto restart;
2843         }
2844
2845         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
2846                 goto restart;
2847
2848         TRACE_EXIT();
2849         return;
2850 }
2851
2852 static inline int test_init_cmd_list(void)
2853 {
2854         int res = (!list_empty(&scst_init_cmd_list) &&
2855                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
2856                   unlikely(kthread_should_stop()) ||
2857                   (scst_init_poll_cnt > 0);
2858         return res;
2859 }
2860
2861 int scst_init_cmd_thread(void *arg)
2862 {
2863         TRACE_ENTRY();
2864
2865         PRINT_INFO("Init thread started, PID %d", current->pid);
2866
2867         current->flags |= PF_NOFREEZE;
2868
2869         set_user_nice(current, -20);
2870
2871         spin_lock_irq(&scst_init_lock);
2872         while(!kthread_should_stop()) {
2873                 wait_queue_t wait;
2874                 init_waitqueue_entry(&wait, current);
2875
2876                 if (!test_init_cmd_list()) {
2877                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
2878                                                  &wait);
2879                         for (;;) {
2880                                 set_current_state(TASK_INTERRUPTIBLE);
2881                                 if (test_init_cmd_list())
2882                                         break;
2883                                 spin_unlock_irq(&scst_init_lock);
2884                                 schedule();
2885                                 spin_lock_irq(&scst_init_lock);
2886                         }
2887                         set_current_state(TASK_RUNNING);
2888                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
2889                 }
2890                 scst_do_job_init();
2891         }
2892         spin_unlock_irq(&scst_init_lock);
2893
2894         /*
2895          * If kthread_should_stop() is true, we are guaranteed to be
2896          * on the module unload, so scst_init_cmd_list must be empty.
2897          */
2898         sBUG_ON(!list_empty(&scst_init_cmd_list));
2899
2900         PRINT_INFO("Init thread PID %d finished", current->pid);
2901
2902         TRACE_EXIT();
2903         return 0;
2904 }
2905
2906 /* Called with no locks held */
2907 void scst_process_active_cmd(struct scst_cmd *cmd, int context)
2908 {
2909         int res;
2910
2911         TRACE_ENTRY();
2912
2913         EXTRACHECKS_BUG_ON(in_irq());
2914
2915         cmd->atomic = (context == SCST_CONTEXT_DIRECT_ATOMIC);
2916
2917         do {
2918                 switch (cmd->state) {
2919                 case SCST_CMD_STATE_PRE_PARSE:
2920                         res = scst_pre_parse(cmd);
2921                         break;
2922
2923                 case SCST_CMD_STATE_DEV_PARSE:
2924                         res = scst_parse_cmd(cmd);
2925                         break;
2926
2927                 case SCST_CMD_STATE_PREPARE_SPACE:
2928                         res = scst_prepare_space(cmd);
2929                         break;
2930
2931                 case SCST_CMD_STATE_RDY_TO_XFER:
2932                         res = scst_rdy_to_xfer(cmd);
2933                         break;
2934
2935                 case SCST_CMD_STATE_TGT_PRE_EXEC:
2936                         res = scst_tgt_pre_exec(cmd);
2937                         break;
2938
2939                 case SCST_CMD_STATE_SEND_TO_MIDLEV:
2940                         if (tm_dbg_check_cmd(cmd) != 0) {
2941                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2942                                 TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
2943                                         "because of TM DBG delay", cmd,
2944                                         cmd->tag);
2945                                 break;
2946                         }
2947                         res = scst_send_to_midlev(&cmd);
2948                         /* !! At this point cmd, sess & tgt_dev can be already freed !! */
2949                         break;
2950
2951                 case SCST_CMD_STATE_PRE_DEV_DONE:
2952                         res = scst_pre_dev_done(cmd);
2953                         break;
2954
2955                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2956                         res = scst_mode_select_checks(cmd);
2957                         break;
2958
2959                 case SCST_CMD_STATE_DEV_DONE:
2960                         res = scst_dev_done(cmd);
2961                         break;
2962
2963                 case SCST_CMD_STATE_PRE_XMIT_RESP:
2964                         res = scst_pre_xmit_response(cmd);
2965                         break;
2966
2967                 case SCST_CMD_STATE_XMIT_RESP:
2968                         res = scst_xmit_response(cmd);
2969                         break;
2970
2971                 case SCST_CMD_STATE_FINISHED:
2972                         res = scst_finish_cmd(cmd);
2973                         break;
2974
2975                 default:
2976                         PRINT_ERROR("cmd (%p) in state %d, but shouldn't be",
2977                                cmd, cmd->state);
2978                         sBUG();
2979                         res = SCST_CMD_STATE_RES_CONT_NEXT;
2980                         break;
2981                 }
2982         } while(res == SCST_CMD_STATE_RES_CONT_SAME);
2983
2984         if (res == SCST_CMD_STATE_RES_CONT_NEXT) {
2985                 /* None */
2986         } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
2987                 spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
2988                 switch (cmd->state) {
2989                 case SCST_CMD_STATE_PRE_PARSE:
2990                 case SCST_CMD_STATE_DEV_PARSE:
2991                 case SCST_CMD_STATE_PREPARE_SPACE:
2992                 case SCST_CMD_STATE_RDY_TO_XFER:
2993                 case SCST_CMD_STATE_TGT_PRE_EXEC:
2994                 case SCST_CMD_STATE_SEND_TO_MIDLEV:
2995                 case SCST_CMD_STATE_PRE_DEV_DONE:
2996                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2997                 case SCST_CMD_STATE_DEV_DONE:
2998                 case SCST_CMD_STATE_PRE_XMIT_RESP:
2999                 case SCST_CMD_STATE_XMIT_RESP:
3000                 case SCST_CMD_STATE_FINISHED:
3001                         TRACE_DBG("Adding cmd %p to head of active cmd list", cmd);
3002                         list_add(&cmd->cmd_list_entry,
3003                                 &cmd->cmd_lists->active_cmd_list);
3004                         break;
3005 #ifdef EXTRACHECKS
3006                 /* not very valid commands */
3007                 case SCST_CMD_STATE_DEFAULT:
3008                 case SCST_CMD_STATE_NEED_THREAD_CTX:
3009                         PRINT_ERROR("cmd %p is in state %d, not putting on "
3010                                 "useful list (left on scst cmd list)", cmd, 
3011                                 cmd->state);
3012                         spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3013                         sBUG();
3014                         spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3015                         break;
3016 #endif
3017                 default:
3018                         break;
3019                 }
3020                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3021                 spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3022         } else
3023                 sBUG();
3024
3025         TRACE_EXIT();
3026         return;
3027 }
3028
3029 /* Called under cmd_list_lock and IRQs disabled */
3030 static void scst_do_job_active(struct list_head *cmd_list,
3031         spinlock_t *cmd_list_lock, int context)
3032 {
3033         TRACE_ENTRY();
3034
3035 #ifdef EXTRACHECKS
3036         WARN_ON((context != SCST_CONTEXT_DIRECT_ATOMIC) && 
3037                 (context != SCST_CONTEXT_DIRECT));
3038 #endif
3039
3040         while (!list_empty(cmd_list)) {
3041                 struct scst_cmd *cmd = list_entry(cmd_list->next, typeof(*cmd),
3042                                         cmd_list_entry);
3043                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
3044                 list_del(&cmd->cmd_list_entry);
3045                 spin_unlock_irq(cmd_list_lock);
3046                 scst_process_active_cmd(cmd, context);
3047                 spin_lock_irq(cmd_list_lock);
3048         }
3049
3050         TRACE_EXIT();
3051         return;
3052 }
3053
3054 static inline int test_cmd_lists(struct scst_cmd_lists *p_cmd_lists)
3055 {
3056         int res = !list_empty(&p_cmd_lists->active_cmd_list) ||
3057             unlikely(kthread_should_stop()) ||
3058             tm_dbg_is_release();
3059         return res;
3060 }
3061
3062 int scst_cmd_thread(void *arg)
3063 {
3064         struct scst_cmd_lists *p_cmd_lists = (struct scst_cmd_lists*)arg;
3065
3066         TRACE_ENTRY();
3067
3068         PRINT_INFO("Processing thread started, PID %d", current->pid);
3069
3070 #if 0
3071         set_user_nice(current, 10);
3072 #endif
3073         current->flags |= PF_NOFREEZE;
3074
3075         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3076         while (!kthread_should_stop()) {
3077                 wait_queue_t wait;
3078                 init_waitqueue_entry(&wait, current);
3079
3080                 if (!test_cmd_lists(p_cmd_lists)) {
3081                         add_wait_queue_exclusive(&p_cmd_lists->cmd_list_waitQ,
3082                                 &wait);
3083                         for (;;) {
3084                                 set_current_state(TASK_INTERRUPTIBLE);
3085                                 if (test_cmd_lists(p_cmd_lists))
3086                                         break;
3087                                 spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3088                                 schedule();
3089                                 spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3090                         }
3091                         set_current_state(TASK_RUNNING);
3092                         remove_wait_queue(&p_cmd_lists->cmd_list_waitQ, &wait);
3093                 }
3094
3095                 if (tm_dbg_is_release()) {
3096                         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3097                         tm_dbg_check_released_cmds();
3098                         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3099                 }
3100
3101                 scst_do_job_active(&p_cmd_lists->active_cmd_list,
3102                         &p_cmd_lists->cmd_list_lock, SCST_CONTEXT_DIRECT);
3103         }
3104         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3105
3106 #ifdef EXTRACHECKS
3107         /*
3108          * If kthread_should_stop() is true, we are guaranteed to be either
3109          * on the module unload, or there must be at least one other thread to
3110          * process the commands lists.
3111          */
3112         if (p_cmd_lists == &scst_main_cmd_lists) {
3113                 sBUG_ON((scst_threads_info.nr_cmd_threads == 1) &&
3114                          !list_empty(&scst_main_cmd_lists.active_cmd_list));
3115         }
3116 #endif
3117
3118         PRINT_INFO("Processing thread PID %d finished", current->pid);
3119
3120         TRACE_EXIT();
3121         return 0;
3122 }
3123
3124 void scst_cmd_tasklet(long p)
3125 {
3126         struct scst_tasklet *t = (struct scst_tasklet*)p;
3127
3128         TRACE_ENTRY();
3129
3130         spin_lock_irq(&t->tasklet_lock);
3131         scst_do_job_active(&t->tasklet_cmd_list, &t->tasklet_lock,
3132                 SCST_CONTEXT_DIRECT_ATOMIC);
3133         spin_unlock_irq(&t->tasklet_lock);
3134
3135         TRACE_EXIT();
3136         return;
3137 }
3138
3139 /*
3140  * Returns 0 on success, < 0 if there is no device handler or
3141  * > 0 if SCST_FLAG_SUSPENDED set and SCST_FLAG_SUSPENDING - not.
3142  * No locks, protection is done by the suspended activity.
3143  */
3144 static int scst_mgmt_translate_lun(struct scst_mgmt_cmd *mcmd)
3145 {
3146         struct scst_tgt_dev *tgt_dev = NULL;
3147         struct list_head *sess_tgt_dev_list_head;
3148         int res = -1;
3149
3150         TRACE_ENTRY();
3151
3152         TRACE_DBG("Finding tgt_dev for mgmt cmd %p (lun %Ld)", mcmd,
3153               (uint64_t)mcmd->lun);
3154
3155         __scst_get(1);
3156
3157         if (unlikely(test_bit(SCST_FLAG_SUSPENDED, &scst_flags) &&
3158                      !test_bit(SCST_FLAG_SUSPENDING, &scst_flags))) {
3159                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3160                 __scst_put();
3161                 res = 1;
3162                 goto out;
3163         }
3164
3165         sess_tgt_dev_list_head =
3166                 &mcmd->sess->sess_tgt_dev_list_hash[HASH_VAL(mcmd->lun)];
3167         list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3168                         sess_tgt_dev_list_entry) {
3169                 if (tgt_dev->lun == mcmd->lun) {
3170                         TRACE_DBG("tgt_dev %p found", tgt_dev);
3171                         mcmd->mcmd_tgt_dev = tgt_dev;
3172                         res = 0;
3173                         break;
3174                 }
3175         }
3176         if (mcmd->mcmd_tgt_dev == NULL)
3177                 __scst_put();
3178
3179 out:
3180         TRACE_EXIT_HRES(res);
3181         return res;
3182 }
3183
3184 /* No locks */
3185 void scst_complete_cmd_mgmt(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd)
3186 {
3187         TRACE_ENTRY();
3188
3189         spin_lock_irq(&scst_mcmd_lock);
3190
3191         TRACE_MGMT_DBG("cmd %p completed (tag %llu, mcmd %p, "
3192                 "mcmd->cmd_wait_count %d)", cmd, cmd->tag, mcmd,
3193                 mcmd->cmd_wait_count);
3194
3195         cmd->mgmt_cmnd = NULL;
3196
3197         if (cmd->completed)
3198                 mcmd->completed_cmd_count++;
3199
3200         mcmd->cmd_wait_count--;
3201         if (mcmd->cmd_wait_count > 0) {
3202                 TRACE_MGMT_DBG("cmd_wait_count(%d) not 0, skipping",
3203                         mcmd->cmd_wait_count);
3204                 goto out_unlock;
3205         }
3206
3207         mcmd->state = SCST_MGMT_CMD_STATE_DONE;
3208
3209         if (mcmd->completed) {
3210                 TRACE_MGMT_DBG("Adding mgmt cmd %p to active mgmt cmd list",
3211                         mcmd);
3212                 list_add_tail(&mcmd->mgmt_cmd_list_entry,
3213                         &scst_active_mgmt_cmd_list);
3214         }
3215
3216         spin_unlock_irq(&scst_mcmd_lock);
3217
3218         wake_up(&scst_mgmt_cmd_list_waitQ);
3219
3220 out:
3221         TRACE_EXIT();
3222         return;
3223
3224 out_unlock:
3225         spin_unlock_irq(&scst_mcmd_lock);
3226         goto out;
3227 }
3228
3229 static int scst_call_dev_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
3230         struct scst_tgt_dev *tgt_dev, int set_status)
3231 {
3232         int res = SCST_DEV_TM_NOT_COMPLETED;
3233         struct scst_dev_type *h = tgt_dev->dev->handler;
3234
3235         if (h->task_mgmt_fn) {
3236                 TRACE_MGMT_DBG("Calling dev handler %s task_mgmt_fn(fn=%d)",
3237                         h->name, mcmd->fn);
3238                 EXTRACHECKS_BUG_ON(in_irq());
3239                 res = h->task_mgmt_fn(mcmd, tgt_dev);
3240                 TRACE_MGMT_DBG("Dev handler %s task_mgmt_fn() returned %d",
3241                       h->name, res);
3242                 if (set_status && (res != SCST_DEV_TM_NOT_COMPLETED))
3243                         mcmd->status = res;
3244         }
3245         return res;
3246 }
3247
3248 static inline int scst_is_strict_mgmt_fn(int mgmt_fn)
3249 {
3250         switch(mgmt_fn) {
3251 #ifdef ABORT_CONSIDER_FINISHED_TASKS_AS_NOT_EXISTING
3252                 case SCST_ABORT_TASK:
3253 #endif
3254 #if 0
3255                 case SCST_ABORT_TASK_SET:
3256                 case SCST_CLEAR_TASK_SET:
3257 #endif
3258                         return 1;
3259                 default:
3260                         return 0;
3261         }
3262 }
3263
3264 /* 
3265  * Might be called under sess_list_lock and IRQ off + BHs also off
3266  * Returns -1 if command is being executed (ABORT failed), 0 otherwise
3267  */
3268 void scst_abort_cmd(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd,
3269         int other_ini, int call_dev_task_mgmt_fn)
3270 {
3271         TRACE_ENTRY();
3272
3273         TRACE(((mcmd != NULL) && (mcmd->fn == SCST_ABORT_TASK)) ? TRACE_MGMT_MINOR : TRACE_MGMT,
3274                 "Aborting cmd %p (tag %llu)", cmd, cmd->tag);
3275
3276         if (other_ini) {
3277                 set_bit(SCST_CMD_ABORTED_OTHER, &cmd->cmd_flags);
3278                 smp_mb__after_set_bit();
3279         } else {
3280                 /* Might be necessary if command aborted several times */
3281                 clear_bit(SCST_CMD_ABORTED_OTHER, &cmd->cmd_flags);
3282         }
3283         set_bit(SCST_CMD_ABORTED, &cmd->cmd_flags);
3284         smp_mb__after_set_bit();
3285
3286         if (cmd->tgt_dev == NULL) {
3287                 unsigned long flags;
3288                 spin_lock_irqsave(&scst_init_lock, flags);
3289                 scst_init_poll_cnt++;
3290                 spin_unlock_irqrestore(&scst_init_lock, flags);
3291                 wake_up(&scst_init_cmd_list_waitQ);
3292         }
3293
3294         if (call_dev_task_mgmt_fn && (cmd->tgt_dev != NULL)) {
3295                 EXTRACHECKS_BUG_ON(irqs_disabled());
3296                 scst_call_dev_task_mgmt_fn(mcmd, cmd->tgt_dev, 1);
3297         }
3298
3299         if (mcmd) {
3300                 unsigned long flags;
3301                 /*
3302                  * Delay the response until the command's finish in
3303                  * order to guarantee that "no further responses from
3304                  * the task are sent to the SCSI initiator port" after
3305                  * response from the TM function is sent (SAM). Plus,
3306                  * we must wait here to be sure that we won't receive
3307                  * double commands with the same tag.
3308                  */
3309                 TRACE((mcmd->fn == SCST_ABORT_TASK) ? TRACE_MGMT_MINOR : TRACE_MGMT,
3310                         "cmd %p (tag %llu) being executed/xmitted (state %d), "
3311                         "deferring ABORT...", cmd, cmd->tag, cmd->state);
3312 #ifdef EXTRACHECKS
3313                 if (cmd->mgmt_cmnd) {
3314                         printk(KERN_ALERT "cmd %p (tag %llu, state %d) "
3315                                 "has non-NULL mgmt_cmnd %p!!! Current "
3316                                 "mcmd %p\n", cmd, cmd->tag, cmd->state,
3317                                 cmd->mgmt_cmnd, mcmd);
3318                 }
3319 #endif
3320                 sBUG_ON(cmd->mgmt_cmnd);
3321                 spin_lock_irqsave(&scst_mcmd_lock, flags);
3322                 mcmd->cmd_wait_count++;
3323                 spin_unlock_irqrestore(&scst_mcmd_lock, flags);
3324                 /* cmd can't die here or sess_list_lock already taken */
3325                 cmd->mgmt_cmnd = mcmd;
3326         }
3327
3328         tm_dbg_release_cmd(cmd);
3329
3330         TRACE_EXIT();
3331         return;
3332 }
3333
3334 /* No locks */
3335 static int scst_set_mcmd_next_state(struct scst_mgmt_cmd *mcmd)
3336 {
3337         int res;
3338         spin_lock_irq(&scst_mcmd_lock);
3339         if (mcmd->cmd_wait_count != 0) {
3340                 TRACE_MGMT_DBG("cmd_wait_count(%d) not 0, preparing to "
3341                         "wait", mcmd->cmd_wait_count);
3342                 mcmd->state = SCST_MGMT_CMD_STATE_EXECUTING;
3343                 res = -1;
3344         } else {
3345                 mcmd->state = SCST_MGMT_CMD_STATE_DONE;
3346                 res = 0;
3347         }
3348         mcmd->completed = 1;
3349         spin_unlock_irq(&scst_mcmd_lock);
3350         return res;
3351 }
3352
3353 static int __scst_check_unblock_aborted_cmd(struct scst_cmd *cmd)
3354 {
3355         int res;
3356         if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
3357                 TRACE_MGMT_DBG("Adding aborted blocked cmd %p to active cmd "
3358                         "list", cmd);
3359                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
3360                 list_add_tail(&cmd->cmd_list_entry,
3361                         &cmd->cmd_lists->active_cmd_list);
3362                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3363                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
3364                 res = 1;
3365         } else
3366                 res = 0;
3367         return res;
3368 }
3369
3370 static void scst_unblock_aborted_cmds(int scst_mutex_held)
3371 {
3372         struct scst_device *dev;
3373
3374         TRACE_ENTRY();
3375
3376         if (!scst_mutex_held)
3377                 mutex_lock(&scst_mutex);
3378
3379         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
3380                 struct scst_cmd *cmd, *tcmd;
3381                 struct scst_tgt_dev *tgt_dev;
3382                 spin_lock_bh(&dev->dev_lock);
3383                 local_irq_disable();
3384                 list_for_each_entry_safe(cmd, tcmd, &dev->blocked_cmd_list,
3385                                         blocked_cmd_list_entry) {
3386                         if (__scst_check_unblock_aborted_cmd(cmd))
3387                                 list_del(&cmd->blocked_cmd_list_entry);
3388                 }
3389                 local_irq_enable();
3390                 spin_unlock_bh(&dev->dev_lock);
3391
3392                 local_irq_disable();
3393                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
3394                                          dev_tgt_dev_list_entry) {
3395                         spin_lock(&tgt_dev->sn_lock);
3396                         list_for_each_entry_safe(cmd, tcmd,
3397                                         &tgt_dev->deferred_cmd_list,
3398                                         sn_cmd_list_entry) {
3399                                 if (__scst_check_unblock_aborted_cmd(cmd)) {
3400                                         TRACE_MGMT_DBG("Deleting aborted SN "
3401                                                 "cmd %p from SN list", cmd);
3402                                         tgt_dev->def_cmd_count--;
3403                                         list_del(&cmd->sn_cmd_list_entry);
3404                                 }
3405                         }
3406                         spin_unlock(&tgt_dev->sn_lock);
3407                 }
3408                 local_irq_enable();
3409         }
3410
3411         if (!scst_mutex_held)
3412                 mutex_unlock(&scst_mutex);
3413
3414         TRACE_EXIT();
3415         return;
3416 }
3417
3418 static void __scst_abort_task_set(struct scst_mgmt_cmd *mcmd,
3419         struct scst_tgt_dev *tgt_dev, int other_ini, int scst_mutex_held)
3420 {
3421         struct scst_cmd *cmd;
3422         struct scst_session *sess = tgt_dev->sess;
3423
3424         TRACE_ENTRY();
3425
3426         spin_lock_irq(&sess->sess_list_lock);
3427
3428         TRACE_DBG("Searching in search cmd list (sess=%p)", sess);
3429         list_for_each_entry(cmd, &sess->search_cmd_list, 
3430                         search_cmd_list_entry) {
3431                 if ((cmd->tgt_dev == tgt_dev) ||
3432                     ((cmd->tgt_dev == NULL) && 
3433                      (cmd->lun == tgt_dev->lun))) {
3434                         if (mcmd->cmd_sn_set) {
3435                                 sBUG_ON(!cmd->tgt_sn_set);
3436                                 if (scst_sn_before(mcmd->cmd_sn, cmd->tgt_sn) ||
3437                                     (mcmd->cmd_sn == cmd->tgt_sn))
3438                                         continue;
3439                         }
3440                         scst_abort_cmd(cmd, mcmd, other_ini, 0);
3441                 }
3442         }
3443         spin_unlock_irq(&sess->sess_list_lock);
3444
3445         scst_unblock_aborted_cmds(scst_mutex_held);
3446
3447         TRACE_EXIT();
3448         return;
3449 }
3450
3451 /* Returns 0 if the command processing should be continued, <0 otherwise */
3452 static int scst_abort_task_set(struct scst_mgmt_cmd *mcmd)
3453 {
3454         int res;
3455         struct scst_tgt_dev *tgt_dev = mcmd->mcmd_tgt_dev;
3456         struct scst_device *dev = tgt_dev->dev;
3457
3458         TRACE(TRACE_MGMT, "Aborting task set (lun=%Ld, mcmd=%p)",
3459                 tgt_dev->lun, mcmd);
3460
3461         mcmd->needs_unblocking = 1;
3462
3463         spin_lock_bh(&dev->dev_lock);
3464         __scst_block_dev(dev);
3465         spin_unlock_bh(&dev->dev_lock);
3466
3467         __scst_abort_task_set(mcmd, tgt_dev, 0, 0);
3468         scst_call_dev_task_mgmt_fn(mcmd, tgt_dev, 0);
3469
3470         res = scst_set_mcmd_next_state(mcmd);
3471
3472         TRACE_EXIT_RES(res);
3473         return res;
3474 }
3475
3476 static int scst_is_cmd_belongs_to_dev(struct scst_cmd *cmd,
3477         struct scst_device *dev)
3478 {
3479         struct scst_tgt_dev *tgt_dev = NULL;
3480         struct list_head *sess_tgt_dev_list_head;
3481         int res = 0;
3482
3483         TRACE_ENTRY();
3484
3485         TRACE_DBG("Finding match for dev %p and cmd %p (lun %Ld)", dev, cmd,
3486               (uint64_t)cmd->lun);
3487
3488         sess_tgt_dev_list_head =
3489                 &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
3490         list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3491                         sess_tgt_dev_list_entry) {
3492                 if (tgt_dev->lun == cmd->lun) {
3493                         TRACE_DBG("dev %p found", tgt_dev->dev);
3494                         res = (tgt_dev->dev == dev);
3495                         goto out;
3496                 }
3497         }
3498
3499 out:
3500         TRACE_EXIT_HRES(res);
3501         return res;
3502 }
3503
3504 /* Returns 0 if the command processing should be continued, <0 otherwise */
3505 static int scst_clear_task_set(struct scst_mgmt_cmd *mcmd)
3506 {
3507         int res;
3508         struct scst_device *dev = mcmd->mcmd_tgt_dev->dev;
3509         struct scst_tgt_dev *tgt_dev;
3510         LIST_HEAD(UA_tgt_devs);
3511
3512         TRACE_ENTRY();
3513
3514         TRACE(TRACE_MGMT, "Clearing task set (lun=%Ld, mcmd=%p)",
3515                 (uint64_t)mcmd->lun, mcmd);
3516
3517         mcmd->needs_unblocking = 1;
3518
3519         spin_lock_bh(&dev->dev_lock);
3520         __scst_block_dev(dev);
3521         spin_unlock_bh(&dev->dev_lock);
3522
3523         __scst_abort_task_set(mcmd, mcmd->mcmd_tgt_dev, 0, 0);
3524
3525         mutex_lock(&scst_mutex);
3526
3527         list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list, 
3528                         dev_tgt_dev_list_entry) {
3529                 struct scst_session *sess = tgt_dev->sess;
3530                 struct scst_cmd *cmd;
3531                 int aborted = 0;
3532
3533                 if (tgt_dev == mcmd->mcmd_tgt_dev)
3534                         continue;
3535
3536                 spin_lock_irq(&sess->sess_list_lock);
3537
3538                 TRACE_DBG("Searching in search cmd list (sess=%p)", sess);
3539                 list_for_each_entry(cmd, &sess->search_cmd_list, 
3540                                 search_cmd_list_entry) {
3541                         if ((cmd->dev == dev) ||
3542                             ((cmd->dev == NULL) && 
3543                              scst_is_cmd_belongs_to_dev(cmd, dev))) {
3544                                 scst_abort_cmd(cmd, mcmd, 1, 0);
3545                                 aborted = 1;
3546                         }