d2d617d231dc57d2cce2b1228407cb4edfa0f1ed
[mirror/scst/.git] / scst / src / scst_targ.c
1 /*
2  *  scst_targ.c
3  *  
4  *  Copyright (C) 2004-2007 Vladislav Bolkhovitin <vst@vlnb.net>
5  *                 and Leonid Stoljar
6  *  
7  *  This program is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU General Public License
9  *  as published by the Free Software Foundation, version 2
10  *  of the License.
11  * 
12  *  This program is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  *  GNU General Public License for more details.
16  */
17
18 #include <linux/init.h>
19 #include <linux/kernel.h>
20 #include <linux/errno.h>
21 #include <linux/list.h>
22 #include <linux/spinlock.h>
23 #include <linux/slab.h>
24 #include <linux/sched.h>
25 #include <linux/smp_lock.h>
26 #include <asm/unistd.h>
27 #include <asm/string.h>
28 #include <linux/kthread.h>
29 #include <linux/delay.h>
30
31 #include "scsi_tgt.h"
32 #include "scst_priv.h"
33
34 static void scst_cmd_set_sn(struct scst_cmd *cmd);
35 static int __scst_init_cmd(struct scst_cmd *cmd);
36
37 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
38 {
39         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
40         unsigned long flags;
41
42         spin_lock_irqsave(&t->tasklet_lock, flags);
43         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
44                 smp_processor_id());
45         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
46         spin_unlock_irqrestore(&t->tasklet_lock, flags);
47
48         tasklet_schedule(&t->tasklet);
49 }
50
51 /* 
52  * Must not be called in parallel with scst_unregister_session() for the 
53  * same sess
54  */
55 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
56                              const uint8_t *lun, int lun_len,
57                              const uint8_t *cdb, int cdb_len, int atomic)
58 {
59         struct scst_cmd *cmd;
60
61         TRACE_ENTRY();
62
63 #ifdef EXTRACHECKS
64         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
65                 PRINT_ERROR("%s", "New cmd while shutting down the session");
66                 sBUG();
67         }
68 #endif
69
70         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
71         if (cmd == NULL)
72                 goto out;
73
74         cmd->sess = sess;
75         cmd->tgt = sess->tgt;
76         cmd->tgtt = sess->tgt->tgtt;
77
78         /* 
79          * For both wrong lun and CDB defer the error reporting for
80          * scst_cmd_init_done()
81          */
82
83         cmd->lun = scst_unpack_lun(lun, lun_len);
84
85         if (cdb_len <= SCST_MAX_CDB_SIZE) {
86                 memcpy(cmd->cdb, cdb, cdb_len);
87                 cmd->cdb_len = cdb_len;
88         }
89
90         TRACE_DBG("cmd %p, sess %p", cmd, sess);
91         scst_sess_get(sess);
92
93 out:
94         TRACE_EXIT();
95         return cmd;
96 }
97
98 static int scst_init_cmd(struct scst_cmd *cmd, int context)
99 {
100         int rc;
101
102         TRACE_ENTRY();
103
104         /* See the comment in scst_do_job_init() */
105         if (unlikely(!list_empty(&scst_init_cmd_list))) {
106                 TRACE_MGMT_DBG("%s", "init cmd list busy");
107                 goto out_redirect;
108         }
109         smp_rmb();
110
111         rc = __scst_init_cmd(cmd);
112         if (unlikely(rc > 0))
113                 goto out_redirect;
114         else if (unlikely(rc != 0))
115                 goto out;
116
117         /* Small context optimization */
118         if (((context == SCST_CONTEXT_TASKLET) ||
119              (context == SCST_CONTEXT_DIRECT_ATOMIC)) && 
120             scst_cmd_is_expected_set(cmd)) {
121                 if (cmd->expected_data_direction == SCST_DATA_WRITE) {
122                         if ( !test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
123                                         &cmd->tgt_dev->tgt_dev_flags))
124                                 context = SCST_CONTEXT_THREAD;
125                 } else {
126                         if ( !test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
127                                         &cmd->tgt_dev->tgt_dev_flags))
128                                 context = SCST_CONTEXT_THREAD;
129                 }
130         }
131
132 out:
133         TRACE_EXIT_RES(context);
134         return context;
135
136 out_redirect:
137         if (cmd->preprocessing_only) {
138                 /*
139                  * Poor man solution for single threaded targets, where 
140                  * blocking receiver at least sometimes means blocking all.
141                  */
142                 sBUG_ON(context != SCST_CONTEXT_DIRECT);
143                 scst_set_busy(cmd);
144                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
145                 /* Keep initiator away from too many BUSY commands */
146                 if (!in_interrupt() && !in_atomic())
147                         msleep(50);
148                 else
149                         WARN_ON_ONCE(1);
150         } else {
151                 unsigned long flags;
152                 spin_lock_irqsave(&scst_init_lock, flags);
153                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
154                         "%d)", cmd, atomic_read(&scst_cmd_count));
155                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
156                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
157                         scst_init_poll_cnt++;
158                 spin_unlock_irqrestore(&scst_init_lock, flags);
159                 wake_up(&scst_init_cmd_list_waitQ);
160                 context = -1;
161         }
162         goto out;
163 }
164
165 void scst_cmd_init_done(struct scst_cmd *cmd, int pref_context)
166 {
167         unsigned long flags;
168         struct scst_session *sess = cmd->sess;
169
170         TRACE_ENTRY();
171
172         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
173         TRACE(TRACE_SCSI, "tag=%llu, lun=%Ld, CDB len=%d", cmd->tag, 
174                 (uint64_t)cmd->lun, cmd->cdb_len);
175         TRACE_BUFF_FLAG(TRACE_SCSI|TRACE_RECV_BOT, "Recieving CDB",
176                 cmd->cdb, cmd->cdb_len);
177
178 #ifdef EXTRACHECKS
179         if (unlikely(in_irq()) && ((pref_context == SCST_CONTEXT_DIRECT) ||
180                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
181         {
182                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
183                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
184                         cmd->tgtt->name);
185                 pref_context = SCST_CONTEXT_TASKLET;
186         }
187 #endif
188
189         atomic_inc(&sess->sess_cmd_count);
190
191         spin_lock_irqsave(&sess->sess_list_lock, flags);
192
193         list_add_tail(&cmd->search_cmd_list_entry, &sess->search_cmd_list);
194
195         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
196                 switch(sess->init_phase) {
197                 case SCST_SESS_IPH_SUCCESS:
198                         break;
199                 case SCST_SESS_IPH_INITING:
200                         TRACE_DBG("Adding cmd %p to init deferred cmd list", cmd);
201                         list_add_tail(&cmd->cmd_list_entry, 
202                                 &sess->init_deferred_cmd_list);
203                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
204                         goto out;
205                 case SCST_SESS_IPH_FAILED:
206                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
207                         scst_set_busy(cmd);
208                         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
209                         goto active;
210                 default:
211                         sBUG();
212                 }
213         }
214
215         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
216
217         if (unlikely(cmd->lun == (lun_t)-1)) {
218                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
219                 scst_set_cmd_error(cmd,
220                         SCST_LOAD_SENSE(scst_sense_lun_not_supported));
221                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
222                 goto active;
223         }
224
225         if (unlikely(cmd->cdb_len == 0)) {
226                 PRINT_ERROR("Wrong CDB len %d, finishing cmd", 0);
227                 scst_set_cmd_error(cmd,
228                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
229                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
230                 goto active;
231         }
232
233         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
234                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
235                 scst_set_cmd_error(cmd,
236                         SCST_LOAD_SENSE(scst_sense_invalid_message));
237                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
238                 goto active;
239         }
240
241         cmd->state = SCST_CMD_STATE_INIT;
242         /* cmd must be inited here to keep the order */
243         pref_context = scst_init_cmd(cmd, pref_context);
244         if (unlikely(pref_context < 0))
245                 goto out;
246
247 active:
248         /* Here cmd must not be in any cmd list, no locks */
249         switch (pref_context) {
250         case SCST_CONTEXT_TASKLET:
251                 scst_schedule_tasklet(cmd);
252                 break;
253
254         case SCST_CONTEXT_DIRECT:
255         case SCST_CONTEXT_DIRECT_ATOMIC:
256                 scst_process_active_cmd(cmd, pref_context);
257                 /* For *NEED_THREAD wake_up() is already done */
258                 break;
259
260         default:
261                 PRINT_ERROR("Context %x is undefined, using the thread one",
262                         pref_context);
263                 /* go through */
264         case SCST_CONTEXT_THREAD:
265                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
266                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
267                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
268                         list_add(&cmd->cmd_list_entry,
269                                 &cmd->cmd_lists->active_cmd_list);
270                 else
271                         list_add_tail(&cmd->cmd_list_entry,
272                                 &cmd->cmd_lists->active_cmd_list);
273                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
274                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
275                 break;
276         }
277
278 out:
279         TRACE_EXIT();
280         return;
281 }
282
283 static int scst_pre_parse(struct scst_cmd *cmd)
284 {
285         int res = SCST_CMD_STATE_RES_CONT_SAME;
286         struct scst_device *dev = cmd->dev;
287         struct scst_info_cdb cdb_info;
288
289         TRACE_ENTRY();
290
291         cmd->inc_expected_sn_on_done = !dev->has_own_order_mgmt;
292
293         sBUG_ON(cmd->internal);
294
295         /*
296          * Expected transfer data supplied by the SCSI transport via the
297          * target driver are untrusted, so we prefer to fetch them from CDB.
298          * Additionally, not all transports support supplying the expected
299          * transfer data.
300          */
301
302         if (unlikely(scst_get_cdb_info(cmd->cdb, dev->handler->type, 
303                         &cdb_info) != 0)) {
304                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
305                         "Should you update scst_scsi_op_table?",
306                         cmd->cdb[0], dev->handler->name);
307 #ifdef USE_EXPECTED_VALUES
308                 if (scst_cmd_is_expected_set(cmd)) {
309                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
310                                 "direction %d, transfer_len %d",
311                                 cmd->expected_data_direction,
312                                 cmd->expected_transfer_len);
313                         cmd->data_direction = cmd->expected_data_direction;
314                         
315                         cmd->bufflen = cmd->expected_transfer_len;
316                         /* Restore (likely) lost CDB length */
317                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
318                         if (cmd->cdb_len == -1) {
319                                 PRINT_ERROR("Unable to get CDB length for "
320                                         "opcode 0x%02x. Returning INVALID "
321                                         "OPCODE", cmd->cdb[0]);
322                                 scst_set_cmd_error(cmd,
323                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
324                                 goto out_xmit;
325                         }
326                 } else {
327                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
328                              "target %s not supplied expected values",
329                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
330                         scst_set_cmd_error(cmd,
331                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
332                         goto out_xmit;
333                 }
334 #else
335                 scst_set_cmd_error(cmd,
336                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
337                 goto out_xmit;
338 #endif
339         } else {
340                 TRACE(TRACE_SCSI, "op_name <%s>, direction=%d (expected %d, "
341                         "set %s), transfer_len=%d (expected len %d), flags=%d",
342                         cdb_info.op_name, cdb_info.direction,
343                         cmd->expected_data_direction,
344                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
345                         cdb_info.transfer_len, cmd->expected_transfer_len,
346                         cdb_info.flags);
347
348                 cmd->data_direction = cdb_info.direction;
349
350                 if (unlikely((cdb_info.flags & SCST_UNKNOWN_LENGTH) != 0)) {
351                         if (scst_cmd_is_expected_set(cmd)) {
352                                 /*
353                                  * Command data length can't be easily
354                                  * determined from the CDB. ToDo, that should 
355                                  * be fixed. Until it's fixed, get it from
356                                  * the supplied expected value, but
357                                  * limit it to some reasonable value (15MB).
358                                  */
359                                 cmd->bufflen = min(cmd->expected_transfer_len,
360                                                         15*1024*1024);
361                         } else
362                                 cmd->bufflen = 0;
363                 } else
364                         cmd->bufflen = cdb_info.transfer_len;
365
366                 /* Restore (likely) lost CDB length */
367                 cmd->cdb_len = cdb_info.cdb_len;
368         }
369
370         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
371                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
372                             "(opcode 0x%02x)", cmd->cdb[0]);
373                 scst_set_cmd_error(cmd,
374                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
375                 goto out_xmit;
376         }
377
378         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
379                 PRINT_ERROR("Linked commands are not supported "
380                             "(opcode 0x%02x)", cmd->cdb[0]);
381                 scst_set_cmd_error(cmd,
382                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
383                 goto out_xmit;
384         }
385
386         cmd->state = SCST_CMD_STATE_DEV_PARSE;
387
388 out:
389         TRACE_EXIT_RES(res);
390         return res;
391
392 out_xmit:
393         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
394         res = SCST_CMD_STATE_RES_CONT_SAME;
395         goto out;
396 }
397
398 static int scst_parse_cmd(struct scst_cmd *cmd)
399 {
400         int res = SCST_CMD_STATE_RES_CONT_SAME;
401         int state;
402         struct scst_device *dev = cmd->dev;
403         struct scst_info_cdb cdb_info;
404         int atomic = scst_cmd_atomic(cmd);
405         int orig_bufflen;
406
407         TRACE_ENTRY();
408
409         if (atomic && !dev->handler->parse_atomic) {
410                 TRACE_DBG("Dev handler %s parse() can not be "
411                       "called in atomic context, rescheduling to the thread",
412                       dev->handler->name);
413                 res = SCST_CMD_STATE_RES_NEED_THREAD;
414                 goto out;
415         }
416
417         orig_bufflen = cmd->bufflen;
418
419         if (likely(!scst_is_cmd_local(cmd))) {
420                 TRACE_DBG("Calling dev handler %s parse(%p)",
421                       dev->handler->name, cmd);
422                 TRACE_BUFF_FLAG(TRACE_SEND_BOT, "Parsing: ", cmd->cdb, cmd->cdb_len);
423                 state = dev->handler->parse(cmd, &cdb_info);
424                 /* Caution: cmd can be already dead here */
425                 TRACE_DBG("Dev handler %s parse() returned %d",
426                         dev->handler->name, state);
427
428                 switch (state) {
429                 case SCST_CMD_STATE_NEED_THREAD_CTX:
430                         TRACE_DBG("Dev handler %s parse() requested thread "
431                               "context, rescheduling", dev->handler->name);
432                         res = SCST_CMD_STATE_RES_NEED_THREAD;
433                         goto out;
434
435                 case SCST_CMD_STATE_STOP:
436                         TRACE_DBG("Dev handler %s parse() requested stop "
437                                 "processing", dev->handler->name);
438                         res = SCST_CMD_STATE_RES_CONT_NEXT;
439                         goto out;
440                 }
441
442                 if (state == SCST_CMD_STATE_DEFAULT)
443                         state = SCST_CMD_STATE_PREPARE_SPACE;
444         } else
445                 state = SCST_CMD_STATE_PREPARE_SPACE;
446
447         if (cmd->data_len == -1)
448                 cmd->data_len = cmd->bufflen;
449
450         if (cmd->data_buf_alloced && unlikely((orig_bufflen > cmd->bufflen))) {
451                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
452                         "is less, than required (size %d)", cmd->bufflen,
453                         orig_bufflen);
454                 goto out_error;
455         }
456
457         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
458                 goto set_res;
459
460 #ifdef EXTRACHECKS
461         if ((cmd->bufflen != 0) &&
462             ((cmd->data_direction == SCST_DATA_NONE) ||
463              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
464                 PRINT_ERROR("Dev handler %s parse() returned "
465                         "invalid cmd data_direction %d, bufflen %d, state %d "
466                         "or sg %p (opcode 0x%x)", dev->handler->name, 
467                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
468                         cmd->cdb[0]);
469                 goto out_error;
470         }
471 #endif
472
473         if (scst_cmd_is_expected_set(cmd)) {
474 #ifdef USE_EXPECTED_VALUES
475 #       ifdef EXTRACHECKS
476                 if ((cmd->data_direction != cmd->expected_data_direction) ||
477                     (cmd->bufflen != cmd->expected_transfer_len)) {
478                         PRINT_ERROR("Expected values don't match decoded ones: "
479                                 "data_direction %d, expected_data_direction %d, "
480                                 "bufflen %d, expected_transfer_len %d",
481                                 cmd->data_direction, cmd->expected_data_direction,
482                                 cmd->bufflen, cmd->expected_transfer_len);
483                 }
484 #       endif
485                 cmd->data_direction = cmd->expected_data_direction;
486                 cmd->bufflen = cmd->expected_transfer_len;
487 #else
488                 if (unlikely(cmd->data_direction != cdb_info.direction)) {
489                         PRINT_ERROR("Expected data direction %d for opcode "
490                                 "0x%02x (handler %s, target %s) doesn't match "
491                                 "decoded value %d", cmd->data_direction,
492                                 cmd->cdb[0], dev->handler->name,
493                                 cmd->tgtt->name, cdb_info.direction);
494                         scst_set_cmd_error(cmd,
495                                 SCST_LOAD_SENSE(scst_sense_invalid_message));
496                         goto out_dev_done;
497                 }
498                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
499                         PRINT_INFO("Warning: expected transfer length %d for "
500                                 "opcode 0x%02x (handler %s, target %s) doesn't "
501                                 "match decoded value %d. Faulty initiator "
502                                 "(e.g. VMware is known to be such) or "
503                                 "scst_scsi_op_table should be updated?",
504                                 cmd->expected_transfer_len, cmd->cdb[0],
505                                 dev->handler->name, cmd->tgtt->name,
506                                 cmd->bufflen);
507                         PRINT_BUFFER("Suspicious CDB", cmd->cdb, cmd->cdb_len);
508                 }
509 #endif
510         }
511
512         if ((cmd->data_direction == SCST_DATA_UNKNOWN) ||
513             ((cmd->bufflen == 0) && (cmd->data_direction != SCST_DATA_NONE))) {
514                 PRINT_ERROR("Wrong data direction (%d) or/and buffer "
515                         "length (%d). Opcode 0x%x, handler %s, target %s",
516                         cmd->data_direction, cmd->bufflen, cmd->cdb[0],
517                         dev->handler->name, cmd->tgtt->name);
518                 goto out_error;
519         }
520
521 set_res:
522         switch (state) {
523         case SCST_CMD_STATE_PREPARE_SPACE:
524         case SCST_CMD_STATE_PRE_PARSE:
525         case SCST_CMD_STATE_DEV_PARSE:
526         case SCST_CMD_STATE_RDY_TO_XFER:
527         case SCST_CMD_STATE_TGT_PRE_EXEC:
528         case SCST_CMD_STATE_SEND_TO_MIDLEV:
529         case SCST_CMD_STATE_PRE_DEV_DONE:
530         case SCST_CMD_STATE_DEV_DONE:
531         case SCST_CMD_STATE_PRE_XMIT_RESP:
532         case SCST_CMD_STATE_XMIT_RESP:
533         case SCST_CMD_STATE_FINISHED:
534                 cmd->state = state;
535                 res = SCST_CMD_STATE_RES_CONT_SAME;
536                 break;
537
538         default:
539                 if (state >= 0) {
540                         PRINT_ERROR("Dev handler %s parse() returned "
541                              "invalid cmd state %d (opcode %d)", 
542                              dev->handler->name, state, cmd->cdb[0]);
543                 } else {
544                         PRINT_ERROR("Dev handler %s parse() returned "
545                                 "error %d (opcode %d)", dev->handler->name, 
546                                 state, cmd->cdb[0]);
547                 }
548                 goto out_error;
549         }
550
551         if (cmd->resp_data_len == -1) {
552                 if (cmd->data_direction == SCST_DATA_READ)
553                         cmd->resp_data_len = cmd->bufflen;
554                 else
555                          cmd->resp_data_len = 0;
556         }
557         
558 out:
559         TRACE_EXIT_HRES(res);
560         return res;
561
562 out_error:
563         /* dev_done() will be called as part of the regular cmd's finish */
564         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
565
566 #ifndef USE_EXPECTED_VALUES
567 out_dev_done:
568 #endif
569         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
570         res = SCST_CMD_STATE_RES_CONT_SAME;
571         goto out;
572 }
573
574 static int scst_prepare_space(struct scst_cmd *cmd)
575 {
576         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
577
578         TRACE_ENTRY();
579
580         if (cmd->data_direction == SCST_DATA_NONE)
581                 goto prep_done;
582
583         if (cmd->data_buf_tgt_alloc) {
584                 int orig_bufflen = cmd->bufflen;
585
586                 TRACE_MEM("%s", "Custom tgt data buf allocation requested");
587
588                 r = cmd->tgtt->alloc_data_buf(cmd);
589                 if (r > 0)
590                         goto alloc;
591                 else if (r == 0) {
592                         cmd->data_buf_alloced = 1;
593                         if (unlikely(orig_bufflen < cmd->bufflen)) {
594                                 PRINT_ERROR("Target driver allocated data "
595                                         "buffer (size %d), is less, than "
596                                         "required (size %d)", orig_bufflen,
597                                         cmd->bufflen);
598                                 goto out_error;
599                         }
600                 } else
601                         goto check;
602         }
603
604 alloc:
605         if (!cmd->data_buf_alloced) {
606                 r = scst_alloc_space(cmd);
607         } else {
608                 TRACE_MEM("%s", "data_buf_alloced set, returning");
609         }
610         
611 check:
612         if (r != 0) {
613                 if (scst_cmd_atomic(cmd)) {
614                         TRACE_MEM("%s", "Atomic memory allocation failed, "
615                               "rescheduling to the thread");
616                         res = SCST_CMD_STATE_RES_NEED_THREAD;
617                         goto out;
618                 } else
619                         goto out_no_space;
620         }
621
622 prep_done:
623         if (cmd->preprocessing_only) {
624                 if (scst_cmd_atomic(cmd) && 
625                     !cmd->tgtt->preprocessing_done_atomic) {
626                         TRACE_DBG("%s", "preprocessing_done() can not be "
627                               "called in atomic context, rescheduling to "
628                               "the thread");
629                         res = SCST_CMD_STATE_RES_NEED_THREAD;
630                         goto out;
631                 }
632
633                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
634                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
635                                 "cmd %p", cmd);
636                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
637                         res = SCST_CMD_STATE_RES_CONT_SAME;
638                         goto out;
639                 }
640
641                 res = SCST_CMD_STATE_RES_CONT_NEXT;
642                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
643
644                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
645                 cmd->tgtt->preprocessing_done(cmd);
646                 TRACE_DBG("%s", "preprocessing_done() returned");
647                 goto out;
648
649         }
650
651         switch (cmd->data_direction) {
652         case SCST_DATA_WRITE:
653                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
654                 break;
655
656         default:
657                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
658                 break;
659         }
660
661 out:
662         TRACE_EXIT_HRES(res);
663         return res;
664
665 out_no_space:
666         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
667                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
668         scst_set_busy(cmd);
669         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
670         res = SCST_CMD_STATE_RES_CONT_SAME;
671         goto out;
672
673 out_error:
674         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
675         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
676         res = SCST_CMD_STATE_RES_CONT_SAME;
677         goto out;
678 }
679
680 void scst_restart_cmd(struct scst_cmd *cmd, int status, int pref_context)
681 {
682         TRACE_ENTRY();
683
684         TRACE_DBG("Preferred context: %d", pref_context);
685         TRACE_DBG("tag=%llu, status=%#x", scst_cmd_get_tag(cmd), status);
686
687 #ifdef EXTRACHECKS
688         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
689                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
690         {
691                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
692                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
693                         cmd->tgtt->name);
694                 pref_context = SCST_CONTEXT_TASKLET;
695         }
696 #endif
697
698         switch (status) {
699         case SCST_PREPROCESS_STATUS_SUCCESS:
700                 switch (cmd->data_direction) {
701                 case SCST_DATA_WRITE:
702                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
703                         break;
704                 default:
705                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
706                         break;
707                 }
708                 if (cmd->set_sn_on_restart_cmd)
709                         scst_cmd_set_sn(cmd);
710                 /* Small context optimization */
711                 if ((pref_context == SCST_CONTEXT_TASKLET) || 
712                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) {
713                         if (cmd->data_direction == SCST_DATA_WRITE) {
714                                 if ( !test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
715                                                 &cmd->tgt_dev->tgt_dev_flags))
716                                         pref_context = SCST_CONTEXT_THREAD;
717                         } else {
718                                 if ( !test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
719                                                 &cmd->tgt_dev->tgt_dev_flags))
720                                         pref_context = SCST_CONTEXT_THREAD;
721                         }
722                 }
723                 break;
724
725         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
726                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
727                 break;
728
729         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
730                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
731                 /* go through */
732         case SCST_PREPROCESS_STATUS_ERROR:
733                 scst_set_cmd_error(cmd,
734                            SCST_LOAD_SENSE(scst_sense_hardw_error));
735                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
736                 break;
737
738         default:
739                 PRINT_ERROR("%s() received unknown status %x", __func__,
740                         status);
741                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
742                 break;
743         }
744
745         scst_proccess_redirect_cmd(cmd, pref_context, 1);
746
747         TRACE_EXIT();
748         return;
749 }
750
751 /* No locks */
752 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
753 {
754         struct scst_tgt *tgt = cmd->sess->tgt;
755         int res = 0;
756         unsigned long flags;
757
758         TRACE_ENTRY();
759
760         spin_lock_irqsave(&tgt->tgt_lock, flags);
761         tgt->retry_cmds++;
762         smp_mb();
763         TRACE(TRACE_RETRY, "TGT QUEUE FULL: incrementing retry_cmds %d",
764               tgt->retry_cmds);
765         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
766                 /* At least one cmd finished, so try again */
767                 tgt->retry_cmds--;
768                 TRACE(TRACE_RETRY, "Some command(s) finished, direct retry "
769                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
770                       "retry_cmds=%d)", finished_cmds,
771                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
772                 res = -1;
773                 goto out_unlock_tgt;
774         }
775
776         TRACE(TRACE_RETRY, "Adding cmd %p to retry cmd list", cmd);
777         list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
778
779         if (!tgt->retry_timer_active) {
780                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
781                 add_timer(&tgt->retry_timer);
782                 tgt->retry_timer_active = 1;
783         }
784
785 out_unlock_tgt:
786         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
787
788         TRACE_EXIT_RES(res);
789         return res;
790 }
791
792 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
793 {
794         int res, rc;
795         int atomic = scst_cmd_atomic(cmd);
796
797         TRACE_ENTRY();
798
799         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
800                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
801                 goto out_dev_done;
802         }
803
804         if (cmd->tgtt->rdy_to_xfer == NULL) {
805                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
806                 res = SCST_CMD_STATE_RES_CONT_SAME;
807                 goto out;
808         }
809
810         if (atomic && !cmd->tgtt->rdy_to_xfer_atomic) {
811                 TRACE_DBG("%s", "rdy_to_xfer() can not be "
812                       "called in atomic context, rescheduling to the thread");
813                 res = SCST_CMD_STATE_RES_NEED_THREAD;
814                 goto out;
815         }
816
817         while (1) {
818                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
819
820                 res = SCST_CMD_STATE_RES_CONT_NEXT;
821                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
822
823                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
824 #ifdef DEBUG_RETRY
825                 if (((scst_random() % 100) == 75))
826                         rc = SCST_TGT_RES_QUEUE_FULL;
827                 else
828 #endif
829                         rc = cmd->tgtt->rdy_to_xfer(cmd);
830                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
831
832                 if (likely(rc == SCST_TGT_RES_SUCCESS))
833                         goto out;
834
835                 /* Restore the previous state */
836                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
837
838                 switch (rc) {
839                 case SCST_TGT_RES_QUEUE_FULL:
840                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
841                                 break;
842                         else
843                                 continue;
844
845                 case SCST_TGT_RES_NEED_THREAD_CTX:
846                         TRACE_DBG("Target driver %s "
847                               "rdy_to_xfer() requested thread "
848                               "context, rescheduling", cmd->tgtt->name);
849                         res = SCST_CMD_STATE_RES_NEED_THREAD;
850                         break;
851
852                 default:
853                         goto out_error_rc;
854                 }
855                 break;
856         }
857
858 out:
859         TRACE_EXIT_HRES(res);
860         return res;
861
862 out_error_rc:
863         if (rc == SCST_TGT_RES_FATAL_ERROR) {
864                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
865                      "fatal error", cmd->tgtt->name);
866         } else {
867                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
868                             "value %d", cmd->tgtt->name, rc);
869         }
870         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
871
872 out_dev_done:
873         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
874         res = SCST_CMD_STATE_RES_CONT_SAME;
875         goto out;
876 }
877
878 /* No locks, but might be in IRQ */
879 void scst_proccess_redirect_cmd(struct scst_cmd *cmd, int context,
880         int check_retries)
881 {
882         unsigned long flags;
883
884         TRACE_ENTRY();
885
886         TRACE_DBG("Context: %d", context);
887
888         switch(context) {
889         case SCST_CONTEXT_DIRECT:
890         case SCST_CONTEXT_DIRECT_ATOMIC:
891                 if (check_retries)
892                         scst_check_retries(cmd->tgt);
893                 scst_process_active_cmd(cmd, context);
894                 break;
895
896         default:
897                 PRINT_ERROR("Context %x is unknown, using the thread one",
898                             context);
899                 /* go through */
900         case SCST_CONTEXT_THREAD:
901                 if (check_retries)
902                         scst_check_retries(cmd->tgt);
903                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
904                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
905                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
906                         list_add(&cmd->cmd_list_entry,
907                                 &cmd->cmd_lists->active_cmd_list);
908                 else
909                         list_add_tail(&cmd->cmd_list_entry,
910                                 &cmd->cmd_lists->active_cmd_list);
911                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
912                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
913                 break;
914
915         case SCST_CONTEXT_TASKLET:
916                 if (check_retries)
917                         scst_check_retries(cmd->tgt);
918                 scst_schedule_tasklet(cmd);
919                 break;
920         }
921
922         TRACE_EXIT();
923         return;
924 }
925
926 void scst_rx_data(struct scst_cmd *cmd, int status, int pref_context)
927 {
928         TRACE_ENTRY();
929
930         TRACE_DBG("Preferred context: %d", pref_context);
931         TRACE(TRACE_SCSI, "tag=%llu status=%#x", scst_cmd_get_tag(cmd), status);
932
933 #ifdef EXTRACHECKS
934         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
935                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
936         {
937                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
938                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
939                         cmd->tgtt->name);
940                 pref_context = SCST_CONTEXT_TASKLET;
941         }
942 #endif
943
944         switch (status) {
945         case SCST_RX_STATUS_SUCCESS:
946                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
947                 /* Small context optimization */
948                 if ((pref_context == SCST_CONTEXT_TASKLET) || 
949                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)) {
950                         if ( !test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC, 
951                                         &cmd->tgt_dev->tgt_dev_flags))
952                                 pref_context = SCST_CONTEXT_THREAD;
953                 }
954                 break;
955
956         case SCST_RX_STATUS_ERROR_SENSE_SET:
957                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
958                 break;
959
960         case SCST_RX_STATUS_ERROR_FATAL:
961                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
962                 /* go through */
963         case SCST_RX_STATUS_ERROR:
964                 scst_set_cmd_error(cmd,
965                            SCST_LOAD_SENSE(scst_sense_hardw_error));
966                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
967                 break;
968
969         default:
970                 PRINT_ERROR("scst_rx_data() received unknown status %x",
971                         status);
972                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
973                 break;
974         }
975
976         scst_proccess_redirect_cmd(cmd, pref_context, 1);
977
978         TRACE_EXIT();
979         return;
980 }
981
982 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
983 {
984         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
985
986         TRACE_ENTRY();
987
988         cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
989
990         if (cmd->tgtt->pre_exec == NULL)
991                 goto out;
992
993         TRACE_DBG("Calling pre_exec(%p)", cmd);
994         rc = cmd->tgtt->pre_exec(cmd);
995         TRACE_DBG("pre_exec() returned %d", rc);
996
997         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
998                 switch(rc) {
999                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
1000                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1001                         break;
1002                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
1003                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1004                         /* go through */
1005                 case SCST_PREPROCESS_STATUS_ERROR:
1006                         scst_set_cmd_error(cmd,
1007                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1008                         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1009                         break;
1010                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1011                         TRACE_DBG("Target driver's %s pre_exec() requested "
1012                                 "thread context, rescheduling", cmd->tgtt->name);
1013                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1014                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1015                         break;
1016                 default:
1017                         sBUG();
1018                         break;
1019                 }
1020         }
1021
1022 out:
1023         TRACE_EXIT_RES(res);
1024         return res;
1025 }
1026
1027 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1028         const uint8_t *rq_sense, int rq_sense_len, int resid)
1029 {
1030         TRACE_ENTRY();
1031
1032         cmd->status = result & 0xff;
1033         cmd->msg_status = msg_byte(result);
1034         cmd->host_status = host_byte(result);
1035         cmd->driver_status = driver_byte(result);
1036         if (unlikely(resid != 0)) {
1037 #ifdef EXTRACHECKS
1038                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1039                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1040                                 "op %x)", resid, cmd->resp_data_len,
1041                                 cmd->cdb[0]);
1042                 } else
1043 #endif
1044                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1045         }
1046
1047         /* 
1048          * We checked that rq_sense_len < sizeof(cmd->sense_buffer)
1049          * in init_scst()
1050          */
1051         memcpy(cmd->sense_buffer, rq_sense, rq_sense_len);
1052         memset(&cmd->sense_buffer[rq_sense_len], 0,
1053                 sizeof(cmd->sense_buffer) - rq_sense_len);
1054
1055         TRACE(TRACE_SCSI, "result=%x, cmd->status=%x, resid=%d, "
1056               "cmd->msg_status=%x, cmd->host_status=%x, "
1057               "cmd->driver_status=%x", result, cmd->status, resid,
1058               cmd->msg_status, cmd->host_status, cmd->driver_status);
1059
1060         cmd->completed = 1;
1061
1062         TRACE_EXIT();
1063         return;
1064 }
1065
1066 /* For small context optimization */
1067 static inline int scst_optimize_post_exec_context(struct scst_cmd *cmd,
1068         int context)
1069 {
1070         if ((context == SCST_CONTEXT_TASKLET) || 
1071             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1072                 if ( !test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC, 
1073                                 &cmd->tgt_dev->tgt_dev_flags))
1074                         context = SCST_CONTEXT_THREAD;
1075         }
1076         return context;
1077 }
1078
1079 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18)
1080 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1081                                             struct scsi_request **req)
1082 {
1083         struct scst_cmd *cmd = NULL;
1084
1085         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1086                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1087
1088         if (cmd == NULL) {
1089                 PRINT_ERROR("%s", "Request with NULL cmd");
1090                 if (*req)
1091                         scsi_release_request(*req);
1092         }
1093
1094         return cmd;
1095 }
1096
1097 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1098 {
1099         struct scsi_request *req = NULL;
1100         struct scst_cmd *cmd;
1101
1102         TRACE_ENTRY();
1103
1104         cmd = scst_get_cmd(scsi_cmd, &req);
1105         if (cmd == NULL)
1106                 goto out;
1107
1108         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1109                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1110
1111         /* Clear out request structure */
1112         req->sr_use_sg = 0;
1113         req->sr_sglist_len = 0;
1114         req->sr_bufflen = 0;
1115         req->sr_buffer = NULL;
1116         req->sr_underflow = 0;
1117         req->sr_request->rq_disk = NULL; /* disown request blk */
1118
1119         scst_release_request(cmd);
1120
1121         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1122
1123         scst_proccess_redirect_cmd(cmd,
1124                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1125
1126 out:
1127         TRACE_EXIT();
1128         return;
1129 }
1130 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) */
1131 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1132 {
1133         struct scst_cmd *cmd;
1134
1135         TRACE_ENTRY();
1136
1137         cmd = (struct scst_cmd *)data;
1138         if (cmd == NULL)
1139                 goto out;
1140
1141         scst_do_cmd_done(cmd, result, sense, SCST_SENSE_BUFFERSIZE, resid);
1142
1143         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1144
1145         scst_proccess_redirect_cmd(cmd,
1146                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1147
1148 out:
1149         TRACE_EXIT();
1150         return;
1151 }
1152 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) */
1153
1154 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state)
1155 {
1156         TRACE_ENTRY();
1157
1158         if (next_state == SCST_CMD_STATE_DEFAULT)
1159                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1160
1161 #if defined(DEBUG) || defined(TRACING)
1162         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1163                 if (cmd->sg) {
1164                         int i;
1165                         struct scatterlist *sg = cmd->sg;
1166                         TRACE(TRACE_RECV_TOP, 
1167                               "Exec'd %d S/G(s) at %p sg[0].page at %p",
1168                               cmd->sg_cnt, sg, (void*)sg[0].page);
1169                         for(i = 0; i < cmd->sg_cnt; ++i) {
1170                                 TRACE_BUFF_FLAG(TRACE_RECV_TOP, 
1171                                         "Exec'd sg", page_address(sg[i].page),
1172                                         sg[i].length);
1173                         }
1174                 }
1175         }
1176 #endif
1177
1178
1179 #ifdef EXTRACHECKS
1180         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1181             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1182             (next_state != SCST_CMD_STATE_FINISHED)) 
1183         {
1184                 PRINT_ERROR("scst_cmd_done_local() received invalid cmd "
1185                             "state %d (opcode %d)", next_state, cmd->cdb[0]);
1186                 scst_set_cmd_error(cmd,
1187                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1188                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1189         }
1190 #endif
1191         cmd->state = next_state;
1192
1193         scst_proccess_redirect_cmd(cmd,
1194                 scst_optimize_post_exec_context(cmd, scst_get_context()), 0);
1195
1196         TRACE_EXIT();
1197         return;
1198 }
1199
1200 static int scst_report_luns_local(struct scst_cmd *cmd)
1201 {
1202         int rc;
1203         int dev_cnt = 0;
1204         int buffer_size;
1205         int i;
1206         struct scst_tgt_dev *tgt_dev = NULL;
1207         uint8_t *buffer;
1208         int offs, overflow = 0;
1209
1210         TRACE_ENTRY();
1211
1212         rc = scst_check_local_events(cmd);
1213         if (unlikely(rc != 0))
1214                 goto out_done;
1215
1216         cmd->status = 0;
1217         cmd->msg_status = 0;
1218         cmd->host_status = DID_OK;
1219         cmd->driver_status = 0;
1220
1221         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1222                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1223                         "LUNS command", cmd->cdb[2]);
1224                 goto out_err;
1225         }
1226
1227         buffer_size = scst_get_buf_first(cmd, &buffer);
1228         if (unlikely(buffer_size <= 0))
1229                 goto out_err;
1230
1231         if (buffer_size < 16)
1232                 goto out_put_err;
1233
1234         memset(buffer, 0, buffer_size);
1235         offs = 8;
1236
1237         /* sess->sess_tgt_dev_list_hash is protected by suspended activity */
1238         for(i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1239                 struct list_head *sess_tgt_dev_list_head =
1240                         &cmd->sess->sess_tgt_dev_list_hash[i];
1241                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1242                                 sess_tgt_dev_list_entry) {
1243                         if (!overflow) {
1244                                 if (offs >= buffer_size) {
1245                                         scst_put_buf(cmd, buffer);
1246                                         buffer_size = scst_get_buf_next(cmd, &buffer);
1247                                         if (buffer_size > 0) {
1248                                                 memset(buffer, 0, buffer_size);
1249                                                 offs = 0;
1250                                         } else {
1251                                                 overflow = 1;
1252                                                 goto inc_dev_cnt;
1253                                         }
1254                                 }
1255                                 if ((buffer_size - offs) < 8) {
1256                                         PRINT_ERROR("Buffer allocated for REPORT "
1257                                                 "LUNS command doesn't allow to fit 8 "
1258                                                 "byte entry (buffer_size=%d)",
1259                                                 buffer_size);
1260                                         goto out_put_hw_err;
1261                                 }
1262                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1263                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1264                                 offs += 8;
1265                         }
1266 inc_dev_cnt:
1267                         dev_cnt++;
1268                 }
1269         }
1270         if (!overflow)
1271                 scst_put_buf(cmd, buffer);
1272
1273         /* Set the response header */
1274         buffer_size = scst_get_buf_first(cmd, &buffer);
1275         if (unlikely(buffer_size <= 0))
1276                 goto out_err;
1277         dev_cnt *= 8;
1278         buffer[0] = (dev_cnt >> 24) & 0xff;
1279         buffer[1] = (dev_cnt >> 16) & 0xff;
1280         buffer[2] = (dev_cnt >> 8) & 0xff;
1281         buffer[3] = dev_cnt & 0xff;
1282         scst_put_buf(cmd, buffer);
1283
1284         dev_cnt += 8;
1285         if (dev_cnt < cmd->resp_data_len)
1286                 scst_set_resp_data_len(cmd, dev_cnt);
1287
1288 out_compl:
1289         cmd->completed = 1;
1290
1291 out_done:
1292         /* Report the result */
1293         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1294
1295         TRACE_EXIT();
1296         return SCST_EXEC_COMPLETED;
1297         
1298 out_put_err:
1299         scst_put_buf(cmd, buffer);
1300
1301 out_err:
1302         scst_set_cmd_error(cmd,
1303                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1304         goto out_compl;
1305
1306 out_put_hw_err:
1307         scst_put_buf(cmd, buffer);
1308         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1309         goto out_compl;
1310 }
1311
1312 static int scst_pre_select(struct scst_cmd *cmd)
1313 {
1314         int res = SCST_EXEC_NOT_COMPLETED;
1315
1316         TRACE_ENTRY();
1317
1318         if (scst_cmd_atomic(cmd)) {
1319                 res = SCST_EXEC_NEED_THREAD;
1320                 goto out;
1321         }
1322
1323         if (cmd->local_exec_done)
1324                 goto out;
1325
1326         cmd->local_exec_done = 1;
1327
1328         scst_block_dev_cmd(cmd, 1);
1329
1330         /* Check for local events will be done when cmd will be executed */
1331
1332 out:
1333         TRACE_EXIT_RES(res);
1334         return res;
1335 }
1336
1337 static int scst_reserve_local(struct scst_cmd *cmd)
1338 {
1339         int res = SCST_EXEC_NOT_COMPLETED, rc;
1340         struct scst_device *dev;
1341         struct scst_tgt_dev *tgt_dev_tmp;
1342
1343         TRACE_ENTRY();
1344
1345         if (scst_cmd_atomic(cmd)) {
1346                 res = SCST_EXEC_NEED_THREAD;
1347                 goto out;
1348         }
1349
1350         if (cmd->local_exec_done)
1351                 goto out;
1352
1353         cmd->local_exec_done = 1;
1354
1355         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1356                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1357                      "(lun=%Ld)", (uint64_t)cmd->lun);
1358                 scst_set_cmd_error(cmd,
1359                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1360                 goto out_done;
1361         }
1362
1363         dev = cmd->dev;
1364
1365         scst_block_dev_cmd(cmd, 1);
1366
1367         rc = scst_check_local_events(cmd);
1368         if (unlikely(rc != 0))
1369                 goto out_done;
1370
1371         spin_lock_bh(&dev->dev_lock);
1372
1373         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1374                 spin_unlock_bh(&dev->dev_lock);
1375                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1376                 goto out_done;
1377         }
1378
1379         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1380                             dev_tgt_dev_list_entry) 
1381         {
1382                 if (cmd->tgt_dev != tgt_dev_tmp)
1383                         set_bit(SCST_TGT_DEV_RESERVED, 
1384                                 &tgt_dev_tmp->tgt_dev_flags);
1385         }
1386         dev->dev_reserved = 1;
1387
1388         spin_unlock_bh(&dev->dev_lock);
1389         
1390 out:
1391         TRACE_EXIT_RES(res);
1392         return res;
1393
1394 out_done:
1395         /* Report the result */
1396         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1397         res = SCST_EXEC_COMPLETED;
1398         goto out;
1399 }
1400
1401 static int scst_release_local(struct scst_cmd *cmd)
1402 {
1403         int res = SCST_EXEC_NOT_COMPLETED, rc;
1404         struct scst_tgt_dev *tgt_dev_tmp;
1405         struct scst_device *dev;
1406
1407         TRACE_ENTRY();
1408
1409         if (scst_cmd_atomic(cmd)) {
1410                 res = SCST_EXEC_NEED_THREAD;
1411                 goto out;
1412         }
1413
1414         if (cmd->local_exec_done)
1415                 goto out;
1416
1417         cmd->local_exec_done = 1;
1418
1419         dev = cmd->dev;
1420
1421         scst_block_dev_cmd(cmd, 1);
1422
1423         rc = scst_check_local_events(cmd);
1424         if (unlikely(rc != 0))
1425                 goto out_done;
1426
1427         spin_lock_bh(&dev->dev_lock);
1428
1429         /* 
1430          * The device could be RELEASED behind us, if RESERVING session 
1431          * is closed (see scst_free_tgt_dev()), but this actually doesn't 
1432          * matter, so use lock and no retest for DEV_RESERVED bits again
1433          */
1434         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1435                 res = SCST_EXEC_COMPLETED;
1436                 cmd->status = 0;
1437                 cmd->msg_status = 0;
1438                 cmd->host_status = DID_OK;
1439                 cmd->driver_status = 0;
1440                 cmd->completed = 1;
1441         } else {
1442                 list_for_each_entry(tgt_dev_tmp,
1443                                     &dev->dev_tgt_dev_list,
1444                                     dev_tgt_dev_list_entry) {
1445                         clear_bit(SCST_TGT_DEV_RESERVED, 
1446                                 &tgt_dev_tmp->tgt_dev_flags);
1447                 }
1448                 dev->dev_reserved = 0;
1449         }
1450
1451         spin_unlock_bh(&dev->dev_lock);
1452
1453         if (res == SCST_EXEC_COMPLETED)
1454                 goto out_done;
1455
1456 out:
1457         TRACE_EXIT_RES(res);
1458         return res;
1459
1460 out_done:
1461         res = SCST_EXEC_COMPLETED;
1462         /* Report the result */
1463         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1464         goto out;
1465 }
1466
1467 /* No locks, no IRQ or IRQ-safe context allowed */
1468 int scst_check_local_events(struct scst_cmd *cmd)
1469 {
1470         int res, rc;
1471         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1472
1473         TRACE_ENTRY();
1474
1475         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1476                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1477                 goto out_uncomplete;
1478         }
1479
1480         /* Reserve check before Unit Attention */
1481         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED, &tgt_dev->tgt_dev_flags))) {
1482                 if ((cmd->cdb[0] != INQUIRY) && (cmd->cdb[0] != REPORT_LUNS) &&
1483                     (cmd->cdb[0] != RELEASE) && (cmd->cdb[0] != RELEASE_10) &&
1484                     (cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER) &&
1485                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL || (cmd->cdb[4] & 3)) &&
1486                     (cmd->cdb[0] != LOG_SENSE) && (cmd->cdb[0] != REQUEST_SENSE)) {
1487                         scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1488                         goto out_complete;
1489                 }
1490         }
1491
1492         /* If we had internal bus reset, set the command error unit attention */
1493         if ((cmd->dev->scsi_dev != NULL) &&
1494             unlikely(cmd->dev->scsi_dev->was_reset)) {
1495                 if (scst_is_ua_command(cmd)) {
1496                         struct scst_device *dev = cmd->dev;
1497                         int done = 0;
1498                         /* Prevent more than 1 cmd to be triggered by was_reset */
1499                         spin_lock_bh(&dev->dev_lock);
1500                         barrier(); /* to reread was_reset */
1501                         if (dev->scsi_dev->was_reset) {
1502                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1503                                 scst_set_cmd_error(cmd,
1504                                            SCST_LOAD_SENSE(scst_sense_reset_UA));
1505                                 /* It looks like it is safe to clear was_reset here */
1506                                 dev->scsi_dev->was_reset = 0;
1507                                 smp_mb();
1508                                 done = 1;
1509                         }
1510                         spin_unlock_bh(&dev->dev_lock);
1511
1512                         if (done)
1513                                 goto out_complete;
1514                 }
1515         }
1516
1517         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING, 
1518                         &cmd->tgt_dev->tgt_dev_flags))) {
1519                 if (scst_is_ua_command(cmd)) {
1520                         rc = scst_set_pending_UA(cmd);
1521                         if (rc == 0)
1522                                 goto out_complete;
1523                 }
1524         }
1525
1526         res = 0;
1527
1528 out:
1529         TRACE_EXIT_RES(res);
1530         return res;
1531
1532 out_complete:
1533         res = 1;
1534         sBUG_ON(!cmd->completed);
1535         goto out;
1536
1537 out_uncomplete:
1538         res = -1;
1539         goto out;
1540 }
1541
1542 /* 
1543  * The result of cmd execution, if any, should be reported 
1544  * via scst_cmd_done_local() 
1545  */
1546 static int scst_pre_exec(struct scst_cmd *cmd)
1547 {
1548         int res = SCST_EXEC_NOT_COMPLETED;
1549         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1550
1551         TRACE_ENTRY();
1552
1553         /*
1554          * This function can be called several times for the same cmd, so it
1555          * can't change any state in a non-reentrable way or use something
1556          * like local_exec_done!!
1557          */
1558
1559         /* Check READ_ONLY device status */
1560         if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
1561             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
1562              cmd->cdb[0] == WRITE_10 ||
1563              cmd->cdb[0] == WRITE_12 ||
1564              cmd->cdb[0] == WRITE_16 ||
1565              cmd->cdb[0] == WRITE_VERIFY ||
1566              cmd->cdb[0] == WRITE_VERIFY_12 ||
1567              cmd->cdb[0] == WRITE_VERIFY_16 ||
1568              (cmd->dev->handler->type == TYPE_TAPE &&
1569               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
1570                 scst_set_cmd_error(cmd,
1571                            SCST_LOAD_SENSE(scst_sense_data_protect));
1572                 goto out_done;
1573         }
1574
1575 out:
1576         TRACE_EXIT_RES(res);
1577         return res;
1578
1579 out_done:
1580         res = SCST_EXEC_COMPLETED;
1581         /* Report the result */
1582         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1583         goto out;
1584 }
1585
1586 /* 
1587  * The result of cmd execution, if any, should be reported 
1588  * via scst_cmd_done_local() 
1589  */
1590 static inline int scst_local_exec(struct scst_cmd *cmd)
1591 {
1592         int res = SCST_EXEC_NOT_COMPLETED;
1593
1594         TRACE_ENTRY();
1595
1596         /*
1597          * Adding new commands here don't forget to update
1598          * scst_is_cmd_local() in scsi_tgt.h, if necessary
1599          */
1600
1601         switch (cmd->cdb[0]) {
1602         case MODE_SELECT:
1603         case MODE_SELECT_10:
1604         case LOG_SELECT:
1605                 res = scst_pre_select(cmd);
1606                 break;
1607         case RESERVE:
1608         case RESERVE_10:
1609                 res = scst_reserve_local(cmd);
1610                 break;
1611         case RELEASE:
1612         case RELEASE_10:
1613                 res = scst_release_local(cmd);
1614                 break;
1615         case REPORT_LUNS:
1616                 res = scst_report_luns_local(cmd);
1617                 break;
1618         }
1619
1620         TRACE_EXIT_RES(res);
1621         return res;
1622 }
1623
1624 static int scst_do_send_to_midlev(struct scst_cmd *cmd)
1625 {
1626         int rc = SCST_EXEC_NOT_COMPLETED;
1627
1628         TRACE_ENTRY();
1629
1630         /* Check here to let an out of SN cmd be queued w/o context switch */
1631         if (scst_cmd_atomic(cmd) && !cmd->dev->handler->exec_atomic) {
1632                 TRACE_DBG("Dev handler %s exec() can not be "
1633                       "called in atomic context, rescheduling to the thread",
1634                       cmd->dev->handler->name);
1635                 rc = SCST_EXEC_NEED_THREAD;
1636                 goto out;
1637         }
1638
1639         cmd->sent_to_midlev = 1;
1640         cmd->state = SCST_CMD_STATE_EXECUTING;
1641         cmd->scst_cmd_done = scst_cmd_done_local;
1642
1643         rc = scst_pre_exec(cmd);
1644         /* !! At this point cmd, sess & tgt_dev can be already freed !! */
1645         if (rc != SCST_EXEC_NOT_COMPLETED) {
1646                 if (rc == SCST_EXEC_COMPLETED)
1647                         goto out;
1648                 else if (rc == SCST_EXEC_NEED_THREAD)
1649                         goto out_clear;
1650                 else
1651                         goto out_rc_error;
1652         }
1653
1654         rc = scst_local_exec(cmd);
1655         /* !! At this point cmd, sess & tgt_dev can be already freed !! */
1656         if (rc != SCST_EXEC_NOT_COMPLETED) {
1657                 if (rc == SCST_EXEC_COMPLETED)
1658                         goto out;
1659                 else if (rc == SCST_EXEC_NEED_THREAD)
1660                         goto out_clear;
1661                 else
1662                         goto out_rc_error;
1663         }
1664
1665         if (cmd->dev->handler->exec) {
1666                 struct scst_device *dev = cmd->dev;
1667                 TRACE_DBG("Calling dev handler %s exec(%p)",
1668                       dev->handler->name, cmd);
1669                 TRACE_BUFF_FLAG(TRACE_SEND_TOP, "Execing: ", cmd->cdb, cmd->cdb_len);
1670                 cmd->scst_cmd_done = scst_cmd_done_local;
1671                 rc = dev->handler->exec(cmd);
1672                 /* !! At this point cmd, sess & tgt_dev can be already freed !! */
1673                 TRACE_DBG("Dev handler %s exec() returned %d",
1674                       dev->handler->name, rc);
1675                 if (rc == SCST_EXEC_COMPLETED)
1676                         goto out;
1677                 else if (rc == SCST_EXEC_NEED_THREAD)
1678                         goto out_clear;
1679                 else if (rc != SCST_EXEC_NOT_COMPLETED)
1680                         goto out_rc_error;
1681         }
1682
1683         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1684         
1685         if (unlikely(cmd->dev->scsi_dev == NULL)) {
1686                 PRINT_ERROR("Command for virtual device must be "
1687                         "processed by device handler (lun %Ld)!",
1688                         (uint64_t)cmd->lun);
1689                 goto out_error;
1690         }
1691
1692         rc = scst_check_local_events(cmd);
1693         if (unlikely(rc != 0))
1694                 goto out_done;
1695
1696 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) 
1697         if (unlikely(scst_alloc_request(cmd) != 0)) {
1698                 if (scst_cmd_atomic(cmd)) {
1699                         rc = SCST_EXEC_NEED_THREAD;
1700                         goto out_clear;
1701                 } else {
1702                         PRINT_INFO("%s", "Unable to allocate request, "
1703                                 "sending BUSY status");
1704                         goto out_busy;
1705                 }
1706         }
1707         
1708         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1709                     (void *)cmd->scsi_req->sr_buffer,
1710                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1711                     cmd->retries);
1712 #else
1713         rc = scst_exec_req(cmd->dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1714                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1715                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1716                         scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
1717         if (unlikely(rc != 0)) {
1718                 if (scst_cmd_atomic(cmd)) {
1719                         rc = SCST_EXEC_NEED_THREAD;
1720                         goto out_clear;
1721                 } else {
1722                         PRINT_INFO("scst_exec_req() failed: %d", rc);
1723                         goto out_error;
1724                 }
1725         }
1726 #endif
1727
1728         rc = SCST_EXEC_COMPLETED;
1729
1730 out:
1731         TRACE_EXIT();
1732         return rc;
1733
1734 out_clear:
1735         /* Restore the state */
1736         cmd->sent_to_midlev = 0;
1737         cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
1738         goto out;
1739
1740 out_rc_error:
1741         PRINT_ERROR("Dev handler %s exec() or scst_local_exec() returned "
1742                     "invalid code %d", cmd->dev->handler->name, rc);
1743         /* go through */
1744
1745 out_error:
1746         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1747
1748 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) 
1749 out_busy:
1750         scst_set_busy(cmd);
1751         /* go through */
1752 #endif
1753
1754 out_done:
1755         rc = SCST_EXEC_COMPLETED;
1756         /* Report the result */
1757         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1758         goto out;
1759 }
1760
1761 /* No locks */
1762 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1763 {
1764         if (slot == NULL)
1765                 goto inc;
1766
1767         /* Optimized for lockless fast path */
1768
1769         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1770                 atomic_read(slot));
1771
1772         if (!atomic_dec_and_test(slot))
1773                 goto out;
1774
1775         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1776                 tgt_dev->num_free_sn_slots);
1777         if (tgt_dev->num_free_sn_slots != ARRAY_SIZE(tgt_dev->sn_slots)) {
1778                 spin_lock_irq(&tgt_dev->sn_lock);
1779                 if (tgt_dev->num_free_sn_slots != ARRAY_SIZE(tgt_dev->sn_slots)) {
1780                         tgt_dev->num_free_sn_slots++;
1781                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1782                                 tgt_dev->num_free_sn_slots);
1783                         if (tgt_dev->num_free_sn_slots == 0)
1784                                 tgt_dev->cur_sn_slot = slot;
1785                 }
1786                 spin_unlock_irq(&tgt_dev->sn_lock);
1787         }
1788
1789 inc:
1790         /*
1791          * No locks is needed, because only one thread at time can 
1792          * be here (serialized by sn). Also it is supposed that there
1793          * could not be half-incremented halves.
1794          */
1795         tgt_dev->expected_sn++;
1796         smp_mb(); /* write must be before def_cmd_count read */
1797         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1798
1799 out:
1800         return;
1801 }
1802
1803 static int scst_send_to_midlev(struct scst_cmd **active_cmd)
1804 {
1805         int res, rc;
1806         struct scst_cmd *cmd = *active_cmd;
1807         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1808         struct scst_device *dev = cmd->dev;
1809         typeof(tgt_dev->expected_sn) expected_sn;
1810         int count;
1811
1812         TRACE_ENTRY();
1813
1814         res = SCST_CMD_STATE_RES_CONT_NEXT;
1815
1816         __scst_get(0); /* protect dev & tgt_dev */
1817
1818         if (unlikely(cmd->internal || cmd->retry)) {
1819                 rc = scst_do_send_to_midlev(cmd);
1820                 /* !! At this point cmd, sess & tgt_dev can be already freed !! */
1821                 if (rc == SCST_EXEC_NEED_THREAD) {
1822                         TRACE_DBG("%s", "scst_do_send_to_midlev() requested "
1823                               "thread context, rescheduling");
1824                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1825                         scst_dec_on_dev_cmd(cmd);
1826                         goto out_put;
1827                 } else {
1828                         sBUG_ON(rc != SCST_EXEC_COMPLETED);
1829                         goto out_unplug;
1830                 }
1831         }
1832
1833         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
1834                 goto out_put;
1835
1836         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
1837                 goto exec;
1838
1839         sBUG_ON(!cmd->sn_set);
1840
1841         expected_sn = tgt_dev->expected_sn;
1842         /* Optimized for lockless fast path */
1843         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
1844                 spin_lock_irq(&tgt_dev->sn_lock);
1845                 tgt_dev->def_cmd_count++;
1846                 smp_mb();
1847                 barrier(); /* to reread expected_sn & hq_cmd_count */
1848                 expected_sn = tgt_dev->expected_sn;
1849                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
1850                         /* We are under IRQ lock, but dev->dev_lock is BH one */
1851                         int cmd_blocking = scst_pre_dec_on_dev_cmd(cmd);
1852                         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1853                                 /* Necessary to allow aborting out of sn cmds */
1854                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p (tag %llu)",
1855                                         cmd, cmd->tag);
1856                                 tgt_dev->def_cmd_count--;
1857                                 cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1858                                 res = SCST_CMD_STATE_RES_CONT_SAME;
1859                         } else {
1860                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
1861                                         "expected_sn=%ld)", cmd, cmd->sn,
1862                                         cmd->sn_set, expected_sn);
1863                                 list_add_tail(&cmd->sn_cmd_list_entry,
1864                                               &tgt_dev->deferred_cmd_list);
1865                         }
1866                         spin_unlock_irq(&tgt_dev->sn_lock);
1867                         /* !! At this point cmd can be already freed !! */
1868                         __scst_dec_on_dev_cmd(dev, cmd_blocking);
1869                         goto out_put;
1870                 } else {
1871                         TRACE_SN("Somebody incremented expected_sn %ld, "
1872                                 "continuing", expected_sn);
1873                         tgt_dev->def_cmd_count--;
1874                         spin_unlock_irq(&tgt_dev->sn_lock);
1875                 }
1876         }
1877
1878 exec:
1879         count = 0;
1880         while(1) {
1881                 atomic_t *slot = cmd->sn_slot;
1882                 int inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1883                                       cmd->sn_set;
1884                 rc = scst_do_send_to_midlev(cmd);
1885                 if (rc == SCST_EXEC_NEED_THREAD) {
1886                         TRACE_DBG("%s", "scst_do_send_to_midlev() requested "
1887                               "thread context, rescheduling");
1888                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1889                         scst_dec_on_dev_cmd(cmd);
1890                         *active_cmd = cmd;
1891                         if (count != 0)
1892                                 goto out_unplug;
1893                         else
1894                                 goto out_put;
1895                 }
1896                 sBUG_ON(rc != SCST_EXEC_COMPLETED);
1897                 /* !! At this point cmd can be already freed !! */
1898                 count++;
1899                 if (inc_expected_sn)
1900                         scst_inc_expected_sn(tgt_dev, slot);
1901                 cmd = scst_check_deferred_commands(tgt_dev);
1902                 if (cmd == NULL)
1903                         break;
1904                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
1905                         break;
1906         }
1907
1908 out_unplug:
1909         if (dev->scsi_dev != NULL)
1910                 generic_unplug_device(dev->scsi_dev->request_queue);
1911
1912 out_put:
1913         __scst_put();
1914         /* !! At this point sess, dev and tgt_dev can be already freed !! */
1915
1916         TRACE_EXIT_HRES(res);
1917         return res;
1918 }
1919
1920 /* No locks supposed to be held */
1921 static int scst_check_sense(struct scst_cmd *cmd)
1922 {
1923         int res = 0;
1924         struct scst_device *dev = cmd->dev;
1925         int dbl_ua_possible, ua_sent = 0;
1926
1927         TRACE_ENTRY();
1928
1929         /* If we had internal bus reset behind us, set the command error UA */
1930         if ((dev->scsi_dev != NULL) &&
1931             unlikely(cmd->host_status == DID_RESET) &&
1932             scst_is_ua_command(cmd)) {
1933                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
1934                       dev->scsi_dev->was_reset, cmd->host_status);
1935                 scst_set_cmd_error(cmd,
1936                    SCST_LOAD_SENSE(scst_sense_reset_UA));
1937                 /* just in case */
1938                 cmd->ua_ignore = 0;
1939                 /* It looks like it is safe to clear was_reset here */
1940                 dev->scsi_dev->was_reset = 0;
1941                 smp_mb();
1942         }
1943
1944         dbl_ua_possible = dev->dev_double_ua_possible;
1945         TRACE_DBG("cmd %p dbl_ua_possible %d", cmd, dbl_ua_possible);
1946         if (unlikely(dbl_ua_possible)) {
1947                 spin_lock_bh(&dev->dev_lock);
1948                 barrier(); /* to reread dev_double_ua_possible */
1949                 dbl_ua_possible = dev->dev_double_ua_possible;
1950                 if (dbl_ua_possible)
1951                         ua_sent = dev->dev_reset_ua_sent;
1952                 else
1953                         spin_unlock_bh(&dev->dev_lock);
1954         }
1955
1956         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) && 
1957             SCST_SENSE_VALID(cmd->sense_buffer)) {
1958                 TRACE_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense_buffer,
1959                         sizeof(cmd->sense_buffer));
1960                 /* Check Unit Attention Sense Key */
1961                 if (scst_is_ua_sense(cmd->sense_buffer)) {
1962                         if (cmd->sense_buffer[12] == SCST_SENSE_ASC_UA_RESET) {
1963                                 if (dbl_ua_possible) {
1964                                         if (ua_sent) {
1965                                                 TRACE(TRACE_MGMT_MINOR, "%s", 
1966                                                         "Double UA detected");
1967                                                 /* Do retry */
1968                                                 TRACE(TRACE_MGMT_MINOR, "Retrying cmd %p "
1969                                                         "(tag %llu)", cmd, cmd->tag);
1970                                                 cmd->status = 0;
1971                                                 cmd->msg_status = 0;
1972                                                 cmd->host_status = DID_OK;
1973                                                 cmd->driver_status = 0;
1974                                                 memset(cmd->sense_buffer, 0,
1975                                                         sizeof(cmd->sense_buffer));
1976                                                 cmd->retry = 1;
1977                                                 cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
1978                                                 res = 1;
1979                                                 /* 
1980                                                  * Dev is still blocked by this cmd, so
1981                                                  * it's OK to clear SCST_DEV_SERIALIZED
1982                                                  * here.
1983                                                  */
1984                                                 dev->dev_double_ua_possible = 0;
1985                                                 dev->dev_serialized = 0;
1986                                                 dev->dev_reset_ua_sent = 0;
1987                                                 goto out_unlock;
1988                                         } else
1989                                                 dev->dev_reset_ua_sent = 1;
1990                                 }
1991                         }
1992                         if (cmd->ua_ignore == 0) {
1993                                 if (unlikely(dbl_ua_possible)) {
1994                                         __scst_dev_check_set_UA(dev, cmd,
1995                                                 cmd->sense_buffer,
1996                                                 sizeof(cmd->sense_buffer));
1997                                 } else {
1998                                         scst_dev_check_set_UA(dev, cmd,
1999                                                 cmd->sense_buffer,
2000                                                 sizeof(cmd->sense_buffer));
2001                                 }
2002                         }
2003                 }
2004         }
2005
2006         if (unlikely(dbl_ua_possible)) {
2007                 if (ua_sent && scst_is_ua_command(cmd)) {
2008                         TRACE_MGMT_DBG("%s", "Clearing dbl_ua_possible flag");
2009                         dev->dev_double_ua_possible = 0;
2010                         dev->dev_serialized = 0;
2011                         dev->dev_reset_ua_sent = 0;
2012                 }
2013                 spin_unlock_bh(&dev->dev_lock);
2014         }
2015
2016 out:
2017         TRACE_EXIT_RES(res);
2018         return res;
2019
2020 out_unlock:
2021         spin_unlock_bh(&dev->dev_lock);
2022         goto out;
2023 }
2024
2025 static int scst_check_auto_sense(struct scst_cmd *cmd)
2026 {
2027         int res = 0;
2028
2029         TRACE_ENTRY();
2030
2031         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2032             (!SCST_SENSE_VALID(cmd->sense_buffer) ||
2033              SCST_NO_SENSE(cmd->sense_buffer))) {
2034                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2035                       "cmd->status=%x, cmd->msg_status=%x, "
2036                       "cmd->host_status=%x, cmd->driver_status=%x", cmd->status,
2037                       cmd->msg_status, cmd->host_status, cmd->driver_status);
2038                 res = 1;
2039         } else if (unlikely(cmd->host_status)) {
2040                 if ((cmd->host_status == DID_REQUEUE) ||
2041                     (cmd->host_status == DID_IMM_RETRY) ||
2042                     (cmd->host_status == DID_SOFT_ERROR) ||
2043                     (cmd->host_status == DID_ABORT)) {
2044                         scst_set_busy(cmd);
2045                 } else {
2046                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2047                                 "received, returning HARDWARE ERROR instead",
2048                                 cmd->host_status);
2049                         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2050                 }
2051         }
2052
2053         TRACE_EXIT_RES(res);
2054         return res;
2055 }
2056
2057 static int scst_done_cmd_check(struct scst_cmd *cmd, int *pres)
2058 {
2059         int res = 0, rc;
2060
2061         TRACE_ENTRY();
2062
2063         if (unlikely(cmd->cdb[0] == REQUEST_SENSE)) {
2064                 if (cmd->internal)
2065                         cmd = scst_complete_request_sense(cmd);
2066         } else if (unlikely(scst_check_auto_sense(cmd))) {
2067                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2068                             "without sense data (opcode 0x%x), issuing "
2069                             "REQUEST SENSE", cmd->cdb[0]);
2070                 rc = scst_prepare_request_sense(cmd);
2071                 if (rc > 0) {
2072                         *pres = rc;
2073                         res = 1;
2074                         goto out;
2075                 } else {
2076                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2077                                     "returning HARDWARE ERROR");
2078                         scst_set_cmd_error(cmd,
2079                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2080                 }
2081         } else if (unlikely(scst_check_sense(cmd))) {
2082                 *pres = SCST_CMD_STATE_RES_CONT_SAME;
2083                 res = 1;
2084                 goto out;
2085         }
2086
2087         if (likely(scsi_status_is_good(cmd->status))) {
2088                 unsigned char type = cmd->dev->handler->type;
2089                 if ((cmd->cdb[0] == MODE_SENSE || cmd->cdb[0] == MODE_SENSE_10) &&
2090                     cmd->tgt_dev->acg_dev->rd_only_flag &&
2091                     (type == TYPE_DISK || type == TYPE_WORM || type == TYPE_MOD ||
2092                      type == TYPE_TAPE)) {
2093                         int32_t length;
2094                         uint8_t *address;
2095
2096                         length = scst_get_buf_first(cmd, &address);
2097                         if (length <= 0)
2098                                 goto out;
2099                         if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2100                                 address[2] |= 0x80;   /* Write Protect*/
2101                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2102                                 address[3] |= 0x80;   /* Write Protect*/
2103                         scst_put_buf(cmd, address);
2104                 }
2105
2106                 /* 
2107                  * Check and clear NormACA option for the device, if necessary,
2108                  * since we don't support ACA
2109                  */
2110                 if ((cmd->cdb[0] == INQUIRY) &&
2111                     !(cmd->cdb[1] & SCST_INQ_EVPD/* Std INQUIRY data (no EVPD) */) &&
2112                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2113                         uint8_t *buffer;
2114                         int buflen;
2115
2116                         /* ToDo: all pages ?? */
2117                         buflen = scst_get_buf_first(cmd, &buffer);
2118                         if (buflen > SCST_INQ_BYTE3) {
2119 #ifdef EXTRACHECKS
2120                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2121                                         PRINT_INFO("NormACA set for device: "
2122                                             "lun=%Ld, type 0x%02x", 
2123                                             (uint64_t)cmd->lun, buffer[0]);
2124                                 }
2125 #endif
2126                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2127                         } else {
2128                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2129                                     "buffer");
2130                                 scst_set_cmd_error(cmd,
2131                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
2132                         }
2133                         if (buflen > 0)
2134                                 scst_put_buf(cmd, buffer);
2135                 }
2136
2137                 if (unlikely((cmd->cdb[0] == MODE_SELECT) || 
2138                     (cmd->cdb[0] == MODE_SELECT_10) ||
2139                     (cmd->cdb[0] == LOG_SELECT))) {
2140                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded (LUN %Ld)",
2141                                 (uint64_t)cmd->lun);
2142                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2143                         *pres = SCST_CMD_STATE_RES_CONT_SAME;
2144                         res = 1;
2145                         goto out;
2146                 }
2147         } else {
2148                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2149                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2150                                         &cmd->tgt_dev->tgt_dev_flags)) {
2151                                 struct scst_tgt_dev *tgt_dev_tmp;
2152                                 TRACE(TRACE_SCSI, "Real RESERVE failed lun=%Ld, status=%x",
2153                                       (uint64_t)cmd->lun, cmd->status);
2154                                 TRACE_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense_buffer,
2155                                              sizeof(cmd->sense_buffer));
2156                                 /* Clearing the reservation */
2157                                 list_for_each_entry(tgt_dev_tmp, &cmd->dev->dev_tgt_dev_list,
2158                                                     dev_tgt_dev_list_entry) {
2159                                         clear_bit(SCST_TGT_DEV_RESERVED, 
2160                                                 &tgt_dev_tmp->tgt_dev_flags);
2161                                 }
2162                                 cmd->dev->dev_reserved = 0;
2163                         }
2164                 }
2165
2166                 /* Check for MODE PARAMETERS CHANGED UA */
2167                 if ((cmd->dev->scsi_dev != NULL) &&
2168                     (cmd->status == SAM_STAT_CHECK_CONDITION) && 
2169                     SCST_SENSE_VALID(cmd->sense_buffer) &&
2170                     scst_is_ua_sense(cmd->sense_buffer) &&
2171                     (cmd->sense_buffer[12] == 0x2a) &&
2172                     (cmd->sense_buffer[13] == 0x01)) {
2173                         TRACE(TRACE_SCSI, "MODE PARAMETERS CHANGED UA (lun %Ld)",
2174                                 (uint64_t)cmd->lun);
2175                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2176                         *pres = SCST_CMD_STATE_RES_CONT_SAME;
2177                         res = 1;
2178                         goto out;
2179                 }
2180         }
2181
2182 out:
2183         TRACE_EXIT_RES(res);
2184         return res;
2185 }
2186
2187 static int scst_pre_dev_done(struct scst_cmd *cmd)
2188 {
2189         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2190
2191         TRACE_ENTRY();
2192
2193         rc = scst_done_cmd_check(cmd, &res);
2194         if (rc)
2195                 goto out;
2196
2197         cmd->state = SCST_CMD_STATE_DEV_DONE;
2198
2199 out:
2200         TRACE_EXIT_HRES(res);
2201         return res;
2202 }
2203
2204 static int scst_mode_select_checks(struct scst_cmd *cmd)
2205 {
2206         int res = SCST_CMD_STATE_RES_CONT_SAME;
2207         int atomic = scst_cmd_atomic(cmd);
2208
2209         TRACE_ENTRY();
2210
2211         if (likely(scsi_status_is_good(cmd->status))) {
2212                 if (unlikely((cmd->cdb[0] == MODE_SELECT) || 
2213                     (cmd->cdb[0] == MODE_SELECT_10) ||
2214                     (cmd->cdb[0] == LOG_SELECT))) {
2215                         if (atomic && (cmd->dev->scsi_dev != NULL)) {
2216                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2217                                         "context required");
2218                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2219                                 goto out;
2220                         }
2221
2222                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2223                                 "setting the SELECT UA (lun=%Ld)", 
2224                                 (uint64_t)cmd->lun);
2225
2226                         spin_lock_bh(&scst_temp_UA_lock);
2227                         if (cmd->cdb[0] == LOG_SELECT) {
2228                                 scst_set_sense(scst_temp_UA,
2229                                         sizeof(scst_temp_UA),
2230                                         UNIT_ATTENTION, 0x2a, 0x02);
2231                         } else {
2232                                 scst_set_sense(scst_temp_UA,
2233                                         sizeof(scst_temp_UA),
2234                                         UNIT_ATTENTION, 0x2a, 0x01);
2235                         }
2236                         scst_dev_check_set_local_UA(cmd->dev, cmd, scst_temp_UA,
2237                                 sizeof(scst_temp_UA));
2238                         spin_unlock_bh(&scst_temp_UA_lock);
2239
2240                         if (cmd->dev->scsi_dev != NULL)
2241                                 scst_obtain_device_parameters(cmd->dev);
2242                 }
2243         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) && 
2244                     SCST_SENSE_VALID(cmd->sense_buffer) &&
2245                     scst_is_ua_sense(cmd->sense_buffer) &&
2246                     (cmd->sense_buffer[12] == 0x2a) &&
2247                     (cmd->sense_buffer[13] == 0x01)) {
2248                 if (atomic) {
2249                         TRACE_DBG("%s", "MODE PARAMETERS CHANGED UA: thread "
2250                                 "context required");
2251                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2252                         goto out;
2253                 }
2254
2255                 TRACE(TRACE_SCSI, "MODE PARAMETERS CHANGED UA (lun %Ld): "
2256                         "getting new parameters", (uint64_t)cmd->lun);
2257
2258                 scst_obtain_device_parameters(cmd->dev);
2259         } else
2260                 sBUG();
2261
2262         cmd->state = SCST_CMD_STATE_DEV_DONE;
2263
2264 out:
2265         TRACE_EXIT_HRES(res);
2266         return res;
2267 }
2268
2269 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2270 {
2271         struct scst_cmd *c;
2272
2273         if (likely(cmd->sn_set))
2274                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2275
2276         c = scst_check_deferred_commands(cmd->tgt_dev);
2277         if (c != NULL) {
2278                 unsigned long flags;
2279                 spin_lock_irqsave(&c->cmd_lists->cmd_list_lock, flags);
2280                 TRACE_SN("Adding cmd %p to active cmd list", c);
2281                 list_add_tail(&c->cmd_list_entry,
2282                         &c->cmd_lists->active_cmd_list);
2283                 wake_up(&c->cmd_lists->cmd_list_waitQ);
2284                 spin_unlock_irqrestore(&c->cmd_lists->cmd_list_lock, flags);
2285         }
2286 }
2287
2288 static int scst_dev_done(struct scst_cmd *cmd)
2289 {
2290         int res = SCST_CMD_STATE_RES_CONT_SAME;
2291         int state;
2292         int atomic = scst_cmd_atomic(cmd);
2293
2294         TRACE_ENTRY();
2295
2296         if (atomic && !cmd->dev->handler->dev_done_atomic) {
2297                 TRACE_DBG("Dev handler %s dev_done() can not be "
2298                       "called in atomic context, rescheduling to the thread",
2299                       cmd->dev->handler->name);
2300                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2301                 goto out;
2302         }
2303
2304         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2305         if (likely(!scst_is_cmd_local(cmd)) && 
2306             likely(cmd->dev->handler->dev_done != NULL))
2307         {
2308                 int rc;
2309                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2310                       cmd->dev->handler->name, cmd);
2311                 rc = cmd->dev->handler->dev_done(cmd);
2312                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2313                       cmd->dev->handler->name, rc);
2314                 if (rc != SCST_CMD_STATE_DEFAULT)
2315                         state = rc;
2316         }
2317
2318         switch (state) {
2319         case SCST_CMD_STATE_PRE_XMIT_RESP:
2320         case SCST_CMD_STATE_DEV_PARSE:
2321         case SCST_CMD_STATE_PRE_PARSE:
2322         case SCST_CMD_STATE_PREPARE_SPACE:
2323         case SCST_CMD_STATE_RDY_TO_XFER:
2324         case SCST_CMD_STATE_TGT_PRE_EXEC:
2325         case SCST_CMD_STATE_SEND_TO_MIDLEV:
2326         case SCST_CMD_STATE_PRE_DEV_DONE:
2327         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2328         case SCST_CMD_STATE_DEV_DONE:
2329         case SCST_CMD_STATE_XMIT_RESP:
2330         case SCST_CMD_STATE_FINISHED:
2331                 cmd->state = state;
2332                 res = SCST_CMD_STATE_RES_CONT_SAME;
2333                 break;
2334
2335         case SCST_CMD_STATE_NEED_THREAD_CTX:
2336                 TRACE_DBG("Dev handler %s dev_done() requested "
2337                       "thread context, rescheduling",
2338                       cmd->dev->handler->name);
2339                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2340                 break;
2341
2342         default:
2343                 if (state >= 0) {
2344                         PRINT_ERROR("Dev handler %s dev_done() returned "
2345                                 "invalid cmd state %d", 
2346                                 cmd->dev->handler->name, state);
2347                 } else {
2348                         PRINT_ERROR("Dev handler %s dev_done() returned "
2349                                 "error %d", cmd->dev->handler->name, 
2350                                 state);
2351                 }
2352                 scst_set_cmd_error(cmd,
2353                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2354                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2355                 res = SCST_CMD_STATE_RES_CONT_SAME;
2356                 break;
2357         }
2358
2359         if (cmd->needs_unblocking)
2360                 scst_unblock_dev_cmd(cmd);
2361
2362         if (likely(cmd->dec_on_dev_needed))
2363                 scst_dec_on_dev_cmd(cmd);
2364
2365         if (cmd->inc_expected_sn_on_done && cmd->sent_to_midlev)
2366                 scst_inc_check_expected_sn(cmd);
2367
2368 out:
2369         TRACE_EXIT_HRES(res);
2370         return res;
2371 }
2372
2373 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2374 {
2375         int res;
2376
2377         TRACE_ENTRY();
2378
2379         if (cmd->tgt_dev != NULL) {
2380                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2381                         scst_on_hq_cmd_response(cmd);
2382
2383                 if (unlikely(!cmd->sent_to_midlev)) {
2384                         TRACE_SN("cmd %p was not sent to mid-lev (sn %ld, set %d)",
2385                                 cmd, cmd->sn, cmd->sn_set);
2386                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2387                         cmd->sent_to_midlev = 1;
2388                 }
2389         }
2390
2391         /*
2392          * If we don't remove cmd from the search list here, before
2393          * submitting it for transmittion, we will have a race, when for
2394          * some reason cmd's release is delayed after transmittion and
2395          * initiator sends cmd with the same tag => it is possible that
2396          * a wrong cmd will be found by find() functions.
2397          */
2398         spin_lock_irq(&cmd->sess->sess_list_lock);
2399         list_del(&cmd->search_cmd_list_entry);
2400         spin_unlock_irq(&cmd->sess->sess_list_lock);
2401
2402         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2403                 scst_xmit_process_aborted_cmd(cmd);
2404
2405         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2406                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu), skipping",
2407                         cmd, cmd->tag);
2408                 cmd->state = SCST_CMD_STATE_FINISHED;
2409                 res = SCST_CMD_STATE_RES_CONT_SAME;
2410                 goto out;
2411         }
2412
2413         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2414         res = SCST_CMD_STATE_RES_CONT_SAME;
2415
2416 out:
2417         TRACE_EXIT_HRES(res);
2418         return res;
2419 }
2420
2421 static int scst_xmit_response(struct scst_cmd *cmd)
2422 {
2423         int res, rc;
2424         int atomic = scst_cmd_atomic(cmd);
2425
2426         TRACE_ENTRY();
2427
2428         if (atomic && !cmd->tgtt->xmit_response_atomic) {
2429                 TRACE_DBG("%s", "xmit_response() can not be "
2430                       "called in atomic context, rescheduling to the thread");
2431                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2432                 goto out;
2433         }
2434
2435 #ifdef DEBUG_TM
2436         if (cmd->tm_dbg_delayed && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2437                 if (atomic && !cmd->tgtt->xmit_response_atomic) {
2438                         TRACE_MGMT_DBG("%s", "DEBUG_TM delayed cmd needs a thread");
2439                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2440                         goto out;
2441                 }
2442                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2443                         cmd, cmd->tag);
2444                 schedule_timeout_uninterruptible(HZ);
2445         }
2446 #endif
2447
2448         while (1) {
2449                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2450
2451                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2452                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2453
2454                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2455
2456 #if defined(DEBUG) || defined(TRACING)
2457                 if (cmd->sg) {
2458                         int i;
2459                         struct scatterlist *sg = cmd->sg;
2460                         TRACE(TRACE_SEND_BOT,
2461                               "Xmitting %d S/G(s) at %p sg[0].page at %p",
2462                               cmd->sg_cnt, sg, (void*)sg[0].page);
2463                         for(i = 0; i < cmd->sg_cnt; ++i) {
2464                                 TRACE_BUFF_FLAG(TRACE_SEND_BOT,
2465                                     "Xmitting sg", page_address(sg[i].page),
2466                                     sg[i].length);
2467                         }
2468                 }
2469 #endif
2470
2471 #ifdef DEBUG_RETRY
2472                 if (((scst_random() % 100) == 77))
2473                         rc = SCST_TGT_RES_QUEUE_FULL;
2474                 else
2475 #endif
2476                         rc = cmd->tgtt->xmit_response(cmd);
2477                 TRACE_DBG("xmit_response() returned %d", rc);
2478
2479                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2480                         goto out;
2481
2482                 /* Restore the previous state */
2483                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2484
2485                 switch (rc) {
2486                 case SCST_TGT_RES_QUEUE_FULL:
2487                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2488                                 break;
2489                         else
2490                                 continue;
2491
2492                 case SCST_TGT_RES_NEED_THREAD_CTX:
2493                         TRACE_DBG("Target driver %s xmit_response() "
2494                               "requested thread context, rescheduling",
2495                               cmd->tgtt->name);
2496                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2497                         break;
2498
2499                 default:
2500                         goto out_error;
2501                 }
2502                 break;
2503         }
2504
2505 out:
2506         /* Caution: cmd can be already dead here */
2507         TRACE_EXIT_HRES(res);
2508         return res;
2509
2510 out_error:
2511         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2512                 PRINT_ERROR("Target driver %s xmit_response() returned "
2513                         "fatal error", cmd->tgtt->name);
2514         } else {
2515                 PRINT_ERROR("Target driver %s xmit_response() returned "
2516                         "invalid value %d", cmd->tgtt->name, rc);
2517         }
2518         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2519         cmd->state = SCST_CMD_STATE_FINISHED;
2520         res = SCST_CMD_STATE_RES_CONT_SAME;
2521         goto out;
2522 }
2523
2524 void scst_tgt_cmd_done(struct scst_cmd *cmd)
2525 {
2526         TRACE_ENTRY();
2527
2528         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2529
2530         cmd->state = SCST_CMD_STATE_FINISHED;
2531         scst_proccess_redirect_cmd(cmd, scst_get_context(), 1);
2532
2533         TRACE_EXIT();
2534         return;
2535 }
2536
2537 static int scst_finish_cmd(struct scst_cmd *cmd)
2538 {
2539         int res;
2540
2541         TRACE_ENTRY();
2542
2543         atomic_dec(&cmd->sess->sess_cmd_count);
2544
2545         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2546                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
2547                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
2548                         atomic_read(&scst_cmd_count));
2549         }
2550
2551         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
2552                 if ((cmd->tgt_dev != NULL) &&
2553                     scst_is_ua_sense(cmd->sense_buffer)) {
2554                         /* This UA delivery failed, so requeue it */
2555                         scst_check_set_UA(cmd->tgt_dev, cmd->sense_buffer,
2556                                         sizeof(cmd->sense_buffer), 1);
2557                 }
2558         }
2559
2560         scst_cmd_put(cmd);
2561
2562         res = SCST_CMD_STATE_RES_CONT_NEXT;
2563
2564         TRACE_EXIT_HRES(res);
2565         return res;
2566 }
2567
2568 /*
2569  * No locks, but it must be externally serialized (see comment for
2570  * scst_cmd_init_done() in scsi_tgt.h)
2571  */
2572 static void scst_cmd_set_sn(struct scst_cmd *cmd)
2573 {
2574         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2575         unsigned long flags;
2576
2577         if (scst_is_implicit_hq(cmd)) {
2578                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "Implicit HQ cmd %p", cmd);
2579                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
2580         }
2581
2582         /* Optimized for lockless fast path */
2583
2584         scst_check_debug_sn(cmd);
2585
2586         if (cmd->dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER)
2587                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
2588
2589         switch(cmd->queue_type) {
2590         case SCST_CMD_QUEUE_SIMPLE:
2591         case SCST_CMD_QUEUE_UNTAGGED:
2592 #if 1 /* temporary, ToDo */
2593                 if (scst_cmd_is_expected_set(cmd)) {
2594                         if (cmd->expected_data_direction == SCST_DATA_READ)
2595                                 goto ordered;
2596                 } else
2597                         goto ordered;
2598 #endif
2599                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
2600                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
2601                                 tgt_dev->curr_sn++;
2602                                 TRACE_SN("Incremented curr_sn %ld",
2603                                         tgt_dev->curr_sn);
2604                         }
2605                         cmd->sn_slot = tgt_dev->cur_sn_slot;
2606                         cmd->sn = tgt_dev->curr_sn;
2607                         
2608                         tgt_dev->prev_cmd_ordered = 0;
2609                 } else {
2610                         TRACE(TRACE_MINOR, "%s", "Not enough SN slots");
2611                         goto ordered;
2612                 }
2613                 break;
2614
2615         case SCST_CMD_QUEUE_ORDERED:
2616                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "ORDERED cmd %p "
2617                         "(op %x)", cmd, cmd->cdb[0]);
2618 ordered:
2619                 if (!tgt_dev->prev_cmd_ordered) {
2620                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2621                         tgt_dev->num_free_sn_slots--;
2622                         smp_mb();
2623                         if ((tgt_dev->num_free_sn_slots >= 0) &&
2624                             (atomic_read(tgt_dev->cur_sn_slot) > 0)) {
2625                                 do {
2626                                         tgt_dev->cur_sn_slot++;
2627                                         if (tgt_dev->cur_sn_slot == 
2628                                                 tgt_dev->sn_slots +
2629                                                 ARRAY_SIZE(tgt_dev->sn_slots))
2630                                             tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
2631                                 } while(atomic_read(tgt_dev->cur_sn_slot) != 0);
2632                                 TRACE_SN("New cur SN slot %zd",
2633                                         tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
2634                         } else
2635                                 tgt_dev->num_free_sn_slots++;
2636                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2637                 }
2638                 tgt_dev->prev_cmd_ordered = 1;
2639                 tgt_dev->curr_sn++;
2640                 cmd->sn = tgt_dev->curr_sn;
2641                 break;
2642
2643         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
2644                 TRACE(TRACE_SCSI|TRACE_SCSI_SERIALIZING, "HQ cmd %p "
2645                         "(op %x)", cmd, cmd->cdb[0]);
2646                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2647                 tgt_dev->hq_cmd_count++;
2648                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
2649                 goto out;
2650
2651         default:
2652                 sBUG();
2653         }
2654
2655         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
2656                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
2657                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
2658                 atomic_read(tgt_dev->cur_sn_slot), 
2659                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
2660                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
2661
2662         cmd->sn_set = 1;
2663 out:
2664         return;
2665 }
2666
2667 /*
2668  * Returns 0 on success, > 0 when we need to wait for unblock,
2669  * < 0 if there is no device (lun) or device type handler.
2670  *
2671  * No locks, but might be on IRQ, protection is done by the
2672  * suspended activity.
2673  */
2674 static int scst_translate_lun(struct scst_cmd *cmd)
2675 {
2676         struct scst_tgt_dev *tgt_dev = NULL;
2677         int res;
2678
2679         TRACE_ENTRY();
2680
2681         __scst_get(1);
2682
2683         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
2684                 struct list_head *sess_tgt_dev_list_head =
2685                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
2686                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %Ld)", cmd,
2687                         (uint64_t)cmd->lun);
2688                 res = -1;
2689                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
2690                                 sess_tgt_dev_list_entry) {
2691                         if (tgt_dev->lun == cmd->lun) {
2692                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
2693
2694                                 if (unlikely(tgt_dev->dev->handler == &scst_null_devtype)) {
2695                                         PRINT_INFO("Dev handler for device "
2696                                           "%Ld is NULL, the device will not be "
2697                                           "visible remotely", (uint64_t)cmd->lun);
2698                                         break;
2699                                 }
2700                                 
2701                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
2702                                 cmd->tgt_dev = tgt_dev;
2703                                 cmd->dev = tgt_dev->dev;
2704
2705                                 res = 0;
2706                                 break;
2707                         }
2708                 }
2709                 if (res != 0) {
2710                         TRACE(TRACE_MINOR, "tgt_dev for lun %Ld not found, command to "
2711                                 "unexisting LU?", (uint64_t)cmd->lun);
2712                         __scst_put();
2713                 }
2714         } else {
2715                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
2716                 __scst_put();
2717                 res = 1;
2718         }
2719
2720         TRACE_EXIT_RES(res);
2721         return res;
2722 }
2723
2724 /*
2725  * No locks, but might be on IRQ
2726  *
2727  * Returns 0 on success, > 0 when we need to wait for unblock,
2728  * < 0 if there is no device (lun) or device type handler.
2729  */
2730 static int __scst_init_cmd(struct scst_cmd *cmd)
2731 {
2732         int res = 0;
2733
2734         TRACE_ENTRY();
2735
2736         res = scst_translate_lun(cmd);
2737         if (likely(res == 0)) {
2738                 int cnt;
2739                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
2740                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
2741                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
2742                         TRACE(TRACE_MGMT_MINOR, "Too many pending commands (%d) in "
2743                                 "session, returning BUSY to initiator \"%s\"",
2744                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
2745                                   "Anonymous" : cmd->sess->initiator_name);
2746                         goto out_busy;
2747                 }
2748                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
2749                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
2750                         TRACE(TRACE_MGMT_MINOR, "Too many pending device commands "
2751                                 "(%d), returning BUSY to initiator \"%s\"",
2752                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
2753                                   "Anonymous" : cmd->sess->initiator_name);
2754                         goto out_busy;
2755                 }
2756                 if (!cmd->set_sn_on_restart_cmd)
2757                         scst_cmd_set_sn(cmd);
2758         } else if (res < 0) {
2759                 TRACE_DBG("Finishing cmd %p", cmd);
2760                 scst_set_cmd_error(cmd,
2761                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
2762                 cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2763         } else
2764                 goto out;
2765
2766 out:
2767         TRACE_EXIT_RES(res);
2768         return res;
2769
2770 out_busy:
2771         scst_set_busy(cmd);
2772         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2773         goto out;
2774 }
2775
2776 /* Called under scst_init_lock and IRQs disabled */
2777 static void scst_do_job_init(void)
2778 {
2779         struct scst_cmd *cmd;
2780         int susp;
2781
2782         TRACE_ENTRY();
2783
2784 restart:
2785         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
2786         if (scst_init_poll_cnt > 0)
2787                 scst_init_poll_cnt--;
2788
2789         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
2790                 int rc;
2791                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
2792                         continue;
2793                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2794                         spin_unlock_irq(&scst_init_lock);
2795                         rc = __scst_init_cmd(cmd);
2796                         spin_lock_irq(&scst_init_lock);
2797                         if (rc > 0) {
2798                                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, restarting");
2799                                 goto restart;
2800                         }
2801                 } else {
2802                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
2803                                 cmd, cmd->tag);
2804                         cmd->state = SCST_CMD_STATE_PRE_XMIT_RESP;
2805                 }
2806
2807                 /*
2808                  * Deleting cmd from init cmd list after __scst_init_cmd()
2809                  * is necessary to keep the check in scst_init_cmd() correct
2810                  * to preserve the commands order.
2811                  *
2812                  * We don't care about the race, when init cmd list is empty
2813                  * and one command detected that it just was not empty, so
2814                  * it's inserting to it, but another command at the same time
2815                  * seeing init cmd list empty and goes directly, because it
2816                  * could affect only commands from the same initiator to the
2817                  * same tgt_dev, but init_cmd_done() doesn't guarantee the order
2818                  * in case of simultaneous such calls anyway.
2819                  */
2820                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
2821                 list_del(&cmd->cmd_list_entry);
2822                 spin_unlock(&scst_init_lock);
2823
2824                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
2825                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
2826                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2827                         list_add(&cmd->cmd_list_entry,
2828                                 &cmd->cmd_lists->active_cmd_list);
2829                 else
2830                         list_add_tail(&cmd->cmd_list_entry,
2831                                 &cmd->cmd_lists->active_cmd_list);
2832                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
2833                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
2834
2835                 spin_lock(&scst_init_lock);
2836                 goto restart;
2837         }
2838
2839         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
2840                 goto restart;
2841
2842         TRACE_EXIT();
2843         return;
2844 }
2845
2846 static inline int test_init_cmd_list(void)
2847 {
2848         int res = (!list_empty(&scst_init_cmd_list) &&
2849                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
2850                   unlikely(kthread_should_stop()) ||
2851                   (scst_init_poll_cnt > 0);
2852         return res;
2853 }
2854
2855 int scst_init_cmd_thread(void *arg)
2856 {
2857         TRACE_ENTRY();
2858
2859         current->flags |= PF_NOFREEZE;
2860
2861         set_user_nice(current, -20);
2862
2863         spin_lock_irq(&scst_init_lock);
2864         while(!kthread_should_stop()) {
2865                 wait_queue_t wait;
2866                 init_waitqueue_entry(&wait, current);
2867
2868                 if (!test_init_cmd_list()) {
2869                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
2870                                                  &wait);
2871                         for (;;) {
2872                                 set_current_state(TASK_INTERRUPTIBLE);
2873                                 if (test_init_cmd_list())
2874                                         break;
2875                                 spin_unlock_irq(&scst_init_lock);
2876                                 schedule();
2877                                 spin_lock_irq(&scst_init_lock);
2878                         }
2879                         set_current_state(TASK_RUNNING);
2880                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
2881                 }
2882                 scst_do_job_init();
2883         }
2884         spin_unlock_irq(&scst_init_lock);
2885
2886         /*
2887          * If kthread_should_stop() is true, we are guaranteed to be
2888          * on the module unload, so scst_init_cmd_list must be empty.
2889          */
2890         sBUG_ON(!list_empty(&scst_init_cmd_list));
2891
2892         TRACE_EXIT();
2893         return 0;
2894 }
2895
2896 /* Called with no locks held */
2897 void scst_process_active_cmd(struct scst_cmd *cmd, int context)
2898 {
2899         int res;
2900
2901         TRACE_ENTRY();
2902
2903         EXTRACHECKS_BUG_ON(in_irq());
2904
2905         cmd->atomic = (context == SCST_CONTEXT_DIRECT_ATOMIC);
2906
2907         do {
2908                 switch (cmd->state) {
2909                 case SCST_CMD_STATE_PRE_PARSE:
2910                         res = scst_pre_parse(cmd);
2911                         break;
2912
2913                 case SCST_CMD_STATE_DEV_PARSE:
2914                         res = scst_parse_cmd(cmd);
2915                         break;
2916
2917                 case SCST_CMD_STATE_PREPARE_SPACE:
2918                         res = scst_prepare_space(cmd);
2919                         break;
2920
2921                 case SCST_CMD_STATE_RDY_TO_XFER:
2922                         res = scst_rdy_to_xfer(cmd);
2923                         break;
2924
2925                 case SCST_CMD_STATE_TGT_PRE_EXEC:
2926                         res = scst_tgt_pre_exec(cmd);
2927                         break;
2928
2929                 case SCST_CMD_STATE_SEND_TO_MIDLEV:
2930                         if (tm_dbg_check_cmd(cmd) != 0) {
2931                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2932                                 TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
2933                                         "because of TM DBG delay", cmd,
2934                                         cmd->tag);
2935                                 break;
2936                         }
2937                         res = scst_send_to_midlev(&cmd);
2938                         /* !! At this point cmd, sess & tgt_dev can be already freed !! */
2939                         break;
2940
2941                 case SCST_CMD_STATE_PRE_DEV_DONE:
2942                         res = scst_pre_dev_done(cmd);
2943                         break;
2944
2945                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2946                         res = scst_mode_select_checks(cmd);
2947                         break;
2948
2949                 case SCST_CMD_STATE_DEV_DONE:
2950                         res = scst_dev_done(cmd);
2951                         break;
2952
2953                 case SCST_CMD_STATE_PRE_XMIT_RESP:
2954                         res = scst_pre_xmit_response(cmd);
2955                         break;
2956
2957                 case SCST_CMD_STATE_XMIT_RESP:
2958                         res = scst_xmit_response(cmd);
2959                         break;
2960
2961                 case SCST_CMD_STATE_FINISHED:
2962                         res = scst_finish_cmd(cmd);
2963                         break;
2964
2965                 default:
2966                         PRINT_ERROR("cmd (%p) in state %d, but shouldn't be",
2967                                cmd, cmd->state);
2968                         sBUG();
2969                         res = SCST_CMD_STATE_RES_CONT_NEXT;
2970                         break;
2971                 }
2972         } while(res == SCST_CMD_STATE_RES_CONT_SAME);
2973
2974         if (res == SCST_CMD_STATE_RES_CONT_NEXT) {
2975                 /* None */
2976         } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
2977                 spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
2978                 switch (cmd->state) {
2979                 case SCST_CMD_STATE_PRE_PARSE:
2980                 case SCST_CMD_STATE_DEV_PARSE:
2981                 case SCST_CMD_STATE_PREPARE_SPACE:
2982                 case SCST_CMD_STATE_RDY_TO_XFER:
2983                 case SCST_CMD_STATE_TGT_PRE_EXEC:
2984                 case SCST_CMD_STATE_SEND_TO_MIDLEV:
2985                 case SCST_CMD_STATE_PRE_DEV_DONE:
2986                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2987                 case SCST_CMD_STATE_DEV_DONE:
2988                 case SCST_CMD_STATE_PRE_XMIT_RESP:
2989                 case SCST_CMD_STATE_XMIT_RESP:
2990                 case SCST_CMD_STATE_FINISHED:
2991                         TRACE_DBG("Adding cmd %p to head of active cmd list", cmd);
2992                         list_add(&cmd->cmd_list_entry,
2993                                 &cmd->cmd_lists->active_cmd_list);
2994                         break;
2995 #ifdef EXTRACHECKS
2996                 /* not very valid commands */
2997                 case SCST_CMD_STATE_DEFAULT:
2998                 case SCST_CMD_STATE_NEED_THREAD_CTX:
2999                         PRINT_ERROR("cmd %p is in state %d, not putting on "
3000                                 "useful list (left on scst cmd list)", cmd, 
3001                                 cmd->state);
3002                         spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3003                         sBUG();
3004                         spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3005                         break;
3006 #endif
3007                 default:
3008                         break;
3009                 }
3010                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3011                 spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3012         } else
3013                 sBUG();
3014
3015         TRACE_EXIT();
3016         return;
3017 }
3018
3019 /* Called under cmd_list_lock and IRQs disabled */
3020 static void scst_do_job_active(struct list_head *cmd_list,
3021         spinlock_t *cmd_list_lock, int context)
3022 {
3023         TRACE_ENTRY();
3024
3025 #ifdef EXTRACHECKS
3026         WARN_ON((context != SCST_CONTEXT_DIRECT_ATOMIC) && 
3027                 (context != SCST_CONTEXT_DIRECT));
3028 #endif
3029
3030         while (!list_empty(cmd_list)) {
3031                 struct scst_cmd *cmd = list_entry(cmd_list->next, typeof(*cmd),
3032                                         cmd_list_entry);
3033                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
3034                 list_del(&cmd->cmd_list_entry);
3035                 spin_unlock_irq(cmd_list_lock);
3036                 scst_process_active_cmd(cmd, context);
3037                 spin_lock_irq(cmd_list_lock);
3038         }
3039
3040         TRACE_EXIT();
3041         return;
3042 }
3043
3044 static inline int test_cmd_lists(struct scst_cmd_lists *p_cmd_lists)
3045 {
3046         int res = !list_empty(&p_cmd_lists->active_cmd_list) ||
3047             unlikely(kthread_should_stop()) ||
3048             tm_dbg_is_release();
3049         return res;
3050 }
3051
3052 int scst_cmd_thread(void *arg)
3053 {
3054         struct scst_cmd_lists *p_cmd_lists = (struct scst_cmd_lists*)arg;
3055
3056         TRACE_ENTRY();
3057
3058 #if 0
3059         set_user_nice(current, 10);
3060 #endif
3061         current->flags |= PF_NOFREEZE;
3062
3063         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3064         while (!kthread_should_stop()) {
3065                 wait_queue_t wait;
3066                 init_waitqueue_entry(&wait, current);
3067
3068                 if (!test_cmd_lists(p_cmd_lists)) {
3069                         add_wait_queue_exclusive(&p_cmd_lists->cmd_list_waitQ,
3070                                 &wait);
3071                         for (;;) {
3072                                 set_current_state(TASK_INTERRUPTIBLE);
3073                                 if (test_cmd_lists(p_cmd_lists))
3074                                         break;
3075                                 spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3076                                 schedule();
3077                                 spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3078                         }
3079                         set_current_state(TASK_RUNNING);
3080                         remove_wait_queue(&p_cmd_lists->cmd_list_waitQ, &wait);
3081                 }
3082
3083                 if (tm_dbg_is_release()) {
3084                         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3085                         tm_dbg_check_released_cmds();
3086                         spin_lock_irq(&p_cmd_lists->cmd_list_lock);
3087                 }
3088
3089                 scst_do_job_active(&p_cmd_lists->active_cmd_list,
3090                         &p_cmd_lists->cmd_list_lock, SCST_CONTEXT_DIRECT);
3091         }
3092         spin_unlock_irq(&p_cmd_lists->cmd_list_lock);
3093
3094 #ifdef EXTRACHECKS
3095         /*
3096          * If kthread_should_stop() is true, we are guaranteed to be either
3097          * on the module unload, or there must be at least one other thread to
3098          * process the commands lists.
3099          */
3100         if (p_cmd_lists == &scst_main_cmd_lists) {
3101                 sBUG_ON((scst_threads_info.nr_cmd_threads == 1) &&
3102                          !list_empty(&scst_main_cmd_lists.active_cmd_list));
3103         }
3104 #endif
3105
3106         TRACE_EXIT();
3107         return 0;
3108 }
3109
3110 void scst_cmd_tasklet(long p)
3111 {
3112         struct scst_tasklet *t = (struct scst_tasklet*)p;
3113
3114         TRACE_ENTRY();
3115
3116         spin_lock_irq(&t->tasklet_lock);
3117         scst_do_job_active(&t->tasklet_cmd_list, &t->tasklet_lock,
3118                 SCST_CONTEXT_DIRECT_ATOMIC);
3119         spin_unlock_irq(&t->tasklet_lock);
3120
3121         TRACE_EXIT();
3122         return;
3123 }
3124
3125 /*
3126  * Returns 0 on success, < 0 if there is no device handler or
3127  * > 0 if SCST_FLAG_SUSPENDED set and SCST_FLAG_SUSPENDING - not.
3128  * No locks, protection is done by the suspended activity.
3129  */
3130 static int scst_mgmt_translate_lun(struct scst_mgmt_cmd *mcmd)
3131 {
3132         struct scst_tgt_dev *tgt_dev = NULL;
3133         struct list_head *sess_tgt_dev_list_head;
3134         int res = -1;
3135
3136         TRACE_ENTRY();
3137
3138         TRACE_DBG("Finding tgt_dev for mgmt cmd %p (lun %Ld)", mcmd,
3139               (uint64_t)mcmd->lun);
3140
3141         __scst_get(1);
3142
3143         if (unlikely(test_bit(SCST_FLAG_SUSPENDED, &scst_flags) &&
3144                      !test_bit(SCST_FLAG_SUSPENDING, &scst_flags))) {
3145                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3146                 __scst_put();
3147                 res = 1;
3148                 goto out;
3149         }
3150
3151         sess_tgt_dev_list_head =
3152                 &mcmd->sess->sess_tgt_dev_list_hash[HASH_VAL(mcmd->lun)];
3153         list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3154                         sess_tgt_dev_list_entry) {
3155                 if (tgt_dev->lun == mcmd->lun) {
3156                         TRACE_DBG("tgt_dev %p found", tgt_dev);
3157                         mcmd->mcmd_tgt_dev = tgt_dev;
3158                         res = 0;
3159                         break;
3160                 }
3161         }
3162         if (mcmd->mcmd_tgt_dev == NULL)
3163                 __scst_put();
3164
3165 out:
3166         TRACE_EXIT_HRES(res);
3167         return res;
3168 }
3169
3170 /* No locks */
3171 void scst_complete_cmd_mgmt(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd)
3172 {
3173         TRACE_ENTRY();
3174
3175         spin_lock_irq(&scst_mcmd_lock);
3176
3177         TRACE_MGMT_DBG("cmd %p completed (tag %llu, mcmd %p, "
3178                 "mcmd->cmd_wait_count %d)", cmd, cmd->tag, mcmd,
3179                 mcmd->cmd_wait_count);
3180
3181         cmd->mgmt_cmnd = NULL;
3182
3183         if (cmd->completed)
3184                 mcmd->completed_cmd_count++;
3185
3186         mcmd->cmd_wait_count--;
3187         if (mcmd->cmd_wait_count > 0) {
3188                 TRACE_MGMT_DBG("cmd_wait_count(%d) not 0, skipping",
3189                         mcmd->cmd_wait_count);
3190                 goto out_unlock;
3191         }
3192
3193         mcmd->state = SCST_MGMT_CMD_STATE_DONE;
3194
3195         if (mcmd->completed) {
3196                 TRACE_MGMT_DBG("Adding mgmt cmd %p to active mgmt cmd list",
3197                         mcmd);
3198                 list_add_tail(&mcmd->mgmt_cmd_list_entry,
3199                         &scst_active_mgmt_cmd_list);
3200         }
3201
3202         spin_unlock_irq(&scst_mcmd_lock);
3203
3204         wake_up(&scst_mgmt_cmd_list_waitQ);
3205
3206 out:
3207         TRACE_EXIT();
3208         return;
3209
3210 out_unlock:
3211         spin_unlock_irq(&scst_mcmd_lock);
3212         goto out;
3213 }
3214
3215 static int scst_call_dev_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
3216         struct scst_tgt_dev *tgt_dev, int set_status)
3217 {
3218         int res = SCST_DEV_TM_NOT_COMPLETED;
3219         struct scst_dev_type *h = tgt_dev->dev->handler;
3220
3221         if (h->task_mgmt_fn) {
3222                 TRACE_MGMT_DBG("Calling dev handler %s task_mgmt_fn(fn=%d)",
3223                         h->name, mcmd->fn);
3224                 EXTRACHECKS_BUG_ON(in_irq());
3225                 res = h->task_mgmt_fn(mcmd, tgt_dev);
3226                 TRACE_MGMT_DBG("Dev handler %s task_mgmt_fn() returned %d",
3227                       h->name, res);
3228                 if (set_status && (res != SCST_DEV_TM_NOT_COMPLETED))
3229                         mcmd->status = res;
3230         }
3231         return res;
3232 }
3233
3234 static inline int scst_is_strict_mgmt_fn(int mgmt_fn)
3235 {
3236         switch(mgmt_fn) {
3237                 case SCST_ABORT_TASK:
3238 #if 0
3239                 case SCST_ABORT_TASK_SET:
3240                 case SCST_CLEAR_TASK_SET:
3241 #endif
3242                         return 1;
3243                 default:
3244                         return 0;
3245         }
3246 }
3247
3248 /* 
3249  * Might be called under sess_list_lock and IRQ off + BHs also off
3250  * Returns -1 if command is being executed (ABORT failed), 0 otherwise
3251  */
3252 void scst_abort_cmd(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd,
3253         int other_ini, int call_dev_task_mgmt_fn)
3254 {
3255         TRACE_ENTRY();
3256
3257         TRACE(((mcmd != NULL) && (mcmd->fn == SCST_ABORT_TASK)) ? TRACE_MGMT_MINOR : TRACE_MGMT,
3258                 "Aborting cmd %p (tag %llu)", cmd, cmd->tag);
3259
3260         if (other_ini) {
3261                 set_bit(SCST_CMD_ABORTED_OTHER, &cmd->cmd_flags);
3262                 smp_mb__after_set_bit();
3263         } else {
3264                 /* Might be necessary if command aborted several times */
3265                 clear_bit(SCST_CMD_ABORTED_OTHER, &cmd->cmd_flags);
3266         }
3267         set_bit(SCST_CMD_ABORTED, &cmd->cmd_flags);
3268         smp_mb__after_set_bit();
3269
3270         if (cmd->tgt_dev == NULL) {
3271                 unsigned long flags;
3272                 spin_lock_irqsave(&scst_init_lock, flags);
3273                 scst_init_poll_cnt++;
3274                 spin_unlock_irqrestore(&scst_init_lock, flags);
3275                 wake_up(&scst_init_cmd_list_waitQ);
3276         }
3277
3278         if (call_dev_task_mgmt_fn && (cmd->tgt_dev != NULL)) {
3279                 EXTRACHECKS_BUG_ON(irqs_disabled());
3280                 scst_call_dev_task_mgmt_fn(mcmd, cmd->tgt_dev, 1);
3281         }
3282
3283         if (mcmd) {
3284                 unsigned long flags;
3285                 /*
3286                  * Delay the response until the command's finish in
3287                  * order to guarantee that "no further responses from
3288                  * the task are sent to the SCSI initiator port" after
3289                  * response from the TM function is sent (SAM). Plus,
3290                  * we must wait here to be sure that we won't receive
3291                  * double commands with the same tag.
3292                  */
3293                 TRACE((mcmd->fn == SCST_ABORT_TASK) ? TRACE_MGMT_MINOR : TRACE_MGMT,
3294                         "cmd %p (tag %llu) being executed/xmitted (state %d), "
3295                         "deferring ABORT...", cmd, cmd->tag, cmd->state);
3296 #ifdef EXTRACHECKS
3297                 if (cmd->mgmt_cmnd) {
3298                         printk(KERN_ALERT "cmd %p (tag %llu, state %d) "
3299                                 "has non-NULL mgmt_cmnd %p!!! Current "
3300                                 "mcmd %p\n", cmd, cmd->tag, cmd->state,
3301                                 cmd->mgmt_cmnd, mcmd);
3302                 }
3303 #endif
3304                 sBUG_ON(cmd->mgmt_cmnd);
3305                 spin_lock_irqsave(&scst_mcmd_lock, flags);
3306                 mcmd->cmd_wait_count++;
3307                 spin_unlock_irqrestore(&scst_mcmd_lock, flags);
3308                 /* cmd can't die here or sess_list_lock already taken */
3309                 cmd->mgmt_cmnd = mcmd;
3310         }
3311
3312         tm_dbg_release_cmd(cmd);
3313
3314         TRACE_EXIT();
3315         return;
3316 }
3317
3318 /* No locks */
3319 static int scst_set_mcmd_next_state(struct scst_mgmt_cmd *mcmd)
3320 {
3321         int res;
3322         spin_lock_irq(&scst_mcmd_lock);
3323         if (mcmd->cmd_wait_count != 0) {
3324                 TRACE_MGMT_DBG("cmd_wait_count(%d) not 0, preparing to "
3325                         "wait", mcmd->cmd_wait_count);
3326                 mcmd->state = SCST_MGMT_CMD_STATE_EXECUTING;
3327                 res = -1;
3328         } else {
3329                 mcmd->state = SCST_MGMT_CMD_STATE_DONE;
3330                 res = 0;
3331         }
3332         mcmd->completed = 1;
3333         spin_unlock_irq(&scst_mcmd_lock);
3334         return res;
3335 }
3336
3337 static int __scst_check_unblock_aborted_cmd(struct scst_cmd *cmd)
3338 {
3339         int res;
3340         if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
3341                 TRACE_MGMT_DBG("Adding aborted blocked cmd %p to active cmd "
3342                         "list", cmd);
3343                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
3344                 list_add_tail(&cmd->cmd_list_entry,
3345                         &cmd->cmd_lists->active_cmd_list);
3346                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3347                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
3348                 res = 1;
3349         } else
3350                 res = 0;
3351         return res;
3352 }
3353
3354 static void scst_unblock_aborted_cmds(int scst_mutex_held)
3355 {
3356         struct scst_device *dev;
3357
3358         TRACE_ENTRY();
3359
3360         if (!scst_mutex_held)
3361                 mutex_lock(&scst_mutex);
3362
3363         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
3364                 struct scst_cmd *cmd, *tcmd;
3365                 struct scst_tgt_dev *tgt_dev;
3366                 spin_lock_bh(&dev->dev_lock);
3367                 local_irq_disable();
3368                 list_for_each_entry_safe(cmd, tcmd, &dev->blocked_cmd_list,
3369                                         blocked_cmd_list_entry) {
3370                         if (__scst_check_unblock_aborted_cmd(cmd))
3371                                 list_del(&cmd->blocked_cmd_list_entry);
3372                 }
3373                 local_irq_enable();
3374                 spin_unlock_bh(&dev->dev_lock);
3375
3376                 local_irq_disable();
3377                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
3378                                          dev_tgt_dev_list_entry) {
3379                         spin_lock(&tgt_dev->sn_lock);
3380                         list_for_each_entry_safe(cmd, tcmd,
3381                                         &tgt_dev->deferred_cmd_list,
3382                                         sn_cmd_list_entry) {
3383                                 if (__scst_check_unblock_aborted_cmd(cmd)) {
3384                                         TRACE_MGMT_DBG("Deleting aborted SN "
3385                                                 "cmd %p from SN list", cmd);
3386                                         tgt_dev->def_cmd_count--;
3387                                         list_del(&cmd->sn_cmd_list_entry);
3388                                 }
3389                         }
3390                         spin_unlock(&tgt_dev->sn_lock);
3391                 }
3392                 local_irq_enable();
3393         }
3394
3395         if (!scst_mutex_held)
3396                 mutex_unlock(&scst_mutex);
3397
3398         TRACE_EXIT();
3399         return;
3400 }
3401
3402 static void __scst_abort_task_set(struct scst_mgmt_cmd *mcmd,
3403         struct scst_tgt_dev *tgt_dev, int other_ini, int scst_mutex_held)
3404 {
3405         struct scst_cmd *cmd;
3406         struct scst_session *sess = tgt_dev->sess;
3407
3408         TRACE_ENTRY();
3409
3410         spin_lock_irq(&sess->sess_list_lock);
3411
3412         TRACE_DBG("Searching in search cmd list (sess=%p)", sess);
3413         list_for_each_entry(cmd, &sess->search_cmd_list, 
3414                         search_cmd_list_entry) {
3415                 if ((cmd->tgt_dev == tgt_dev) ||
3416                     ((cmd->tgt_dev == NULL) && 
3417                      (cmd->lun == tgt_dev->lun))) {
3418                         if (mcmd->cmd_sn_set) {
3419                                 sBUG_ON(!cmd->tgt_sn_set);
3420                                 if (scst_sn_before(mcmd->cmd_sn, cmd->tgt_sn) ||
3421                                     (mcmd->cmd_sn == cmd->tgt_sn))
3422                                         continue;
3423                         }
3424                         scst_abort_cmd(cmd, mcmd, other_ini, 0);
3425                 }
3426         }
3427         spin_unlock_irq(&sess->sess_list_lock);
3428
3429         scst_unblock_aborted_cmds(scst_mutex_held);
3430
3431         TRACE_EXIT();
3432         return;
3433 }
3434
3435 /* Returns 0 if the command processing should be continued, <0 otherwise */
3436 static int scst_abort_task_set(struct scst_mgmt_cmd *mcmd)
3437 {
3438         int res;
3439         struct scst_tgt_dev *tgt_dev = mcmd->mcmd_tgt_dev;
3440         struct scst_device *dev = tgt_dev->dev;
3441
3442         TRACE(TRACE_MGMT, "Aborting task set (lun=%Ld, mcmd=%p)",
3443                 tgt_dev->lun, mcmd);
3444
3445         mcmd->needs_unblocking = 1;
3446
3447         spin_lock_bh(&dev->dev_lock);
3448         __scst_block_dev(dev);
3449         spin_unlock_bh(&dev->dev_lock);
3450
3451         __scst_abort_task_set(mcmd, tgt_dev, 0, 0);
3452         scst_call_dev_task_mgmt_fn(mcmd, tgt_dev, 0);
3453
3454         res = scst_set_mcmd_next_state(mcmd);
3455
3456         TRACE_EXIT_RES(res);
3457         return res;
3458 }
3459
3460 static int scst_is_cmd_belongs_to_dev(struct scst_cmd *cmd,
3461         struct scst_device *dev)
3462 {
3463         struct scst_tgt_dev *tgt_dev = NULL;
3464         struct list_head *sess_tgt_dev_list_head;
3465         int res = 0;
3466
3467         TRACE_ENTRY();
3468
3469         TRACE_DBG("Finding match for dev %p and cmd %p (lun %Ld)", dev, cmd,
3470               (uint64_t)cmd->lun);
3471
3472         sess_tgt_dev_list_head =
3473                 &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
3474         list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3475                         sess_tgt_dev_list_entry) {
3476                 if (tgt_dev->lun == cmd->lun) {
3477                         TRACE_DBG("dev %p found", tgt_dev->dev);
3478                         res = (tgt_dev->dev == dev);
3479                         goto out;
3480                 }
3481         }
3482
3483 out:
3484         TRACE_EXIT_HRES(res);
3485         return res;
3486 }
3487
3488 /* Returns 0 if the command processing should be continued, <0 otherwise */
3489 static int scst_clear_task_set(struct scst_mgmt_cmd *mcmd)
3490 {
3491         int res;
3492         struct scst_device *dev = mcmd->mcmd_tgt_dev->dev;
3493         struct scst_tgt_dev *tgt_dev;
3494         LIST_HEAD(UA_tgt_devs);
3495
3496         TRACE_ENTRY();
3497
3498         TRACE(TRACE_MGMT, "Clearing task set (lun=%Ld, mcmd=%p)",
3499                 (uint64_t)mcmd->lun, mcmd);
3500
3501         mcmd->needs_unblocking = 1;
3502
3503         spin_lock_bh(&dev->dev_lock);
3504         __scst_block_dev(dev);
3505         spin_unlock_bh(&dev->dev_lock);
3506
3507         __scst_abort_task_set(mcmd, mcmd->mcmd_tgt_dev, 0, 0);
3508
3509         list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list, 
3510                 dev_tgt_dev_list_entry) 
3511         {
3512                 struct scst_session *sess = tgt_dev->sess;
3513                 struct scst_cmd *cmd;
3514                 int aborted = 0;
3515
3516                 if (tgt_dev == mcmd->mcmd_tgt_dev)
3517                         continue;
3518
3519                 spin_lock_irq(&sess->sess_list_lock);
3520
3521                 TRACE_DBG("Searching in search cmd list (sess=%p)", sess);
3522                 list_for_each_entry(cmd, &sess->search_cmd_list, 
3523                                 search_cmd_list_entry) {
3524                         if ((cmd->dev == dev) ||
3525                             ((cmd->dev == NULL) && 
3526                              scst_is_cmd_belongs_to_dev(cmd, dev))) {
3527                                 scst_abort_cmd(cmd, mcmd, 1, 0);
3528                                 aborted = 1;
3529                         }
3530                 }
3531                 spin_unlock_irq(&sess->sess_list_lock);
3532
3533                 if (aborted)
3534                         list_add_tail(&tgt_dev->extra_tgt_dev_list_entry,
3535                                         &UA_tgt_devs);
3536         }
3537
3538         scst_unblock_aborted_cmds(0);
3539
3540         if (!dev->tas) {
3541      &n