Various changes and fixes
[mirror/scst/.git] / scst / src / scst_targ.c
1 /*
2  *  scst_targ.c
3  *  
4  *  Copyright (C) 2004-2006 Vladislav Bolkhovitin <vst@vlnb.net>
5  *                 and Leonid Stoljar
6  *  
7  *  This program is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU General Public License
9  *  as published by the Free Software Foundation, version 2
10  *  of the License.
11  * 
12  *  This program is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  *  GNU General Public License for more details.
16  */
17
18 #include <linux/init.h>
19 #include <linux/kernel.h>
20 #include <linux/errno.h>
21 #include <linux/list.h>
22 #include <linux/spinlock.h>
23 #include <linux/slab.h>
24 #include <linux/sched.h>
25 #include <linux/smp_lock.h>
26 #include <asm/unistd.h>
27 #include <asm/string.h>
28
29 #include "scst_debug.h"
30 #include "scsi_tgt.h"
31 #include "scst_priv.h"
32
33 static int scst_do_job_init(void);
34 static int scst_process_init_cmd(struct scst_cmd *cmd);
35
36 static int __scst_process_active_cmd(struct scst_cmd *cmd, int context,
37         int left_locked);
38
39 static void scst_complete_cmd_mgmt(struct scst_cmd *cmd,
40         struct scst_mgmt_cmd *mcmd);
41
42 /* scst_list_lock assumed to be held */
43 static inline int scst_process_active_cmd(struct scst_cmd *cmd, int context,
44         unsigned long *pflags, int left_locked)
45 {
46         int res;
47
48         TRACE_ENTRY();
49
50         TRACE_DBG("Moving cmd %p to cmd list", cmd);
51         list_move_tail(&cmd->cmd_list_entry, &scst_cmd_list);
52
53         /* This is an inline func., so unneeded code will be optimized out */
54         if (pflags)
55                 spin_unlock_irqrestore(&scst_list_lock, *pflags);
56         else
57                 spin_unlock_irq(&scst_list_lock);
58
59         res = __scst_process_active_cmd(cmd, context, left_locked);
60
61         TRACE_EXIT_RES(res);
62         return res;
63 }
64
65 /* Called under scst_list_lock and IRQs disabled */
66 static inline void scst_cmd_set_sn(struct scst_cmd *cmd)
67 {
68         /* ToDo: cmd->queue_type */
69
70         /* scst_list_lock is enough to protect that */
71         cmd->sn = cmd->tgt_dev->next_sn;
72         cmd->tgt_dev->next_sn++;
73         cmd->no_sn = 0;
74
75         TRACE(TRACE_DEBUG/*TRACE_SCSI_SERIALIZING*/, "cmd(%p)->sn: %d",
76                 cmd, cmd->sn);
77 }
78
79 static inline void scst_schedule_tasklet(void)
80 {
81         struct tasklet_struct *t = &scst_tasklets[smp_processor_id()];
82
83 #if 0 /* Looks like #else is better for performance */
84         if ((!test_bit(TASKLET_STATE_SCHED, &t->state)) || (scst_num_cpus == 1))
85                 tasklet_schedule(t);
86         else {
87                 /* 
88                  * We suppose that other CPU(s) are rather idle, so we
89                  * ask one of them to help
90                  */
91                 TRACE_DBG("Tasklet on CPU %d busy, waking up the thread "
92                         "instead", smp_processor_id());
93                 wake_up(&scst_list_waitQ);
94         }
95 #else
96         tasklet_schedule(t);
97 #endif
98 }
99
100 /* 
101  * Must not been called in parallel with scst_unregister_session() for the 
102  * same sess
103  */
104 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
105                              const uint8_t *lun, int lun_len,
106                              const uint8_t *cdb, int cdb_len, int atomic)
107 {
108         struct scst_cmd *cmd;
109
110         TRACE_ENTRY();
111
112 #ifdef EXTRACHECKS
113         if (unlikely(sess->shutting_down)) {
114                 PRINT_ERROR_PR("%s", "New cmd while shutting down the session");
115                 sBUG();
116         }
117 #endif
118
119         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
120         if (cmd == NULL)
121                 goto out;
122
123         cmd->sess = sess;
124         cmd->tgt = sess->tgt;
125         cmd->tgtt = sess->tgt->tgtt;
126         cmd->state = SCST_CMD_STATE_INIT_WAIT;
127
128         /* 
129          * For both wrong lun and CDB defer the error reporting for
130          * scst_cmd_init_done()
131          */
132
133         cmd->lun = scst_unpack_lun(lun, lun_len);
134
135         if (cdb_len <= MAX_COMMAND_SIZE) {
136                 memcpy(cmd->cdb, cdb, cdb_len);
137                 cmd->cdb_len = cdb_len;
138         }
139
140         TRACE_DBG("cmd %p, sess %p", cmd, sess);
141         scst_sess_get(sess);
142
143 out:
144         TRACE_EXIT();
145         return cmd;
146 }
147
148 static void scst_setup_to_active(struct scst_cmd *cmd)
149 {
150         cmd->state = SCST_CMD_STATE_XMIT_RESP;
151         TRACE_DBG("Adding cmd %p to active cmd list", cmd);
152         list_add_tail(&cmd->cmd_list_entry, &scst_active_cmd_list);
153 }
154
155 void scst_cmd_init_done(struct scst_cmd *cmd, int pref_context)
156 {
157         int res = 0;
158         unsigned long flags = 0;
159         struct scst_session *sess = cmd->sess;
160
161         TRACE_ENTRY();
162
163         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
164         TRACE(TRACE_SCSI, "tag=%d, lun=%Ld, CDB len=%d", cmd->tag, 
165                 (uint64_t)cmd->lun, cmd->cdb_len);
166         TRACE_BUFF_FLAG(TRACE_SCSI|TRACE_RECV_BOT, "Recieving CDB",
167                 cmd->cdb, cmd->cdb_len);
168
169 #ifdef EXTRACHECKS
170         if (unlikely(in_irq()) && ((pref_context == SCST_CONTEXT_DIRECT) ||
171                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
172         {
173                 PRINT_ERROR_PR("Wrong context %d in IRQ from target %s, use "
174                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
175                         cmd->tgtt->name);
176                 pref_context = SCST_CONTEXT_TASKLET;
177         }
178 #endif
179
180         spin_lock_irqsave(&scst_list_lock, flags);
181
182         /* Let's make it here, this will save us a lock or atomic */
183         sess->sess_cmd_count++;
184
185         list_add_tail(&cmd->search_cmd_list_entry, &sess->search_cmd_list);
186
187         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
188                 switch(sess->init_phase) {
189                 case SCST_SESS_IPH_SUCCESS:
190                         break;
191                 case SCST_SESS_IPH_INITING:
192                         TRACE_DBG("Adding cmd %p to init deferred cmd list", cmd);
193                         list_add_tail(&cmd->cmd_list_entry, 
194                                 &sess->init_deferred_cmd_list);
195                         goto out_unlock_flags;
196                 case SCST_SESS_IPH_FAILED:
197                         scst_set_busy(cmd);
198                         scst_setup_to_active(cmd);
199                         goto active;
200                 default:
201                         sBUG();
202                 }
203         }
204
205 #ifdef EXTRACHECKS
206         if (unlikely(cmd->lun == (lun_t)-1)) {
207                 PRINT_ERROR_PR("Wrong LUN %d, finishing cmd", -1);
208                 scst_set_cmd_error(cmd,
209                         SCST_LOAD_SENSE(scst_sense_lun_not_supported));
210                 scst_setup_to_active(cmd);
211                 goto active;
212         }
213
214         if (unlikely(cmd->cdb_len == 0)) {
215                 PRINT_ERROR_PR("Wrong CDB len %d, finishing cmd", 0);
216                 scst_set_cmd_error(cmd,
217                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
218                 scst_setup_to_active(cmd);
219                 goto active;
220         }
221 #endif
222
223         TRACE_DBG("Adding cmd %p to init cmd list", cmd);
224         list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
225
226         cmd->state = SCST_CMD_STATE_INIT;
227
228         switch (pref_context) {
229         case SCST_CONTEXT_TASKLET:
230                 scst_schedule_tasklet();
231                 goto out_unlock_flags;
232
233         case SCST_CONTEXT_DIRECT:
234         case SCST_CONTEXT_DIRECT_ATOMIC:
235                 if (cmd->no_sn)
236                         res = scst_process_init_cmd(cmd);
237                 else
238                         res = scst_do_job_init();
239                 if (unlikely(res > 0))
240                         goto out_unlock_flags;
241                 break;
242
243         case SCST_CONTEXT_THREAD:
244                 goto out_thread_unlock_flags;
245
246         default:
247                 PRINT_ERROR_PR("Context %x is undefined, using the thread one",
248                         pref_context);
249                 goto out_thread_unlock_flags;
250         }
251
252 active:
253         /* Here cmd must be in active cmd list */
254         switch (pref_context) {
255         case SCST_CONTEXT_TASKLET:
256                 scst_schedule_tasklet();
257                 goto out_unlock_flags;
258
259         case SCST_CONTEXT_DIRECT:
260         case SCST_CONTEXT_DIRECT_ATOMIC:
261                 scst_process_active_cmd(cmd, pref_context, &flags, 0);
262                 /* For *NEED_THREAD wake_up() is already done */
263                 break;
264
265         case SCST_CONTEXT_THREAD:
266                 goto out_thread_unlock_flags;
267
268         default:
269                 PRINT_ERROR_PR("Context %x is undefined, using the thread one",
270                         pref_context);
271                 goto out_thread_unlock_flags;
272         }
273
274 out:
275         TRACE_EXIT();
276         return;
277
278 out_unlock_flags:
279         spin_unlock_irqrestore(&scst_list_lock, flags);
280         goto out;
281
282 out_thread_unlock_flags:
283         cmd->non_atomic_only = 1;
284         spin_unlock_irqrestore(&scst_list_lock, flags);
285         wake_up(&scst_list_waitQ);
286         goto out;
287 }
288
289 static int scst_parse_cmd(struct scst_cmd *cmd)
290 {
291         int res = SCST_CMD_STATE_RES_CONT_SAME;
292         int state;
293         struct scst_tgt_dev *tgt_dev_saved = cmd->tgt_dev;
294         struct scst_device *dev = cmd->dev;
295         struct scst_info_cdb cdb_info;
296         int atomic = scst_cmd_atomic(cmd);
297         int orig_bufflen = cmd->bufflen;
298         int set_dir = 1;
299
300         TRACE_ENTRY();
301
302         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
303                 TRACE_DBG("ABORTED set, returning ABORTED "
304                         "for cmd %p", cmd);
305                 goto out_xmit;
306         }
307
308         if (atomic && !dev->handler->parse_atomic) {
309                 TRACE_DBG("Dev handler %s parse() can not be "
310                       "called in atomic context, rescheduling to the thread",
311                       dev->handler->name);
312                 res = SCST_CMD_STATE_RES_NEED_THREAD;
313                 goto out;
314         }
315
316         /*
317          * Expected transfer data supplied by the SCSI transport via the
318          * target driver are untrusted, so we prefer to fetch them from CDB.
319          * Additionally, not all transports support supplying the expected
320          * transfer data.
321          */
322
323         if (unlikely(scst_get_cdb_info(cmd->cdb, dev->handler->type, 
324                         &cdb_info) != 0)) 
325         {
326                 static int t;
327                 if (t < 10) {
328                         t++;
329                         PRINT_INFO_PR("Unknown opcode 0x%02x for %s. "
330                                 "Should you update scst_scsi_op_table?",
331                                 cmd->cdb[0], dev->handler->name);
332                 }
333                 if (scst_cmd_is_expected_set(cmd)) {
334                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
335                                 "direction %d, transfer_len %d",
336                                 cmd->expected_data_direction,
337                                 cmd->expected_transfer_len);
338                         cmd->data_direction = cmd->expected_data_direction;
339                         cmd->bufflen = cmd->expected_transfer_len;
340                         /* Restore (most probably) lost CDB length */
341                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
342                         if (cmd->cdb_len == -1) {
343                                 PRINT_ERROR_PR("Unable to get CDB length for "
344                                         "opcode 0x%02x. Returning INVALID "
345                                         "OPCODE", cmd->cdb[0]);
346                                 scst_set_cmd_error(cmd,
347                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
348                                 goto out_xmit;
349                         }
350                 }
351                 else {
352                         PRINT_ERROR_PR("Unknown opcode 0x%02x for %s and "
353                              "target %s not supplied expected values. "
354                              "Returning INVALID OPCODE.", cmd->cdb[0], 
355                              dev->handler->name, cmd->tgtt->name);
356                         scst_set_cmd_error(cmd,
357                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
358                         goto out_xmit;
359                 }
360         } else {
361                 TRACE(TRACE_SCSI, "op_name <%s>, direction=%d (expected %d, "
362                         "set %s), transfer_len=%d (expected len %d), flags=%d",
363                         cdb_info.op_name, cdb_info.direction,
364                         cmd->expected_data_direction,
365                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
366                         cdb_info.transfer_len, cmd->expected_transfer_len,
367                         cdb_info.flags);
368
369                 /* Restore (most probably) lost CDB length */
370                 cmd->cdb_len = cdb_info.cdb_len;
371
372                 cmd->data_direction = cdb_info.direction;
373                 if (!(cdb_info.flags & SCST_UNKNOWN_LENGTH))
374                         cmd->bufflen = cdb_info.transfer_len;
375                 /* else cmd->bufflen remained as it was inited in 0 */
376         }
377
378         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
379                 PRINT_ERROR_PR("NACA bit in control byte CDB is not supported "
380                             "(opcode 0x%02x)", cmd->cdb[0]);
381                 scst_set_cmd_error(cmd,
382                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
383                 goto out_xmit;
384         }
385
386         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
387                 PRINT_ERROR_PR("Linked commands are not supported "
388                             "(opcode 0x%02x)", cmd->cdb[0]);
389                 scst_set_cmd_error(cmd,
390                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
391                 goto out_xmit;
392         }
393
394         if (likely(!scst_is_cmd_local(cmd))) {
395                 TRACE_DBG("Calling dev handler %s parse(%p)",
396                       dev->handler->name, cmd);
397                 TRACE_BUFF_FLAG(TRACE_SEND_BOT, "Parsing: ", cmd->cdb, cmd->cdb_len);
398                 state = dev->handler->parse(cmd, &cdb_info);
399                 TRACE_DBG("Dev handler %s parse() returned %d",
400                         dev->handler->name, state);
401
402                 if (state == SCST_CMD_STATE_DEFAULT)
403                         state = SCST_CMD_STATE_PREPARE_SPACE;
404         }
405         else
406                 state = SCST_CMD_STATE_PREPARE_SPACE;
407
408         if (scst_cmd_is_expected_set(cmd)) {
409                 if (cmd->expected_transfer_len < cmd->bufflen) {
410                         TRACE(TRACE_SCSI, "cmd->expected_transfer_len(%d) < "
411                                 "cmd->bufflen(%d), using expected_transfer_len "
412                                 "instead", cmd->expected_transfer_len,
413                                 cmd->bufflen);
414                         cmd->bufflen = cmd->expected_transfer_len;
415                 }
416         }
417
418         if (cmd->data_len == -1)
419                 cmd->data_len = cmd->bufflen;
420
421         if (cmd->data_buf_alloced && (orig_bufflen < cmd->bufflen)) {
422                 PRINT_ERROR_PR("Target driver supplied data buffer (size %d), "
423                         "is less, than required (size %d)", orig_bufflen,
424                         cmd->bufflen);
425                 goto out_error;
426         }
427
428 #ifdef EXTRACHECKS
429         if (state != SCST_CMD_STATE_NEED_THREAD_CTX) {
430                 if (((cmd->data_direction == SCST_DATA_UNKNOWN) &&
431                         (state != SCST_CMD_STATE_DEV_PARSE)) ||
432                     ((cmd->bufflen != 0) && 
433                         (cmd->data_direction == SCST_DATA_NONE)) ||
434                     ((cmd->bufflen == 0) && 
435                         (cmd->data_direction != SCST_DATA_NONE)) ||
436                     ((cmd->bufflen != 0) && (cmd->sg == NULL) &&
437                         (state > SCST_CMD_STATE_PREPARE_SPACE))) 
438                 {
439                         PRINT_ERROR_PR("Dev handler %s parse() returned "
440                                        "invalid cmd data_direction %d, "
441                                        "bufflen %zd or state %d (opcode 0x%x)",
442                                        dev->handler->name, 
443                                        cmd->data_direction, cmd->bufflen,
444                                        state, cmd->cdb[0]);
445                         goto out_error;
446                 }
447         }
448 #endif
449
450         switch (state) {
451         case SCST_CMD_STATE_PREPARE_SPACE:
452         case SCST_CMD_STATE_DEV_PARSE:
453         case SCST_CMD_STATE_RDY_TO_XFER:
454         case SCST_CMD_STATE_SEND_TO_MIDLEV:
455         case SCST_CMD_STATE_DEV_DONE:
456         case SCST_CMD_STATE_XMIT_RESP:
457         case SCST_CMD_STATE_FINISHED:
458                 cmd->state = state;
459                 res = SCST_CMD_STATE_RES_CONT_SAME;
460                 break;
461
462
463         case SCST_CMD_STATE_NEED_THREAD_CTX:
464                 TRACE_DBG("Dev handler %s parse() requested thread "
465                       "context, rescheduling", dev->handler->name);
466                 res = SCST_CMD_STATE_RES_NEED_THREAD;
467                 set_dir = 0;
468                 break;
469
470         case SCST_CMD_STATE_REINIT:
471                 cmd->tgt_dev_saved = tgt_dev_saved;
472                 cmd->state = state;
473                 res = SCST_CMD_STATE_RES_RESTART;
474                 set_dir = 0;
475                 break;
476
477         default:
478                 if (state >= 0) {
479                         PRINT_ERROR_PR("Dev handler %s parse() returned "
480                              "invalid cmd state %d (opcode %d)", 
481                              dev->handler->name, state, cmd->cdb[0]);
482                 } else {
483                         PRINT_ERROR_PR("Dev handler %s parse() returned "
484                                 "error %d (opcode %d)", dev->handler->name, 
485                                 state, cmd->cdb[0]);
486                 }
487                 goto out_error;
488         }
489
490         if ((cmd->resp_data_len == -1) && set_dir) {
491                 if (cmd->data_direction == SCST_DATA_READ)
492                         cmd->resp_data_len = cmd->bufflen;
493                 else
494                          cmd->resp_data_len = 0;
495         }
496         
497 out:
498         TRACE_EXIT_HRES(res);
499         return res;
500
501 out_error:
502         /* dev_done() will be called as part of the regular cmd's finish */
503         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
504         cmd->state = SCST_CMD_STATE_DEV_DONE;
505         res = SCST_CMD_STATE_RES_CONT_SAME;
506         goto out;
507
508 out_xmit:
509         cmd->state = SCST_CMD_STATE_XMIT_RESP;
510         res = SCST_CMD_STATE_RES_CONT_SAME;
511         goto out;
512 }
513
514 void scst_cmd_mem_work_fn(void *p)
515 {
516         TRACE_ENTRY();
517
518         spin_lock_bh(&scst_cmd_mem_lock);
519
520         scst_cur_max_cmd_mem += (scst_cur_max_cmd_mem >> 3);
521         if (scst_cur_max_cmd_mem < scst_max_cmd_mem) {
522                 TRACE_MGMT_DBG("%s", "Schedule cmd_mem_work");
523                 schedule_delayed_work(&scst_cmd_mem_work, SCST_CMD_MEM_TIMEOUT);
524         } else {
525                 scst_cur_max_cmd_mem = scst_max_cmd_mem;
526                 clear_bit(SCST_FLAG_CMD_MEM_WORK_SCHEDULED, &scst_flags);
527         }
528         TRACE_MGMT_DBG("New max cmd mem %ld Mb", scst_cur_max_cmd_mem >> 20);
529
530         spin_unlock_bh(&scst_cmd_mem_lock);
531
532         TRACE_EXIT();
533         return;
534 }
535
536 int scst_check_mem(struct scst_cmd *cmd)
537 {
538         int res = 0;
539
540         TRACE_ENTRY();
541
542         if (cmd->mem_checked)
543                 goto out;
544
545         spin_lock_bh(&scst_cmd_mem_lock);
546
547         scst_cur_cmd_mem += cmd->bufflen;
548         cmd->mem_checked = 1;
549         if (likely(scst_cur_cmd_mem <= scst_cur_max_cmd_mem))
550                 goto out_unlock;
551
552         TRACE(TRACE_OUT_OF_MEM, "Total memory allocated by commands (%ld Kb) "
553                 "is too big, returning QUEUE FULL to initiator \"%s\" (maximum "
554                 "allowed %ld Kb)", scst_cur_cmd_mem >> 10,
555                 (cmd->sess->initiator_name[0] == '\0') ?
556                   "Anonymous" : cmd->sess->initiator_name,
557                 scst_cur_max_cmd_mem >> 10);
558
559         scst_cur_cmd_mem -= cmd->bufflen;
560         cmd->mem_checked = 0;
561         scst_set_busy(cmd);
562         cmd->state = SCST_CMD_STATE_XMIT_RESP;
563         res = 1;
564
565 out_unlock:
566         spin_unlock_bh(&scst_cmd_mem_lock);
567
568 out:
569         TRACE_EXIT_RES(res);
570         return res;
571 }
572
573 static void scst_low_cur_max_cmd_mem(void)
574 {
575         TRACE_ENTRY();
576
577         if (test_bit(SCST_FLAG_CMD_MEM_WORK_SCHEDULED, &scst_flags)) {
578                 cancel_delayed_work(&scst_cmd_mem_work);
579                 flush_scheduled_work();
580                 clear_bit(SCST_FLAG_CMD_MEM_WORK_SCHEDULED, &scst_flags);
581         }
582
583         spin_lock_bh(&scst_cmd_mem_lock);
584
585         scst_cur_max_cmd_mem = (scst_cur_cmd_mem >> 1) + 
586                                 (scst_cur_cmd_mem >> 2);
587         if (scst_cur_max_cmd_mem < 16*1024*1024)
588                 scst_cur_max_cmd_mem = 16*1024*1024;
589
590         if (!test_bit(SCST_FLAG_CMD_MEM_WORK_SCHEDULED, &scst_flags)) {
591                 TRACE_MGMT_DBG("%s", "Schedule cmd_mem_work");
592                 schedule_delayed_work(&scst_cmd_mem_work, SCST_CMD_MEM_TIMEOUT);
593                 set_bit(SCST_FLAG_CMD_MEM_WORK_SCHEDULED, &scst_flags);
594         }
595
596         spin_unlock_bh(&scst_cmd_mem_lock);
597
598         TRACE_MGMT_DBG("New max cmd mem %ld Mb", scst_cur_max_cmd_mem >> 20);
599
600         TRACE_EXIT();
601         return;
602 }
603
604 static int scst_prepare_space(struct scst_cmd *cmd)
605 {
606         int r, res = SCST_CMD_STATE_RES_CONT_SAME;
607
608         TRACE_ENTRY();
609
610         if (cmd->data_direction == SCST_DATA_NONE)
611                 goto prep_done;
612
613         r = scst_check_mem(cmd);
614         if (unlikely(r != 0))
615                 goto out;
616
617         if (cmd->data_buf_tgt_alloc) {
618                 int orig_bufflen = cmd->bufflen;
619                 TRACE_MEM("%s", "Custom tgt data buf allocation requested");
620                 r = cmd->tgtt->alloc_data_buf(cmd);
621                 if (r > 0)
622                         r = scst_alloc_space(cmd);
623                 else if (r == 0) {
624                         cmd->data_buf_alloced = 1;
625                         if (cmd->data_buf_alloced && 
626                             unlikely(orig_bufflen < cmd->bufflen)) {
627                                 PRINT_ERROR_PR("Target driver allocated data "
628                                         "buffer (size %d), is less, than "
629                                         "required (size %d)", orig_bufflen,
630                                         cmd->bufflen);
631                                 scst_set_cmd_error(cmd,
632                                         SCST_LOAD_SENSE(scst_sense_hardw_error));
633                                 cmd->state = SCST_CMD_STATE_DEV_DONE;
634                                 res = SCST_CMD_STATE_RES_CONT_SAME;
635                                 goto out;
636                         }
637                 }
638         } else
639                 r = scst_alloc_space(cmd);
640
641         if (r != 0) {
642                 if (scst_cmd_atomic(cmd)) {
643                         TRACE_MEM("%s", "Atomic memory allocation failed, "
644                               "rescheduling to the thread");
645                         res = SCST_CMD_STATE_RES_NEED_THREAD;
646                         goto out;
647                 } else
648                         goto out_no_space;
649         }
650
651 prep_done:
652         if (cmd->preprocessing_only) {
653                 if (scst_cmd_atomic(cmd) && 
654                     !cmd->tgtt->preprocessing_done_atomic) {
655                         TRACE_DBG("%s", "preprocessing_done() can not be "
656                               "called in atomic context, rescheduling to "
657                               "the thread");
658                         res = SCST_CMD_STATE_RES_NEED_THREAD;
659                         goto out;
660                 }
661
662                 res = SCST_CMD_STATE_RES_CONT_NEXT;
663                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
664
665                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
666                 cmd->tgtt->preprocessing_done(cmd);
667                 TRACE_DBG("%s", "preprocessing_done() returned");
668                 goto out;
669
670         }
671
672         switch (cmd->data_direction) {
673         case SCST_DATA_WRITE:
674                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
675                 break;
676
677         default:
678                 cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
679                 break;
680         }
681
682 out:
683         TRACE_EXIT_HRES(res);
684         return res;
685
686 out_no_space:
687         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
688                 "(size %zd), sending BUSY or QUEUE FULL status", cmd->bufflen);
689         scst_low_cur_max_cmd_mem();
690         scst_set_busy(cmd);
691         cmd->state = SCST_CMD_STATE_DEV_DONE;
692         res = SCST_CMD_STATE_RES_CONT_SAME;
693         goto out;
694 }
695
696 void scst_restart_cmd(struct scst_cmd *cmd, int status, int pref_context)
697 {
698         TRACE_ENTRY();
699
700         TRACE_DBG("Preferred context: %d", pref_context);
701         TRACE_DBG("tag=%d, status=%#x", scst_cmd_get_tag(cmd), status);
702         cmd->non_atomic_only = 0;
703
704 #ifdef EXTRACHECKS
705         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
706                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
707         {
708                 PRINT_ERROR_PR("Wrong context %d in IRQ from target %s, use "
709                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
710                         cmd->tgtt->name);
711                 pref_context = SCST_CONTEXT_TASKLET;
712         }
713 #endif
714
715         switch (status) {
716         case SCST_PREPROCESS_STATUS_SUCCESS:
717                 switch (cmd->data_direction) {
718                 case SCST_DATA_WRITE:
719                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
720                         break;
721                 default:
722                         cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
723                         break;
724                 }
725                 if (cmd->no_sn) {
726                         unsigned long flags;
727                         int rc;
728                         spin_lock_irqsave(&scst_list_lock, flags);
729                         /* Necessary to keep the command's order */
730                         rc = scst_do_job_init();
731                         if (unlikely(rc > 0)) {
732                                 TRACE_DBG("Adding cmd %p to init cmd list",
733                                         cmd);
734                                 list_add_tail(&cmd->cmd_list_entry,
735                                         &scst_init_cmd_list);
736                                 spin_unlock_irqrestore(&scst_list_lock, flags);
737                                 goto out;
738                         }
739                         scst_cmd_set_sn(cmd);
740                         spin_unlock_irqrestore(&scst_list_lock, flags);
741                 }
742                 break;
743
744         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
745                 cmd->state = SCST_CMD_STATE_DEV_DONE;
746                 break;
747
748         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
749                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
750                 /* go through */
751         case SCST_PREPROCESS_STATUS_ERROR:
752                 scst_set_cmd_error(cmd,
753                            SCST_LOAD_SENSE(scst_sense_hardw_error));
754                 cmd->state = SCST_CMD_STATE_DEV_DONE;
755                 break;
756
757         default:
758                 PRINT_ERROR_PR("scst_rx_data() received unknown status %x",
759                         status);
760                 cmd->state = SCST_CMD_STATE_DEV_DONE;
761                 break;
762         }
763
764         scst_proccess_redirect_cmd(cmd, pref_context, 1);
765
766 out:
767         TRACE_EXIT();
768         return;
769 }
770
771 /* No locks */
772 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
773 {
774         struct scst_tgt *tgt = cmd->sess->tgt;
775         int res = 0;
776         unsigned long flags;
777
778         TRACE_ENTRY();
779
780         spin_lock_irqsave(&tgt->tgt_lock, flags);
781         tgt->retry_cmds++;
782         smp_mb();
783         TRACE(TRACE_RETRY, "TGT QUEUE FULL: incrementing retry_cmds %d",
784               tgt->retry_cmds);
785         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
786                 /* At least one cmd finished, so try again */
787                 tgt->retry_cmds--;
788                 TRACE(TRACE_RETRY, "TGT QUEUE FULL, direct retry "
789                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
790                       "retry_cmds=%d)", finished_cmds,
791                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
792                 res = -1;
793                 goto out_unlock_tgt;
794         }
795
796         TRACE(TRACE_RETRY, "Moving cmd %p to retry cmd list", cmd);
797         /* IRQ already off */
798         spin_lock(&scst_list_lock);
799         list_move_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
800         spin_unlock(&scst_list_lock);
801
802         if (!tgt->retry_timer_active) {
803                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
804                 add_timer(&tgt->retry_timer);
805                 tgt->retry_timer_active = 1;
806         }
807
808 out_unlock_tgt:
809         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
810
811         TRACE_EXIT_RES(res);
812         return res;
813 }
814
815 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
816 {
817         int res, rc;
818         int atomic = scst_cmd_atomic(cmd);
819
820         TRACE_ENTRY();
821
822         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
823         {
824                 TRACE_DBG("ABORTED set, returning ABORTED for "
825                         "cmd %p", cmd);
826                 goto out_dev_done;
827         }
828
829         if (cmd->tgtt->rdy_to_xfer == NULL) {
830                 cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
831                 res = SCST_CMD_STATE_RES_CONT_SAME;
832                 goto out;
833         }
834
835         if (atomic && !cmd->tgtt->rdy_to_xfer_atomic) {
836                 TRACE_DBG("%s", "rdy_to_xfer() can not be "
837                       "called in atomic context, rescheduling to the thread");
838                 res = SCST_CMD_STATE_RES_NEED_THREAD;
839                 goto out;
840         }
841
842         while (1) {
843                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
844
845                 res = SCST_CMD_STATE_RES_CONT_NEXT;
846                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
847
848                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
849 #ifdef DEBUG_RETRY
850                 if (((scst_random() % 100) == 75))
851                         rc = SCST_TGT_RES_QUEUE_FULL;
852                 else
853 #endif
854                         rc = cmd->tgtt->rdy_to_xfer(cmd);
855                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
856
857                 if (likely(rc == SCST_TGT_RES_SUCCESS))
858                         goto out;
859
860                 /* Restore the previous state */
861                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
862
863                 switch (rc) {
864                 case SCST_TGT_RES_QUEUE_FULL:
865                 {
866                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
867                                 break;
868                         else
869                                 continue;
870                 }
871
872                 case SCST_TGT_RES_NEED_THREAD_CTX:
873                 {
874                         TRACE_DBG("Target driver %s "
875                               "rdy_to_xfer() requested thread "
876                               "context, rescheduling", cmd->tgtt->name);
877                         res = SCST_CMD_STATE_RES_NEED_THREAD;
878                         break;
879                 }
880
881                 default:
882                         goto out_error_rc;
883                 }
884                 break;
885         }
886
887 out:
888         TRACE_EXIT_HRES(res);
889         return res;
890
891 out_error_rc:
892         if (rc == SCST_TGT_RES_FATAL_ERROR) {
893                 PRINT_ERROR_PR("Target driver %s rdy_to_xfer() returned "
894                      "fatal error", cmd->tgtt->name);
895         } else {
896                 PRINT_ERROR_PR("Target driver %s rdy_to_xfer() returned invalid "
897                             "value %d", cmd->tgtt->name, rc);
898         }
899         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
900
901 out_dev_done:
902         cmd->state = SCST_CMD_STATE_DEV_DONE;
903         res = SCST_CMD_STATE_RES_CONT_SAME;
904         goto out;
905 }
906
907 void scst_proccess_redirect_cmd(struct scst_cmd *cmd, int context,
908         int check_retries)
909 {
910         unsigned long flags;
911         int rc;
912
913         TRACE_ENTRY();
914
915         TRACE_DBG("Context: %d", context);
916
917         switch(context) {
918         case SCST_CONTEXT_DIRECT:
919         case SCST_CONTEXT_DIRECT_ATOMIC:
920                 if (check_retries)
921                         scst_check_retries(cmd->tgt, 0);
922                 cmd->non_atomic_only = 0;
923                 rc = __scst_process_active_cmd(cmd, context, 0);
924                 if (rc == SCST_CMD_STATE_RES_NEED_THREAD)
925                         goto out_thread;
926                 break;
927
928         default:
929                 PRINT_ERROR_PR("Context %x is unknown, using the thread one",
930                             context);
931                 /* go through */
932         case SCST_CONTEXT_THREAD:
933                 if (check_retries)
934                         scst_check_retries(cmd->tgt, 1);
935                 goto out_thread;
936
937         case SCST_CONTEXT_TASKLET:
938                 if (check_retries)
939                         scst_check_retries(cmd->tgt, 1);
940                 cmd->non_atomic_only = 0;
941                 spin_lock_irqsave(&scst_list_lock, flags);
942                 TRACE_DBG("Moving cmd %p to active cmd list", cmd);
943                 list_move_tail(&cmd->cmd_list_entry, &scst_active_cmd_list);
944                 spin_unlock_irqrestore(&scst_list_lock, flags);
945                 scst_schedule_tasklet();
946                 break;
947         }
948 out:
949         TRACE_EXIT();
950         return;
951
952 out_thread:
953         cmd->non_atomic_only = 1;
954         spin_lock_irqsave(&scst_list_lock, flags);
955         TRACE_DBG("Moving cmd %p to active cmd list", cmd);
956         list_move_tail(&cmd->cmd_list_entry, &scst_active_cmd_list);
957         spin_unlock_irqrestore(&scst_list_lock, flags);
958         wake_up(&scst_list_waitQ);
959         goto out;
960 }
961
962 void scst_rx_data(struct scst_cmd *cmd, int status, int pref_context)
963 {
964         TRACE_ENTRY();
965
966         TRACE_DBG("Preferred context: %d", pref_context);
967         TRACE(TRACE_SCSI, "tag=%d status=%#x", scst_cmd_get_tag(cmd), status);
968         cmd->non_atomic_only = 0;
969
970 #ifdef EXTRACHECKS
971         if (in_irq() && ((pref_context == SCST_CONTEXT_DIRECT) ||
972                          (pref_context == SCST_CONTEXT_DIRECT_ATOMIC)))
973         {
974                 PRINT_ERROR_PR("Wrong context %d in IRQ from target %s, use "
975                         "SCST_CONTEXT_TASKLET instead\n", pref_context,
976                         cmd->tgtt->name);
977                 pref_context = SCST_CONTEXT_TASKLET;
978         }
979 #endif
980
981         switch (status) {
982         case SCST_RX_STATUS_SUCCESS:
983                 cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
984                 break;
985
986         case SCST_RX_STATUS_ERROR_SENSE_SET:
987                 cmd->state = SCST_CMD_STATE_DEV_DONE;
988                 break;
989
990         case SCST_RX_STATUS_ERROR_FATAL:
991                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
992                 /* go through */
993         case SCST_RX_STATUS_ERROR:
994                 scst_set_cmd_error(cmd,
995                            SCST_LOAD_SENSE(scst_sense_hardw_error));
996                 cmd->state = SCST_CMD_STATE_DEV_DONE;
997                 break;
998
999         default:
1000                 PRINT_ERROR_PR("scst_rx_data() received unknown status %x",
1001                         status);
1002                 cmd->state = SCST_CMD_STATE_DEV_DONE;
1003                 break;
1004         }
1005
1006         scst_proccess_redirect_cmd(cmd, pref_context, 1);
1007
1008         TRACE_EXIT();
1009         return;
1010 }
1011
1012 /* No locks supposed to be held */
1013 static void scst_check_sense(struct scst_cmd *cmd, const uint8_t *rq_sense,
1014         int rq_sense_len, int *next_state)
1015 {
1016         int sense_valid;
1017         struct scst_device *dev = cmd->dev;
1018         int dbl_ua_possible, ua_sent = 0;
1019
1020         TRACE_ENTRY();
1021
1022         /* If we had a internal bus reset behind us, set the command error UA */
1023         if ((dev->scsi_dev != NULL) &&
1024             unlikely(cmd->host_status == DID_RESET) &&
1025             scst_is_ua_command(cmd))
1026         {
1027                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
1028                       dev->scsi_dev->was_reset, cmd->host_status);
1029                 scst_set_cmd_error(cmd,
1030                    SCST_LOAD_SENSE(scst_sense_reset_UA));
1031                 /* just in case */
1032                 cmd->ua_ignore = 0;
1033                 /* It looks like it is safe to clear was_reset here */
1034                 dev->scsi_dev->was_reset = 0;
1035                 smp_mb();
1036         }
1037
1038         if (rq_sense != NULL) {
1039                 sense_valid = SCST_SENSE_VALID(rq_sense);
1040                 if (sense_valid) {
1041                         /* 
1042                          * We checked that rq_sense_len < sizeof(cmd->sense_buffer)
1043                          * in init_scst()
1044                          */
1045                         memcpy(cmd->sense_buffer, rq_sense, rq_sense_len);
1046                         memset(&cmd->sense_buffer[rq_sense_len], 0,
1047                                 sizeof(cmd->sense_buffer) - rq_sense_len);
1048                 }
1049         } else
1050                 sense_valid = SCST_SENSE_VALID(cmd->sense_buffer);
1051
1052         dbl_ua_possible = dev->dev_double_ua_possible;
1053         TRACE_DBG("cmd %p dbl_ua_possible %d", cmd, dbl_ua_possible);
1054         if (unlikely(dbl_ua_possible)) {
1055                 spin_lock_bh(&dev->dev_lock);
1056                 barrier(); /* to reread dev_double_ua_possible */
1057                 dbl_ua_possible = dev->dev_double_ua_possible;
1058                 if (dbl_ua_possible)
1059                         ua_sent = dev->dev_reset_ua_sent;
1060                 else
1061                         spin_unlock_bh(&dev->dev_lock);
1062         }
1063
1064         if (sense_valid) {
1065                 TRACE_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense_buffer,
1066                              sizeof(cmd->sense_buffer));
1067                 /* Check Unit Attention Sense Key */
1068                 if (cmd->sense_buffer[2] == UNIT_ATTENTION) {
1069                         if (cmd->sense_buffer[12] == SCST_SENSE_ASC_UA_RESET) {
1070                                 if (dbl_ua_possible) 
1071                                 {
1072                                         if (ua_sent) {
1073                                                 TRACE(TRACE_MGMT, "%s", 
1074                                                         "Double UA detected");
1075                                                 /* Do retry */
1076                                                 TRACE(TRACE_MGMT, "Retrying cmd %p "
1077                                                         "(tag %d)", cmd, cmd->tag);
1078                                                 cmd->status = 0;
1079                                                 cmd->masked_status = 0;
1080                                                 cmd->msg_status = 0;
1081                                                 cmd->host_status = DID_OK;
1082                                                 cmd->driver_status = 0;
1083                                                 memset(cmd->sense_buffer, 0,
1084                                                         sizeof(cmd->sense_buffer));
1085                                                 cmd->retry = 1;
1086                                                 *next_state = SCST_CMD_STATE_SEND_TO_MIDLEV;
1087                                                 /* 
1088                                                  * Dev is still blocked by this cmd, so
1089                                                  * it's OK to clear SCST_DEV_SERIALIZED
1090                                                  * here.
1091                                                  */
1092                                                 dev->dev_double_ua_possible = 0;
1093                                                 dev->dev_serialized = 0;
1094                                                 dev->dev_reset_ua_sent = 0;
1095                                                 goto out_unlock;
1096                                         } else
1097                                                 dev->dev_reset_ua_sent = 1;
1098                                 }
1099                         }
1100                         if (cmd->ua_ignore == 0) {
1101                                 if (unlikely(dbl_ua_possible)) {
1102                                         __scst_process_UA(dev, cmd,
1103                                                 cmd->sense_buffer,
1104                                                 sizeof(cmd->sense_buffer), 0);
1105                                 } else {
1106                                         scst_process_UA(dev, cmd,
1107                                                 cmd->sense_buffer,
1108                                                 sizeof(cmd->sense_buffer), 0);
1109                                 }
1110                         }
1111                 }
1112         }
1113
1114         if (unlikely(dbl_ua_possible)) {
1115                 if (ua_sent && scst_is_ua_command(cmd)) {
1116                         TRACE_MGMT_DBG("%s", "Clearing dbl_ua_possible flag");
1117                         dev->dev_double_ua_possible = 0;
1118                         dev->dev_serialized = 0;
1119                         dev->dev_reset_ua_sent = 0;
1120                 }
1121                 spin_unlock_bh(&dev->dev_lock);
1122         }
1123
1124 out:
1125         TRACE_EXIT();
1126         return;
1127
1128 out_unlock:
1129         spin_unlock_bh(&dev->dev_lock);
1130         goto out;
1131 }
1132
1133 static int scst_check_auto_sense(struct scst_cmd *cmd)
1134 {
1135         int res = 0;
1136
1137         TRACE_ENTRY();
1138
1139         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
1140             (!SCST_SENSE_VALID(cmd->sense_buffer) ||
1141              SCST_NO_SENSE(cmd->sense_buffer)))
1142         {
1143                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
1144                       "cmd->status=%x, cmd->masked_status=%x, "
1145                       "cmd->msg_status=%x, cmd->host_status=%x, "
1146                       "cmd->driver_status=%x", cmd->status, cmd->masked_status, 
1147                       cmd->msg_status, cmd->host_status, cmd->driver_status);
1148                 res = 1;
1149         } else if (unlikely(cmd->host_status)) {
1150                 if ((cmd->host_status == DID_REQUEUE) ||
1151                     (cmd->host_status == DID_IMM_RETRY) ||
1152                     (cmd->host_status == DID_SOFT_ERROR)) {
1153                         scst_set_busy(cmd);
1154                 } else {
1155                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
1156                                 "received, returning HARDWARE ERROR instead",
1157                                 cmd->host_status);
1158                         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1159                 }
1160         }
1161
1162         TRACE_EXIT_RES(res);
1163         return res;
1164 }
1165
1166 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1167         const uint8_t *rq_sense, int rq_sense_len, int resid,
1168         int *next_state)
1169 {
1170         unsigned char type;
1171
1172         TRACE_ENTRY();
1173
1174         cmd->status = result & 0xff;
1175         cmd->masked_status = status_byte(result);
1176         cmd->msg_status = msg_byte(result);
1177         cmd->host_status = host_byte(result);
1178         cmd->driver_status = driver_byte(result);
1179         if (unlikely(resid != 0)) {
1180 #ifdef EXTRACHECKS
1181                 if ((resid < 0) || (resid >= cmd->resp_data_len)) {
1182                         PRINT_ERROR_PR("Wrong resid %d (cmd->resp_data_len=%d)",
1183                                 resid, cmd->resp_data_len);
1184                 } else
1185 #endif
1186                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1187         }
1188
1189         TRACE(TRACE_SCSI, "result=%x, cmd->status=%x, resid=%d, "
1190               "cmd->masked_status=%x, cmd->msg_status=%x, cmd->host_status=%x, "
1191               "cmd->driver_status=%x", result, cmd->status, resid,
1192               cmd->masked_status, cmd->msg_status, cmd->host_status,
1193               cmd->driver_status);
1194
1195         cmd->completed = 1;
1196
1197         scst_dec_on_dev_cmd(cmd);
1198
1199         type = cmd->dev->handler->type;
1200         if ((cmd->cdb[0] == MODE_SENSE || cmd->cdb[0] == MODE_SENSE_10) &&
1201             cmd->tgt_dev->acg_dev->rd_only_flag &&
1202             (type == TYPE_DISK || type == TYPE_WORM || type == TYPE_MOD ||
1203              type == TYPE_TAPE)) {
1204                 int32_t length;
1205                 uint8_t *address;
1206
1207                 length = scst_get_buf_first(cmd, &address);
1208                 TRACE_DBG("length %d", length);
1209                 if (unlikely(length <= 0)) {
1210                         PRINT_ERROR_PR("%s: scst_get_buf_first() failed",
1211                                 __func__);
1212                         goto next;
1213                 }
1214                 if (length > 2 && cmd->cdb[0] == MODE_SENSE) {
1215                         address[2] |= 0x80;   /* Write Protect*/
1216                 }
1217                 else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10) {
1218                         address[3] |= 0x80;   /* Write Protect*/
1219                 }
1220                 scst_put_buf(cmd, address);
1221         }
1222
1223 next:
1224         scst_check_sense(cmd, rq_sense, rq_sense_len, next_state);
1225
1226         TRACE_EXIT();
1227         return;
1228 }
1229
1230 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18)
1231 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1232                                             struct scsi_request **req)
1233 {
1234         struct scst_cmd *cmd = NULL;
1235
1236         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1237                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1238
1239         if (cmd == NULL) {
1240                 PRINT_ERROR_PR("%s", "Request with NULL cmd");
1241                 if (*req)
1242                         scsi_release_request(*req);
1243         }
1244
1245         return cmd;
1246 }
1247
1248 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1249 {
1250         struct scsi_request *req = NULL;
1251         struct scst_cmd *cmd;
1252         int next_state;
1253
1254         TRACE_ENTRY();
1255
1256         WARN_ON(in_irq());
1257
1258         cmd = scst_get_cmd(scsi_cmd, &req);
1259         if (cmd == NULL)
1260                 goto out;
1261
1262         next_state = SCST_CMD_STATE_DEV_DONE;
1263         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1264                 sizeof(req->sr_sense_buffer), scsi_cmd->resid, &next_state);
1265
1266         /* Clear out request structure */
1267         req->sr_use_sg = 0;
1268         req->sr_sglist_len = 0;
1269         req->sr_bufflen = 0;
1270         req->sr_buffer = NULL;
1271         req->sr_underflow = 0;
1272         req->sr_request->rq_disk = NULL; /* disown request blk */
1273
1274         cmd->bufflen = req->sr_bufflen; //??
1275
1276         scst_release_request(cmd);
1277
1278         cmd->state = next_state;
1279         cmd->non_atomic_only = 0;
1280
1281         scst_proccess_redirect_cmd(cmd, scst_get_context(), 0);
1282
1283 out:
1284         TRACE_EXIT();
1285         return;
1286 }
1287 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) */
1288 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1289 {
1290         struct scst_cmd *cmd;
1291         int next_state;
1292
1293         TRACE_ENTRY();
1294
1295         WARN_ON(in_irq());
1296
1297         cmd = (struct scst_cmd *)data;
1298         if (cmd == NULL)
1299                 goto out;
1300
1301         next_state = SCST_CMD_STATE_DEV_DONE;
1302         scst_do_cmd_done(cmd, result, sense, SCSI_SENSE_BUFFERSIZE, resid,
1303                 &next_state);
1304
1305         cmd->state = next_state;
1306         cmd->non_atomic_only = 0;
1307
1308         scst_proccess_redirect_cmd(cmd, scst_get_context(), 0);
1309
1310 out:
1311         TRACE_EXIT();
1312         return;
1313 }
1314 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) */
1315
1316 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state)
1317 {
1318         TRACE_ENTRY();
1319
1320         sBUG_ON(in_irq());
1321
1322         scst_dec_on_dev_cmd(cmd);
1323
1324         if (next_state == SCST_CMD_STATE_DEFAULT)
1325                 next_state = SCST_CMD_STATE_DEV_DONE;
1326
1327 #if defined(DEBUG) || defined(TRACING)
1328         if (next_state == SCST_CMD_STATE_DEV_DONE) {
1329                 if (cmd->sg) {
1330                         int i;
1331                         struct scatterlist *sg = cmd->sg;
1332                         TRACE(TRACE_RECV_TOP, 
1333                               "Exec'd %d S/G(s) at %p sg[0].page at %p",
1334                               cmd->sg_cnt, sg, (void*)sg[0].page);
1335                         for(i = 0; i < cmd->sg_cnt; ++i) {
1336                                 TRACE_BUFF_FLAG(TRACE_RECV_TOP, 
1337                                         "Exec'd sg", page_address(sg[i].page),
1338                                         sg[i].length);
1339                         }
1340                 }
1341         }
1342 #endif
1343
1344
1345 #ifdef EXTRACHECKS
1346         if ((next_state != SCST_CMD_STATE_DEV_DONE) &&
1347             (next_state != SCST_CMD_STATE_XMIT_RESP) &&
1348             (next_state != SCST_CMD_STATE_FINISHED)) 
1349         {
1350                 PRINT_ERROR_PR("scst_cmd_done_local() received invalid cmd "
1351                             "state %d (opcode %d)", next_state, cmd->cdb[0]);
1352                 scst_set_cmd_error(cmd,
1353                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1354                 next_state = SCST_CMD_STATE_DEV_DONE;
1355         }
1356
1357         if (scst_check_auto_sense(cmd)) {
1358                 PRINT_ERROR_PR("CHECK_CONDITION, but no valid sense for "
1359                         "opcode %d", cmd->cdb[0]);
1360         }
1361 #endif
1362
1363         scst_check_sense(cmd, NULL, 0, &next_state);
1364
1365         cmd->state = next_state;
1366         cmd->non_atomic_only = 0;
1367
1368         scst_proccess_redirect_cmd(cmd, scst_get_context(), 0);
1369
1370         TRACE_EXIT();
1371         return;
1372 }
1373
1374 static int scst_report_luns_local(struct scst_cmd *cmd)
1375 {
1376         int res = SCST_EXEC_COMPLETED;
1377         int dev_cnt = 0;
1378         int buffer_size;
1379         struct scst_tgt_dev *tgt_dev = NULL;
1380         uint8_t *buffer;
1381         int offs, overflow = 0;
1382
1383         TRACE_ENTRY();
1384
1385         cmd->status = 0;
1386         cmd->masked_status = 0;
1387         cmd->msg_status = 0;
1388         cmd->host_status = DID_OK;
1389         cmd->driver_status = 0;
1390
1391         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1392                 PRINT_ERROR_PR("Unsupported SELECT REPORT value %x in REPORT "
1393                         "LUNS command", cmd->cdb[2]);
1394                 goto out_err;
1395         }
1396
1397         buffer_size = scst_get_buf_first(cmd, &buffer);
1398         if (unlikely(buffer_size <= 0))
1399                 goto out_err;
1400
1401         if (buffer_size < 16)
1402                 goto out_put_err;
1403
1404         memset(buffer, 0, buffer_size);
1405         offs = 8;
1406
1407         /* sess->sess_tgt_dev_list is protected by suspended activity */
1408         list_for_each_entry(tgt_dev, &cmd->sess->sess_tgt_dev_list,
1409                             sess_tgt_dev_list_entry) 
1410         {
1411                 if (!overflow) {
1412                         if (offs >= buffer_size) {
1413                                 scst_put_buf(cmd, buffer);
1414                                 buffer_size = scst_get_buf_next(cmd, &buffer);
1415                                 if (buffer_size > 0) {
1416                                         memset(buffer, 0, buffer_size);
1417                                         offs = 0;
1418                                 } else {
1419                                         overflow = 1;
1420                                         goto inc_dev_cnt;
1421                                 }
1422                         }
1423                         if ((buffer_size - offs) < 8) {
1424                                 PRINT_ERROR_PR("Buffer allocated for REPORT "
1425                                         "LUNS command doesn't allow to fit 8 "
1426                                         "byte entry (buffer_size=%d)",
1427                                         buffer_size);
1428                                 goto out_put_hw_err;
1429                         }
1430                         buffer[offs] = (tgt_dev->acg_dev->lun >> 8) & 0xff;
1431                         buffer[offs+1] = tgt_dev->acg_dev->lun & 0xff;
1432                         offs += 8;
1433                 }
1434 inc_dev_cnt:
1435                 dev_cnt++;
1436         }
1437         if (!overflow)
1438                 scst_put_buf(cmd, buffer);
1439
1440         /* Set the response header */
1441         buffer_size = scst_get_buf_first(cmd, &buffer);
1442         if (unlikely(buffer_size <= 0))
1443                 goto out_err;
1444         dev_cnt *= 8;
1445         buffer[0] = (dev_cnt >> 24) & 0xff;
1446         buffer[1] = (dev_cnt >> 16) & 0xff;
1447         buffer[2] = (dev_cnt >> 8) & 0xff;
1448         buffer[3] = dev_cnt & 0xff;
1449         scst_put_buf(cmd, buffer);
1450
1451         dev_cnt += 8;
1452         if (dev_cnt < cmd->resp_data_len)
1453                 scst_set_resp_data_len(cmd, dev_cnt);
1454
1455 out_done:
1456         cmd->completed = 1;
1457
1458         /* Report the result */
1459         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1460
1461         TRACE_EXIT_RES(res);
1462         return res;
1463         
1464 out_put_err:
1465         scst_put_buf(cmd, buffer);
1466
1467 out_err:
1468         scst_set_cmd_error(cmd,
1469                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1470         goto out_done;
1471
1472 out_put_hw_err:
1473         scst_put_buf(cmd, buffer);
1474         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1475         goto out_done;
1476 }
1477
1478 static int scst_pre_select(struct scst_cmd *cmd)
1479 {
1480         int res = SCST_EXEC_NOT_COMPLETED;
1481
1482         TRACE_ENTRY();
1483
1484         if (scst_cmd_atomic(cmd)) {
1485                 res = SCST_EXEC_NEED_THREAD;
1486                 goto out;
1487         }
1488
1489         scst_block_dev(cmd->dev, 1);
1490         /* Device will be unblocked in scst_done_cmd_check() */
1491
1492         if (test_bit(SCST_TGT_DEV_UA_PENDING, &cmd->tgt_dev->tgt_dev_flags)) {
1493                 int rc = scst_set_pending_UA(cmd);
1494                 if (rc == 0) {
1495                         res = SCST_EXEC_COMPLETED;
1496                         cmd->completed = 1;
1497                         /* Report the result */
1498                         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1499                         goto out;
1500                 }
1501         }
1502
1503 out:
1504         TRACE_EXIT_RES(res);
1505         return res;
1506 }
1507
1508 static inline void scst_report_reserved(struct scst_cmd *cmd)
1509 {
1510         TRACE_ENTRY();
1511
1512         scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1513         cmd->completed = 1;
1514         /* Report the result */
1515         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1516
1517         TRACE_EXIT();
1518         return;
1519 }
1520
1521 static int scst_reserve_local(struct scst_cmd *cmd)
1522 {
1523         int res = SCST_EXEC_NOT_COMPLETED;
1524         struct scst_device *dev;
1525         struct scst_tgt_dev *tgt_dev_tmp;
1526
1527         TRACE_ENTRY();
1528
1529         if (scst_cmd_atomic(cmd)) {
1530                 res = SCST_EXEC_NEED_THREAD;
1531                 goto out;
1532         }
1533
1534         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1535                 PRINT_ERROR_PR("RESERVE_10: 3rdPty RESERVE not implemented "
1536                      "(lun=%Ld)", (uint64_t)cmd->lun);
1537                 scst_set_cmd_error(cmd,
1538                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1539                 cmd->completed = 1;
1540                 res = SCST_EXEC_COMPLETED;
1541                 goto out;
1542         }
1543
1544         dev = cmd->dev;
1545         scst_block_dev(dev, 1);
1546         /* Device will be unblocked in scst_done_cmd_check() */
1547
1548         spin_lock_bh(&dev->dev_lock);
1549
1550         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1551                 scst_report_reserved(cmd);
1552                 /* !! At this point cmd, sess & tgt_dev can be already freed !! */
1553                 res = SCST_EXEC_COMPLETED;
1554                 goto out_unlock;
1555         }
1556
1557         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1558                             dev_tgt_dev_list_entry) 
1559         {
1560                 if (cmd->tgt_dev != tgt_dev_tmp)
1561                         set_bit(SCST_TGT_DEV_RESERVED, 
1562                                 &tgt_dev_tmp->tgt_dev_flags);
1563         }
1564         dev->dev_reserved = 1;
1565
1566 out_unlock:
1567         spin_unlock_bh(&dev->dev_lock);
1568         
1569 out:
1570         TRACE_EXIT_RES(res);
1571         return res;
1572 }
1573
1574 static int scst_release_local(struct scst_cmd *cmd)
1575 {
1576         int res = SCST_EXEC_NOT_COMPLETED;
1577         struct scst_tgt_dev *tgt_dev_tmp;
1578         struct scst_device *dev;
1579
1580         TRACE_ENTRY();
1581
1582         dev = cmd->dev;
1583
1584         scst_block_dev(dev, 1);
1585         cmd->blocking = 1;
1586         TRACE_MGMT_DBG("Blocking cmd %p (tag %d)", cmd, cmd->tag);
1587
1588         spin_lock_bh(&dev->dev_lock);
1589
1590         /* 
1591          * The device could be RELEASED behind us, if RESERVING session 
1592          * is closed (see scst_free_tgt_dev()), but this actually doesn't 
1593          * matter, so use lock and no retest for DEV_RESERVED bits again
1594          */
1595         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1596                 res = SCST_EXEC_COMPLETED;
1597                 cmd->status = 0;
1598                 cmd->masked_status = 0;
1599                 cmd->msg_status = 0;
1600                 cmd->host_status = DID_OK;
1601                 cmd->driver_status = 0;
1602         } else {
1603                 list_for_each_entry(tgt_dev_tmp,
1604                                     &dev->dev_tgt_dev_list,
1605                                     dev_tgt_dev_list_entry) 
1606                 {
1607                         clear_bit(SCST_TGT_DEV_RESERVED, 
1608                                 &tgt_dev_tmp->tgt_dev_flags);
1609                 }
1610                 dev->dev_reserved = 0;
1611         }
1612
1613         spin_unlock_bh(&dev->dev_lock);
1614
1615         if (res == SCST_EXEC_COMPLETED) {
1616                 cmd->completed = 1;
1617                 /* Report the result */
1618                 scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1619         }
1620
1621         TRACE_EXIT_RES(res);
1622         return res;
1623 }
1624
1625 /* 
1626  * The result of cmd execution, if any, should be reported 
1627  * via scst_cmd_done_local() 
1628  */
1629 static int scst_pre_exec(struct scst_cmd *cmd)
1630 {
1631         int res = SCST_EXEC_NOT_COMPLETED, rc;
1632         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1633
1634         TRACE_ENTRY();
1635
1636         /* Reserve check before Unit Attention */
1637         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED, &tgt_dev->tgt_dev_flags))) {
1638                 if ((cmd->cdb[0] != INQUIRY) && (cmd->cdb[0] != REPORT_LUNS) &&
1639                     (cmd->cdb[0] != RELEASE) && (cmd->cdb[0] != RELEASE_10) &&
1640                     (cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER) &&
1641                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL || (cmd->cdb[4] & 3)) &&
1642                     (cmd->cdb[0] != LOG_SENSE) && (cmd->cdb[0] != REQUEST_SENSE))
1643                 {
1644                         scst_report_reserved(cmd);
1645                         res = SCST_EXEC_COMPLETED;
1646                         goto out;
1647                 }
1648         }
1649
1650         /* If we had a internal bus reset, set the command error unit attention */
1651         if ((cmd->dev->scsi_dev != NULL) &&
1652             unlikely(cmd->dev->scsi_dev->was_reset)) {
1653                 if (scst_is_ua_command(cmd)) 
1654                 {
1655                         struct scst_device *dev = cmd->dev;
1656                         int done = 0;
1657                         /* Prevent more than 1 cmd to be triggered by was_reset */
1658                         spin_lock_bh(&dev->dev_lock);
1659                         barrier(); /* to reread was_reset */
1660                         if (dev->scsi_dev->was_reset) {
1661                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1662                                 scst_set_cmd_error(cmd,
1663                                            SCST_LOAD_SENSE(scst_sense_reset_UA));
1664                                 /* It looks like it is safe to clear was_reset here */
1665                                 dev->scsi_dev->was_reset = 0;
1666                                 smp_mb();
1667                                 done = 1;
1668                         }
1669                         spin_unlock_bh(&dev->dev_lock);
1670
1671                         if (done)
1672                                 goto out_done;
1673                 }
1674         }
1675
1676         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING, 
1677                         &cmd->tgt_dev->tgt_dev_flags))) {
1678                 if (scst_is_ua_command(cmd)) 
1679                 {
1680                         rc = scst_set_pending_UA(cmd);
1681                         if (rc == 0)
1682                                 goto out_done;
1683                 }
1684         }
1685
1686         /* Check READ_ONLY device status */
1687         if (tgt_dev->acg_dev->rd_only_flag &&
1688             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
1689              cmd->cdb[0] == WRITE_10 ||
1690              cmd->cdb[0] == WRITE_12 ||
1691              cmd->cdb[0] == WRITE_16 ||
1692              cmd->cdb[0] == WRITE_VERIFY ||
1693              cmd->cdb[0] == WRITE_VERIFY_12 ||
1694              cmd->cdb[0] == WRITE_VERIFY_16 ||
1695              (cmd->dev->handler->type == TYPE_TAPE &&
1696               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS))))
1697         {
1698                 scst_set_cmd_error(cmd,
1699                            SCST_LOAD_SENSE(scst_sense_data_protect));
1700                 goto out_done;
1701         }
1702 out:
1703         TRACE_EXIT_RES(res);
1704         return res;
1705
1706 out_done:
1707         res = SCST_EXEC_COMPLETED;
1708         cmd->completed = 1;
1709         /* Report the result */
1710         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1711         goto out;
1712 }
1713
1714 /* 
1715  * The result of cmd execution, if any, should be reported 
1716  * via scst_cmd_done_local() 
1717  */
1718 static inline int scst_local_exec(struct scst_cmd *cmd)
1719 {
1720         int res = SCST_EXEC_NOT_COMPLETED;
1721
1722         TRACE_ENTRY();
1723
1724         /*
1725          * Adding new commands here don't forget to update
1726          * scst_is_cmd_local() in scsi_tgt.h, if necessary
1727          */
1728
1729         switch (cmd->cdb[0]) {
1730         case MODE_SELECT:
1731         case MODE_SELECT_10:
1732         case LOG_SELECT:
1733                 res = scst_pre_select(cmd);
1734                 break;
1735         case RESERVE:
1736         case RESERVE_10:
1737                 res = scst_reserve_local(cmd);
1738                 break;
1739         case RELEASE:
1740         case RELEASE_10:
1741                 res = scst_release_local(cmd);
1742                 break;
1743         case REPORT_LUNS:
1744                 res = scst_report_luns_local(cmd);
1745                 break;
1746         }
1747
1748         TRACE_EXIT_RES(res);
1749         return res;
1750 }
1751
1752 static int scst_do_send_to_midlev(struct scst_cmd *cmd)
1753 {
1754         int rc = SCST_EXEC_NOT_COMPLETED;
1755
1756         TRACE_ENTRY();
1757
1758         cmd->sent_to_midlev = 1;
1759         cmd->state = SCST_CMD_STATE_EXECUTING;
1760         cmd->scst_cmd_done = scst_cmd_done_local;
1761
1762         set_bit(SCST_CMD_EXECUTING, &cmd->cmd_flags);
1763         smp_mb__after_set_bit();
1764
1765         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1766                 TRACE_DBG("ABORTED set, aborting cmd %p", cmd);
1767                 goto out_aborted;
1768         }
1769
1770         rc = scst_pre_exec(cmd);
1771         /* !! At this point cmd, sess & tgt_dev can be already freed !! */
1772         if (rc != SCST_EXEC_NOT_COMPLETED) {
1773                 if (rc == SCST_EXEC_COMPLETED)
1774                         goto out;
1775                 else if (rc == SCST_EXEC_NEED_THREAD)
1776                         goto out_clear;
1777                 else
1778                         goto out_rc_error;
1779         }
1780
1781         rc = scst_local_exec(cmd);
1782         /* !! At this point cmd, sess & tgt_dev can be already freed !! */
1783         if (rc != SCST_EXEC_NOT_COMPLETED) {
1784                 if (rc == SCST_EXEC_COMPLETED)
1785                         goto out;
1786                 else if (rc == SCST_EXEC_NEED_THREAD)
1787                         goto out_clear;
1788                 else
1789                         goto out_rc_error;
1790         }
1791
1792         if (cmd->dev->handler->exec) {
1793                 struct scst_device *dev = cmd->dev;
1794                 TRACE_DBG("Calling dev handler %s exec(%p)",
1795                       dev->handler->name, cmd);
1796                 TRACE_BUFF_FLAG(TRACE_SEND_TOP, "Execing: ", cmd->cdb, cmd->cdb_len);
1797                 cmd->scst_cmd_done = scst_cmd_done_local;
1798                 rc = dev->handler->exec(cmd);
1799                 /* !! At this point cmd, sess & tgt_dev can be already freed !! */
1800                 TRACE_DBG("Dev handler %s exec() returned %d",
1801                       dev->handler->name, rc);
1802                 if (rc == SCST_EXEC_COMPLETED)
1803                         goto out;
1804                 else if (rc == SCST_EXEC_NEED_THREAD)
1805                         goto out_clear;
1806                 else if (rc != SCST_EXEC_NOT_COMPLETED)
1807                         goto out_rc_error;
1808         }
1809
1810         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1811         
1812         if (unlikely(cmd->dev->scsi_dev == NULL)) {
1813                 PRINT_ERROR_PR("Command for virtual device must be "
1814                         "processed by device handler (lun %Ld)!",
1815                         (uint64_t)cmd->lun);
1816                 goto out_error;
1817         }
1818
1819 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) 
1820         if (unlikely(scst_alloc_request(cmd) != 0)) {
1821                 if (scst_cmd_atomic(cmd)) {
1822                         rc = SCST_EXEC_NEED_THREAD;
1823                         goto out_clear;
1824                 } else {
1825                         PRINT_INFO_PR("%s", "Unable to allocate request, "
1826                                 "sending BUSY status");
1827                         goto out_busy;
1828                 }
1829         }
1830         
1831         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1832                     (void *)cmd->scsi_req->sr_buffer,
1833                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1834                     cmd->retries);
1835 #else
1836         rc = scst_exec_req(cmd->dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1837                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1838                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1839                         scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
1840         if (unlikely(rc != 0)) {
1841                 if (scst_cmd_atomic(cmd)) {
1842                         rc = SCST_EXEC_NEED_THREAD;
1843                         goto out_clear;
1844                 } else {
1845                         PRINT_INFO_PR("scst_exec_req() failed: %d", rc);
1846                         goto out_error;
1847                 }
1848         }
1849 #endif
1850
1851         rc = SCST_EXEC_COMPLETED;
1852
1853 out:
1854         TRACE_EXIT();
1855         return rc;
1856
1857 out_clear:
1858         /* Restore the state */
1859         cmd->sent_to_midlev = 0;
1860         cmd->state = SCST_CMD_STATE_SEND_TO_MIDLEV;
1861         goto out;
1862
1863 out_rc_error:
1864         PRINT_ERROR_PR("Dev handler %s exec() or scst_local_exec() returned "
1865                     "invalid code %d", cmd->dev->handler->name, rc);
1866         /* go through */
1867
1868 out_error:
1869         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1870         cmd->completed = 1;
1871         cmd->state = SCST_CMD_STATE_DEV_DONE;
1872         rc = SCST_EXEC_COMPLETED;
1873         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1874         goto out;
1875
1876 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18) 
1877 out_busy:
1878         scst_set_busy(cmd);
1879         cmd->completed = 1;
1880         cmd->state = SCST_CMD_STATE_DEV_DONE;
1881         rc = SCST_EXEC_COMPLETED;
1882         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1883         goto out;
1884 #endif
1885
1886 out_aborted:
1887         rc = SCST_EXEC_COMPLETED;
1888         /* Report the result. The cmd is not completed */
1889         scst_cmd_done_local(cmd, SCST_CMD_STATE_DEFAULT);
1890         goto out;
1891 }
1892
1893 static int scst_send_to_midlev(struct scst_cmd *cmd)
1894 {
1895         int res, rc;
1896         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1897         struct scst_device *dev = cmd->dev;
1898         int expected_sn;
1899         int count;
1900         int atomic = scst_cmd_atomic(cmd);
1901
1902         TRACE_ENTRY();
1903
1904         res = SCST_CMD_STATE_RES_CONT_NEXT;
1905
1906         if (atomic && dev->handler->exec && !dev->handler->exec_atomic) {
1907                 TRACE_DBG("Dev handler %s exec() can not be "
1908                       "called in atomic context, rescheduling to the thread",
1909                       dev->handler->name);
1910                 res = SCST_CMD_STATE_RES_NEED_THREAD;
1911                 goto out;
1912         }
1913
1914         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
1915                 goto out;
1916
1917         scst_inc_cmd_count(); /* protect dev & tgt_dev */
1918
1919         if (unlikely(cmd->internal) || unlikely(cmd->retry)) {
1920                 rc = scst_do_send_to_midlev(cmd);
1921                 /* !! At this point cmd, sess & tgt_dev can be already freed !! */
1922                 if (rc == SCST_EXEC_NEED_THREAD) {
1923                         TRACE_DBG("%s", "scst_do_send_to_midlev() requested "
1924                               "thread context, rescheduling");
1925                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1926                         scst_dec_on_dev_cmd(cmd);
1927                         goto out_dec_cmd_count;
1928                 } else {
1929                         sBUG_ON(rc != SCST_EXEC_COMPLETED);
1930                         goto out_unplug;
1931                 }
1932         }
1933
1934         EXTRACHECKS_BUG_ON(cmd->no_sn);
1935
1936         expected_sn = tgt_dev->expected_sn;
1937         if (cmd->sn != expected_sn) {
1938                 spin_lock_bh(&tgt_dev->sn_lock);
1939                 tgt_dev->def_cmd_count++;
1940                 smp_mb();
1941                 barrier(); /* to reread expected_sn */
1942                 expected_sn = tgt_dev->expected_sn;
1943                 if (cmd->sn != expected_sn) {
1944                         scst_dec_on_dev_cmd(cmd);
1945                         TRACE(TRACE_SCSI_SERIALIZING, "Delaying cmd %p (sn=%d, "
1946                               "expected_sn=%d)", cmd, cmd->sn, expected_sn);
1947                         list_add_tail(&cmd->sn_cmd_list_entry,
1948                                       &tgt_dev->deferred_cmd_list);
1949                         spin_unlock_bh(&tgt_dev->sn_lock);
1950                         /* !! At this point cmd can be already freed !! */
1951                         goto out_dec_cmd_count;
1952                 } else {
1953                         TRACE(TRACE_SCSI_SERIALIZING, "Somebody incremented "
1954                               "expected_sn %d, continuing", expected_sn);
1955                         tgt_dev->def_cmd_count--;
1956                         spin_unlock_bh(&tgt_dev->sn_lock);
1957                 }
1958         }
1959
1960         count = 0;
1961         while(1) {
1962                 rc = scst_do_send_to_midlev(cmd);
1963                 if (rc == SCST_EXEC_NEED_THREAD) {
1964                         TRACE_DBG("%s", "scst_do_send_to_midlev() requested "
1965                               "thread context, rescheduling");
1966                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1967                         scst_dec_on_dev_cmd(cmd);
1968                         if (count != 0)
1969                                 goto out_unplug;
1970                         else
1971                                 goto out_dec_cmd_count;
1972                 }
1973                 sBUG_ON(rc != SCST_EXEC_COMPLETED);
1974                 /* !! At this point cmd can be already freed !! */
1975                 count++;
1976                 expected_sn = __scst_inc_expected_sn(tgt_dev);
1977                 cmd = scst_check_deferred_commands(tgt_dev, expected_sn);
1978                 if (cmd == NULL)
1979                         break;
1980                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
1981                         break;
1982         }
1983
1984 out_unplug:
1985         if (dev->scsi_dev != NULL)
1986                 generic_unplug_device(dev->scsi_dev->request_queue);
1987
1988 out_dec_cmd_count:
1989         scst_dec_cmd_count();
1990         /* !! At this point sess, dev and tgt_dev can be already freed !! */
1991
1992 out:
1993         TRACE_EXIT_HRES(res);
1994         return res;
1995 }
1996
1997 static int scst_done_cmd_check(struct scst_cmd *cmd, int *pres)
1998 {
1999         int res = 0, rc;
2000         unsigned char type;
2001
2002         TRACE_ENTRY();
2003
2004         if (unlikely(cmd->cdb[0] == REQUEST_SENSE)) {
2005                 if (cmd->internal)
2006                         cmd = scst_complete_request_sense(cmd);
2007         } else if (unlikely(scst_check_auto_sense(cmd))) {
2008                 PRINT_INFO_PR("Command finished with CHECK CONDITION, but "
2009                             "without sense data (opcode 0x%x), issuing "
2010                             "REQUEST SENSE", cmd->cdb[0]);
2011                 rc = scst_prepare_request_sense(cmd);
2012                 if (res > 0) {
2013                         *pres = rc;
2014                         res = 1;
2015                         goto out;
2016                 } else {
2017                         PRINT_ERROR_PR("%s", "Unable to issue REQUEST SENSE, "
2018                                     "returning HARDWARE ERROR");
2019                         scst_set_cmd_error(cmd,
2020                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2021                 }
2022         }
2023
2024         type = cmd->dev->handler->type;
2025         if ((cmd->cdb[0] == MODE_SENSE || cmd->cdb[0] == MODE_SENSE_10) &&
2026             cmd->tgt_dev->acg_dev->rd_only_flag &&
2027             (type == TYPE_DISK || type == TYPE_WORM || type == TYPE_MOD ||
2028              type == TYPE_TAPE))
2029         {
2030                 int32_t length;
2031                 uint8_t *address;
2032
2033                 length = scst_get_buf_first(cmd, &address);
2034                 if (length <= 0)
2035                         goto out;
2036                 if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2037                         address[2] |= 0x80;   /* Write Protect*/
2038                 else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2039                         address[3] |= 0x80;   /* Write Protect*/
2040                 scst_put_buf(cmd, address);
2041         }
2042
2043         /* 
2044          * Check and clear NormACA option for the device, if necessary,
2045          * since we don't support ACA
2046          */
2047         if ((cmd->cdb[0] == INQUIRY) &&
2048             !(cmd->cdb[1] & SCST_INQ_EVPD/* Std INQUIRY data (no EVPD) */) &&
2049             (cmd->resp_data_len > SCST_INQ_BYTE3))
2050         {
2051                 uint8_t *buffer;
2052                 int buflen;
2053
2054                 /* ToDo: all pages ?? */
2055                 buflen = scst_get_buf_first(cmd, &buffer);
2056                 if (buflen > 0) {
2057                         if (buflen > SCST_INQ_BYTE3) {
2058 #ifdef EXTRACHECKS
2059                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2060                                         PRINT_INFO_PR("NormACA set for device: "
2061                                             "lun=%Ld, type 0x%02x", 
2062                                             (uint64_t)cmd->lun, buffer[0]);
2063                                 }
2064 #endif
2065                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2066                         } else
2067                                 scst_set_cmd_error(cmd,
2068                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
2069
2070                         scst_put_buf(cmd, buffer);
2071                 }
2072         }
2073
2074         if (unlikely((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10))) {
2075                 if ((cmd->status != 0) && !test_bit(SCST_TGT_DEV_RESERVED,
2076                                                 &cmd->tgt_dev->tgt_dev_flags)) {
2077                         struct scst_tgt_dev *tgt_dev_tmp;
2078                         TRACE(TRACE_SCSI, "Real RESERVE failed lun=%Ld, status=%x",
2079                               (uint64_t)cmd->lun, cmd->masked_status);
2080                         TRACE_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense_buffer,
2081                                      sizeof(cmd->sense_buffer));
2082                         /* Clearing the reservation */
2083                         list_for_each_entry(tgt_dev_tmp, &cmd->dev->dev_tgt_dev_list,
2084                                             dev_tgt_dev_list_entry) {
2085                                 clear_bit(SCST_TGT_DEV_RESERVED, 
2086                                         &tgt_dev_tmp->tgt_dev_flags);
2087                         }
2088                         cmd->dev->dev_reserved = 0;
2089                 }
2090                 scst_unblock_dev(cmd->dev);
2091         }
2092         
2093         if (unlikely((cmd->cdb[0] == MODE_SELECT) || 
2094                      (cmd->cdb[0] == MODE_SELECT_10) ||
2095                      (cmd->cdb[0] == LOG_SELECT)))
2096         {
2097                 if (cmd->status == 0) {
2098                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2099                                 "setting the SELECT UA (lun=%Ld)", 
2100                                 (uint64_t)cmd->lun);
2101                         spin_lock_bh(&scst_temp_UA_lock);
2102                         if (cmd->cdb[0] == LOG_SELECT) {
2103                                 scst_set_sense(scst_temp_UA,
2104                                         sizeof(scst_temp_UA),
2105                                         UNIT_ATTENTION, 0x2a, 0x02);
2106                         } else {
2107                                 scst_set_sense(scst_temp_UA,
2108                                         sizeof(scst_temp_UA),
2109                                         UNIT_ATTENTION, 0x2a, 0x01);
2110                         }
2111                         scst_process_UA(cmd->dev, cmd, scst_temp_UA,
2112                                 sizeof(scst_temp_UA), 1);
2113                         spin_unlock_bh(&scst_temp_UA_lock);
2114                 }
2115                 scst_unblock_dev(cmd->dev);
2116         }
2117
2118 out:
2119         TRACE_EXIT_RES(res);
2120         return res;
2121 }
2122
2123 static int scst_dev_done(struct scst_cmd *cmd)
2124 {
2125         int res = SCST_CMD_STATE_RES_CONT_SAME;
2126         int state;
2127         int atomic = scst_cmd_atomic(cmd);
2128
2129         TRACE_ENTRY();
2130
2131         if (atomic && !cmd->dev->handler->dev_done_atomic &&
2132             cmd->dev->handler->dev_done) 
2133         {
2134                 TRACE_DBG("Dev handler %s dev_done() can not be "
2135                       "called in atomic context, rescheduling to the thread",
2136                       cmd->dev->handler->name);
2137                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2138                 goto out;
2139         }
2140
2141         if (scst_done_cmd_check(cmd, &res))
2142                 goto out;
2143
2144         state = SCST_CMD_STATE_XMIT_RESP;
2145         if (likely(!scst_is_cmd_local(cmd)) && 
2146             likely(cmd->dev->handler->dev_done != NULL))
2147         {
2148                 int rc;
2149                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2150                       cmd->dev->handler->name, cmd);
2151                 rc = cmd->dev->handler->dev_done(cmd);
2152                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2153                       cmd->dev->handler->name, rc);
2154                 if (rc != SCST_CMD_STATE_DEFAULT)
2155                         state = rc;
2156         }
2157
2158         switch (state) {
2159         case SCST_CMD_STATE_REINIT:
2160                 cmd->state = state;
2161                 res = SCST_CMD_STATE_RES_RESTART;
2162                 break;
2163
2164         case SCST_CMD_STATE_DEV_PARSE:
2165         case SCST_CMD_STATE_PREPARE_SPACE:
2166         case SCST_CMD_STATE_RDY_TO_XFER:
2167         case SCST_CMD_STATE_SEND_TO_MIDLEV:
2168         case SCST_CMD_STATE_DEV_DONE:
2169         case SCST_CMD_STATE_XMIT_RESP:
2170         case SCST_CMD_STATE_FINISHED:
2171                 cmd->state = state;
2172                 res = SCST_CMD_STATE_RES_CONT_SAME;
2173                 break;
2174
2175         case SCST_CMD_STATE_NEED_THREAD_CTX:
2176                 TRACE_DBG("Dev handler %s dev_done() requested "
2177                       "thread context, rescheduling",
2178                       cmd->dev->handler->name);
2179                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2180                 break;
2181
2182         default:
2183                 if (state >= 0) {
2184                         PRINT_ERROR_PR("Dev handler %s dev_done() returned "
2185                                 "invalid cmd state %d", 
2186                                 cmd->dev->handler->name, state);
2187                 } else {
2188                         PRINT_ERROR_PR("Dev handler %s dev_done() returned "
2189                                 "error %d", cmd->dev->handler->name, 
2190                                 state);
2191                 }
2192                 scst_set_cmd_error(cmd,
2193                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2194                 cmd->state = SCST_CMD_STATE_XMIT_RESP;
2195                 res = SCST_CMD_STATE_RES_CONT_SAME;
2196                 break;
2197         }
2198
2199 out:
2200         TRACE_EXIT_HRES(res);
2201         return res;
2202 }
2203
2204 static int scst_xmit_response(struct scst_cmd *cmd)
2205 {
2206         int res, rc;
2207         int atomic = scst_cmd_atomic(cmd);
2208
2209         TRACE_ENTRY();
2210
2211         /* 
2212          * Check here also in order to avoid unnecessary delays of other
2213          * commands.
2214          */
2215         if (unlikely(cmd->sent_to_midlev == 0) &&
2216             (cmd->tgt_dev != NULL))
2217         {
2218                 TRACE(TRACE_SCSI_SERIALIZING,
2219                       "cmd %p was not sent to mid-lev (sn %d)", cmd, cmd->sn);
2220                 scst_inc_expected_sn_unblock(cmd->tgt_dev, cmd, 0);
2221                 cmd->sent_to_midlev = 1;
2222         }
2223
2224         if (atomic && !cmd->tgtt->xmit_response_atomic) {
2225                 TRACE_DBG("%s", "xmit_response() can not be "
2226                       "called in atomic context, rescheduling to the thread");
2227                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2228                 goto out;
2229         }
2230
2231         set_bit(SCST_CMD_XMITTING, &cmd->cmd_flags);
2232         smp_mb__after_set_bit();
2233
2234         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2235                 if (test_bit(SCST_CMD_ABORTED_OTHER, &cmd->cmd_flags)) {
2236                         TRACE_MGMT_DBG("Flag ABORTED OTHER set for cmd %p "
2237                                 "(tag %d), returning TASK ABORTED", cmd, cmd->tag);
2238                         scst_set_cmd_error_status(cmd, SAM_STAT_TASK_ABORTED);
2239                 }
2240         }
2241
2242         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2243                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %d), skipping",
2244                         cmd, cmd->tag);
2245                 cmd->state = SCST_CMD_STATE_FINISHED;
2246                 res = SCST_CMD_STATE_RES_CONT_SAME;
2247                 goto out;
2248         }
2249
2250 #ifdef DEBUG_TM
2251         if (cmd->tm_dbg_delayed && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2252                 if (atomic && !cmd->tgtt->xmit_response_atomic) {
2253                         TRACE_MGMT_DBG("%s", "DEBUG_TM delayed cmd needs a thread");
2254                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2255                         goto out;
2256                 }
2257                 TRACE_MGMT_DBG("Delaying cmd %p (tag %d) for 1 second",
2258                         cmd, cmd->tag);
2259                 schedule_timeout_uninterruptible(HZ);
2260         }
2261 #endif
2262
2263         while (1) {
2264                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2265
2266                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2267                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2268
2269                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2270
2271 #if defined(DEBUG) || defined(TRACING)
2272                 if (cmd->sg) {
2273                         int i;
2274                         struct scatterlist *sg = cmd->sg;
2275                         TRACE(TRACE_SEND_BOT,
2276                               "Xmitting %d S/G(s) at %p sg[0].page at %p",
2277                               cmd->sg_cnt, sg, (void*)sg[0].page);
2278                         for(i = 0; i < cmd->sg_cnt; ++i) {
2279                                 TRACE_BUFF_FLAG(TRACE_SEND_BOT,
2280                                     "Xmitting sg", page_address(sg[i].page),
2281                                     sg[i].length);
2282                         }
2283                 }
2284 #endif
2285
2286 #ifdef DEBUG_RETRY
2287                 if (((scst_random() % 100) == 77))
2288                         rc = SCST_TGT_RES_QUEUE_FULL;
2289                 else
2290 #endif
2291                         rc = cmd->tgtt->xmit_response(cmd);
2292                 TRACE_DBG("xmit_response() returned %d", rc);
2293
2294                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2295                         goto out;
2296
2297                 /* Restore the previous state */
2298                 cmd->state = SCST_CMD_STATE_XMIT_RESP;
2299
2300                 switch (rc) {
2301                 case SCST_TGT_RES_QUEUE_FULL:
2302                 {
2303                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2304                                 break;
2305                         else
2306                                 continue;
2307                 }
2308
2309                 case SCST_TGT_RES_NEED_THREAD_CTX:
2310                 {
2311                         TRACE_DBG("Target driver %s xmit_response() "
2312                               "requested thread context, rescheduling",
2313                               cmd->tgtt->name);
2314                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2315                         break;
2316                 }
2317
2318                 default:
2319                         goto out_error;
2320                 }
2321                 break;
2322         }
2323
2324 out:
2325         /* Caution: cmd can be already dead here */
2326         TRACE_EXIT_HRES(res);
2327         return res;
2328
2329 out_error:
2330         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2331                 PRINT_ERROR_PR("Target driver %s xmit_response() returned "
2332                         "fatal error", cmd->tgtt->name);
2333         } else {
2334                 PRINT_ERROR_PR("Target driver %s xmit_response() returned "
2335                         "invalid value %d", cmd->tgtt->name, rc);
2336         }
2337         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2338         cmd->state = SCST_CMD_STATE_FINISHED;
2339         res = SCST_CMD_STATE_RES_CONT_SAME;
2340         goto out;
2341 }
2342
2343 void scst_tgt_cmd_done(struct scst_cmd *cmd)
2344 {
2345         TRACE_ENTRY();
2346
2347         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2348
2349         cmd->state = SCST_CMD_STATE_FINISHED;
2350         scst_proccess_redirect_cmd(cmd, scst_get_context(), 1);
2351
2352         TRACE_EXIT();
2353         return;
2354 }
2355
2356 static int scst_finish_cmd(struct scst_cmd *cmd)
2357 {
2358         int res;
2359
2360         TRACE_ENTRY();
2361
2362         if (cmd->mem_checked) {
2363                 spin_lock_bh(&scst_cmd_mem_lock);
2364                 scst_cur_cmd_mem -= cmd->bufflen;
2365                 spin_unlock_bh(&scst_cmd_mem_lock);
2366         }
2367
2368         spin_lock_irq(&scst_list_lock);
2369
2370         TRACE_DBG("Deleting cmd %p from cmd list", cmd);
2371         list_del(&cmd->cmd_list_entry);
2372
2373         if (cmd->mgmt_cmnd)
2374                 scst_complete_cmd_mgmt(cmd, cmd->mgmt_cmnd);
2375
2376         if (likely(cmd->tgt_dev != NULL))
2377                 cmd->tgt_dev->cmd_count--;
2378
2379         cmd->sess->sess_cmd_count--;
2380
2381         list_del(&cmd->search_cmd_list_entry);
2382
2383         spin_unlock_irq(&scst_list_lock);
2384
2385         scst_free_cmd(cmd);
2386
2387         res = SCST_CMD_STATE_RES_CONT_NEXT;
2388
2389         TRACE_EXIT_HRES(res);
2390         return res;
2391 }
2392
2393 /*
2394  * Returns 0 on success, > 0 when we need to wait for unblock,
2395  * < 0 if there is no device (lun) or device type handler.
2396  * Called under scst_list_lock and IRQs disabled
2397  */
2398 static int scst_translate_lun(struct scst_cmd *cmd)
2399 {
2400         struct scst_tgt_dev *tgt_dev = NULL;
2401         int res;
2402
2403         TRACE_ENTRY();
2404
2405         scst_inc_cmd_count();   
2406
2407         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
2408                 res = -1;
2409                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %Ld)", cmd,
2410                       (uint64_t)cmd->lun);
2411                 list_for_each_entry(tgt_dev, &cmd->sess->sess_tgt_dev_list,
2412                                     sess_tgt_dev_list_entry) 
2413                 {
2414                         if (tgt_dev->acg_dev->lun == cmd->lun) {
2415                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
2416
2417                                 if (unlikely(tgt_dev->acg_dev->dev->handler == NULL)) {
2418                                         PRINT_INFO_PR("Dev handler for device "
2419                                           "%Ld is NULL, the device will not be "
2420                                           "visible remotely", (uint64_t)cmd->lun);
2421                                         break;
2422                                 }
2423                                 
2424                                 if (cmd->state == SCST_CMD_STATE_REINIT) {
2425                                         cmd->tgt_dev_saved->cmd_count--;
2426                                         TRACE(TRACE_SCSI_SERIALIZING,
2427                                               "SCST_CMD_STATE_REINIT: "
2428                                               "incrementing expected_sn on tgt_dev_saved %p",
2429                                               cmd->tgt_dev_saved);
2430                                         scst_inc_expected_sn_unblock(
2431                                                 cmd->tgt_dev_saved, cmd, 1);
2432                                 }
2433                                 cmd->tgt_dev = tgt_dev;
2434                                 tgt_dev->cmd_count++;
2435                                 cmd->dev = tgt_dev->acg_dev->dev;
2436
2437                                 res = 0;
2438                                 break;
2439                         }
2440                 }
2441                 if (res != 0) {
2442                         TRACE_DBG("tgt_dev for lun %Ld not found, command to "
2443                                 "unexisting LU?", (uint64_t)cmd->lun);
2444                         scst_dec_cmd_count();
2445                 }
2446         } else {
2447                 if ( !cmd->sess->waiting) {
2448                         TRACE_DBG("Adding session %p to scst_dev_wait_sess_list",
2449                               cmd->sess);
2450                         list_add_tail(&cmd->sess->dev_wait_sess_list_entry,
2451                                       &scst_dev_wait_sess_list);
2452                         cmd->sess->waiting = 1;
2453                 }
2454                 scst_dec_cmd_count();
2455                 res = 1;
2456         }
2457
2458         TRACE_EXIT_RES(res);
2459         return res;
2460 }
2461
2462 /* Called under scst_list_lock and IRQs disabled */
2463 static int scst_process_init_cmd(struct scst_cmd *cmd)
2464 {
2465         int res = 0;
2466
2467         TRACE_ENTRY();
2468
2469         if (unlikely(cmd->tgt_dev)) {
2470                 scst_cmd_set_sn(cmd);
2471                 goto out_move;
2472         }
2473
2474         res = scst_translate_lun(cmd);
2475         if (likely(res == 0)) {
2476                 cmd->state = SCST_CMD_STATE_DEV_PARSE;
2477                 if (cmd->tgt_dev->cmd_count > SCST_MAX_DEVICE_COMMANDS) {
2478                         TRACE(TRACE_RETRY, "Too many pending commands in "
2479                                 "session, returning BUSY to initiator \"%s\"",
2480                                 (cmd->sess->initiator_name[0] == '\0') ?
2481                                   "Anonymous" : cmd->sess->initiator_name);
2482                         scst_set_busy(cmd);
2483                         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2484                 } else if (!cmd->no_sn)
2485                         scst_cmd_set_sn(cmd);
2486         } else if (res < 0) {
2487                 TRACE_DBG("Finishing cmd %p", cmd);
2488                 scst_set_cmd_error(cmd,
2489                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
2490                 cmd->state = SCST_CMD_STATE_XMIT_RESP;
2491         } else
2492                 goto out;
2493
2494 out_move:
2495         TRACE_DBG("Moving cmd %p to active cmd list", cmd);
2496         list_move_tail(&cmd->cmd_list_entry, &scst_active_cmd_list);
2497
2498 out:
2499         TRACE_EXIT_RES(res);
2500         return res;
2501 }
2502
2503 /* 
2504  * Called under scst_list_lock and IRQs disabled
2505  * We don't drop it anywhere inside, because command execution
2506  * have to be serialized, i.e. commands must be executed in order
2507  * of their arrival, and we set this order inside scst_translate_lun().
2508  */
2509 static int scst_do_job_init(void)
2510 {
2511         int res = 0;
2512
2513         TRACE_ENTRY();
2514
2515         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
2516                 while (!list_empty(&scst_init_cmd_list)) {
2517                         struct scst_cmd *cmd = list_entry(
2518                                 scst_init_cmd_list.next, typeof(*cmd),
2519                                                           cmd_list_entry);
2520                         res = scst_process_init_cmd(cmd);
2521                         if (res > 0)
2522                                 break;
2523                         /* For DIRECT context the cmd is always the last */
2524                 }
2525         } else
2526                 res = 1;
2527
2528         TRACE_EXIT_RES(res);
2529         return res;
2530 }
2531
2532 /* Called with no locks held */
2533 static int __scst_process_active_cmd(struct scst_cmd *cmd, int context,
2534         int left_locked)
2535 {
2536         int res;
2537
2538         TRACE_ENTRY();
2539
2540         EXTRACHECKS_BUG_ON(in_irq());
2541
2542         cmd->atomic = ((context & ~SCST_PROCESSIBLE_ENV) == 
2543                         SCST_CONTEXT_DIRECT_ATOMIC);
2544         cmd->processible_env = (context & SCST_PROCESSIBLE_ENV) != 0;
2545
2546         do {
2547                 switch (cmd->state) {
2548                 case SCST_CMD_STATE_DEV_PARSE:
2549                         res = scst_parse_cmd(cmd);
2550                         break;
2551
2552                 case SCST_CMD_STATE_PREPARE_SPACE:
2553                         res = scst_prepare_space(cmd);
2554                         break;
2555
2556                 case SCST_CMD_STATE_RDY_TO_XFER:
2557                         res = scst_rdy_to_xfer(cmd);
2558                         break;
2559
2560                 case SCST_CMD_STATE_SEND_TO_MIDLEV:
2561                         res = scst_send_to_midlev(cmd);
2562                         /* !! At this point cmd, sess & tgt_dev can be already freed !! */
2563                         break;
2564
2565                 case SCST_CMD_STATE_DEV_DONE:
2566                         res = scst_dev_done(cmd);
2567                         break;
2568
2569                 case SCST_CMD_STATE_XMIT_RESP:
2570                         res = scst_xmit_response(cmd);
2571                         break;
2572
2573                 case SCST_CMD_STATE_FINISHED:
2574                         res = scst_finish_cmd(cmd);
2575                         break;
2576
2577                 default:
2578                         PRINT_ERROR_PR("cmd (%p) in state %d, but shouldn't be",
2579                                cmd, cmd->state);
2580                         sBUG();
2581                         res = SCST_CMD_STATE_RES_CONT_NEXT;
2582                         break;
2583                 }
2584         } while(res == SCST_CMD_STATE_RES_CONT_SAME);
2585
2586         if (res == SCST_CMD_STATE_RES_CONT_NEXT) {
2587                 if (left_locked)
2588                         spin_lock_irq(&scst_list_lock);
2589         } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
2590                 spin_lock_irq(&scst_list_lock);
2591
2592                 switch (cmd->state) {
2593                 case SCST_CMD_STATE_DEV_PARSE:
2594                 case SCST_CMD_STATE_PREPARE_SPACE:
2595                 case SCST_CMD_STATE_RDY_TO_XFER:
2596                 case SCST_CMD_STATE_SEND_TO_MIDLEV:
2597                 case SCST_CMD_STATE_DEV_DONE:
2598                 case SCST_CMD_STATE_XMIT_RESP:
2599                 case SCST_CMD_STATE_FINISHED:
2600                         TRACE_DBG("Moving cmd %p to active cmd list", cmd);
2601                         list_move(&cmd->cmd_list_entry, &scst_active_cmd_list);
2602                         break;
2603 #ifdef EXTRACHECKS
2604                 /* not very valid commands */
2605                 case SCST_CMD_STATE_DEFAULT:
2606                 case SCST_CMD_STATE_NEED_THREAD_CTX:
2607                         PRINT_ERROR_PR("cmd %p is in state %d, not putting on "
2608                                 "useful list (left on scst cmd list)", cmd, 
2609                                 cmd->state);
2610                         spin_unlock_irq(&scst_list_lock);
2611                         sBUG();
2612                         spin_lock_irq(&scst_list_lock);
2613                         break;
2614 #endif
2615                 default:
2616                         break;
2617                 }
2618                 cmd->non_atomic_only = 1;
2619                 if (!left_locked)
2620                         spin_unlock_irq(&scst_list_lock);
2621                 wake_up(&scst_list_waitQ);
2622         } else if (res == SCST_CMD_STATE_RES_RESTART) {
2623                 if (cmd->state == SCST_CMD_STATE_REINIT) {
2624                         spin_lock_irq(&scst_list_lock);
2625                         TRACE_DBG("Moving cmd %p to head of init cmd list", cmd);
2626                         list_move(&cmd->cmd_list_entry, &scst_init_cmd_list);
2627                         if (!left_locked)
2628                                 spin_unlock_irq(&scst_list_lock);
2629                 } else
2630                         sBUG();
2631         } else
2632                 sBUG();
2633
2634         TRACE_EXIT_RES(res);
2635         return res;
2636 }
2637
2638 /* Called under scst_list_lock and IRQs disabled */
2639 static void scst_do_job_active(int context)
2640 {
2641         int res;
2642         struct scst_cmd *cmd;
2643         int atomic = ((context & ~SCST_PROCESSIBLE_ENV) == 
2644                         SCST_CONTEXT_DIRECT_ATOMIC);
2645
2646         TRACE_ENTRY();
2647
2648 #ifdef EXTRACHECKS
2649         {
2650                 int c = (context & ~SCST_PROCESSIBLE_ENV);
2651                 WARN_ON((c != SCST_CONTEXT_DIRECT_ATOMIC) && 
2652                         (c != SCST_CONTEXT_DIRECT));
2653         }
2654 #endif
2655
2656         tm_dbg_check_released_cmds();
2657
2658 restart:
2659         list_for_each_entry(cmd, &scst_active_cmd_list, cmd_list_entry) {
2660                 if (atomic && cmd->non_atomic_only) {
2661                         TRACE(TRACE_DEBUG, "Skipping non-atomic cmd %p", cmd);
2662                         continue;
2663                 }
2664                 if (tm_dbg_check_cmd(cmd) != 0)
2665                         goto restart;
2666                 res = scst_process_active_cmd(cmd, context, NULL, 1);
2667                 if (res == SCST_CMD_STATE_RES_CONT_NEXT) {
2668                         goto restart;
2669                 } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
2670                         goto restart;
2671                 } else if (res == SCST_CMD_STATE_RES_RESTART) {
2672                         break;
2673                 } else
2674                         sBUG();
2675         }
2676
2677         TRACE_EXIT();
2678         return;
2679 }
2680
2681 static inline int test_cmd_lists(void)
2682 {
2683         int res = !list_empty(&scst_active_cmd_list) ||
2684             (!list_empty(&scst_init_cmd_list) &&
2685              !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
2686             test_bit(SCST_FLAG_SHUTDOWN, &scst_flags) ||
2687             unlikely(scst_shut_threads_count > 0) ||
2688             tm_dbg_is_release();
2689         return res;
2690 }
2691
2692 int scst_cmd_thread(void *arg)
2693 {
2694         static spinlock_t lock = SPIN_LOCK_UNLOCKED;
2695         int n;
2696
2697         TRACE_ENTRY();
2698
2699         spin_lock(&lock);
2700         n = scst_thread_num++;
2701         spin_unlock(&lock);
2702         daemonize("scsi_tgt%d", n);
2703         recalc_sigpending();
2704         set_user_nice(current, 10);
2705         current->flags |= PF_NOFREEZE;
2706
2707         spin_lock_irq(&scst_list_lock);
2708         while (1) {
2709                 wait_queue_t wait;
2710                 init_waitqueue_entry(&wait, current);
2711
2712                 if (!test_cmd_lists()) {
2713                         add_wait_queue_exclusive(&scst_list_waitQ, &wait);
2714                         for (;;) {
2715                                 set_current_state(TASK_INTERRUPTIBLE);
2716                                 if (test_cmd_lists())
2717                                         break;
2718                                 spin_unlock_irq(&scst_list_lock);
2719                                 schedule();
2720                                 spin_lock_irq(&scst_list_lock);
2721                         }
2722                         set_current_state(TASK_RUNNING);
2723                         remove_wait_queue(&scst_list_waitQ, &wait);
2724                 }
2725
2726                 scst_do_job_init();
2727                 scst_do_job_active(SCST_CONTEXT_DIRECT|SCST_PROCESSIBLE_ENV);
2728
2729                 if (unlikely(test_bit(SCST_FLAG_SHUTDOWN, &scst_flags)) &&
2730                     list_empty(&scst_cmd_list) &&
2731                     list_empty(&scst_active_cmd_list) &&
2732                     list_empty(&scst_init_cmd_list)) {
2733                         break;
2734                 }
2735                 
2736                 if (unlikely(scst_shut_threads_count > 0)) {
2737                         scst_shut_threads_count--;
2738                         break;
2739                 }
2740         }
2741         spin_unlock_irq(&scst_list_lock);
2742
2743         if (atomic_dec_and_test(&scst_threads_count) && scst_shutdown_mutex) {
2744                 smp_mb__after_atomic_dec();
2745                 TRACE_DBG("%s", "Releasing scst_shutdown_mutex");
2746                 up(scst_shutdown_mutex);
2747         }
2748
2749         TRACE_EXIT();
2750         return 0;
2751 }
2752
2753 void scst_cmd_tasklet(long p)
2754 {
2755         TRACE_ENTRY();
2756
2757         spin_lock_irq(&scst_list_lock);
2758
2759         scst_do_job_init();
2760         scst_do_job_active(SCST_CONTEXT_DIRECT_ATOMIC|SCST_PROCESSIBLE_ENV);
2761
2762         spin_unlock_irq(&scst_list_lock);
2763
2764         TRACE_EXIT();
2765         return;
2766 }
2767
2768 /*
2769  * Returns 0 on success, < 0 if there is no device handler or
2770  * > 0 if SCST_FLAG_SUSPENDED set.
2771  */
2772 static int scst_mgmt_translate_lun(struct scst_mgmt_cmd *mcmd)
2773 {
2774         struct scst_tgt_dev *tgt_dev = NULL;
2775         int res = -1;
2776
2777         TRACE_ENTRY();
2778
2779         TRACE_DBG("Finding tgt_dev for mgmt cmd %p (lun %Ld)", mcmd,
2780               (uint64_t)mcmd->lun);
2781
2782         spin_lock_irq(&scst_list_lock);
2783         scst_inc_cmd_count();   
2784         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
2785                 list_for_each_entry(tgt_dev, &mcmd->sess->sess_tgt_dev_list,
2786                                     sess_tgt_dev_list_entry) 
2787                 {
2788                         if (tgt_dev->acg_dev->lun == mcmd->lun) {
2789                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
2790                                 mcmd->mcmd_tgt_dev = tgt_dev;
2791                                 res = 0;
2792                                 break;
2793                         }
2794                 }
2795                 if (mcmd->mcmd_tgt_dev == NULL)
2796                         scst_dec_cmd_count();
2797         } else {
2798                 if ( !mcmd->sess->waiting) {
2799                         TRACE_DBG("Adding session %p to scst_dev_wait_sess_list",
2800                               mcmd->sess);
2801                         list_add_tail(&mcmd->sess->dev_wait_sess_list_entry,
2802                                       &scst_dev_wait_sess_list);
2803                         mcmd->sess->waiting = 1;
2804                 }
2805                 scst_dec_cmd_count();
2806                 res = 1;
2807         }
2808         spin_unlock_irq(&scst_list_lock);
2809
2810         TRACE_EXIT_HRES(res);
2811         return res;
2812 }
2813
2814 /* Called under scst_list_lock and IRQ off */
2815 static void scst_complete_cmd_mgmt(struct scst_cmd *cmd,
2816         struct scst_mgmt_cmd *mcmd)
2817 {
2818         TRACE_ENTRY();
2819
2820         TRACE_MGMT_DBG("cmd %p completed (tag %d, mcmd %p, "
2821                 "mcmd->cmd_wait_count %d)", cmd, cmd->tag, mcmd,
2822                 mcmd->cmd_wait_count);
2823
2824         cmd->mgmt_cmnd = NULL;
2825
2826         if (cmd->completed)
2827                 mcmd->completed_cmd_count++;
2828
2829         mcmd->cmd_wait_count--;
2830         if (mcmd->cmd_wait_count > 0) {
2831                 TRACE_MGMT_DBG("cmd_wait_count(%d) not 0, skipping",
2832                         mcmd->cmd_wait_count);
2833                 goto out;
2834         }
2835
2836         mcmd->state = SCST_MGMT_CMD_STATE_DONE;
2837
2838         if (mcmd->completed) {
2839                 TRACE_MGMT_DBG("Moving mgmt cmd %p to active mgmt cmd list",
2840                         mcmd);
2841                 list_move_tail(&mcmd->mgmt_cmd_list_entry,
2842                         &scst_active_mgmt_cmd_list);
2843         }
2844
2845         wake_up(&scst_mgmt_cmd_list_waitQ);
2846
2847 out:
2848         TRACE_EXIT();
2849         return;
2850 }
2851
2852 static int scst_call_dev_task_mgmt_fn(struct scst_mgmt_cmd *mcmd,
2853         struct scst_tgt_dev *tgt_dev, int set_status)
2854 {
2855         int res = SCST_DEV_TM_NOT_COMPLETED;
2856         if (tgt_dev->acg_dev->dev->handler->task_mgmt_fn) {
2857                 int irq = irqs_disabled();
2858                 TRACE_MGMT_DBG("Calling dev handler %s task_mgmt_fn(fn=%d)",
2859                         tgt_dev->acg_dev->dev->handler->name, mcmd->fn);
2860                 EXTRACHECKS_BUG_ON(in_irq());
2861                 if (!irq)
2862                         local_bh_disable();
2863                 res = tgt_dev->acg_dev->dev->handler->task_mgmt_fn(mcmd, 
2864                         tgt_dev);
2865                 if (!irq)
2866                         local_bh_enable();
2867                 TRACE_MGMT_DBG("Dev handler %s task_mgmt_fn() returned %d",
2868                       tgt_dev->acg_dev->dev->handler->name, res);
2869                 if (set_status && (res != SCST_DEV_TM_NOT_COMPLETED))
2870                         mcmd->status = res;
2871         }
2872         return res;
2873 }
2874
2875 static inline int scst_is_strict_mgmt_fn(int mgmt_fn)
2876 {
2877         switch(mgmt_fn) {
2878                 case SCST_ABORT_TASK:
2879                 case SCST_ABORT_TASK_SET:
2880                 case SCST_CLEAR_TASK_SET:
2881                         return 1;
2882                 default:
2883                         return 0;
2884         }
2885 }
2886
2887 /* 
2888  * Called under scst_list_lock and IRQ off (to protect cmd
2889  * from being destroyed) + BHs also off
2890  * Returns -1 if command is being executed (ABORT failed), 0 otherwise
2891  */
2892 void scst_abort_cmd(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd,
2893         int other_ini, int call_dev_task_mgmt_fn)
2894 {
2895         TRACE_ENTRY();
2896
2897         TRACE(TRACE_MGMT, "Aborting cmd %p (tag %d)", cmd, cmd->tag);
2898
2899         if (other_ini) {
2900                 set_bit(SCST_CMD_ABORTED_OTHER, &cmd->cmd_flags);
2901                 smp_mb__after_set_bit();
2902         }
2903         set_bit(SCST_CMD_ABORTED, &cmd->cmd_flags);
2904         smp_mb__after_set_bit();
2905
2906         if (call_dev_task_mgmt_fn && cmd->tgt_dev)
2907                 scst_call_dev_task_mgmt_fn(mcmd, cmd->tgt_dev, 1);
2908
2909         if (mcmd) {
2910                 int defer;
2911                 if (cmd->tgtt->tm_sync_reply)
2912                         defer = 1;
2913                 else {
2914                         if (scst_is_strict_mgmt_fn(mcmd->fn))
2915                                 defer = test_bit(SCST_CMD_EXECUTING,
2916                                         &cmd->cmd_flags);
2917                         else
2918                                 defer = test_bit(SCST_CMD_XMITTING,
2919                                         &cmd->cmd_flags);
2920                 }
2921
2922                 if (defer) {
2923                         /*
2924                          * Delay the response until the command's finish in
2925                          * order to guarantee that "no further responses from
2926                          * the task are sent to the SCSI initiator port" after
2927                          * response from the TM function is sent (SAM)
2928                          */
2929                         TRACE(TRACE_MGMT, "cmd %p (tag %d) being executed/"
2930                                 "xmitted (state %d), deferring ABORT...", cmd,
2931                                 cmd->tag, cmd->state);
2932 #ifdef EXTRACHECKS
2933                         if (cmd->mgmt_cmnd) {
2934                                 printk(KERN_ALERT "cmd %p (tag %d, state %d) "
2935                                         "has non-NULL mgmt_cmnd %p!!! Current "
2936                                         "mcmd %p\n", cmd, cmd->tag, cmd->state,
2937                                         cmd->mgmt_cmnd, mcmd);
2938                         }
2939 #endif
2940                         sBUG_ON(cmd->mgmt_cmnd);
2941                         mcmd->cmd_wait_count++;
2942                         cmd->mgmt_cmnd = mcmd;
2943                 }
2944         }
2945
2946         tm_dbg_release_cmd(cmd);
2947
2948         TRACE_EXIT();
2949         return;
2950 }
2951
2952 /* Called under scst_list_lock and IRQ off */
2953 static int scst_set_mcmd_next_state(struct scst_mgmt_cmd *mcmd)
2954 {
2955         int res;
2956         if (mcmd->cmd_wait_count != 0) {
2957                 TRACE_MGMT_DBG("cmd_wait_count(%d) not 0, preparing to "
2958                         "wait", mcmd->cmd_wait_count);
2959                 mcmd->state = SCST_MGMT_CMD_STATE_EXECUTING;
2960                 res = -1;
2961         } else {
2962                 mcmd->state = SCST_MGMT_CMD_STATE_DONE;
2963                 res = 0;
2964         }
2965         mcmd->completed = 1;
2966         return res;
2967 }
2968
2969 static void scst_unblock_aborted_cmds(int scst_mutex_held)
2970 {
2971         struct scst_device *dev;
2972         int wake = 0;
2973
2974         TRACE_ENTRY();
2975
2976         if (!scst_mutex_held)
2977                 down(&scst_mutex);
2978
2979         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
2980                 struct scst_cmd *cmd, *tcmd;
2981                 spin_lock_bh(&dev->dev_lock);
2982                 list_for_each_entry_safe(cmd, tcmd, &dev->blocked_cmd_list,
2983                                         blocked_cmd_list_entry) {
2984                         if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2985                                 list_del(&cmd->blocked_cmd_list_entry);
2986                                 TRACE_MGMT_DBG("Moving aborted blocked cmd %p "
2987                                         "to active cmd list", cmd);
2988                                 spin_lock_irq(&scst_list_lock);
2989                                 list_move_tail(&cmd->cmd_list_entry,
2990                                         &scst_active_cmd_list);
2991                                 spin_unlock_irq(&scst_list_lock);
2992                                 wake = 1;
2993                         }
2994                 }
2995                 spin_unlock_bh(&dev->dev_lock);
2996         }
2997
2998         if (!scst_mutex_held)
2999                 up(&scst_mutex);
3000
3001         if (wake)
3002                 wake_up(&scst_list_waitQ);
3003
3004         TRACE_EXIT();
3005         return;
3006 }
3007
3008 /* Returns 0 if the command processing should be continued, <0 otherwise */
3009 static void __scst_abort_task_set(struct scst_mgmt_cmd *mcmd,
3010         struct scst_tgt_dev *tgt_dev, int other_ini, int scst_mutex_held)
3011 {
3012         struct scst_cmd *cmd;
3013         struct scst_session *sess = tgt_dev->sess;
3014
3015         TRACE_ENTRY();
3016
3017         local_bh_disable();
3018         spin_lock_irq(&scst_list_lock);
3019
3020         TRACE_DBG("Searching in search cmd list (sess=%p)", sess);
3021         list_for_each_entry(cmd, &sess->search_cmd_list, 
3022                         search_cmd_list_entry) {
3023                 if ((cmd->tgt_dev == NULL) && 
3024                     (cmd->lun == tgt_dev->acg_dev->lun))
3025                         continue;
3026                 if (cmd->tgt_dev != tgt_dev)
3027                         continue;
3028                 scst_abort_cmd(cmd, mcmd, other_ini, 0);
3029         }
3030         spin_unlock_irq(&scst_list_lock);
3031         local_bh_enable();
3032
3033         scst_unblock_aborted_cmds(scst_mutex_held);
3034
3035         TRACE_EXIT();
3036         return;
3037 }
3038
3039 /* Returns 0 if the command processing should be continued, <0 otherwise */
3040 static int scst_abort_task_set(struct scst_mgmt_cmd *mcmd)
3041 {
3042         int res;
3043         struct scst_tgt_dev *tgt_dev = mcmd->mcmd_tgt_dev;
3044         struct scst_device *dev = tgt_dev->acg_dev->dev;
3045
3046         TRACE(TRACE_MGMT, "Aborting task set (lun=%d, mcmd=%p)",
3047                 tgt_dev->acg_dev->lun, mcmd);
3048
3049         spin_lock_bh(&dev->dev_lock);
3050         __scst_block_dev(dev);
3051         spin_unlock_bh(&dev->dev_lock);
3052
3053         __scst_abort_task_set(mcmd, tgt_dev, 0, 0);
3054         scst_call_dev_task_mgmt_fn(mcmd, tgt_dev, 0);
3055
3056         res = scst_set_mcmd_next_state(mcmd);
3057
3058         TRACE_EXIT_RES(res);
3059         return res;
3060 }
3061
3062 static int scst_check_delay_mgmt_cmd(struct scst_mgmt_cmd *mcmd, int locked)
3063 {
3064         /*
3065          * No need for special protection for SCST_FLAG_TM_ACTIVE, since
3066          * we could be called from the only thread.
3067          */
3068         if (test_bit(SCST_FLAG_TM_ACTIVE, &scst_flags)) {
3069                 TRACE_MGMT_DBG("Moving mgmt cmd %p to delayed mgmt cmd list",
3070                         mcmd);
3071                 if (!locked)
3072                         spin_lock_irq(&scst_list_lock);
3073                 list_move_tail(&mcmd->mgmt_cmd_list_entry, 
3074                         &scst_delayed_mgmt_cmd_list);
3075                 if (!locked)
3076                         spin_unlock_irq(&scst_list_lock);
3077                 return -1;
3078         } else {
3079                 set_bit(SCST_FLAG_TM_ACTIVE, &scst_flags);
3080                 return 0;
3081         }
3082 }
3083
3084 /* Returns 0 if the command processing should be continued, 
3085  * >0, if it should be requeued, <0 otherwise */
3086 static int scst_mgmt_cmd_init(struct scst_mgmt_cmd *mcmd)
3087 {
3088         int res = 0;
3089
3090         TRACE_ENTRY();
3091
3092         res = scst_check_delay_mgmt_cmd(mcmd, 1);
3093         if (res != 0)
3094                 goto out;
3095
3096         if (mcmd->fn == SCST_ABORT_TASK) {
3097                 struct scst_session *sess = mcmd->sess;
3098                 struct scst_cmd *cmd;
3099
3100                 local_bh_disable();
3101                 spin_lock_irq(&scst_list_lock);
3102                 cmd = __scst_find_cmd_by_tag(sess, mcmd->tag);
3103                 if (cmd == NULL) {
3104                         TRACE(TRACE_MGMT, "ABORT TASK failed: command for "
3105                                 "tag %d not found", mcmd->tag);
3106                         mcmd->status = SCST_MGMT_STATUS_TASK_NOT_EXIST;
3107                         mcmd->state = SCST_MGMT_CMD_STATE_DONE;
3108                 } else {
3109                         TRACE(TRACE_MGMT, "Cmd %p for tag %d (sn %d) found, "
3110                                 "aborting it", cmd, mcmd->tag, cmd->sn);
3111                         mcmd->cmd_to_abort = cmd;
3112                         scst_abort_cmd(cmd, mcmd, 0, 1);
3113                         res = scst_set_mcmd_next_state(mcmd);
3114                         mcmd->cmd_to_abort = NULL; /* just in case */
3115                 }
3116                 spin_unlock_irq(&scst_list_lock);
3117                 local_bh_enable();
3118         } else {
3119                 int rc;
3120                 rc = scst_mgmt_translate_lun(mcmd);
3121                 if (rc < 0) {
3122                         PRINT_ERROR_PR("Corresponding device for lun %Ld not "
3123                                 "found", (uint64_t)mcmd->lun);
3124                         mcmd->status = SCST_MGMT_STATUS_LUN_NOT_EXIST;
3125                         mcmd->state = SCST_MGMT_CMD_STATE_DONE;
3126                 } else if (rc == 0)
3127                         mcmd->state = SCST_MGMT_CMD_STATE_READY;
3128                 else
3129                         res = rc;
3130         }
3131
3132 out:
3133         TRACE_EXIT_RES(res);
3134         return res;
3135 }
3136
3137 /* Returns 0 if the command processing should be continued, <0 otherwise */
3138 static int scst_target_reset(struct scst_mgmt_cmd *mcmd)
3139 {
3140         int res, rc;
3141         struct scst_device *dev, *d;
3142         struct scst_tgt_dev *tgt_dev;
3143         int cont, c;
3144         LIST_HEAD(host_devs);
3145
3146         TRACE_ENTRY();
3147
3148         TRACE(TRACE_MGMT, "Target reset (mcmd %p, cmd count %d)",
3149                 mcmd, mcmd->sess->sess_cmd_count);
3150
3151         down(&scst_mutex);
3152
3153         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
3154                 int found = 0;
3155
3156                 spin_lock_bh(&dev->dev_lock);
3157                 __scst_block_dev(dev);
3158                 scst_process_reset(dev, mcmd->sess, NULL, mcmd);
3159                 spin_unlock_bh(&dev->dev_lock);
3160
3161                 cont = 0;
3162                 c = 0;
3163                 list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
3164                         dev_tgt_dev_list_entry) 
3165                 {
3166                         cont = 1;
3167                         rc = scst_call_dev_task_mgmt_fn(mcmd, tgt_dev, 0);
3168                         if (rc == SCST_DEV_TM_NOT_COMPLETED) 
3169                                 c = 1;
3170                         else if ((rc < 0) &&
3171                                  (mcmd->status == SCST_MGMT_STATUS_SUCCESS))
3172                                 mcmd->status = rc;
3173                 }
3174                 if (cont && !c)
3175                         continue;
3176                 
3177                 if (dev->scsi_dev == NULL)
3178                         continue;
3179
3180                 list_for_each_entry(d, &host_devs, reset_dev_list_entry) {
3181                         if (dev->scsi_dev->host->host_no ==
3182                                     d->scsi_dev->host->host_no) 
3183                         {
3184                                 found = 1;
3185                                 break;
3186                         }
3187                 }
3188                 if (!found)
3189                         list_add_tail(&dev->reset_dev_list_entry, &host_devs);
3190         }
3191
3192         /*
3193          * We suppose here that for all commands that already on devices
3194          * on/after scsi_reset_provider() completion callbacks will be called.
3195          */
3196
3197         list_for_each_entry(dev, &host_devs, reset_dev_list_entry) {
3198                 /* dev->scsi_dev must be non-NULL here */
3199                 TRACE(TRACE_MGMT, "Resetting host %d bus ",
3200                       dev->scsi_dev->host->host_no);
3201                 rc = scsi_reset_provider(dev->scsi_dev, SCSI_TRY_RESET_BUS);
3202                 TRACE(TRACE_MGMT, "Result of host %d bus reset: %s",
3203                       dev->scsi_dev->host->host_no,
3204                       (rc == SUCCESS) ? "SUCCESS" : "FAILED");
3205                 if ((rc != SUCCESS) &&
3206                     (mcmd->status == SCST_MGMT_STATUS_SUCCESS)) {
3207                         /* SCSI_TRY_RESET_BUS is also done by scsi_reset_provider() */
3208                         mcmd->status = SCST_MGMT_STATUS_FAILED;
3209                 }
3210         }
3211
3212         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
3213                 if (dev->scsi_dev != NULL)
3214                         dev->scsi_dev->was_reset = 0;
3215         }
3216
3217         up(&scst_mutex);
3218
3219         spin_lock_irq(&scst_list_lock);
3220         tm_dbg_task_mgmt("TARGET RESET");
3221         res = scst_set_mcmd_next_state(mcmd);
3222         spin_unlock_irq(&scst_list_lock);
3223
3224         TRACE_EXIT_RES(res);
3225         return res;
3226 }
3227
3228 /* Returns 0 if the command processing should be continued, <0 otherwise */
3229 static int scst_lun_reset(struct scst_mgmt_cmd *mcmd)
3230 {
3231         int res, rc;
3232         struct scst_tgt_dev *tgt_dev = mcmd->mcmd_tgt_dev;
3233         struct scst_device *dev = tgt_dev->acg_dev->dev;
3234
3235         TRACE_ENTRY();
3236
3237         TRACE(TRACE_MGMT, "Resetting lun %d (mcmd %p)", tgt_dev->acg_dev->lun,
3238                 mcmd);
3239
3240         spin_lock_bh(&dev->dev_lock);
3241         __scst_block_dev(dev);
3242         scst_process_reset(dev, mcmd->sess, NULL, mcmd);
3243         spin_unlock_bh(&dev->dev_lock);
3244
3245         rc = scst_call_dev_task_mgmt_fn(mcmd, tgt_dev, 1);
3246         if (rc != SCST_DEV_TM_NOT_COMPLETED)
3247                 goto out_tm_dbg;
3248
3249         if (dev->scsi_dev != NULL) {
3250                 TRACE(TRACE_MGMT, "Resetting host %d bus ",
3251                       dev->scsi_dev->host->host_no);
3252                 rc = scsi_reset_provider(dev->scsi_dev, SCSI_TRY_RESET_DEVICE);
3253                 if ((rc != SUCCESS) && (mcmd->status == SCST_MGMT_STATUS_SUCCESS))
3254                         mcmd->status = SCST_MGMT_STATUS_FAILED;
3255                 dev->scsi_dev->was_reset = 0;
3256         }
3257
3258 out_tm_dbg:
3259         spin_lock_irq(&scst_list_lock);
3260         tm_dbg_task_mgmt("LUN RESET");
3261         res = scst_set_mcmd_next_state(mcmd);
3262         spin_unlock_irq(&scst_list_lock);
3263
3264         TRACE_EXIT_RES(res);
3265         return res;
3266 }
3267
3268 /* Returns 0 if the command processing should be continued, <0 otherwise */
3269 static int scst_abort_all_nexus_loss_sess(struct scst_mgmt_cmd *mcmd,
3270         int nexus_loss)
3271 {
3272         int res;
3273         struct scst_session *sess = mcmd->sess;
3274         struct scst_tgt_dev *tgt_dev;
3275
3276         TRACE_ENTRY();
3277
3278         if (nexus_loss) {
3279                 TRACE(TRACE_MGMT, "Nexus loss for sess %p (mcmd %p)", sess,
3280                         mcmd);
3281         } else {
3282                 TRACE(TRACE_MGMT, "Aborting all from sess %p (mcmd %p)", sess,
3283                         mcmd);
3284         }
3285
3286         down(&scst_mutex);
3287         list_for_each_entry(tgt_dev, &sess->sess_tgt_dev_list,
3288                 sess_tgt_dev_list_entry) 
3289         {
3290                 struct scst_device *dev = tgt_dev->acg_dev->dev;
3291                 int rc;
3292
3293                 spin_lock_bh(&dev->dev_lock);
3294                 __scst_block_dev(dev);
3295                 spin_unlock_bh(&dev->dev_lock);
3296
3297                 rc = scst_call_dev_task_mgmt_fn(mcmd, tgt_dev, 0);
3298                 if ((rc < 0) && (mcmd->status == SCST_MGMT_STATUS_SUCCESS))
3299                         mcmd->status = rc;
3300
3301                 __scst_abort_task_set(mcmd, tgt_dev, !nexus_loss, 1);
3302                 if (nexus_loss)
3303                         scst_reset_tgt_dev(tgt_dev, 1);
3304         }
3305         up(&scst_mutex);
3306
3307         spin_lock_irq(&scst_list_lock);
3308         res = scst_set_mcmd_next_state(mcmd);
3309         spin_unlock_irq(&scst_list_lock);
3310
3311         TRACE_EXIT_RES(res);
3312         return res;
3313 }
3314
3315 /* Returns 0 if the command processing should be continued, <0 otherwise */
3316 static int scst_abort_all_nexus_loss_tgt(struct scst_mgmt_cmd *mcmd,
3317         int nexus_loss)
3318 {
3319         int res;
3320         struct scst_tgt *tgt = mcmd->sess->tgt;
3321         struct scst_session *sess;
3322         struct scst_device *dev;
3323         struct scst_tgt_dev *tgt_dev;
3324
3325         TRACE_ENTRY();
3326
3327         if (nexus_loss) {
3328                 TRACE(TRACE_MGMT, "I_T Nexus loss (tgt %p, mcmd %p)", tgt,
3329                         mcmd);
3330         } else {
3331                 TRACE(TRACE_MGMT, "Aborting all from tgt %p (mcmd %p)", tgt,
3332                         mcmd);
3333         }
3334
3335         down(&scst_mutex);
3336
3337         list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
3338                 spin_lock_bh(&dev->dev_lock);
3339                 __scst_block_dev(dev);
3340                 spin_unlock_bh(&dev->dev_lock);
3341         }
3342
3343         list_for_each_entry(sess, &tgt->sess_list, sess_list_entry) {
3344                 list_for_each_entry(tgt_dev, &sess->sess_tgt_dev_list,
3345                         sess_tgt_dev_list_entry) 
3346                 {
3347                         int rc;
3348
3349                         rc = scst_call_dev_task_mgmt_fn(mcmd, tgt_dev, 0);
3350                         if ((rc < 0) &&
3351                             (mcmd->status == SCST_MGMT_STATUS_SUCCESS))
3352                                 mcmd->status = rc;
3353
3354                         __scst_abort_task_set(mcmd, tgt_dev, !nexus_loss, 1);
3355                         if (nexus_loss)
3356                                 scst_reset_tgt_dev(tgt_dev, 1);
3357                 }
3358         }
3359
3360         up(&scst_mutex);
3361
3362         spin_lock_irq(&scst_list_lock);
3363         res = scst_set_mcmd_next_state(mcmd);
3364         spin_unlock_irq(&scst_list_lock);
3365
3366         TRACE_EXIT_RES(res);
3367         return res;
3368 }
3369
3370 /* Returns 0 if the command processing should be continued, <0 otherwise */
3371 static int scst_mgmt_cmd_exec(struct scst_mgmt_cmd *mcmd)
3372 {
3373         int res = 0;
3374
3375         TRACE_ENTRY();
3376
3377         mcmd->status = SCST_MGMT_STATUS_SUCCESS;
3378
3379         switch (mcmd->fn) {
3380         case SCST_ABORT_TASK_SET:
3381         case SCST_CLEAR_TASK_SET:
3382                 res = scst_abort_task_set(mcmd);
3383                 break;
3384
3385         case SCST_LUN_RESET:
3386                 res = scst_lun_reset(mcmd);
3387                 break;
3388
3389         case SCST_TARGET_RESET:
3390                 res = scst_target_reset(mcmd);
3391                 break;
3392
3393         case SCST_ABORT_ALL_TASKS_SESS:
3394                 res = scst_abort_all_nexus_loss_sess(mcmd, 0);
3395                 break;
3396
3397         case SCST_NEXUS_LOSS_SESS:
3398                 res = scst_abort_all_nexus_loss_sess(mcmd, 1);
3399                 break;
3400
3401         case SCST_ABORT_ALL_TASKS:
3402                 res = scst_abort_all_nexus_loss_tgt(mcmd, 0);
3403                 break;
3404
3405         case SCST_NEXUS_LOSS:
3406                 res = scst_abort_all_nexus_loss_tgt(mcmd, 1);
3407                 break;
3408
3409         case SCST_CLEAR_ACA:
3410                 if (scst_call_dev_task_mgmt_fn(mcmd, mcmd->mcmd_tgt_dev, 1) ==
3411                                 SCST_DEV_TM_NOT_COMPLETED) {
3412                         mcmd->status = SCST_MGMT_STATUS_FN_NOT_SUPPORTED;
3413                         /* Nothing to do (yet) */
3414                 }
3415                 break;
3416
3417         default:
3418                 PRINT_ERROR_PR("Unknown task management function %d", mcmd->fn);
3419                 mcmd->status = SCST_MGMT_STATUS_REJECTED;
3420                 break;
3421         }
3422
3423         TRACE_EXIT_RES(res);
3424         return res;
3425 }
3426
3427 static void scst_mgmt_cmd_send_done(struct scst_mgmt_cmd *mcmd)
3428 {
3429         struct scst_device *dev;
3430         struct scst_tgt_dev *tgt_dev;
3431
3432         TRACE_ENTRY();
3433
3434         clear_bit(SCST_FLAG_TM_ACTIVE, &scst_flags);
3435         if (!list_empty(&scst_delayed_mgmt_cmd_list)) {
3436                 struct scst_mgmt_cmd *m;
3437                 spin_lock_irq(&scst_list_lock);
3438                 m = list_entry(scst_delayed_mgmt_cmd_list.next, typeof(*m),
3439                                 mgmt_cmd_list_entry);
3440                 TRACE_MGMT_DBG("Moving delayed mgmt cmd %p to active mgmt "
3441                         "cmd list", m);
3442                 list_move(&m->mgmt_cmd_list_entry, &scst_active_mgmt_cmd_list);
3443                 spin_unlock_irq(&scst_list_lock);
3444         }
3445
3446         mcmd->state = SCST_MGMT_CMD_STATE_FINISHED;
3447         if (scst_is_strict_mgmt_fn(mcmd->fn) && (mcmd->completed_cmd_count > 0))
3448                 mcmd->status = SCST_MGMT_STATUS_FAILED;
3449
3450         if (mcmd->sess->tgt->tgtt->task_mgmt_fn_done) {
3451                 TRACE_DBG("Calling target %s task_mgmt_fn_done()",
3452                       mcmd->sess->tgt->tgtt->name);
3453                 mcmd->sess->tgt->tgtt->task_mgmt_fn_done(mcmd);
3454                 TRACE_MGMT_DBG("Dev handler %s task_mgmt_fn_done() returned",
3455                       mcmd->sess->tgt->tgtt->name);
3456         }
3457
3458         switch (mcmd->fn) {
3459         case SCST_ABORT_TASK_SET:
3460         case SCST_CLEAR_TASK_SET:
3461         case SCST_LUN_RESET:
3462                 scst_unblock_dev(mcmd->mcmd_tgt_dev->acg_dev->dev);
3463                 break;
3464
3465         case SCST_TARGET_RESET:
3466         case SCST_ABORT_ALL_TASKS:
3467         case SCST_NEXUS_LOSS:
3468                 down(&scst_mutex);
3469                 list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
3470                         scst_unblock_dev(dev);
3471                 }
3472                 up(&scst_mutex);
3473                 break;
3474
3475         case SCST_NEXUS_LOSS_SESS:
3476         case SCST_ABORT_ALL_TASKS_SESS:
3477                 down(&scst_mutex);
3478                 list_for_each_entry(tgt_dev, &mcmd->sess->sess_tgt_dev_list,
3479                                 sess_tgt_dev_list_entry) {
3480                         scst_unblock_dev(tgt_dev->acg_dev->dev);
3481                 }
3482                 up(&scst_mutex);
3483                 break;
3484
3485         case SCST_CLEAR_ACA:
3486         default:
3487                 break;
3488         }
3489
3490         mcmd->tgt_priv = NULL;
3491
3492         TRACE_EXIT();
3493         return;
3494 }
3495
3496 /* Returns >0, if cmd should be requeued */
3497 static int scst_process_mgmt_cmd(struct scst_mgmt_cmd *mcmd)
3498 {
3499         int res = 0;
3500
3501         TRACE_ENTRY();
3502
3503         TRACE_DBG("mcmd %p, state %d", mcmd, mcmd->state);
3504
3505         while (1) {
3506                 switch (mcmd->state) {
3507                 case SCST_MGMT_CMD_STATE_INIT:
3508                         res = scst_mgmt_cmd_init(mcmd);
3509                         if (res)
3510                                 goto out;
3511                         break;
3512
3513                 case SCST_MGMT_CMD_STATE_READY:
3514                         if (scst_mgmt_cmd_exec(mcmd))
3515                                 goto out;
3516                         break;
3517
3518                 case SCST_MGMT_CMD_STATE_DONE:
3519                         scst_mgmt_cmd_send_done(mcmd);
3520                         break;
3521
3522                 case SCST_MGMT_CMD_STATE_FINISHED:
3523                         goto out_free;
3524
3525 #ifdef EXTRACHECKS
3526                 case SCST_MGMT_CMD_STATE_EXECUTING:
3527                         sBUG();
3528 #endif
3529
3530                 default:
3531                         PRINT_ERROR_PR("Unknown state %d of management command",
3532                                     mcmd->state);
3533                         res = -1;
3534                         goto out_free;
3535                 }
3536         }
3537
3538 out:
3539         TRACE_EXIT_RES(res);
3540         return res;
3541
3542 out_free:
3543         scst_free_mgmt_cmd(mcmd, 1);
3544         goto out;
3545 }
3546
3547 static inline int test_mgmt_cmd_list(void)
3548 {
3549         int res = (!list_empty(&scst_active_mgmt_cmd_list) &&
3550                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
3551                   test_bit(SCST_FLAG_SHUTDOWN, &scst_flags);
3552         return res;
3553 }
3554
3555 int scst_mgmt_cmd_thread(void *arg)
3556 {
3557         struct scst_mgmt_cmd *mcmd;
3558
3559         TRACE_ENTRY();
3560
3561         daemonize("scsi_tgt_mc");
3562         recalc_sigpending();
3563         current->flags |= PF_NOFREEZE;
3564
3565         spin_lock_irq(&scst_list_lock);
3566         while (1) {
3567                 wait_queue_t wait;
3568                 init_waitqueue_entry(&wait, current);
3569
3570                 if (!test_mgmt_cmd_list()) {
3571                         add_wait_queue_exclusive(&scst_mgmt_cmd_list_waitQ,
3572                                                  &wait);
3573                         for (;;) {
3574                                 set_current_state(TASK_INTERRUPTIBLE);
3575                                 if (test_mgmt_cmd_list())
3576                                         break;
3577                                 spin_unlock_irq(&scst_list_lock);
3578                                 schedule();
3579                                 spin_lock_irq(&scst_list_lock);
3580                         }
3581                         set_current_state(TASK_RUNNING);
3582                         remove_wait_queue(&scst_mgmt_cmd_list_waitQ, &wait);
3583                 }
3584
3585                 while (!list_empty(&scst_active_mgmt_cmd_list) &&
3586                        !test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
3587                 {
3588                         int rc;
3589                         mcmd = list_entry(scst_active_mgmt_cmd_list.next,
3590                                           typeof(*mcmd), mgmt_cmd_list_entry);
3591                         TRACE_MGMT_DBG("Moving mgmt cmd %p to mgmt cmd list",
3592                               mcmd);
3593                         list_move_tail(&mcmd->mgmt_cmd_list_entry,
3594                                        &scst_mgmt_cmd_list);
3595                         spin_unlock_irq(&scst_list_lock);
3596                         rc = scst_process_mgmt_cmd(mcmd);
3597                         spin_lock_irq(&scst_list_lock);
3598                         if (rc > 0) {
3599                                 TRACE_MGMT_DBG("Moving mgmt cmd %p to head "
3600                                         "of active mgmt cmd list", mcmd);
3601                                 list_move(&mcmd->mgmt_cmd_list_entry,
3602