Heavily modified patch from Gennadiy Nerubayev <parakie@gmail.com>.
[mirror/scst/.git] / scst / src / scst_targ.c
1 /*
2  *  scst_targ.c
3  *
4  *  Copyright (C) 2004 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2004 - 2005 Leonid Stoljar
6  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
7  *
8  *  This program is free software; you can redistribute it and/or
9  *  modify it under the terms of the GNU General Public License
10  *  as published by the Free Software Foundation, version 2
11  *  of the License.
12  *
13  *  This program is distributed in the hope that it will be useful,
14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  *  GNU General Public License for more details.
17  */
18
19 #include <linux/init.h>
20 #include <linux/kernel.h>
21 #include <linux/errno.h>
22 #include <linux/list.h>
23 #include <linux/spinlock.h>
24 #include <linux/slab.h>
25 #include <linux/sched.h>
26 #include <linux/smp_lock.h>
27 #include <linux/unistd.h>
28 #include <linux/string.h>
29 #include <linux/kthread.h>
30 #include <linux/delay.h>
31
32 #include "scst.h"
33 #include "scst_priv.h"
34
35 static void scst_cmd_set_sn(struct scst_cmd *cmd);
36 static int __scst_init_cmd(struct scst_cmd *cmd);
37 static void scst_finish_cmd_mgmt(struct scst_cmd *cmd);
38 static struct scst_cmd *__scst_find_cmd_by_tag(struct scst_session *sess,
39         uint64_t tag);
40 static void scst_proccess_redirect_cmd(struct scst_cmd *cmd,
41         enum scst_exec_context context, int check_retries);
42
43 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
44 {
45         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
46         unsigned long flags;
47
48         spin_lock_irqsave(&t->tasklet_lock, flags);
49         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
50                 smp_processor_id());
51         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
52         spin_unlock_irqrestore(&t->tasklet_lock, flags);
53
54         tasklet_schedule(&t->tasklet);
55 }
56
57 /*
58  * Must not be called in parallel with scst_unregister_session_ex() for the
59  * same sess
60  */
61 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
62                              const uint8_t *lun, int lun_len,
63                              const uint8_t *cdb, int cdb_len, int atomic)
64 {
65         struct scst_cmd *cmd;
66
67         TRACE_ENTRY();
68
69 #ifdef CONFIG_SCST_EXTRACHECKS
70         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
71                 PRINT_CRIT_ERROR("%s",
72                         "New cmd while shutting down the session");
73                 sBUG();
74         }
75 #endif
76
77         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
78         if (cmd == NULL)
79                 goto out;
80
81         cmd->sess = sess;
82         cmd->tgt = sess->tgt;
83         cmd->tgtt = sess->tgt->tgtt;
84
85         /*
86          * For both wrong lun and CDB defer the error reporting for
87          * scst_cmd_init_done()
88          */
89
90         cmd->lun = scst_unpack_lun(lun, lun_len);
91
92         if (cdb_len <= SCST_MAX_CDB_SIZE) {
93                 memcpy(cmd->cdb, cdb, cdb_len);
94                 cmd->cdb_len = cdb_len;
95         }
96
97         TRACE_DBG("cmd %p, sess %p", cmd, sess);
98         scst_sess_get(sess);
99
100 out:
101         TRACE_EXIT();
102         return cmd;
103 }
104 EXPORT_SYMBOL(scst_rx_cmd);
105
106 /*
107  * No locks, but might be on IRQ. Returns 0 on success, <0 if processing of
108  * this command should be stopped.
109  */
110 static int scst_init_cmd(struct scst_cmd *cmd, enum scst_exec_context *context)
111 {
112         int rc, res = 0;
113
114         TRACE_ENTRY();
115
116         /* See the comment in scst_do_job_init() */
117         if (unlikely(!list_empty(&scst_init_cmd_list))) {
118                 TRACE_MGMT_DBG("%s", "init cmd list busy");
119                 goto out_redirect;
120         }
121         /*
122          * Memory barrier isn't necessary here, because CPU appears to
123          * be self-consistent and we don't care about the race, described
124          * in comment in scst_do_job_init().
125          */
126
127         rc = __scst_init_cmd(cmd);
128         if (unlikely(rc > 0))
129                 goto out_redirect;
130         else if (unlikely(rc != 0))
131                 goto out;
132
133         /* Small context optimization */
134         if (((*context == SCST_CONTEXT_TASKLET) ||
135              (*context == SCST_CONTEXT_DIRECT_ATOMIC) ||
136              ((*context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd))) &&
137               scst_cmd_is_expected_set(cmd)) {
138                 if (cmd->expected_data_direction == SCST_DATA_WRITE) {
139                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
140                                         &cmd->tgt_dev->tgt_dev_flags))
141                                 *context = SCST_CONTEXT_THREAD;
142                 } else {
143                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
144                                         &cmd->tgt_dev->tgt_dev_flags))
145                                 *context = SCST_CONTEXT_THREAD;
146                 }
147         }
148
149 out:
150         TRACE_EXIT_RES(res);
151         return res;
152
153 out_redirect:
154         if (cmd->preprocessing_only) {
155                 /*
156                  * Poor man solution for single threaded targets, where
157                  * blocking receiver at least sometimes means blocking all.
158                  */
159                 sBUG_ON(*context != SCST_CONTEXT_DIRECT);
160                 scst_set_busy(cmd);
161                 scst_set_cmd_abnormal_done_state(cmd);
162                 /* Keep initiator away from too many BUSY commands */
163                 msleep(50);
164         } else {
165                 unsigned long flags;
166                 spin_lock_irqsave(&scst_init_lock, flags);
167                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
168                         "%d)", cmd, atomic_read(&scst_cmd_count));
169                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
170                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
171                         scst_init_poll_cnt++;
172                 spin_unlock_irqrestore(&scst_init_lock, flags);
173                 wake_up(&scst_init_cmd_list_waitQ);
174                 res = -1;
175         }
176         goto out;
177 }
178
179 #ifdef CONFIG_SCST_MEASURE_LATENCY
180 static inline uint64_t scst_sec_to_nsec(time_t sec)
181 {
182         return (uint64_t)sec * 1000000000;
183 }
184 #endif
185
186 void scst_cmd_init_done(struct scst_cmd *cmd,
187         enum scst_exec_context pref_context)
188 {
189         unsigned long flags;
190         struct scst_session *sess = cmd->sess;
191         int rc;
192
193         TRACE_ENTRY();
194
195 #ifdef CONFIG_SCST_MEASURE_LATENCY
196         {
197                 struct timespec ts;
198                 getnstimeofday(&ts);
199                 cmd->start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
200                 TRACE_DBG("cmd %p (sess %p): start %lld (tv_sec %ld, "
201                         "tv_nsec %ld)", cmd, sess, cmd->start, ts.tv_sec,
202                         ts.tv_nsec);
203         }
204 #endif
205
206         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
207         TRACE(TRACE_SCSI, "tag=%llu, lun=%lld, CDB len=%d, queue_type=%x "
208                 "(cmd %p)", (long long unsigned int)cmd->tag,
209                 (long long unsigned int)cmd->lun, cmd->cdb_len,
210                 cmd->queue_type, cmd);
211         PRINT_BUFF_FLAG(TRACE_SCSI|TRACE_RCV_BOT, "Recieving CDB",
212                 cmd->cdb, cmd->cdb_len);
213
214 #ifdef CONFIG_SCST_EXTRACHECKS
215         if (unlikely((in_irq() || irqs_disabled())) &&
216             ((pref_context == SCST_CONTEXT_DIRECT) ||
217              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
218                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
219                         "SCST_CONTEXT_THREAD instead\n", pref_context,
220                         cmd->tgtt->name);
221                 pref_context = SCST_CONTEXT_THREAD;
222         }
223 #endif
224
225         atomic_inc(&sess->sess_cmd_count);
226
227         spin_lock_irqsave(&sess->sess_list_lock, flags);
228
229         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
230                 /*
231                  * We have to always keep command in the search list from the
232                  * very beginning, because otherwise it can be missed during
233                  * TM processing. This check is needed because there might be
234                  * old, i.e. deferred, commands and new, i.e. just coming, ones.
235                  */
236                 if (cmd->search_cmd_list_entry.next == NULL)
237                         list_add_tail(&cmd->search_cmd_list_entry,
238                                 &sess->search_cmd_list);
239                 switch (sess->init_phase) {
240                 case SCST_SESS_IPH_SUCCESS:
241                         break;
242                 case SCST_SESS_IPH_INITING:
243                         TRACE_DBG("Adding cmd %p to init deferred cmd list",
244                                   cmd);
245                         list_add_tail(&cmd->cmd_list_entry,
246                                 &sess->init_deferred_cmd_list);
247                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
248                         goto out;
249                 case SCST_SESS_IPH_FAILED:
250                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
251                         scst_set_busy(cmd);
252                         scst_set_cmd_abnormal_done_state(cmd);
253                         goto active;
254                 default:
255                         sBUG();
256                 }
257         } else
258                 list_add_tail(&cmd->search_cmd_list_entry,
259                               &sess->search_cmd_list);
260
261         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
262
263         if (unlikely(cmd->lun == NO_SUCH_LUN)) {
264                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
265                 scst_set_cmd_error(cmd,
266                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
267                 scst_set_cmd_abnormal_done_state(cmd);
268                 goto active;
269         }
270
271         if (unlikely(cmd->cdb_len == 0)) {
272                 PRINT_ERROR("Wrong CDB len %d, finishing cmd", 0);
273                 scst_set_cmd_error(cmd,
274                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
275                 scst_set_cmd_abnormal_done_state(cmd);
276                 goto active;
277         }
278
279         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
280                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
281                 scst_set_cmd_error(cmd,
282                         SCST_LOAD_SENSE(scst_sense_invalid_message));
283                 scst_set_cmd_abnormal_done_state(cmd);
284                 goto active;
285         }
286
287         cmd->state = SCST_CMD_STATE_INIT;
288         /* cmd must be inited here to preserve the order */
289         rc = scst_init_cmd(cmd, &pref_context);
290         if (unlikely(rc < 0))
291                 goto out;
292
293 active:
294         /* Here cmd must not be in any cmd list, no locks */
295         switch (pref_context) {
296         case SCST_CONTEXT_TASKLET:
297                 scst_schedule_tasklet(cmd);
298                 break;
299
300         case SCST_CONTEXT_DIRECT:
301                 scst_process_active_cmd(cmd, false);
302                 /* For *NEED_THREAD wake_up() is already done */
303                 break;
304
305         case SCST_CONTEXT_DIRECT_ATOMIC:
306                 scst_process_active_cmd(cmd, true);
307                 /* For *NEED_THREAD wake_up() is already done */
308                 break;
309
310         default:
311                 PRINT_ERROR("Context %x is undefined, using the thread one",
312                         pref_context);
313                 /* go through */
314         case SCST_CONTEXT_THREAD:
315                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
316                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
317                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
318                         list_add(&cmd->cmd_list_entry,
319                                 &cmd->cmd_lists->active_cmd_list);
320                 else
321                         list_add_tail(&cmd->cmd_list_entry,
322                                 &cmd->cmd_lists->active_cmd_list);
323                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
324                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
325                 break;
326         }
327
328 out:
329         TRACE_EXIT();
330         return;
331 }
332 EXPORT_SYMBOL(scst_cmd_init_done);
333
334 static int scst_pre_parse(struct scst_cmd *cmd)
335 {
336         int res = SCST_CMD_STATE_RES_CONT_SAME;
337         struct scst_device *dev = cmd->dev;
338         int rc;
339
340         TRACE_ENTRY();
341
342         cmd->inc_expected_sn_on_done = dev->handler->exec_sync ||
343              (!dev->has_own_order_mgmt &&
344               (dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER ||
345                cmd->queue_type == SCST_CMD_QUEUE_ORDERED));
346
347         /*
348          * Expected transfer data supplied by the SCSI transport via the
349          * target driver are untrusted, so we prefer to fetch them from CDB.
350          * Additionally, not all transports support supplying the expected
351          * transfer data.
352          */
353
354         rc = scst_get_cdb_info(cmd);
355         if (unlikely(rc != 0)) {
356                 if (rc > 0) {
357                         PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
358                         goto out_xmit;
359                 }
360                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
361                         "Should you update scst_scsi_op_table?",
362                         cmd->cdb[0], dev->handler->name);
363                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
364 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
365                 if (scst_cmd_is_expected_set(cmd)) {
366                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
367                                 "direction %d, transfer_len %d",
368                                 cmd->expected_data_direction,
369                                 cmd->expected_transfer_len);
370                         cmd->data_direction = cmd->expected_data_direction;
371
372                         cmd->bufflen = cmd->expected_transfer_len;
373                         /* Restore (likely) lost CDB length */
374                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
375                         if (cmd->cdb_len == -1) {
376                                 PRINT_ERROR("Unable to get CDB length for "
377                                         "opcode 0x%02x. Returning INVALID "
378                                         "OPCODE", cmd->cdb[0]);
379                                 scst_set_cmd_error(cmd,
380                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
381                                 goto out_xmit;
382                         }
383                 } else {
384                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
385                              "target %s not supplied expected values",
386                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
387                         scst_set_cmd_error(cmd,
388                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
389                         goto out_xmit;
390                 }
391 #else
392                 scst_set_cmd_error(cmd,
393                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
394                 goto out_xmit;
395 #endif
396         } else {
397                 TRACE(TRACE_SCSI, "op_name <%s> (cmd %p), direction=%d "
398                         "(expected %d, set %s), transfer_len=%d (expected "
399                         "len %d), flags=%d", cmd->op_name, cmd,
400                         cmd->data_direction, cmd->expected_data_direction,
401                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
402                         cmd->bufflen, cmd->expected_transfer_len,
403                         cmd->op_flags);
404
405                 if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
406                         if (scst_cmd_is_expected_set(cmd)) {
407                                 /*
408                                  * Command data length can't be easily
409                                  * determined from the CDB. ToDo, all such
410                                  * commands processing should be fixed. Until
411                                  * it's done, get the length from the supplied
412                                  * expected value, but limit it to some
413                                  * reasonable value (15MB).
414                                  */
415                                 cmd->bufflen = min(cmd->expected_transfer_len,
416                                                         15*1024*1024);
417                                 cmd->op_flags &= ~SCST_UNKNOWN_LENGTH;
418                         } else
419                                 cmd->bufflen = 0;
420                 }
421         }
422
423         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
424                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
425                             "(opcode 0x%02x)", cmd->cdb[0]);
426                 scst_set_cmd_error(cmd,
427                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
428                 goto out_xmit;
429         }
430
431         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
432                 PRINT_ERROR("Linked commands are not supported "
433                             "(opcode 0x%02x)", cmd->cdb[0]);
434                 scst_set_cmd_error(cmd,
435                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
436                 goto out_xmit;
437         }
438
439         cmd->state = SCST_CMD_STATE_DEV_PARSE;
440
441 out:
442         TRACE_EXIT_RES(res);
443         return res;
444
445 out_xmit:
446         scst_set_cmd_abnormal_done_state(cmd);
447         res = SCST_CMD_STATE_RES_CONT_SAME;
448         goto out;
449 }
450
451 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
452 static bool scst_is_allowed_to_mismatch_cmd(struct scst_cmd *cmd)
453 {
454         bool res = false;
455
456         switch (cmd->cdb[0]) {
457         case TEST_UNIT_READY:
458                 /* Crazy VMware people sometimes do TUR with READ direction */
459                 res = true;
460                 break;
461         case VERIFY:
462         case VERIFY_6:
463         case VERIFY_12:
464         case VERIFY_16:
465                 /* VERIFY commands with BYTCHK unset shouldn't fail here */
466                 if ((cmd->op_flags & SCST_VERIFY_BYTCHK_MISMATCH_ALLOWED) &&
467                     (cmd->cdb[1] & BYTCHK) == 0)
468                         res = true;
469                 break;
470         }
471
472         return res;
473 }
474 #endif
475
476 static int scst_parse_cmd(struct scst_cmd *cmd)
477 {
478         int res = SCST_CMD_STATE_RES_CONT_SAME;
479         int state;
480         struct scst_device *dev = cmd->dev;
481         int orig_bufflen = cmd->bufflen;
482
483         TRACE_ENTRY();
484
485         if (likely(!scst_is_cmd_local(cmd))) {
486                 if (unlikely(!dev->handler->parse_atomic &&
487                              scst_cmd_atomic(cmd))) {
488                         /*
489                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
490                          * optimization.
491                          */
492                         TRACE_DBG("Dev handler %s parse() needs thread "
493                                 "context, rescheduling", dev->handler->name);
494                         res = SCST_CMD_STATE_RES_NEED_THREAD;
495                         goto out;
496                 }
497
498                 TRACE_DBG("Calling dev handler %s parse(%p)",
499                       dev->handler->name, cmd);
500                 TRACE_BUFF_FLAG(TRACE_SND_BOT, "Parsing: ",
501                                 cmd->cdb, cmd->cdb_len);
502                 state = dev->handler->parse(cmd);
503                 /* Caution: cmd can be already dead here */
504                 TRACE_DBG("Dev handler %s parse() returned %d",
505                         dev->handler->name, state);
506
507                 switch (state) {
508                 case SCST_CMD_STATE_NEED_THREAD_CTX:
509                         TRACE_DBG("Dev handler %s parse() requested thread "
510                               "context, rescheduling", dev->handler->name);
511                         res = SCST_CMD_STATE_RES_NEED_THREAD;
512                         goto out;
513
514                 case SCST_CMD_STATE_STOP:
515                         TRACE_DBG("Dev handler %s parse() requested stop "
516                                 "processing", dev->handler->name);
517                         res = SCST_CMD_STATE_RES_CONT_NEXT;
518                         goto out;
519                 }
520
521                 if (state == SCST_CMD_STATE_DEFAULT)
522                         state = SCST_CMD_STATE_PREPARE_SPACE;
523         } else
524                 state = SCST_CMD_STATE_PREPARE_SPACE;
525
526         if (cmd->data_len == -1)
527                 cmd->data_len = cmd->bufflen;
528
529         if (cmd->bufflen == 0) {
530                 /*
531                  * According to SPC bufflen 0 for data transfer commands isn't
532                  * an error, so we need to fix the transfer direction.
533                  */
534                 cmd->data_direction = SCST_DATA_NONE;
535         }
536
537         if (cmd->dh_data_buf_alloced &&
538             unlikely((orig_bufflen > cmd->bufflen))) {
539                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
540                         "is less, than required (size %d)", cmd->bufflen,
541                         orig_bufflen);
542                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
543                 goto out_error;
544         }
545
546         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
547                 goto set_res;
548
549         if (unlikely((cmd->bufflen == 0) &&
550                      (cmd->op_flags & SCST_UNKNOWN_LENGTH))) {
551                 PRINT_ERROR("Unknown data transfer length for opcode 0x%x "
552                         "(handler %s, target %s)", cmd->cdb[0],
553                         dev->handler->name, cmd->tgtt->name);
554                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
555                 goto out_error;
556         }
557
558 #ifdef CONFIG_SCST_EXTRACHECKS
559         if ((cmd->bufflen != 0) &&
560             ((cmd->data_direction == SCST_DATA_NONE) ||
561              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
562                 PRINT_ERROR("Dev handler %s parse() returned "
563                         "invalid cmd data_direction %d, bufflen %d, state %d "
564                         "or sg %p (opcode 0x%x)", dev->handler->name,
565                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
566                         cmd->cdb[0]);
567                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
568                 goto out_error;
569         }
570 #endif
571
572         if (scst_cmd_is_expected_set(cmd)) {
573 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
574 #       ifdef CONFIG_SCST_EXTRACHECKS
575                 if ((cmd->data_direction != cmd->expected_data_direction) ||
576                     (cmd->bufflen != cmd->expected_transfer_len)) {
577                         PRINT_WARNING("Expected values don't match decoded "
578                                 "ones: data_direction %d, "
579                                 "expected_data_direction %d, "
580                                 "bufflen %d, expected_transfer_len %d",
581                                 cmd->data_direction,
582                                 cmd->expected_data_direction,
583                                 cmd->bufflen, cmd->expected_transfer_len);
584                         PRINT_BUFFER("Suspicious CDB", cmd->cdb, cmd->cdb_len);
585                 }
586 #       endif
587                 cmd->data_direction = cmd->expected_data_direction;
588                 cmd->bufflen = cmd->expected_transfer_len;
589 #else
590                 if (unlikely(cmd->data_direction !=
591                                 cmd->expected_data_direction)) {
592                         if (((cmd->expected_data_direction != SCST_DATA_NONE) ||
593                              (cmd->bufflen != 0)) &&
594                             !scst_is_allowed_to_mismatch_cmd(cmd)) {
595                                 PRINT_ERROR("Expected data direction %d for "
596                                         "opcode 0x%02x (handler %s, target %s) "
597                                         "doesn't match "
598                                         "decoded value %d",
599                                         cmd->expected_data_direction,
600                                         cmd->cdb[0], dev->handler->name,
601                                         cmd->tgtt->name, cmd->data_direction);
602                                 PRINT_BUFFER("Failed CDB",
603                                         cmd->cdb, cmd->cdb_len);
604                                 scst_set_cmd_error(cmd,
605                                    SCST_LOAD_SENSE(scst_sense_invalid_message));
606                                 goto out_dev_done;
607                         }
608                 }
609                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
610                         TRACE(TRACE_MGMT_MINOR, "Warning: expected "
611                                 "transfer length %d for opcode 0x%02x "
612                                 "(handler %s, target %s) doesn't match "
613                                 "decoded value %d. Faulty initiator "
614                                 "(e.g. VMware is known to be such) or "
615                                 "scst_scsi_op_table should be updated?",
616                                 cmd->expected_transfer_len, cmd->cdb[0],
617                                 dev->handler->name, cmd->tgtt->name,
618                                 cmd->bufflen);
619                         PRINT_BUFF_FLAG(TRACE_MGMT_MINOR, "Suspicious CDB",
620                                 cmd->cdb, cmd->cdb_len);
621                 }
622 #endif
623         }
624
625         if (unlikely(cmd->data_direction == SCST_DATA_UNKNOWN)) {
626                 PRINT_ERROR("Unknown data direction. Opcode 0x%x, handler %s, "
627                         "target %s", cmd->cdb[0], dev->handler->name,
628                         cmd->tgtt->name);
629                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
630                 goto out_error;
631         }
632
633 set_res:
634         switch (state) {
635         case SCST_CMD_STATE_PREPARE_SPACE:
636         case SCST_CMD_STATE_PRE_PARSE:
637         case SCST_CMD_STATE_DEV_PARSE:
638         case SCST_CMD_STATE_RDY_TO_XFER:
639         case SCST_CMD_STATE_TGT_PRE_EXEC:
640         case SCST_CMD_STATE_SEND_FOR_EXEC:
641         case SCST_CMD_STATE_LOCAL_EXEC:
642         case SCST_CMD_STATE_REAL_EXEC:
643         case SCST_CMD_STATE_PRE_DEV_DONE:
644         case SCST_CMD_STATE_DEV_DONE:
645         case SCST_CMD_STATE_PRE_XMIT_RESP:
646         case SCST_CMD_STATE_XMIT_RESP:
647         case SCST_CMD_STATE_FINISHED:
648         case SCST_CMD_STATE_FINISHED_INTERNAL:
649                 cmd->state = state;
650                 res = SCST_CMD_STATE_RES_CONT_SAME;
651                 break;
652
653         default:
654                 if (state >= 0) {
655                         PRINT_ERROR("Dev handler %s parse() returned "
656                              "invalid cmd state %d (opcode %d)",
657                              dev->handler->name, state, cmd->cdb[0]);
658                 } else {
659                         PRINT_ERROR("Dev handler %s parse() returned "
660                                 "error %d (opcode %d)", dev->handler->name,
661                                 state, cmd->cdb[0]);
662                 }
663                 goto out_error;
664         }
665
666         if (cmd->resp_data_len == -1) {
667                 if (cmd->data_direction == SCST_DATA_READ)
668                         cmd->resp_data_len = cmd->bufflen;
669                 else
670                          cmd->resp_data_len = 0;
671         }
672
673 out:
674         TRACE_EXIT_HRES(res);
675         return res;
676
677 out_error:
678         /* dev_done() will be called as part of the regular cmd's finish */
679         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
680
681 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
682 out_dev_done:
683 #endif
684         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
685         res = SCST_CMD_STATE_RES_CONT_SAME;
686         goto out;
687 }
688
689 static int scst_prepare_space(struct scst_cmd *cmd)
690 {
691         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
692
693         TRACE_ENTRY();
694
695         if (cmd->data_direction == SCST_DATA_NONE)
696                 goto prep_done;
697
698         if (cmd->tgt_need_alloc_data_buf) {
699                 int orig_bufflen = cmd->bufflen;
700
701                 TRACE_MEM("Custom tgt data buf allocation requested (cmd %p)",
702                         cmd);
703
704                 r = cmd->tgtt->alloc_data_buf(cmd);
705                 if (r > 0)
706                         goto alloc;
707                 else if (r == 0) {
708                         if (unlikely(cmd->bufflen == 0)) {
709                                 /* See comment in scst_alloc_space() */
710                                 if (cmd->sg == NULL)
711                                         goto alloc;
712                         }
713
714                         cmd->tgt_data_buf_alloced = 1;
715
716                         if (unlikely(orig_bufflen < cmd->bufflen)) {
717                                 PRINT_ERROR("Target driver allocated data "
718                                         "buffer (size %d), is less, than "
719                                         "required (size %d)", orig_bufflen,
720                                         cmd->bufflen);
721                                 goto out_error;
722                         }
723                         TRACE_MEM("tgt_data_buf_alloced (cmd %p)", cmd);
724                 } else
725                         goto check;
726         }
727
728 alloc:
729         if (!cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
730                 r = scst_alloc_space(cmd);
731                 cmd->tgt_sg = cmd->sg;
732                 cmd->tgt_sg_cnt = cmd->sg_cnt;
733         } else if (cmd->dh_data_buf_alloced && !cmd->tgt_data_buf_alloced) {
734                 TRACE_MEM("dh_data_buf_alloced set (cmd %p)", cmd);
735                 cmd->tgt_sg = cmd->sg;
736                 cmd->tgt_sg_cnt = cmd->sg_cnt;
737                 r = 0;
738         } else if (cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
739                 TRACE_MEM("tgt_data_buf_alloced set (cmd %p)", cmd);
740                 cmd->sg = cmd->tgt_sg;
741                 cmd->sg_cnt = cmd->tgt_sg_cnt;
742                 r = 0;
743         } else {
744                 TRACE_MEM("Both *_data_buf_alloced set (cmd %p, sg %p, "
745                         "sg_cnt %d, tgt_sg %p, tgt_sg_cnt %d)", cmd, cmd->sg,
746                         cmd->sg_cnt, cmd->tgt_sg, cmd->tgt_sg_cnt);
747                 r = 0;
748         }
749
750 check:
751         if (r != 0) {
752                 if (scst_cmd_atomic(cmd)) {
753                         TRACE_MEM("%s", "Atomic memory allocation failed, "
754                               "rescheduling to the thread");
755                         res = SCST_CMD_STATE_RES_NEED_THREAD;
756                         goto out;
757                 } else
758                         goto out_no_space;
759         }
760
761 prep_done:
762         if (cmd->preprocessing_only) {
763                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
764                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
765                                 "cmd %p", cmd);
766                         scst_set_cmd_abnormal_done_state(cmd);
767                         res = SCST_CMD_STATE_RES_CONT_SAME;
768                         goto out;
769                 }
770
771                 res = SCST_CMD_STATE_RES_CONT_NEXT;
772                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
773
774                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
775                 cmd->tgtt->preprocessing_done(cmd);
776                 TRACE_DBG("%s", "preprocessing_done() returned");
777                 goto out;
778
779         }
780
781         switch (cmd->data_direction) {
782         case SCST_DATA_WRITE:
783                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
784                 break;
785
786         default:
787                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
788                 break;
789         }
790
791 out:
792         TRACE_EXIT_HRES(res);
793         return res;
794
795 out_no_space:
796         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
797                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
798         scst_set_busy(cmd);
799         scst_set_cmd_abnormal_done_state(cmd);
800         res = SCST_CMD_STATE_RES_CONT_SAME;
801         goto out;
802
803 out_error:
804         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
805         scst_set_cmd_abnormal_done_state(cmd);
806         res = SCST_CMD_STATE_RES_CONT_SAME;
807         goto out;
808 }
809
810 void scst_restart_cmd(struct scst_cmd *cmd, int status,
811         enum scst_exec_context pref_context)
812 {
813         TRACE_ENTRY();
814
815         TRACE_DBG("Preferred context: %d", pref_context);
816         TRACE_DBG("tag=%llu, status=%#x",
817                   (long long unsigned int)scst_cmd_get_tag(cmd),
818                   status);
819
820 #ifdef CONFIG_SCST_EXTRACHECKS
821         if ((in_irq() || irqs_disabled()) &&
822             ((pref_context == SCST_CONTEXT_DIRECT) ||
823              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
824                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
825                         "SCST_CONTEXT_THREAD instead\n", pref_context,
826                         cmd->tgtt->name);
827                 pref_context = SCST_CONTEXT_THREAD;
828         }
829 #endif
830
831         switch (status) {
832         case SCST_PREPROCESS_STATUS_SUCCESS:
833                 switch (cmd->data_direction) {
834                 case SCST_DATA_WRITE:
835                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
836                         break;
837                 default:
838                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
839                         break;
840                 }
841                 if (cmd->set_sn_on_restart_cmd)
842                         scst_cmd_set_sn(cmd);
843                 /* Small context optimization */
844                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
845                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
846                     ((pref_context == SCST_CONTEXT_SAME) &&
847                      scst_cmd_atomic(cmd))) {
848                         if (cmd->data_direction == SCST_DATA_WRITE) {
849                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
850                                                 &cmd->tgt_dev->tgt_dev_flags))
851                                         pref_context = SCST_CONTEXT_THREAD;
852                         } else {
853                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
854                                                 &cmd->tgt_dev->tgt_dev_flags))
855                                         pref_context = SCST_CONTEXT_THREAD;
856                         }
857                 }
858                 break;
859
860         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
861                 scst_set_cmd_abnormal_done_state(cmd);
862                 break;
863
864         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
865                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
866                 /* go through */
867         case SCST_PREPROCESS_STATUS_ERROR:
868                 scst_set_cmd_error(cmd,
869                            SCST_LOAD_SENSE(scst_sense_hardw_error));
870                 scst_set_cmd_abnormal_done_state(cmd);
871                 break;
872
873         default:
874                 PRINT_ERROR("%s() received unknown status %x", __func__,
875                         status);
876                 scst_set_cmd_abnormal_done_state(cmd);
877                 break;
878         }
879
880         scst_proccess_redirect_cmd(cmd, pref_context, 1);
881
882         TRACE_EXIT();
883         return;
884 }
885 EXPORT_SYMBOL(scst_restart_cmd);
886
887 /* No locks */
888 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
889 {
890         struct scst_tgt *tgt = cmd->sess->tgt;
891         int res = 0;
892         unsigned long flags;
893
894         TRACE_ENTRY();
895
896         spin_lock_irqsave(&tgt->tgt_lock, flags);
897         tgt->retry_cmds++;
898         /*
899          * Memory barrier is needed here, because we need the exact order
900          * between the read and write between retry_cmds and finished_cmds to
901          * not miss the case when a command finished while we queuing it for
902          * retry after the finished_cmds check.
903          */
904         smp_mb();
905         TRACE_RETRY("TGT QUEUE FULL: incrementing retry_cmds %d",
906               tgt->retry_cmds);
907         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
908                 /* At least one cmd finished, so try again */
909                 tgt->retry_cmds--;
910                 TRACE_RETRY("Some command(s) finished, direct retry "
911                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
912                       "retry_cmds=%d)", finished_cmds,
913                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
914                 res = -1;
915                 goto out_unlock_tgt;
916         }
917
918         TRACE_RETRY("Adding cmd %p to retry cmd list", cmd);
919         list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
920
921         if (!tgt->retry_timer_active) {
922                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
923                 add_timer(&tgt->retry_timer);
924                 tgt->retry_timer_active = 1;
925         }
926
927 out_unlock_tgt:
928         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
929
930         TRACE_EXIT_RES(res);
931         return res;
932 }
933
934 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
935 {
936         int res, rc;
937
938         TRACE_ENTRY();
939
940         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
941                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
942                 goto out_dev_done;
943         }
944
945         if ((cmd->tgtt->rdy_to_xfer == NULL) || unlikely(cmd->internal)) {
946                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
947                 res = SCST_CMD_STATE_RES_CONT_SAME;
948                 goto out;
949         }
950
951         if (unlikely(!cmd->tgtt->rdy_to_xfer_atomic && scst_cmd_atomic(cmd))) {
952                 /*
953                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
954                  * optimization.
955                  */
956                 TRACE_DBG("Target driver %s rdy_to_xfer() needs thread "
957                               "context, rescheduling", cmd->tgtt->name);
958                 res = SCST_CMD_STATE_RES_NEED_THREAD;
959                 goto out;
960         }
961
962         while (1) {
963                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
964
965                 res = SCST_CMD_STATE_RES_CONT_NEXT;
966                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
967
968                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
969 #ifdef CONFIG_SCST_DEBUG_RETRY
970                 if (((scst_random() % 100) == 75))
971                         rc = SCST_TGT_RES_QUEUE_FULL;
972                 else
973 #endif
974                         rc = cmd->tgtt->rdy_to_xfer(cmd);
975                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
976
977                 if (likely(rc == SCST_TGT_RES_SUCCESS))
978                         goto out;
979
980                 /* Restore the previous state */
981                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
982
983                 switch (rc) {
984                 case SCST_TGT_RES_QUEUE_FULL:
985                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
986                                 break;
987                         else
988                                 continue;
989
990                 case SCST_TGT_RES_NEED_THREAD_CTX:
991                         TRACE_DBG("Target driver %s "
992                               "rdy_to_xfer() requested thread "
993                               "context, rescheduling", cmd->tgtt->name);
994                         res = SCST_CMD_STATE_RES_NEED_THREAD;
995                         break;
996
997                 default:
998                         goto out_error_rc;
999                 }
1000                 break;
1001         }
1002
1003 out:
1004         TRACE_EXIT_HRES(res);
1005         return res;
1006
1007 out_error_rc:
1008         if (rc == SCST_TGT_RES_FATAL_ERROR) {
1009                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
1010                      "fatal error", cmd->tgtt->name);
1011         } else {
1012                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
1013                             "value %d", cmd->tgtt->name, rc);
1014         }
1015         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1016
1017 out_dev_done:
1018         scst_set_cmd_abnormal_done_state(cmd);
1019         res = SCST_CMD_STATE_RES_CONT_SAME;
1020         goto out;
1021 }
1022
1023 /* No locks, but might be in IRQ */
1024 static void scst_proccess_redirect_cmd(struct scst_cmd *cmd,
1025         enum scst_exec_context context, int check_retries)
1026 {
1027         unsigned long flags;
1028
1029         TRACE_ENTRY();
1030
1031         TRACE_DBG("Context: %x", context);
1032
1033         if (context == SCST_CONTEXT_SAME)
1034                 context = scst_cmd_atomic(cmd) ? SCST_CONTEXT_DIRECT_ATOMIC :
1035                                                  SCST_CONTEXT_DIRECT;
1036
1037         switch (context) {
1038         case SCST_CONTEXT_DIRECT_ATOMIC:
1039                 scst_process_active_cmd(cmd, true);
1040                 break;
1041
1042         case SCST_CONTEXT_DIRECT:
1043                 if (check_retries)
1044                         scst_check_retries(cmd->tgt);
1045                 scst_process_active_cmd(cmd, false);
1046                 break;
1047
1048         default:
1049                 PRINT_ERROR("Context %x is unknown, using the thread one",
1050                             context);
1051                 /* go through */
1052         case SCST_CONTEXT_THREAD:
1053                 if (check_retries)
1054                         scst_check_retries(cmd->tgt);
1055                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
1056                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
1057                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
1058                         list_add(&cmd->cmd_list_entry,
1059                                 &cmd->cmd_lists->active_cmd_list);
1060                 else
1061                         list_add_tail(&cmd->cmd_list_entry,
1062                                 &cmd->cmd_lists->active_cmd_list);
1063                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
1064                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
1065                 break;
1066
1067         case SCST_CONTEXT_TASKLET:
1068                 if (check_retries)
1069                         scst_check_retries(cmd->tgt);
1070                 scst_schedule_tasklet(cmd);
1071                 break;
1072         }
1073
1074         TRACE_EXIT();
1075         return;
1076 }
1077
1078 void scst_rx_data(struct scst_cmd *cmd, int status,
1079         enum scst_exec_context pref_context)
1080 {
1081         TRACE_ENTRY();
1082
1083         TRACE_DBG("Preferred context: %d", pref_context);
1084         TRACE(TRACE_SCSI, "cmd %p, status %#x", cmd, status);
1085
1086 #ifdef CONFIG_SCST_EXTRACHECKS
1087         if ((in_irq() || irqs_disabled()) &&
1088             ((pref_context == SCST_CONTEXT_DIRECT) ||
1089              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
1090                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
1091                         "SCST_CONTEXT_THREAD instead\n", pref_context,
1092                         cmd->tgtt->name);
1093                 pref_context = SCST_CONTEXT_THREAD;
1094         }
1095 #endif
1096
1097         switch (status) {
1098         case SCST_RX_STATUS_SUCCESS:
1099 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
1100                 if (cmd->tgt_sg) {
1101                         int i;
1102                         struct scatterlist *sg = cmd->tgt_sg;
1103                         TRACE_RECV_BOT("RX data for cmd %p "
1104                                 "(sg_cnt %d, sg %p, sg[0].page %p)", cmd,
1105                                 cmd->tgt_sg_cnt, sg, (void *)sg_page(&sg[0]));
1106                         for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
1107                                 PRINT_BUFF_FLAG(TRACE_RCV_BOT, "RX sg",
1108                                         sg_virt(&sg[i]), sg[i].length);
1109                         }
1110                 }
1111 #endif
1112                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1113                 /* Small context optimization */
1114                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
1115                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
1116                     ((pref_context == SCST_CONTEXT_SAME) &&
1117                      scst_cmd_atomic(cmd))) {
1118                         if (!test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC,
1119                                         &cmd->tgt_dev->tgt_dev_flags))
1120                                 pref_context = SCST_CONTEXT_THREAD;
1121                 }
1122                 break;
1123
1124         case SCST_RX_STATUS_ERROR_SENSE_SET:
1125                 scst_set_cmd_abnormal_done_state(cmd);
1126                 break;
1127
1128         case SCST_RX_STATUS_ERROR_FATAL:
1129                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1130                 /* go through */
1131         case SCST_RX_STATUS_ERROR:
1132                 scst_set_cmd_error(cmd,
1133                            SCST_LOAD_SENSE(scst_sense_hardw_error));
1134                 scst_set_cmd_abnormal_done_state(cmd);
1135                 break;
1136
1137         default:
1138                 PRINT_ERROR("scst_rx_data() received unknown status %x",
1139                         status);
1140                 scst_set_cmd_abnormal_done_state(cmd);
1141                 break;
1142         }
1143
1144         scst_proccess_redirect_cmd(cmd, pref_context, 1);
1145
1146         TRACE_EXIT();
1147         return;
1148 }
1149 EXPORT_SYMBOL(scst_rx_data);
1150
1151 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
1152 {
1153         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
1154
1155         TRACE_ENTRY();
1156
1157         cmd->state = SCST_CMD_STATE_SEND_FOR_EXEC;
1158
1159         if ((cmd->tgtt->pre_exec == NULL) || unlikely(cmd->internal))
1160                 goto out;
1161
1162         TRACE_DBG("Calling pre_exec(%p)", cmd);
1163         rc = cmd->tgtt->pre_exec(cmd);
1164         TRACE_DBG("pre_exec() returned %d", rc);
1165
1166         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
1167                 switch (rc) {
1168                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
1169                         scst_set_cmd_abnormal_done_state(cmd);
1170                         break;
1171                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
1172                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1173                         /* go through */
1174                 case SCST_PREPROCESS_STATUS_ERROR:
1175                         scst_set_cmd_error(cmd,
1176                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1177                         scst_set_cmd_abnormal_done_state(cmd);
1178                         break;
1179                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1180                         TRACE_DBG("Target driver's %s pre_exec() requested "
1181                                 "thread context, rescheduling",
1182                                 cmd->tgtt->name);
1183                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1184                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1185                         break;
1186                 default:
1187                         sBUG();
1188                         break;
1189                 }
1190         }
1191
1192 out:
1193         TRACE_EXIT_RES(res);
1194         return res;
1195 }
1196
1197 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1198         const uint8_t *rq_sense, int rq_sense_len, int resid)
1199 {
1200         TRACE_ENTRY();
1201
1202 #ifdef CONFIG_SCST_MEASURE_LATENCY
1203         {
1204                 struct timespec ts;
1205                 getnstimeofday(&ts);
1206                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1207                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1208                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1209                         ts.tv_sec, ts.tv_nsec);
1210         }
1211 #endif
1212
1213         cmd->status = result & 0xff;
1214         cmd->msg_status = msg_byte(result);
1215         cmd->host_status = host_byte(result);
1216         cmd->driver_status = driver_byte(result);
1217         if (unlikely(resid != 0)) {
1218 #ifdef CONFIG_SCST_EXTRACHECKS
1219                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1220                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1221                                 "op %x)", resid, cmd->resp_data_len,
1222                                 cmd->cdb[0]);
1223                 } else
1224 #endif
1225                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1226         }
1227
1228         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION)) {
1229                 /* We might have double reset UA here */
1230                 cmd->dbl_ua_orig_resp_data_len = cmd->resp_data_len;
1231                 cmd->dbl_ua_orig_data_direction = cmd->data_direction;
1232
1233                 scst_alloc_set_sense(cmd, 1, rq_sense, rq_sense_len);
1234         }
1235
1236         TRACE(TRACE_SCSI, "cmd %p, result=%x, cmd->status=%x, resid=%d, "
1237               "cmd->msg_status=%x, cmd->host_status=%x, "
1238               "cmd->driver_status=%x (cmd %p)", cmd, result, cmd->status, resid,
1239               cmd->msg_status, cmd->host_status, cmd->driver_status, cmd);
1240
1241         cmd->completed = 1;
1242
1243         TRACE_EXIT();
1244         return;
1245 }
1246
1247 /* For small context optimization */
1248 static inline enum scst_exec_context scst_optimize_post_exec_context(
1249         struct scst_cmd *cmd, enum scst_exec_context context)
1250 {
1251         if (((context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd)) ||
1252             (context == SCST_CONTEXT_TASKLET) ||
1253             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1254                 if (!test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC,
1255                                 &cmd->tgt_dev->tgt_dev_flags))
1256                         context = SCST_CONTEXT_THREAD;
1257         }
1258         return context;
1259 }
1260
1261 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1262 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1263                                             struct scsi_request **req)
1264 {
1265         struct scst_cmd *cmd = NULL;
1266
1267         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1268                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1269
1270         if (cmd == NULL) {
1271                 PRINT_ERROR("%s", "Request with NULL cmd");
1272                 if (*req)
1273                         scsi_release_request(*req);
1274         }
1275
1276         return cmd;
1277 }
1278
1279 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1280 {
1281         struct scsi_request *req = NULL;
1282         struct scst_cmd *cmd;
1283
1284         TRACE_ENTRY();
1285
1286         cmd = scst_get_cmd(scsi_cmd, &req);
1287         if (cmd == NULL)
1288                 goto out;
1289
1290         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1291                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1292
1293         /* Clear out request structure */
1294         req->sr_use_sg = 0;
1295         req->sr_sglist_len = 0;
1296         req->sr_bufflen = 0;
1297         req->sr_buffer = NULL;
1298         req->sr_underflow = 0;
1299         req->sr_request->rq_disk = NULL; /* disown request blk */
1300
1301         scst_release_request(cmd);
1302
1303         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1304
1305         scst_proccess_redirect_cmd(cmd,
1306                 scst_optimize_post_exec_context(cmd, scst_estimate_context()),
1307                                                 0);
1308
1309 out:
1310         TRACE_EXIT();
1311         return;
1312 }
1313 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1314 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1315 {
1316         struct scst_cmd *cmd;
1317
1318         TRACE_ENTRY();
1319
1320         cmd = (struct scst_cmd *)data;
1321         if (cmd == NULL)
1322                 goto out;
1323
1324         scst_do_cmd_done(cmd, result, sense, SCST_SENSE_BUFFERSIZE, resid);
1325
1326         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1327
1328         scst_proccess_redirect_cmd(cmd,
1329                 scst_optimize_post_exec_context(cmd, scst_estimate_context()),
1330                                                 0);
1331
1332 out:
1333         TRACE_EXIT();
1334         return;
1335 }
1336 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1337
1338 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state,
1339         enum scst_exec_context pref_context)
1340 {
1341         TRACE_ENTRY();
1342
1343 #ifdef CONFIG_SCST_MEASURE_LATENCY
1344         {
1345                 struct timespec ts;
1346                 getnstimeofday(&ts);
1347                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1348                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1349                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1350                         ts.tv_sec, ts.tv_nsec);
1351         }
1352 #endif
1353
1354         if (next_state == SCST_CMD_STATE_DEFAULT)
1355                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1356
1357 #if defined(CONFIG_SCST_DEBUG)
1358         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1359                 if (cmd->sg) {
1360                         int i;
1361                         struct scatterlist *sg = cmd->sg;
1362                         TRACE_RECV_TOP("Exec'd %d S/G(s) at %p sg[0].page at "
1363                                 "%p", cmd->sg_cnt, sg, (void *)sg_page(&sg[0]));
1364                         for (i = 0; i < cmd->sg_cnt; ++i) {
1365                                 TRACE_BUFF_FLAG(TRACE_RCV_TOP,
1366                                         "Exec'd sg", sg_virt(&sg[i]),
1367                                         sg[i].length);
1368                         }
1369                 }
1370         }
1371 #endif
1372
1373         cmd->state = next_state;
1374
1375 #ifdef CONFIG_SCST_EXTRACHECKS
1376         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1377             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1378             (next_state != SCST_CMD_STATE_FINISHED) &&
1379             (next_state != SCST_CMD_STATE_FINISHED_INTERNAL)) {
1380                 PRINT_ERROR("%s() received invalid cmd state %d (opcode %d)",
1381                         __func__, next_state, cmd->cdb[0]);
1382                 scst_set_cmd_error(cmd,
1383                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1384                 scst_set_cmd_abnormal_done_state(cmd);
1385         }
1386 #endif
1387         pref_context = scst_optimize_post_exec_context(cmd, pref_context);
1388         scst_proccess_redirect_cmd(cmd, pref_context, 0);
1389
1390         TRACE_EXIT();
1391         return;
1392 }
1393
1394 static int scst_report_luns_local(struct scst_cmd *cmd)
1395 {
1396         int rc;
1397         int dev_cnt = 0;
1398         int buffer_size;
1399         int i;
1400         struct scst_tgt_dev *tgt_dev = NULL;
1401         uint8_t *buffer;
1402         int offs, overflow = 0;
1403
1404         TRACE_ENTRY();
1405
1406         rc = scst_check_local_events(cmd);
1407         if (unlikely(rc != 0))
1408                 goto out_done;
1409
1410         cmd->status = 0;
1411         cmd->msg_status = 0;
1412         cmd->host_status = DID_OK;
1413         cmd->driver_status = 0;
1414
1415         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1416                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1417                         "LUNS command", cmd->cdb[2]);
1418                 goto out_err;
1419         }
1420
1421         buffer_size = scst_get_buf_first(cmd, &buffer);
1422         if (unlikely(buffer_size == 0))
1423                 goto out_compl;
1424         else if (unlikely(buffer_size < 0))
1425                 goto out_err;
1426
1427         if (buffer_size < 16)
1428                 goto out_put_err;
1429
1430         memset(buffer, 0, buffer_size);
1431         offs = 8;
1432
1433         /* sess->sess_tgt_dev_list_hash is protected by suspended activity */
1434         for (i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1435                 struct list_head *sess_tgt_dev_list_head =
1436                         &cmd->sess->sess_tgt_dev_list_hash[i];
1437                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1438                                 sess_tgt_dev_list_entry) {
1439                         if (!overflow) {
1440                                 if (offs >= buffer_size) {
1441                                         scst_put_buf(cmd, buffer);
1442                                         buffer_size = scst_get_buf_next(cmd,
1443                                                                        &buffer);
1444                                         if (buffer_size > 0) {
1445                                                 memset(buffer, 0, buffer_size);
1446                                                 offs = 0;
1447                                         } else {
1448                                                 overflow = 1;
1449                                                 goto inc_dev_cnt;
1450                                         }
1451                                 }
1452                                 if ((buffer_size - offs) < 8) {
1453                                         PRINT_ERROR("Buffer allocated for "
1454                                                 "REPORT LUNS command doesn't "
1455                                                 "allow to fit 8 byte entry "
1456                                                 "(buffer_size=%d)",
1457                                                 buffer_size);
1458                                         goto out_put_hw_err;
1459                                 }
1460                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1461                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1462                                 offs += 8;
1463                         }
1464 inc_dev_cnt:
1465                         dev_cnt++;
1466                 }
1467         }
1468         if (!overflow)
1469                 scst_put_buf(cmd, buffer);
1470
1471         /* Set the response header */
1472         buffer_size = scst_get_buf_first(cmd, &buffer);
1473         if (unlikely(buffer_size == 0))
1474                 goto out_compl;
1475         else if (unlikely(buffer_size < 0))
1476                 goto out_err;
1477
1478         dev_cnt *= 8;
1479         buffer[0] = (dev_cnt >> 24) & 0xff;
1480         buffer[1] = (dev_cnt >> 16) & 0xff;
1481         buffer[2] = (dev_cnt >> 8) & 0xff;
1482         buffer[3] = dev_cnt & 0xff;
1483
1484         scst_put_buf(cmd, buffer);
1485
1486         dev_cnt += 8;
1487         if (dev_cnt < cmd->resp_data_len)
1488                 scst_set_resp_data_len(cmd, dev_cnt);
1489
1490 out_compl:
1491         cmd->completed = 1;
1492
1493 out_done:
1494         /* Report the result */
1495         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1496
1497         TRACE_EXIT();
1498         return SCST_EXEC_COMPLETED;
1499
1500 out_put_err:
1501         scst_put_buf(cmd, buffer);
1502
1503 out_err:
1504         scst_set_cmd_error(cmd,
1505                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1506         goto out_compl;
1507
1508 out_put_hw_err:
1509         scst_put_buf(cmd, buffer);
1510         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1511         goto out_compl;
1512 }
1513
1514 static int scst_pre_select(struct scst_cmd *cmd)
1515 {
1516         int res = SCST_EXEC_NOT_COMPLETED;
1517
1518         TRACE_ENTRY();
1519
1520         if (scst_cmd_atomic(cmd)) {
1521                 res = SCST_EXEC_NEED_THREAD;
1522                 goto out;
1523         }
1524
1525         scst_block_dev_cmd(cmd, 1);
1526
1527         /* Check for local events will be done when cmd will be executed */
1528
1529 out:
1530         TRACE_EXIT_RES(res);
1531         return res;
1532 }
1533
1534 static int scst_reserve_local(struct scst_cmd *cmd)
1535 {
1536         int res = SCST_EXEC_NOT_COMPLETED, rc;
1537         struct scst_device *dev;
1538         struct scst_tgt_dev *tgt_dev_tmp;
1539
1540         TRACE_ENTRY();
1541
1542         if (scst_cmd_atomic(cmd)) {
1543                 res = SCST_EXEC_NEED_THREAD;
1544                 goto out;
1545         }
1546
1547         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1548                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1549                      "(lun=%lld)", (long long unsigned int)cmd->lun);
1550                 scst_set_cmd_error(cmd,
1551                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1552                 goto out_done;
1553         }
1554
1555         dev = cmd->dev;
1556
1557         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1558                 scst_block_dev_cmd(cmd, 1);
1559
1560         rc = scst_check_local_events(cmd);
1561         if (unlikely(rc != 0))
1562                 goto out_done;
1563
1564         spin_lock_bh(&dev->dev_lock);
1565
1566         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1567                 spin_unlock_bh(&dev->dev_lock);
1568                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1569                 goto out_done;
1570         }
1571
1572         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1573                             dev_tgt_dev_list_entry) {
1574                 if (cmd->tgt_dev != tgt_dev_tmp)
1575                         set_bit(SCST_TGT_DEV_RESERVED,
1576                                 &tgt_dev_tmp->tgt_dev_flags);
1577         }
1578         dev->dev_reserved = 1;
1579
1580         spin_unlock_bh(&dev->dev_lock);
1581
1582 out:
1583         TRACE_EXIT_RES(res);
1584         return res;
1585
1586 out_done:
1587         /* Report the result */
1588         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1589         res = SCST_EXEC_COMPLETED;
1590         goto out;
1591 }
1592
1593 static int scst_release_local(struct scst_cmd *cmd)
1594 {
1595         int res = SCST_EXEC_NOT_COMPLETED, rc;
1596         struct scst_tgt_dev *tgt_dev_tmp;
1597         struct scst_device *dev;
1598
1599         TRACE_ENTRY();
1600
1601         if (scst_cmd_atomic(cmd)) {
1602                 res = SCST_EXEC_NEED_THREAD;
1603                 goto out;
1604         }
1605
1606         dev = cmd->dev;
1607
1608         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1609                 scst_block_dev_cmd(cmd, 1);
1610
1611         rc = scst_check_local_events(cmd);
1612         if (unlikely(rc != 0))
1613                 goto out_done;
1614
1615         spin_lock_bh(&dev->dev_lock);
1616
1617         /*
1618          * The device could be RELEASED behind us, if RESERVING session
1619          * is closed (see scst_free_tgt_dev()), but this actually doesn't
1620          * matter, so use lock and no retest for DEV_RESERVED bits again
1621          */
1622         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1623                 res = SCST_EXEC_COMPLETED;
1624                 cmd->status = 0;
1625                 cmd->msg_status = 0;
1626                 cmd->host_status = DID_OK;
1627                 cmd->driver_status = 0;
1628                 cmd->completed = 1;
1629         } else {
1630                 list_for_each_entry(tgt_dev_tmp,
1631                                     &dev->dev_tgt_dev_list,
1632                                     dev_tgt_dev_list_entry) {
1633                         clear_bit(SCST_TGT_DEV_RESERVED,
1634                                 &tgt_dev_tmp->tgt_dev_flags);
1635                 }
1636                 dev->dev_reserved = 0;
1637         }
1638
1639         spin_unlock_bh(&dev->dev_lock);
1640
1641         if (res == SCST_EXEC_COMPLETED)
1642                 goto out_done;
1643
1644 out:
1645         TRACE_EXIT_RES(res);
1646         return res;
1647
1648 out_done:
1649         res = SCST_EXEC_COMPLETED;
1650         /* Report the result */
1651         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1652         goto out;
1653 }
1654
1655 /* No locks, no IRQ or IRQ-safe context allowed */
1656 int scst_check_local_events(struct scst_cmd *cmd)
1657 {
1658         int res, rc;
1659         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1660         struct scst_device *dev = cmd->dev;
1661
1662         TRACE_ENTRY();
1663
1664         /*
1665          * There's no race here, because we need to trace commands sent
1666          * *after* dev_double_ua_possible flag was set.
1667          */
1668         if (unlikely(dev->dev_double_ua_possible))
1669                 cmd->double_ua_possible = 1;
1670
1671         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1672                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1673                 goto out_uncomplete;
1674         }
1675
1676         /* Reserve check before Unit Attention */
1677         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED,
1678                               &tgt_dev->tgt_dev_flags))) {
1679                 if (cmd->cdb[0] != INQUIRY &&
1680                     cmd->cdb[0] != REPORT_LUNS &&
1681                     cmd->cdb[0] != RELEASE &&
1682                     cmd->cdb[0] != RELEASE_10 &&
1683                     cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER &&
1684                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL ||
1685                      (cmd->cdb[4] & 3)) &&
1686                     cmd->cdb[0] != LOG_SENSE &&
1687                     cmd->cdb[0] != REQUEST_SENSE) {
1688                         scst_set_cmd_error_status(cmd,
1689                                 SAM_STAT_RESERVATION_CONFLICT);
1690                         goto out_complete;
1691                 }
1692         }
1693
1694         /* If we had internal bus reset, set the command error unit attention */
1695         if ((dev->scsi_dev != NULL) &&
1696             unlikely(dev->scsi_dev->was_reset)) {
1697                 if (scst_is_ua_command(cmd)) {
1698                         int done = 0;
1699                         /*
1700                          * Prevent more than 1 cmd to be triggered by
1701                          * was_reset.
1702                          */
1703                         spin_lock_bh(&dev->dev_lock);
1704                         if (dev->scsi_dev->was_reset) {
1705                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1706                                 scst_set_cmd_error(cmd,
1707                                           SCST_LOAD_SENSE(scst_sense_reset_UA));
1708                                 /*
1709                                  * It looks like it is safe to clear was_reset
1710                                  * here.
1711                                  */
1712                                 dev->scsi_dev->was_reset = 0;
1713                                 done = 1;
1714                         }
1715                         spin_unlock_bh(&dev->dev_lock);
1716
1717                         if (done)
1718                                 goto out_complete;
1719                 }
1720         }
1721
1722         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING,
1723                         &cmd->tgt_dev->tgt_dev_flags))) {
1724                 if (scst_is_ua_command(cmd)) {
1725                         rc = scst_set_pending_UA(cmd);
1726                         if (rc == 0)
1727                                 goto out_complete;
1728                 }
1729         }
1730
1731         res = 0;
1732
1733 out:
1734         TRACE_EXIT_RES(res);
1735         return res;
1736
1737 out_complete:
1738         res = 1;
1739         sBUG_ON(!cmd->completed);
1740         goto out;
1741
1742 out_uncomplete:
1743         res = -1;
1744         goto out;
1745 }
1746 EXPORT_SYMBOL(scst_check_local_events);
1747
1748 /* No locks */
1749 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1750 {
1751         if (slot == NULL)
1752                 goto inc;
1753
1754         /* Optimized for lockless fast path */
1755
1756         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1757                 atomic_read(slot));
1758
1759         if (!atomic_dec_and_test(slot))
1760                 goto out;
1761
1762         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1763                 tgt_dev->num_free_sn_slots);
1764         if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
1765                 spin_lock_irq(&tgt_dev->sn_lock);
1766                 if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
1767                         if (tgt_dev->num_free_sn_slots < 0)
1768                                 tgt_dev->cur_sn_slot = slot;
1769                         /*
1770                          * To be in-sync with SIMPLE case in scst_cmd_set_sn()
1771                          */
1772                         smp_mb();
1773                         tgt_dev->num_free_sn_slots++;
1774                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1775                                 tgt_dev->num_free_sn_slots);
1776
1777                 }
1778                 spin_unlock_irq(&tgt_dev->sn_lock);
1779         }
1780
1781 inc:
1782         /*
1783          * No protection of expected_sn is needed, because only one thread
1784          * at time can be here (serialized by sn). Also it is supposed that
1785          * there could not be half-incremented halves.
1786          */
1787         tgt_dev->expected_sn++;
1788         /*
1789          * Write must be before def_cmd_count read to be in sync. with
1790          * scst_post_exec_sn(). See comment in scst_send_for_exec().
1791          */
1792         smp_mb();
1793         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1794
1795 out:
1796         return;
1797 }
1798
1799 /* No locks */
1800 static struct scst_cmd *scst_post_exec_sn(struct scst_cmd *cmd,
1801         bool make_active)
1802 {
1803         /* For HQ commands SN is not set */
1804         bool inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1805                                cmd->sn_set && !cmd->retry;
1806         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1807         struct scst_cmd *res;
1808
1809         TRACE_ENTRY();
1810
1811         if (inc_expected_sn)
1812                 scst_inc_expected_sn(tgt_dev, cmd->sn_slot);
1813
1814         if (make_active) {
1815                 scst_make_deferred_commands_active(tgt_dev);
1816                 res = NULL;
1817         } else
1818                 res = scst_check_deferred_commands(tgt_dev);
1819
1820         TRACE_EXIT_HRES(res);
1821         return res;
1822 }
1823
1824 /* cmd must be additionally referenced to not die inside */
1825 static int scst_do_real_exec(struct scst_cmd *cmd)
1826 {
1827         int res = SCST_EXEC_NOT_COMPLETED;
1828 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
1829         int rc;
1830 #endif
1831         struct scst_device *dev = cmd->dev;
1832         struct scst_dev_type *handler = dev->handler;
1833
1834         TRACE_ENTRY();
1835
1836         cmd->state = SCST_CMD_STATE_REAL_EXECUTING;
1837
1838         if (handler->exec) {
1839                 if (unlikely(!dev->handler->exec_atomic &&
1840                              scst_cmd_atomic(cmd))) {
1841                         /*
1842                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
1843                          * optimization.
1844                          */
1845                         TRACE_DBG("Dev handler %s exec() needs thread "
1846                                 "context, rescheduling", dev->handler->name);
1847                         res = SCST_EXEC_NEED_THREAD;
1848                         goto out_restore;
1849                 }
1850
1851                 TRACE_DBG("Calling dev handler %s exec(%p)",
1852                       handler->name, cmd);
1853                 TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb,
1854                         cmd->cdb_len);
1855                 res = handler->exec(cmd);
1856                 TRACE_DBG("Dev handler %s exec() returned %d",
1857                       handler->name, res);
1858
1859                 if (res == SCST_EXEC_COMPLETED)
1860                         goto out_complete;
1861                 else if (res == SCST_EXEC_NEED_THREAD)
1862                         goto out_restore;
1863
1864                 sBUG_ON(res != SCST_EXEC_NOT_COMPLETED);
1865         }
1866
1867         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1868
1869         if (unlikely(dev->scsi_dev == NULL)) {
1870                 PRINT_ERROR("Command for virtual device must be "
1871                         "processed by device handler (lun %lld)!",
1872                         (long long unsigned int)cmd->lun);
1873                 goto out_error;
1874         }
1875
1876         res = scst_check_local_events(cmd);
1877         if (unlikely(res != 0))
1878                 goto out_done;
1879
1880 #ifndef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1881         if (unlikely(scst_cmd_atomic(cmd))) {
1882                 TRACE_DBG("Pass-through exec() can not be called in atomic "
1883                         "context, rescheduling to the thread (handler %s)",
1884                         handler->name);
1885                 res = SCST_EXEC_NEED_THREAD;
1886                 goto out_restore;
1887         }
1888 #endif
1889
1890 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1891         if (unlikely(scst_alloc_request(cmd) != 0)) {
1892                 if (scst_cmd_atomic(cmd)) {
1893                         res = SCST_EXEC_NEED_THREAD;
1894                         goto out_restore;
1895                 } else {
1896                         PRINT_INFO("%s", "Unable to allocate request, "
1897                                 "sending BUSY status");
1898                         goto out_busy;
1899                 }
1900         }
1901
1902         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1903                     (void *)cmd->scsi_req->sr_buffer,
1904                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1905                     cmd->retries);
1906 #else
1907         rc = scst_exec_req(dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1908                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1909                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1910                         scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
1911         if (unlikely(rc != 0)) {
1912                 if (scst_cmd_atomic(cmd)) {
1913                         res = SCST_EXEC_NEED_THREAD;
1914                         goto out_restore;
1915                 } else {
1916                         PRINT_ERROR("scst_exec_req() failed: %d", res);
1917                         goto out_error;
1918                 }
1919         }
1920 #endif
1921
1922 out_complete:
1923         res = SCST_EXEC_COMPLETED;
1924
1925 out:
1926         TRACE_EXIT();
1927         return res;
1928
1929 out_restore:
1930         /* Restore the state */
1931         cmd->state = SCST_CMD_STATE_REAL_EXEC;
1932         goto out;
1933
1934 out_error:
1935         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1936         goto out_done;
1937
1938 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1939 out_busy:
1940         scst_set_busy(cmd);
1941         /* go through */
1942 #endif
1943
1944 out_done:
1945         res = SCST_EXEC_COMPLETED;
1946         /* Report the result */
1947         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1948         goto out_complete;
1949 }
1950
1951 static inline int scst_real_exec(struct scst_cmd *cmd)
1952 {
1953         int res;
1954
1955         TRACE_ENTRY();
1956
1957         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
1958         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
1959         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
1960
1961         __scst_cmd_get(cmd);
1962
1963         res = scst_do_real_exec(cmd);
1964
1965         if (likely(res == SCST_EXEC_COMPLETED)) {
1966                 scst_post_exec_sn(cmd, true);
1967                 if (cmd->dev->scsi_dev != NULL)
1968                         generic_unplug_device(
1969                                 cmd->dev->scsi_dev->request_queue);
1970         } else
1971                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
1972
1973         __scst_cmd_put(cmd);
1974
1975         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
1976
1977         TRACE_EXIT_RES(res);
1978         return res;
1979 }
1980
1981 static int scst_do_local_exec(struct scst_cmd *cmd)
1982 {
1983         int res;
1984         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1985
1986         TRACE_ENTRY();
1987
1988         /* Check READ_ONLY device status */
1989         if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
1990             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
1991              cmd->cdb[0] == WRITE_10 ||
1992              cmd->cdb[0] == WRITE_12 ||
1993              cmd->cdb[0] == WRITE_16 ||
1994              cmd->cdb[0] == WRITE_VERIFY ||
1995              cmd->cdb[0] == WRITE_VERIFY_12 ||
1996              cmd->cdb[0] == WRITE_VERIFY_16 ||
1997              (cmd->dev->handler->type == TYPE_TAPE &&
1998               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
1999                 scst_set_cmd_error(cmd,
2000                            SCST_LOAD_SENSE(scst_sense_data_protect));
2001                 goto out_done;
2002         }
2003
2004         /*
2005          * Adding new commands here don't forget to update
2006          * scst_is_cmd_local() in scst.h, if necessary
2007          */
2008
2009         switch (cmd->cdb[0]) {
2010         case MODE_SELECT:
2011         case MODE_SELECT_10:
2012         case LOG_SELECT:
2013                 res = scst_pre_select(cmd);
2014                 break;
2015         case RESERVE:
2016         case RESERVE_10:
2017                 res = scst_reserve_local(cmd);
2018                 break;
2019         case RELEASE:
2020         case RELEASE_10:
2021                 res = scst_release_local(cmd);
2022                 break;
2023         case REPORT_LUNS:
2024                 res = scst_report_luns_local(cmd);
2025                 break;
2026         default:
2027                 res = SCST_EXEC_NOT_COMPLETED;
2028                 break;
2029         }
2030
2031 out:
2032         TRACE_EXIT_RES(res);
2033         return res;
2034
2035 out_done:
2036         /* Report the result */
2037         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
2038         res = SCST_EXEC_COMPLETED;
2039         goto out;
2040 }
2041
2042 static int scst_local_exec(struct scst_cmd *cmd)
2043 {
2044         int res;
2045
2046         TRACE_ENTRY();
2047
2048         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
2049         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
2050         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
2051
2052         __scst_cmd_get(cmd);
2053
2054         res = scst_do_local_exec(cmd);
2055         if (likely(res == SCST_EXEC_NOT_COMPLETED))
2056                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2057         else if (res == SCST_EXEC_COMPLETED)
2058                 scst_post_exec_sn(cmd, true);
2059         else
2060                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
2061
2062         __scst_cmd_put(cmd);
2063
2064         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
2065         TRACE_EXIT_RES(res);
2066         return res;
2067 }
2068
2069 static int scst_exec(struct scst_cmd **active_cmd)
2070 {
2071         struct scst_cmd *cmd = *active_cmd;
2072         struct scst_cmd *ref_cmd;
2073         struct scst_device *dev = cmd->dev;
2074         int res = SCST_CMD_STATE_RES_CONT_NEXT, count;
2075
2076         TRACE_ENTRY();
2077
2078         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2079                 goto out;
2080
2081         /* To protect tgt_dev */
2082         ref_cmd = cmd;
2083         __scst_cmd_get(ref_cmd);
2084
2085         count = 0;
2086         while (1) {
2087                 int rc;
2088
2089                 cmd->sent_for_exec = 1;
2090                 /*
2091                  * To sync with scst_abort_cmd(). The above assignment must
2092                  * be before SCST_CMD_ABORTED test, done later in
2093                  * scst_check_local_events(). It's far from here, so the order
2094                  * is virtually guaranteed, but let's have it just in case.
2095                  */
2096                 smp_mb();
2097
2098                 cmd->scst_cmd_done = scst_cmd_done_local;
2099                 cmd->state = SCST_CMD_STATE_LOCAL_EXEC;
2100
2101                 if (cmd->tgt_data_buf_alloced && cmd->dh_data_buf_alloced &&
2102                     (cmd->data_direction == SCST_DATA_WRITE))
2103                         scst_copy_sg(cmd, SCST_SG_COPY_FROM_TARGET);
2104
2105                 rc = scst_do_local_exec(cmd);
2106                 if (likely(rc == SCST_EXEC_NOT_COMPLETED))
2107                         /* Nothing to do */;
2108                 else if (rc == SCST_EXEC_NEED_THREAD) {
2109                         TRACE_DBG("%s", "scst_do_local_exec() requested "
2110                                 "thread context, rescheduling");
2111                         scst_dec_on_dev_cmd(cmd);
2112                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2113                         break;
2114                 } else {
2115                         sBUG_ON(rc != SCST_EXEC_COMPLETED);
2116                         goto done;
2117                 }
2118
2119                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2120
2121                 rc = scst_do_real_exec(cmd);
2122                 if (likely(rc == SCST_EXEC_COMPLETED))
2123                         /* Nothing to do */;
2124                 else if (rc == SCST_EXEC_NEED_THREAD) {
2125                         TRACE_DBG("scst_real_exec() requested thread "
2126                                 "context, rescheduling (cmd %p)", cmd);
2127                         scst_dec_on_dev_cmd(cmd);
2128                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2129                         break;
2130                 } else
2131                         sBUG();
2132
2133 done:
2134                 count++;
2135
2136                 cmd = scst_post_exec_sn(cmd, false);
2137                 if (cmd == NULL)
2138                         break;
2139
2140                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2141                         break;
2142
2143                 __scst_cmd_put(ref_cmd);
2144                 ref_cmd = cmd;
2145                 __scst_cmd_get(ref_cmd);
2146         }
2147
2148         *active_cmd = cmd;
2149
2150         if (count == 0)
2151                 goto out_put;
2152
2153         if (dev->scsi_dev != NULL)
2154                 generic_unplug_device(dev->scsi_dev->request_queue);
2155
2156 out_put:
2157         __scst_cmd_put(ref_cmd);
2158         /* !! At this point sess, dev and tgt_dev can be already freed !! */
2159
2160 out:
2161         TRACE_EXIT_RES(res);
2162         return res;
2163 }
2164
2165 static int scst_send_for_exec(struct scst_cmd **active_cmd)
2166 {
2167         int res;
2168         struct scst_cmd *cmd = *active_cmd;
2169         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2170         typeof(tgt_dev->expected_sn) expected_sn;
2171
2172         TRACE_ENTRY();
2173
2174 #ifdef CONFIG_SCST_MEASURE_LATENCY
2175         if (cmd->pre_exec_finish == 0) {
2176                 struct timespec ts;
2177                 getnstimeofday(&ts);
2178                 cmd->pre_exec_finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2179                 TRACE_DBG("cmd %p (sess %p): pre_exec_finish %lld (tv_sec %ld, "
2180                         "tv_nsec %ld)", cmd, cmd->sess, cmd->pre_exec_finish,
2181                         ts.tv_sec, ts.tv_nsec);
2182         }
2183 #endif
2184
2185         if (unlikely(cmd->internal))
2186                 goto exec;
2187
2188         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2189                 goto exec;
2190
2191         sBUG_ON(!cmd->sn_set);
2192
2193         expected_sn = tgt_dev->expected_sn;
2194         /* Optimized for lockless fast path */
2195         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2196                 spin_lock_irq(&tgt_dev->sn_lock);
2197
2198                 tgt_dev->def_cmd_count++;
2199                 /*
2200                  * Memory barrier is needed here to implement lockless fast
2201                  * path. We need the exact order of read and write between
2202                  * def_cmd_count and expected_sn. Otherwise, we can miss case,
2203                  * when expected_sn was changed to be equal to cmd->sn while
2204                  * we are queuing cmd the deferred list after the expected_sn
2205                  * below. It will lead to a forever stuck command. But with
2206                  * the barrier in such case __scst_check_deferred_commands()
2207                  * will be called and it will take sn_lock, so we will be
2208                  * synchronized.
2209                  */
2210                 smp_mb();
2211
2212                 expected_sn = tgt_dev->expected_sn;
2213                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2214                         if (unlikely(test_bit(SCST_CMD_ABORTED,
2215                                               &cmd->cmd_flags))) {
2216                                 /* Necessary to allow aborting out of sn cmds */
2217                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p "
2218                                         "(tag %llu, sn %lu)", cmd,
2219                                         (long long unsigned)cmd->tag, cmd->sn);
2220                                 tgt_dev->def_cmd_count--;
2221                                 scst_set_cmd_abnormal_done_state(cmd);
2222                                 res = SCST_CMD_STATE_RES_CONT_SAME;
2223                         } else {
2224                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
2225                                         "expected_sn=%ld)", cmd, cmd->sn,
2226                                         cmd->sn_set, expected_sn);
2227                                 list_add_tail(&cmd->sn_cmd_list_entry,
2228                                               &tgt_dev->deferred_cmd_list);
2229                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2230                         }
2231                         spin_unlock_irq(&tgt_dev->sn_lock);
2232                         goto out;
2233                 } else {
2234                         TRACE_SN("Somebody incremented expected_sn %ld, "
2235                                 "continuing", expected_sn);
2236                         tgt_dev->def_cmd_count--;
2237                         spin_unlock_irq(&tgt_dev->sn_lock);
2238                 }
2239         }
2240
2241 exec:
2242         res = scst_exec(active_cmd);
2243
2244 out:
2245         TRACE_EXIT_HRES(res);
2246         return res;
2247 }
2248
2249 /* No locks supposed to be held */
2250 static int scst_check_sense(struct scst_cmd *cmd)
2251 {
2252         int res = 0;
2253         struct scst_device *dev = cmd->dev;
2254
2255         TRACE_ENTRY();
2256
2257         if (unlikely(cmd->ua_ignore))
2258                 goto out;
2259
2260         /* If we had internal bus reset behind us, set the command error UA */
2261         if ((dev->scsi_dev != NULL) &&
2262             unlikely(cmd->host_status == DID_RESET) &&
2263             scst_is_ua_command(cmd)) {
2264                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
2265                       dev->scsi_dev->was_reset, cmd->host_status);
2266                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_reset_UA));
2267                 /* It looks like it is safe to clear was_reset here */
2268                 dev->scsi_dev->was_reset = 0;
2269         }
2270
2271         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2272             SCST_SENSE_VALID(cmd->sense)) {
2273                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2274                         SCST_SENSE_BUFFERSIZE);
2275
2276                 /* Check Unit Attention Sense Key */
2277                 if (scst_is_ua_sense(cmd->sense)) {
2278                         if (cmd->sense[12] == SCST_SENSE_ASC_UA_RESET) {
2279                                 if (cmd->double_ua_possible) {
2280                                         TRACE(TRACE_MGMT_MINOR, "Double UA "
2281                                                 "detected for device %p", dev);
2282                                         TRACE(TRACE_MGMT_MINOR, "Retrying cmd"
2283                                                 " %p (tag %llu)", cmd,
2284                                                 (long long unsigned)cmd->tag);
2285
2286                                         cmd->status = 0;
2287                                         cmd->msg_status = 0;
2288                                         cmd->host_status = DID_OK;
2289                                         cmd->driver_status = 0;
2290
2291                                         mempool_free(cmd->sense,
2292                                                      scst_sense_mempool);
2293                                         cmd->sense = NULL;
2294
2295                                         scst_check_restore_sg_buff(cmd);
2296
2297                                         sBUG_ON(cmd->dbl_ua_orig_resp_data_len < 0);
2298                                         cmd->data_direction =
2299                                                 cmd->dbl_ua_orig_data_direction;
2300                                         cmd->resp_data_len =
2301                                                 cmd->dbl_ua_orig_resp_data_len;
2302
2303                                         cmd->state = SCST_CMD_STATE_REAL_EXEC;
2304                                         cmd->retry = 1;
2305                                         res = 1;
2306                                         goto out;
2307                                 }
2308                         }
2309                         scst_dev_check_set_UA(dev, cmd, cmd->sense,
2310                                 SCST_SENSE_BUFFERSIZE);
2311                 }
2312         }
2313
2314         if (unlikely(cmd->double_ua_possible)) {
2315                 if (scst_is_ua_command(cmd)) {
2316                         TRACE_MGMT_DBG("Clearing dbl_ua_possible flag (dev %p, "
2317                                 "cmd %p)", dev, cmd);
2318                         /*
2319                          * Lock used to protect other flags in the bitfield
2320                          * (just in case, actually). Those flags can't be
2321                          * changed in parallel, because the device is
2322                          * serialized.
2323                          */
2324                         spin_lock_bh(&dev->dev_lock);
2325                         dev->dev_double_ua_possible = 0;
2326                         spin_unlock_bh(&dev->dev_lock);
2327                 }
2328         }
2329
2330 out:
2331         TRACE_EXIT_RES(res);
2332         return res;
2333 }
2334
2335 static int scst_check_auto_sense(struct scst_cmd *cmd)
2336 {
2337         int res = 0;
2338
2339         TRACE_ENTRY();
2340
2341         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2342             (!SCST_SENSE_VALID(cmd->sense) ||
2343              SCST_NO_SENSE(cmd->sense))) {
2344                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2345                       "cmd->status=%x, cmd->msg_status=%x, "
2346                       "cmd->host_status=%x, cmd->driver_status=%x (cmd %p)",
2347                       cmd->status, cmd->msg_status, cmd->host_status,
2348                       cmd->driver_status, cmd);
2349                 res = 1;
2350         } else if (unlikely(cmd->host_status)) {
2351                 if ((cmd->host_status == DID_REQUEUE) ||
2352                     (cmd->host_status == DID_IMM_RETRY) ||
2353                     (cmd->host_status == DID_SOFT_ERROR) ||
2354                     (cmd->host_status == DID_ABORT)) {
2355                         scst_set_busy(cmd);
2356                 } else {
2357                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2358                                 "received, returning HARDWARE ERROR instead "
2359                                 "(cmd %p)", cmd->host_status, cmd);
2360                         scst_set_cmd_error(cmd,
2361                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2362                 }
2363         }
2364
2365         TRACE_EXIT_RES(res);
2366         return res;
2367 }
2368
2369 static int scst_pre_dev_done(struct scst_cmd *cmd)
2370 {
2371         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2372
2373         TRACE_ENTRY();
2374
2375         if (unlikely(scst_check_auto_sense(cmd))) {
2376                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2377                             "without sense data (opcode 0x%x), issuing "
2378                             "REQUEST SENSE", cmd->cdb[0]);
2379                 rc = scst_prepare_request_sense(cmd);
2380                 if (rc == 0)
2381                         res = SCST_CMD_STATE_RES_CONT_NEXT;
2382                 else {
2383                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2384                                     "returning HARDWARE ERROR");
2385                         scst_set_cmd_error(cmd,
2386                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2387                 }
2388                 goto out;
2389         } else if (unlikely(scst_check_sense(cmd)))
2390                 goto out;
2391
2392         if (likely(scsi_status_is_good(cmd->status))) {
2393                 unsigned char type = cmd->dev->handler->type;
2394                 if (unlikely((cmd->cdb[0] == MODE_SENSE ||
2395                               cmd->cdb[0] == MODE_SENSE_10)) &&
2396                     cmd->tgt_dev->acg_dev->rd_only_flag &&
2397                     (type == TYPE_DISK ||
2398                      type == TYPE_WORM ||
2399                      type == TYPE_MOD ||
2400                      type == TYPE_TAPE)) {
2401                         int32_t length;
2402                         uint8_t *address;
2403                         bool err = false;
2404
2405                         length = scst_get_buf_first(cmd, &address);
2406                         if (length < 0) {
2407                                 PRINT_ERROR("%s", "Unable to get "
2408                                         "MODE_SENSE buffer");
2409                                 scst_set_cmd_error(cmd,
2410                                         SCST_LOAD_SENSE(
2411                                                 scst_sense_hardw_error));
2412                                 err = true;
2413                         } else if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2414                                 address[2] |= 0x80;   /* Write Protect*/
2415                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2416                                 address[3] |= 0x80;   /* Write Protect*/
2417                         scst_put_buf(cmd, address);
2418
2419                         if (err)
2420                                 goto out;
2421                 }
2422
2423                 /*
2424                  * Check and clear NormACA option for the device, if necessary,
2425                  * since we don't support ACA
2426                  */
2427                 if (unlikely((cmd->cdb[0] == INQUIRY)) &&
2428                     /* Std INQUIRY data (no EVPD) */
2429                     !(cmd->cdb[1] & SCST_INQ_EVPD) &&
2430                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2431                         uint8_t *buffer;
2432                         int buflen;
2433                         bool err = false;
2434
2435                         /* ToDo: all pages ?? */
2436                         buflen = scst_get_buf_first(cmd, &buffer);
2437                         if (buflen > SCST_INQ_BYTE3) {
2438 #ifdef CONFIG_SCST_EXTRACHECKS
2439                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2440                                         PRINT_INFO("NormACA set for device: "
2441                                             "lun=%lld, type 0x%02x. Clear it, "
2442                                             "since it's unsupported.",
2443                                             (long long unsigned int)cmd->lun,
2444                                             buffer[0]);
2445                                 }
2446 #endif
2447                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2448                         } else if (buflen != 0) {
2449                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2450                                     "buffer");
2451                                 scst_set_cmd_error(cmd,
2452                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2453                                 err = true;
2454                         }
2455                         if (buflen > 0)
2456                                 scst_put_buf(cmd, buffer);
2457
2458                         if (err)
2459                                 goto out;
2460                 }
2461
2462                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2463                     (cmd->cdb[0] == MODE_SELECT_10) ||
2464                     (cmd->cdb[0] == LOG_SELECT))) {
2465                         TRACE(TRACE_SCSI,
2466                                 "MODE/LOG SELECT succeeded (LUN %lld)",
2467                                 (long long unsigned int)cmd->lun);
2468                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2469                         goto out;
2470                 }
2471         } else {
2472                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2473                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2474                                         &cmd->tgt_dev->tgt_dev_flags)) {
2475                                 struct scst_tgt_dev *tgt_dev_tmp;
2476                                 struct scst_device *dev = cmd->dev;
2477
2478                                 TRACE(TRACE_SCSI,
2479                                         "Real RESERVE failed lun=%lld, "
2480                                         "status=%x",
2481                                         (long long unsigned int)cmd->lun,
2482                                         cmd->status);
2483                                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2484                                         SCST_SENSE_BUFFERSIZE);
2485
2486                                 /* Clearing the reservation */
2487                                 spin_lock_bh(&dev->dev_lock);
2488                                 list_for_each_entry(tgt_dev_tmp,
2489                                                     &dev->dev_tgt_dev_list,
2490                                                     dev_tgt_dev_list_entry) {
2491                                         clear_bit(SCST_TGT_DEV_RESERVED,
2492                                                 &tgt_dev_tmp->tgt_dev_flags);
2493                                 }
2494                                 dev->dev_reserved = 0;
2495                                 spin_unlock_bh(&dev->dev_lock);
2496                         }
2497                 }
2498
2499                 /* Check for MODE PARAMETERS CHANGED UA */
2500                 if ((cmd->dev->scsi_dev != NULL) &&
2501                     (cmd->status == SAM_STAT_CHECK_CONDITION) &&
2502                     SCST_SENSE_VALID(cmd->sense) &&
2503                     scst_is_ua_sense(cmd->sense) &&
2504                     (cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) {
2505                         TRACE(TRACE_SCSI,
2506                               "MODE PARAMETERS CHANGED UA (lun %lld)",
2507                               (long long unsigned int)cmd->lun);
2508                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2509                         goto out;
2510                 }
2511         }
2512
2513         cmd->state = SCST_CMD_STATE_DEV_DONE;
2514
2515 out:
2516         TRACE_EXIT_RES(res);
2517         return res;
2518 }
2519
2520 static int scst_mode_select_checks(struct scst_cmd *cmd)
2521 {
2522         int res = SCST_CMD_STATE_RES_CONT_SAME;
2523         int atomic = scst_cmd_atomic(cmd);
2524
2525         TRACE_ENTRY();
2526
2527         if (likely(scsi_status_is_good(cmd->status))) {
2528                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2529                     (cmd->cdb[0] == MODE_SELECT_10) ||
2530                     (cmd->cdb[0] == LOG_SELECT))) {
2531                         struct scst_device *dev = cmd->dev;
2532                         if (atomic && (dev->scsi_dev != NULL)) {
2533                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2534                                         "context required");
2535                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2536                                 goto out;
2537                         }
2538
2539                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2540                                 "setting the SELECT UA (lun=%lld)",
2541                                 (long long unsigned int)cmd->lun);
2542
2543                         spin_lock_bh(&dev->dev_lock);
2544                         spin_lock(&scst_temp_UA_lock);
2545                         if (cmd->cdb[0] == LOG_SELECT) {
2546                                 scst_set_sense(scst_temp_UA,
2547                                         sizeof(scst_temp_UA),
2548                                         UNIT_ATTENTION, 0x2a, 0x02);
2549                         } else {
2550                                 scst_set_sense(scst_temp_UA,
2551                                         sizeof(scst_temp_UA),
2552                                         UNIT_ATTENTION, 0x2a, 0x01);
2553                         }
2554                         scst_dev_check_set_local_UA(dev, cmd, scst_temp_UA,
2555                                 sizeof(scst_temp_UA));
2556                         spin_unlock(&scst_temp_UA_lock);
2557                         spin_unlock_bh(&dev->dev_lock);
2558
2559                         if (dev->scsi_dev != NULL)
2560                                 scst_obtain_device_parameters(dev);
2561                 }
2562         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) &&
2563                     SCST_SENSE_VALID(cmd->sense) &&
2564                     scst_is_ua_sense(cmd->sense) &&
2565                     (((cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) ||
2566                      (cmd->sense[12] == 0x29) /* reset */ ||
2567                      (cmd->sense[12] == 0x28) /* medium changed */ ||
2568                      /* cleared by another ini (just in case) */
2569                      (cmd->sense[12] == 0x2F))) {
2570                 if (atomic) {
2571                         TRACE_DBG("Possible parameters changed UA %x: "
2572                                 "thread context required", cmd->sense[12]);
2573                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2574                         goto out;
2575                 }
2576
2577                 TRACE(TRACE_SCSI, "Possible parameters changed UA %x "
2578                         "(lun %lld): getting new parameters", cmd->sense[12],
2579                         (long long unsigned int)cmd->lun);
2580
2581                 scst_obtain_device_parameters(cmd->dev);
2582         } else
2583                 sBUG();
2584
2585         cmd->state = SCST_CMD_STATE_DEV_DONE;
2586
2587 out:
2588         TRACE_EXIT_HRES(res);
2589         return res;
2590 }
2591
2592 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2593 {
2594         if (likely(cmd->sn_set))
2595                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2596
2597         scst_make_deferred_commands_active(cmd->tgt_dev);
2598 }
2599
2600 static int scst_dev_done(struct scst_cmd *cmd)
2601 {
2602         int res = SCST_CMD_STATE_RES_CONT_SAME;
2603         int state;
2604         struct scst_device *dev = cmd->dev;
2605
2606         TRACE_ENTRY();
2607
2608         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2609
2610         if (likely(!scst_is_cmd_local(cmd)) &&
2611             likely(dev->handler->dev_done != NULL)) {
2612                 int rc;
2613
2614                 if (unlikely(!dev->handler->dev_done_atomic &&
2615                              scst_cmd_atomic(cmd))) {
2616                         /*
2617                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2618                          * optimization.
2619                          */
2620                         TRACE_DBG("Dev handler %s dev_done() needs thread "
2621                               "context, rescheduling", dev->handler->name);
2622                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2623                         goto out;
2624                 }
2625
2626                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2627                       dev->handler->name, cmd);
2628                 rc = dev->handler->dev_done(cmd);
2629                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2630                       dev->handler->name, rc);
2631                 if (rc != SCST_CMD_STATE_DEFAULT)
2632                         state = rc;
2633         }
2634
2635         switch (state) {
2636         case SCST_CMD_STATE_PRE_XMIT_RESP:
2637         case SCST_CMD_STATE_DEV_PARSE:
2638         case SCST_CMD_STATE_PRE_PARSE:
2639         case SCST_CMD_STATE_PREPARE_SPACE:
2640         case SCST_CMD_STATE_RDY_TO_XFER:
2641         case SCST_CMD_STATE_TGT_PRE_EXEC:
2642         case SCST_CMD_STATE_SEND_FOR_EXEC:
2643         case SCST_CMD_STATE_LOCAL_EXEC:
2644         case SCST_CMD_STATE_REAL_EXEC:
2645         case SCST_CMD_STATE_PRE_DEV_DONE:
2646         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2647         case SCST_CMD_STATE_DEV_DONE:
2648         case SCST_CMD_STATE_XMIT_RESP:
2649         case SCST_CMD_STATE_FINISHED:
2650         case SCST_CMD_STATE_FINISHED_INTERNAL:
2651                 cmd->state = state;
2652                 break;
2653
2654         case SCST_CMD_STATE_NEED_THREAD_CTX:
2655                 TRACE_DBG("Dev handler %s dev_done() requested "
2656                       "thread context, rescheduling",
2657                       dev->handler->name);
2658                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2659                 break;
2660
2661         default:
2662                 if (state >= 0) {
2663                         PRINT_ERROR("Dev handler %s dev_done() returned "
2664                                 "invalid cmd state %d",
2665                                 dev->handler->name, state);
2666                 } else {
2667                         PRINT_ERROR("Dev handler %s dev_done() returned "
2668                                 "error %d", dev->handler->name,
2669                                 state);
2670                 }
2671                 scst_set_cmd_error(cmd,
2672                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2673                 scst_set_cmd_abnormal_done_state(cmd);
2674                 break;
2675         }
2676
2677         if (cmd->needs_unblocking)
2678                 scst_unblock_dev_cmd(cmd);
2679
2680         if (likely(cmd->dec_on_dev_needed))
2681                 scst_dec_on_dev_cmd(cmd);
2682
2683         if (cmd->inc_expected_sn_on_done && cmd->sent_for_exec)
2684                 scst_inc_check_expected_sn(cmd);
2685
2686         if (unlikely(cmd->internal))
2687                 cmd->state = SCST_CMD_STATE_FINISHED_INTERNAL;
2688
2689 out:
2690         TRACE_EXIT_HRES(res);
2691         return res;
2692 }
2693
2694 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2695 {
2696         int res;
2697
2698         TRACE_ENTRY();
2699
2700         EXTRACHECKS_BUG_ON(cmd->internal);
2701
2702 #ifdef CONFIG_SCST_DEBUG_TM
2703         if (cmd->tm_dbg_delayed &&
2704                         !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2705                 if (scst_cmd_atomic(cmd)) {
2706                         TRACE_MGMT_DBG("%s",
2707                                 "DEBUG_TM delayed cmd needs a thread");
2708                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2709                         return res;
2710                 }
2711                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2712                         cmd, cmd->tag);
2713                 schedule_timeout_uninterruptible(HZ);
2714         }
2715 #endif
2716
2717         if (likely(cmd->tgt_dev != NULL)) {
2718                 atomic_dec(&cmd->tgt_dev->tgt_dev_cmd_count);
2719                 atomic_dec(&cmd->dev->dev_cmd_count);
2720                 /* If expected values not set, expected direction is UNKNOWN */
2721                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
2722                         atomic_dec(&cmd->dev->write_cmd_count);
2723
2724                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2725                         scst_on_hq_cmd_response(cmd);
2726
2727                 if (unlikely(!cmd->sent_for_exec)) {
2728                         TRACE_SN("cmd %p was not sent to mid-lev"
2729                                 " (sn %ld, set %d)",
2730                                 cmd, cmd->sn, cmd->sn_set);
2731                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2732                         cmd->sent_for_exec = 1;
2733                 }
2734         }
2735
2736         /*
2737          * If we don't remove cmd from the search list here, before
2738          * submitting it for transmittion, we will have a race, when for
2739          * some reason cmd's release is delayed after transmittion and
2740          * initiator sends cmd with the same tag => it is possible that
2741          * a wrong cmd will be found by find() functions.
2742          */
2743         spin_lock_irq(&cmd->sess->sess_list_lock);
2744         list_del(&cmd->search_cmd_list_entry);
2745         spin_unlock_irq(&cmd->sess->sess_list_lock);
2746
2747         cmd->done = 1;
2748         smp_mb(); /* to sync with scst_abort_cmd() */
2749
2750         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2751                 scst_xmit_process_aborted_cmd(cmd);
2752
2753         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2754                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu),"
2755                                 " skipping",
2756                                 cmd, (long long unsigned int)cmd->tag);
2757                 cmd->state = SCST_CMD_STATE_FINISHED;
2758                 res = SCST_CMD_STATE_RES_CONT_SAME;
2759                 goto out;
2760         }
2761
2762         if (cmd->tgt_data_buf_alloced && cmd->dh_data_buf_alloced &&
2763             (cmd->data_direction == SCST_DATA_READ))
2764                 scst_copy_sg(cmd, SCST_SG_COPY_TO_TARGET);
2765
2766         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2767         res = SCST_CMD_STATE_RES_CONT_SAME;
2768
2769 out:
2770 #ifdef CONFIG_SCST_MEASURE_LATENCY
2771         {
2772                 struct timespec ts;
2773                 uint64_t finish, scst_time, proc_time;
2774                 struct scst_session *sess = cmd->sess;
2775
2776                 getnstimeofday(&ts);
2777                 finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2778
2779                 spin_lock_bh(&sess->meas_lock);
2780
2781                 scst_time = cmd->pre_exec_finish - cmd->start;
2782                 scst_time += finish - cmd->post_exec_start;
2783                 proc_time = finish - cmd->start;
2784
2785                 sess->scst_time += scst_time;
2786                 sess->processing_time += proc_time;
2787                 sess->processed_cmds++;
2788
2789                 spin_unlock_bh(&sess->meas_lock);
2790
2791                 TRACE_DBG("cmd %p (sess %p): finish %lld (tv_sec %ld, "
2792                         "tv_nsec %ld), scst_time %lld, proc_time %lld",
2793                         cmd, sess, finish, ts.tv_sec, ts.tv_nsec, scst_time,
2794                         proc_time);
2795         }
2796 #endif
2797         TRACE_EXIT_HRES(res);
2798         return res;
2799 }
2800
2801 static int scst_xmit_response(struct scst_cmd *cmd)
2802 {
2803         int res, rc;
2804
2805         TRACE_ENTRY();
2806
2807         EXTRACHECKS_BUG_ON(cmd->internal);
2808
2809         if (unlikely(!cmd->tgtt->xmit_response_atomic &&
2810                      scst_cmd_atomic(cmd))) {
2811                 /*
2812                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2813                  * optimization.
2814                  */
2815                 TRACE_DBG("Target driver %s xmit_response() needs thread "
2816                               "context, rescheduling", cmd->tgtt->name);
2817                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2818                 goto out;
2819         }
2820
2821         while (1) {
2822                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2823
2824                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2825                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2826
2827                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2828
2829 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2830                 if (cmd->tgt_sg) {
2831                         int i;
2832                         struct scatterlist *sg = cmd->tgt_sg;
2833                         TRACE(TRACE_SND_BOT, "Xmitting data for cmd %p "
2834                                 "(sg_cnt %d, sg %p, sg[0].page %p)", cmd,
2835                                 cmd->tgt_sg_cnt, sg, (void *)sg_page(&sg[0]));
2836                         for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
2837                                 PRINT_BUFF_FLAG(TRACE_SND_BOT, "Xmitting sg",
2838                                         sg_virt(&sg[i]), sg[i].length);
2839                         }
2840                 }
2841 #endif
2842
2843 #ifdef CONFIG_SCST_DEBUG_RETRY
2844                 if (((scst_random() % 100) == 77))
2845                         rc = SCST_TGT_RES_QUEUE_FULL;
2846                 else
2847 #endif
2848                         rc = cmd->tgtt->xmit_response(cmd);
2849                 TRACE_DBG("xmit_response() returned %d", rc);
2850
2851                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2852                         goto out;
2853
2854                 /* Restore the previous state */
2855                 cmd->state = SCST_CMD_STATE_XMIT_RESP;
2856
2857                 switch (rc) {
2858                 case SCST_TGT_RES_QUEUE_FULL:
2859                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2860                                 break;
2861                         else
2862                                 continue;
2863
2864                 case SCST_TGT_RES_NEED_THREAD_CTX:
2865                         TRACE_DBG("Target driver %s xmit_response() "
2866                               "requested thread context, rescheduling",
2867                               cmd->tgtt->name);
2868                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2869                         break;
2870
2871                 default:
2872                         goto out_error;
2873                 }
2874                 break;
2875         }
2876
2877 out:
2878         /* Caution: cmd can be already dead here */
2879         TRACE_EXIT_HRES(res);
2880         return res;
2881
2882 out_error:
2883         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2884                 PRINT_ERROR("Target driver %s xmit_response() returned "
2885                         "fatal error", cmd->tgtt->name);
2886         } else {
2887                 PRINT_ERROR("Target driver %s xmit_response() returned "
2888                         "invalid value %d", cmd->tgtt->name, rc);
2889         }
2890         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2891         cmd->state = SCST_CMD_STATE_FINISHED;
2892         res = SCST_CMD_STATE_RES_CONT_SAME;
2893         goto out;
2894 }
2895
2896 void scst_tgt_cmd_done(struct scst_cmd *cmd,
2897         enum scst_exec_context pref_context)
2898 {
2899         TRACE_ENTRY();
2900
2901         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2902
2903         cmd->state = SCST_CMD_STATE_FINISHED;
2904         scst_proccess_redirect_cmd(cmd, pref_context, 1);
2905
2906         TRACE_EXIT();
2907         return;
2908 }
2909 EXPORT_SYMBOL(scst_tgt_cmd_done);
2910
2911 static int scst_finish_cmd(struct scst_cmd *cmd)
2912 {
2913         int res;
2914
2915         TRACE_ENTRY();
2916
2917         atomic_dec(&cmd->sess->sess_cmd_count);
2918
2919         cmd->finished = 1;
2920         smp_mb(); /* to sync with scst_abort_cmd() */
2921
2922         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2923                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
2924                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
2925                         atomic_read(&scst_cmd_count));
2926
2927                 scst_finish_cmd_mgmt(cmd);
2928         }
2929
2930         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
2931                 if ((cmd->tgt_dev != NULL) &&
2932                     scst_is_ua_sense(cmd->sense)) {
2933                         /* This UA delivery failed, so requeue it */
2934                         TRACE_MGMT_DBG("Requeuing UA for delivery failed cmd "
2935                                 "%p", cmd);
2936                         scst_check_set_UA(cmd->tgt_dev, cmd->sense,
2937                                         SCST_SENSE_BUFFERSIZE, 1);
2938                 }
2939         }
2940
2941         __scst_cmd_put(cmd);
2942
2943         res = SCST_CMD_STATE_RES_CONT_NEXT;
2944
2945         TRACE_EXIT_HRES(res);
2946         return res;
2947 }
2948
2949 /*
2950  * No locks, but it must be externally serialized (see comment for
2951  * scst_cmd_init_done() in scst.h)
2952  */
2953 static void scst_cmd_set_sn(struct scst_cmd *cmd)
2954 {
2955         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2956         unsigned long flags;
2957
2958         TRACE_ENTRY();
2959
2960         if (scst_is_implicit_hq(cmd)) {
2961                 TRACE_SN("Implicit HQ cmd %p", cmd);
2962                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
2963         }
2964
2965         EXTRACHECKS_BUG_ON(cmd->sn_set || cmd->hq_cmd_inced);
2966
2967         /* Optimized for lockless fast path */
2968
2969         scst_check_debug_sn(cmd);
2970
2971         if (cmd->dev->queue_alg ==
2972                         SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) {
2973                 /*
2974                  * Not the best way, but well enough until there will be a
2975                  * possibility to specify queue type during pass-through
2976                  * commands submission.
2977                  */
2978                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
2979         }
2980
2981         switch (cmd->queue_type) {
2982         case SCST_CMD_QUEUE_SIMPLE:
2983         case SCST_CMD_QUEUE_UNTAGGED:
2984 #if 0 /* left for future performance investigations */
2985                 if (scst_cmd_is_expected_set(cmd)) {
2986                         if ((cmd->expected_data_direction == SCST_DATA_READ) &&
2987                             (atomic_read(&cmd->dev->write_cmd_count) == 0))
2988                                 goto ordered;
2989                 } else
2990                         goto ordered;
2991 #endif
2992                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
2993                         /*
2994                          * atomic_inc_return() implies memory barrier to sync
2995                          * with scst_inc_expected_sn()
2996                          */
2997                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
2998                                 tgt_dev->curr_sn++;
2999                                 TRACE_SN("Incremented curr_sn %ld",
3000                                         tgt_dev->curr_sn);
3001                         }
3002                         cmd->sn_slot = tgt_dev->cur_sn_slot;
3003                         cmd->sn = tgt_dev->curr_sn;
3004
3005                         tgt_dev->prev_cmd_ordered = 0;
3006                 } else {
3007                         TRACE(TRACE_MINOR, "***WARNING*** Not enough SN slots "
3008                                 "%zd", ARRAY_SIZE(tgt_dev->sn_slots));
3009                         goto ordered;
3010                 }
3011                 break;
3012
3013         case SCST_CMD_QUEUE_ORDERED:
3014                 TRACE_SN("ORDERED cmd %p (op %x)", cmd, cmd->cdb[0]);
3015 ordered:
3016                 if (!tgt_dev->prev_cmd_ordered) {
3017                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
3018                         if (tgt_dev->num_free_sn_slots >= 0) {
3019                                 tgt_dev->num_free_sn_slots--;
3020                                 if (tgt_dev->num_free_sn_slots >= 0) {
3021                                         int i = 0;
3022                                         /* Commands can finish in any order, so
3023                                          * we don't know which slot is empty.
3024                                          */
3025                                         while (1) {
3026                                                 tgt_dev->cur_sn_slot++;
3027                                                 if (tgt_dev->cur_sn_slot ==
3028                                                       tgt_dev->sn_slots + ARRAY_SIZE(tgt_dev->sn_slots))
3029                                                         tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
3030
3031                                                 if (atomic_read(tgt_dev->cur_sn_slot) == 0)
3032                                                         break;
3033
3034                                                 i++;
3035                                                 sBUG_ON(i == ARRAY_SIZE(tgt_dev->sn_slots));
3036                                         }
3037                                         TRACE_SN("New cur SN slot %zd",
3038                                                 tgt_dev->cur_sn_slot -
3039                                                 tgt_dev->sn_slots);
3040                                 }
3041                         }
3042                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
3043                 }
3044                 tgt_dev->prev_cmd_ordered = 1;
3045                 tgt_dev->curr_sn++;
3046                 cmd->sn = tgt_dev->curr_sn;
3047                 break;
3048
3049         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
3050                 TRACE_SN("HQ cmd %p (op %x)", cmd, cmd->cdb[0]);
3051                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
3052                 tgt_dev->hq_cmd_count++;
3053                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
3054                 cmd->hq_cmd_inced = 1;
3055                 goto out;
3056
3057         default:
3058                 sBUG();
3059         }
3060
3061         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
3062                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
3063                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
3064                 atomic_read(tgt_dev->cur_sn_slot),
3065                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
3066                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
3067
3068         cmd->sn_set = 1;
3069
3070 out:
3071         TRACE_EXIT();
3072         return;
3073 }
3074
3075 /*
3076  * Returns 0 on success, > 0 when we need to wait for unblock,
3077  * < 0 if there is no device (lun) or device type handler.
3078  *
3079  * No locks, but might be on IRQ, protection is done by the
3080  * suspended activity.
3081  */
3082 static int scst_translate_lun(struct scst_cmd *cmd)
3083 {
3084         struct scst_tgt_dev *tgt_dev = NULL;
3085         int res;
3086
3087         TRACE_ENTRY();
3088
3089         /* See comment about smp_mb() in scst_suspend_activity() */
3090         __scst_get(1);
3091
3092         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
3093                 struct list_head *sess_tgt_dev_list_head =
3094                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
3095                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %lld)", cmd,
3096                         (long long unsigned int)cmd->lun);
3097                 res = -1;
3098                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3099                                 sess_tgt_dev_list_entry) {
3100                         if (tgt_dev->lun == cmd->lun) {
3101                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
3102
3103                                 if (unlikely(tgt_dev->dev->handler ==
3104                                                 &scst_null_devtype)) {
3105                                         PRINT_INFO("Dev handler for device "
3106                                           "%lld is NULL, the device will not "
3107                                           "be visible remotely",
3108                                            (long long unsigned int)cmd->lun);
3109                                         break;
3110                                 }
3111
3112                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
3113                                 cmd->tgt_dev = tgt_dev;
3114                                 cmd->dev = tgt_dev->dev;
3115
3116                                 res = 0;
3117                                 break;
3118                         }
3119                 }
3120                 if (res != 0) {
3121                         TRACE(TRACE_MINOR,
3122                                 "tgt_dev for lun %lld not found, command to "
3123                                 "unexisting LU?",
3124                                 (long long unsigned int)cmd->lun);
3125                         __scst_put();
3126                 }
3127         } else {
3128                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3129                 __scst_put();
3130                 res = 1;
3131         }
3132
3133         TRACE_EXIT_RES(res);
3134         return res;
3135 }
3136
3137 /*
3138  * No locks, but might be on IRQ
3139  *
3140  * Returns 0 on success, > 0 when we need to wait for unblock,
3141  * < 0 if there is no device (lun) or device type handler.
3142  */
3143 static int __scst_init_cmd(struct scst_cmd *cmd)
3144 {
3145         int res = 0;
3146
3147         TRACE_ENTRY();
3148
3149         res = scst_translate_lun(cmd);
3150         if (likely(res == 0)) {
3151                 int cnt;
3152                 bool failure = false;
3153
3154                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
3155
3156                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
3157                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
3158                         TRACE(TRACE_MGMT_MINOR,
3159                                 "Too many pending commands (%d) in "
3160                                 "session, returning BUSY to initiator \"%s\"",
3161                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
3162                                   "Anonymous" : cmd->sess->initiator_name);
3163                         failure = true;
3164                 }
3165
3166                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
3167                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
3168                         if (!failure) {
3169                                 TRACE(TRACE_MGMT_MINOR,
3170                                         "Too many pending device "
3171                                         "commands (%d), returning BUSY to "
3172                                         "initiator \"%s\"", cnt,
3173                                         (cmd->sess->initiator_name[0] == '\0') ?
3174                                                 "Anonymous" :
3175                                                 cmd->sess->initiator_name);
3176                                 failure = true;
3177                         }
3178                 }
3179
3180                 /* If expected values not set, expected direction is UNKNOWN */
3181                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
3182                         atomic_inc(&cmd->dev->write_cmd_count);
3183
3184                 if (unlikely(failure))
3185                         goto out_busy;
3186
3187                 if (!cmd->set_sn_on_restart_cmd)
3188                         scst_cmd_set_sn(cmd);
3189         } else if (res < 0) {
3190                 TRACE_DBG("Finishing cmd %p", cmd);
3191                 scst_set_cmd_error(cmd,
3192                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
3193                 scst_set_cmd_abnormal_done_state(cmd);
3194         } else
3195                 goto out;
3196
3197 out:
3198         TRACE_EXIT_RES(res);
3199         return res;
3200
3201 out_busy:
3202         scst_set_busy(cmd);
3203         scst_set_cmd_abnormal_done_state(cmd);
3204         goto out;
3205 }
3206
3207 /* Called under scst_init_lock and IRQs disabled */
3208 static void scst_do_job_init(void)
3209         __releases(&scst_init_lock)
3210         __acquires(&scst_init_lock)
3211 {
3212         struct scst_cmd *cmd;
3213         int susp;
3214
3215         TRACE_ENTRY();
3216
3217 restart:
3218         /*
3219          * There is no need for read barrier here, because we don't care where
3220          * this check will be done.
3221          */
3222         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
3223         if (scst_init_poll_cnt > 0)
3224                 scst_init_poll_cnt--;
3225
3226         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
3227                 int rc;
3228                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
3229                         continue;
3230                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
3231                         spin_unlock_irq(&scst_init_lock);
3232                         rc = __scst_init_cmd(cmd);
3233                         spin_lock_irq(&scst_init_lock);
3234                         if (rc > 0) {
3235                                 TRACE_MGMT_DBG("%s",
3236                                         "FLAG SUSPENDED set, restarting");
3237                                 goto restart;
3238                         }
3239                 } else {
3240                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
3241                                        cmd, (long long unsigned int)cmd->tag);
3242                         scst_set_cmd_abnormal_done_state(cmd);
3243                 }
3244
3245                 /*
3246                  * Deleting cmd from init cmd list after __scst_init_cmd()
3247                  * is necessary to keep the check in scst_init_cmd() correct
3248                  * to preserve the commands order.
3249                  *
3250                  * We don't care about the race, when init cmd list is empty
3251                  * and one command detected that it just was not empty, so
3252                  * it's inserting to it, but another command at the same time
3253                  * seeing init cmd list empty and goes directly, because it
3254                  * could affect only commands from the same initiator to the
3255                  * same tgt_dev, but scst_cmd_init_done*() doesn't guarantee
3256                  * the order in case of simultaneous such calls anyway.
3257                  */
3258                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
3259                 smp_wmb(); /* enforce the required order */
3260                 list_del(&cmd->cmd_list_entry);
3261                 spin_unlock(&scst_init_lock);
3262
3263                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
3264                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
3265                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
3266                         list_add(&cmd->cmd_list_entry,
3267                                 &cmd->cmd_lists->active_cmd_list);
3268                 else
3269                         list_add_tail(&cmd->cmd_list_entry,
3270                                 &cmd->cmd_lists->active_cmd_list);
3271                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3272                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
3273
3274                 spin_lock(&scst_init_lock);
3275                 goto restart;
3276         }
3277
3278         /* It isn't really needed, but let's keep it */
3279         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
3280                 goto restart;
3281
3282         TRACE_EXIT();
3283         return;
3284 }
3285
3286 static inline int test_init_cmd_list(void)
3287 {
3288         int res = (!list_empty(&scst_init_cmd_list) &&
3289                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
3290                   unlikely(kthread_should_stop()) ||
3291                   (scst_init_poll_cnt > 0);
3292         return res;
3293 }
3294
3295 int scst_init_cmd_thread(void *arg)
3296 {
3297         TRACE_ENTRY();
3298
3299         PRINT_INFO("Init thread started, PID %d", current->pid);
3300
3301         current->flags |= PF_NOFREEZE;
3302
3303         set_user_nice(current, -10);
3304
3305         spin_lock_irq(&scst_init_lock);
3306         while (!kthread_should_stop()) {
3307                 wait_queue_t wait;
3308                 init_waitqueue_entry(&wait, current);
3309
3310                 if (!test_init_cmd_list()) {
3311                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
3312                                                  &wait);
3313                         for (;;) {
3314                                 set_current_state(TASK_INTERRUPTIBLE);
3315                                 if (test_init_cmd_list())
3316                                         break;
3317                                 spin_unlock_irq(&scst_init_lock);
3318                                 schedule();
3319                                 spin_lock_irq(&scst_init_lock);
3320                         }
3321                         set_current_state(TASK_RUNNING);
3322                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
3323                 }
3324                 scst_do_job_init();
3325         }
3326         spin_unlock_irq(&scst_init_lock);
3327
3328         /*
3329          * If kthread_should_stop() is true, we are guaranteed to be
3330          * on the module unload, so scst_init_cmd_list must be empty.
3331          */
3332         sBUG_ON(!list_empty(&scst_init_cmd_list));
3333
3334         PRINT_INFO("Init thread PID %d finished", current->pid);
3335
3336         TRACE_EXIT();
3337         return 0;
3338 }
3339
3340 /* Called with no locks held */
3341 void scst_process_active_cmd(struct scst_cmd *cmd, bool atomic)
3342 {
3343         int res;
3344
3345         TRACE_ENTRY();
3346
3347         EXTRACHECKS_BUG_ON(in_irq() || irqs_disabled());
3348
3349         cmd->atomic = atomic;
3350
3351         TRACE_DBG("cmd %p, atomic %d", cmd, atomic);
3352
3353         do {
3354                 switch (cmd->state) {
3355                 case SCST_CMD_STATE_PRE_PARSE:
3356                         res = scst_pre_parse(cmd);
3357                         EXTRACHECKS_BUG_ON(res ==
3358                                 SCST_CMD_STATE_RES_NEED_THREAD);
3359                         break;
3360
3361                 case SCST_CMD_STATE_DEV_PARSE:
3362                         res = scst_parse_cmd(cmd);
3363                         break;
3364
3365                 case SCST_CMD_STATE_PREPARE_SPACE:
3366                         res = scst_prepare_space(cmd);
3367                         break;
3368
3369                 case SCST_CMD_STATE_RDY_TO_XFER:
3370                         res = scst_rdy_to_xfer(cmd);
3371                         break;
3372
3373                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3374                         res = scst_tgt_pre_exec(cmd);
3375                         break;
3376
3377                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3378                         if (tm_dbg_check_cmd(cmd) != 0) {
3379                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
3380                                 TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
3381                                         "because of TM DBG delay", cmd,
3382                                         (long long unsigned int)cmd->tag);
3383                                 break;
3384                         }
3385                         res = scst_send_for_exec(&cmd);
3386                         /*
3387                          * !! At this point cmd, sess & tgt_dev can already be
3388                          * freed !!
3389                          */
3390                         break;
3391
3392                 case SCST_CMD_STATE_LOCAL_EXEC:
3393                         res = scst_local_exec(cmd);
3394                         /*
3395                          * !! At this point cmd, sess & tgt_dev can already be
3396                          * freed !!
3397                          */
3398                         break;
3399
3400                 case SCST_CMD_STATE_REAL_EXEC:
3401                         res = scst_real_exec(cmd);
3402                         /*
3403                          * !! At this point cmd, sess & tgt_dev can already be
3404                          * freed !!
3405                          */
3406                         break;
3407
3408                 case SCST_CMD_STATE_PRE_DEV_DONE:
3409                         res = scst_pre_dev_done(cmd);
3410                         EXTRACHECKS_BUG_ON(res ==
3411                                 SCST_CMD_STATE_RES_NEED_THREAD);
3412                         break;
3413
3414                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3415                         res = scst_mode_select_checks(cmd);
3416                         break;
3417
3418                 case SCST_CMD_STATE_DEV_DONE:
3419                         res = scst_dev_done(cmd);
3420                         break;
3421
3422                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3423                         res = scst_pre_xmit_response(cmd);
3424                         EXTRACHECKS_BUG_ON(res ==
3425                                 SCST_CMD_STATE_RES_NEED_THREAD);
3426                         break;
3427
3428                 case SCST_CMD_STATE_XMIT_RESP:
3429                         res = scst_xmit_response(cmd);
3430                         break;
3431
3432                 case SCST_CMD_STATE_FINISHED:
3433                         res = scst_finish_cmd(cmd);
3434                         EXTRACHECKS_BUG_ON(res ==
3435                                 SCST_CMD_STATE_RES_NEED_THREAD);
3436                         break;
3437
3438                 case SCST_CMD_STATE_FINISHED_INTERNAL:
3439                         res = scst_finish_internal_cmd(cmd);
3440                         EXTRACHECKS_BUG_ON(res ==
3441                                 SCST_CMD_STATE_RES_NEED_THREAD);
3442                         break;
3443
3444                 default:
3445                         PRINT_CRIT_ERROR("cmd (%p) in state %d, but shouldn't "
3446                                 "be", cmd, cmd->state);
3447                         sBUG();
3448                         res = SCST_CMD_STATE_RES_CONT_NEXT;
3449                         break;
3450                 }
3451         } while (res == SCST_CMD_STATE_RES_CONT_SAME);
3452
3453         if (res == SCST_CMD_STATE_RES_CONT_NEXT) {
3454                 /* None */
3455         } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
3456                 spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3457                 switch (cmd->state) {
3458                 case SCST_CMD_STATE_PRE_PARSE:
3459                 case SCST_CMD_STATE_DEV_PARSE:
3460                 case SCST_CMD_STATE_PREPARE_SPACE:
3461                 case SCST_CMD_STATE_RDY_TO_XFER:
3462                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3463                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3464                 case SCST_CMD_STATE_LOCAL_EXEC:
3465                 case SCST_CMD_STATE_REAL_EXEC:
3466                 case SCST_CMD_STATE_PRE_DEV_DONE:
3467                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3468                 case SCST_CMD_STATE_DEV_DONE:
3469                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3470                 case SCST_CMD_STATE_XMIT_RESP:
3471                 case SCST_CMD_STATE_FINISHED:
3472                 case SCST_CMD_STATE_FINISHED_INTERNAL:
3473                         TRACE_DBG("Adding cmd %p to head of active cmd list",
3474                                   cmd);
3475                         list_add(&cmd->cmd_list_entry,
3476                                 &cmd->cmd_lists->active_cmd_list);
3477                         break;
3478 #ifdef CONFIG_SCST_EXTRACHECKS
3479                 /* not very valid commands */
3480                 case SCST_CMD_STATE_DEFAULT:
3481                 case SCST_CMD_STATE_NEED_THREAD_CTX:
3482                         PRINT_CRIT_ERROR("cmd %p is in invalid state %d)", cmd,
3483                                 cmd->state);
3484                         spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3485                         sBUG();
3486                         spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3487                         break;
3488 #endif
3489                 default:
3490                         break;
3491                 }
3492                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3493                 spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3494         } else
3495                 sBUG();
3496
3497         TRACE_EXIT();