- Memory barriers cleanup. Comments for them improved
[mirror/scst/.git] / scst / src / scst_targ.c
1 /*
2  *  scst_targ.c
3  *
4  *  Copyright (C) 2004 - 2008 Vladislav Bolkhovitin <vst@vlnb.net>
5  *  Copyright (C) 2004 - 2005 Leonid Stoljar
6  *  Copyright (C) 2007 - 2008 CMS Distribution Limited
7  *
8  *  This program is free software; you can redistribute it and/or
9  *  modify it under the terms of the GNU General Public License
10  *  as published by the Free Software Foundation, version 2
11  *  of the License.
12  *
13  *  This program is distributed in the hope that it will be useful,
14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  *  GNU General Public License for more details.
17  */
18
19 #include <linux/init.h>
20 #include <linux/kernel.h>
21 #include <linux/errno.h>
22 #include <linux/list.h>
23 #include <linux/spinlock.h>
24 #include <linux/slab.h>
25 #include <linux/sched.h>
26 #include <linux/smp_lock.h>
27 #include <linux/unistd.h>
28 #include <linux/string.h>
29 #include <linux/kthread.h>
30 #include <linux/delay.h>
31
32 #include "scst.h"
33 #include "scst_priv.h"
34
35 static void scst_cmd_set_sn(struct scst_cmd *cmd);
36 static int __scst_init_cmd(struct scst_cmd *cmd);
37 static void scst_finish_cmd_mgmt(struct scst_cmd *cmd);
38 static struct scst_cmd *__scst_find_cmd_by_tag(struct scst_session *sess,
39         uint64_t tag);
40 static void scst_proccess_redirect_cmd(struct scst_cmd *cmd,
41         enum scst_exec_context context, int check_retries);
42
43 static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
44 {
45         struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
46         unsigned long flags;
47
48         spin_lock_irqsave(&t->tasklet_lock, flags);
49         TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
50                 smp_processor_id());
51         list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
52         spin_unlock_irqrestore(&t->tasklet_lock, flags);
53
54         tasklet_schedule(&t->tasklet);
55 }
56
57 /*
58  * Must not be called in parallel with scst_unregister_session_ex() for the
59  * same sess
60  */
61 struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
62                              const uint8_t *lun, int lun_len,
63                              const uint8_t *cdb, int cdb_len, int atomic)
64 {
65         struct scst_cmd *cmd;
66
67         TRACE_ENTRY();
68
69 #ifdef CONFIG_SCST_EXTRACHECKS
70         if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
71                 PRINT_CRIT_ERROR("%s",
72                         "New cmd while shutting down the session");
73                 sBUG();
74         }
75 #endif
76
77         cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
78         if (cmd == NULL)
79                 goto out;
80
81         cmd->sess = sess;
82         cmd->tgt = sess->tgt;
83         cmd->tgtt = sess->tgt->tgtt;
84
85         /*
86          * For both wrong lun and CDB defer the error reporting for
87          * scst_cmd_init_done()
88          */
89
90         cmd->lun = scst_unpack_lun(lun, lun_len);
91
92         if (cdb_len <= SCST_MAX_CDB_SIZE) {
93                 memcpy(cmd->cdb, cdb, cdb_len);
94                 cmd->cdb_len = cdb_len;
95         }
96
97         TRACE_DBG("cmd %p, sess %p", cmd, sess);
98         scst_sess_get(sess);
99
100 out:
101         TRACE_EXIT();
102         return cmd;
103 }
104 EXPORT_SYMBOL(scst_rx_cmd);
105
106 /*
107  * No locks, but might be on IRQ. Returns 0 on success, <0 if processing of
108  * this command should be stopped.
109  */
110 static int scst_init_cmd(struct scst_cmd *cmd, enum scst_exec_context *context)
111 {
112         int rc, res = 0;
113
114         TRACE_ENTRY();
115
116         /* See the comment in scst_do_job_init() */
117         if (unlikely(!list_empty(&scst_init_cmd_list))) {
118                 TRACE_MGMT_DBG("%s", "init cmd list busy");
119                 goto out_redirect;
120         }
121         /*
122          * Memory barrier isn't necessary here, because CPU appears to
123          * be self-consistent and we don't care about the race, described
124          * in comment in scst_do_job_init().
125          */
126
127         rc = __scst_init_cmd(cmd);
128         if (unlikely(rc > 0))
129                 goto out_redirect;
130         else if (unlikely(rc != 0))
131                 goto out;
132
133         /* Small context optimization */
134         if (((*context == SCST_CONTEXT_TASKLET) ||
135              (*context == SCST_CONTEXT_DIRECT_ATOMIC) ||
136              ((*context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd))) &&
137               scst_cmd_is_expected_set(cmd)) {
138                 if (cmd->expected_data_direction == SCST_DATA_WRITE) {
139                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
140                                         &cmd->tgt_dev->tgt_dev_flags))
141                                 *context = SCST_CONTEXT_THREAD;
142                 } else {
143                         if (!test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
144                                         &cmd->tgt_dev->tgt_dev_flags))
145                                 *context = SCST_CONTEXT_THREAD;
146                 }
147         }
148
149 out:
150         TRACE_EXIT_RES(res);
151         return res;
152
153 out_redirect:
154         if (cmd->preprocessing_only) {
155                 /*
156                  * Poor man solution for single threaded targets, where
157                  * blocking receiver at least sometimes means blocking all.
158                  */
159                 sBUG_ON(*context != SCST_CONTEXT_DIRECT);
160                 scst_set_busy(cmd);
161                 scst_set_cmd_abnormal_done_state(cmd);
162                 /* Keep initiator away from too many BUSY commands */
163                 msleep(50);
164         } else {
165                 unsigned long flags;
166                 spin_lock_irqsave(&scst_init_lock, flags);
167                 TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
168                         "%d)", cmd, atomic_read(&scst_cmd_count));
169                 list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
170                 if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
171                         scst_init_poll_cnt++;
172                 spin_unlock_irqrestore(&scst_init_lock, flags);
173                 wake_up(&scst_init_cmd_list_waitQ);
174                 res = -1;
175         }
176         goto out;
177 }
178
179 #ifdef CONFIG_SCST_MEASURE_LATENCY
180 static inline uint64_t scst_sec_to_nsec(time_t sec)
181 {
182         return (uint64_t)sec * 1000000000;
183 }
184 #endif
185
186 void scst_cmd_init_done(struct scst_cmd *cmd,
187         enum scst_exec_context pref_context)
188 {
189         unsigned long flags;
190         struct scst_session *sess = cmd->sess;
191         int rc;
192
193         TRACE_ENTRY();
194
195 #ifdef CONFIG_SCST_MEASURE_LATENCY
196         {
197                 struct timespec ts;
198                 getnstimeofday(&ts);
199                 cmd->start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
200                 TRACE_DBG("cmd %p (sess %p): start %lld (tv_sec %ld, "
201                         "tv_nsec %ld)", cmd, sess, cmd->start, ts.tv_sec,
202                         ts.tv_nsec);
203         }
204 #endif
205
206         TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
207         TRACE(TRACE_SCSI, "tag=%llu, lun=%lld, CDB len=%d, queue_type=%x "
208                 "(cmd %p)", (long long unsigned int)cmd->tag,
209                 (long long unsigned int)cmd->lun, cmd->cdb_len,
210                 cmd->queue_type, cmd);
211         PRINT_BUFF_FLAG(TRACE_SCSI|TRACE_RCV_BOT, "Recieving CDB",
212                 cmd->cdb, cmd->cdb_len);
213
214 #ifdef CONFIG_SCST_EXTRACHECKS
215         if (unlikely((in_irq() || irqs_disabled())) &&
216             ((pref_context == SCST_CONTEXT_DIRECT) ||
217              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
218                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
219                         "SCST_CONTEXT_THREAD instead\n", pref_context,
220                         cmd->tgtt->name);
221                 pref_context = SCST_CONTEXT_THREAD;
222         }
223 #endif
224
225         atomic_inc(&sess->sess_cmd_count);
226
227         spin_lock_irqsave(&sess->sess_list_lock, flags);
228
229         if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
230                 /*
231                  * We have to always keep command in the search list from the
232                  * very beginning, because otherwise it can be missed during
233                  * TM processing. This check is needed because there might be
234                  * old, i.e. deferred, commands and new, i.e. just coming, ones.
235                  */
236                 if (cmd->search_cmd_list_entry.next == NULL)
237                         list_add_tail(&cmd->search_cmd_list_entry,
238                                 &sess->search_cmd_list);
239                 switch (sess->init_phase) {
240                 case SCST_SESS_IPH_SUCCESS:
241                         break;
242                 case SCST_SESS_IPH_INITING:
243                         TRACE_DBG("Adding cmd %p to init deferred cmd list",
244                                   cmd);
245                         list_add_tail(&cmd->cmd_list_entry,
246                                 &sess->init_deferred_cmd_list);
247                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
248                         goto out;
249                 case SCST_SESS_IPH_FAILED:
250                         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
251                         scst_set_busy(cmd);
252                         scst_set_cmd_abnormal_done_state(cmd);
253                         goto active;
254                 default:
255                         sBUG();
256                 }
257         } else
258                 list_add_tail(&cmd->search_cmd_list_entry,
259                               &sess->search_cmd_list);
260
261         spin_unlock_irqrestore(&sess->sess_list_lock, flags);
262
263         if (unlikely(cmd->lun == NO_SUCH_LUN)) {
264                 PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
265                 scst_set_cmd_error(cmd,
266                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
267                 scst_set_cmd_abnormal_done_state(cmd);
268                 goto active;
269         }
270
271         if (unlikely(cmd->cdb_len == 0)) {
272                 PRINT_ERROR("Wrong CDB len %d, finishing cmd", 0);
273                 scst_set_cmd_error(cmd,
274                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
275                 scst_set_cmd_abnormal_done_state(cmd);
276                 goto active;
277         }
278
279         if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
280                 PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
281                 scst_set_cmd_error(cmd,
282                         SCST_LOAD_SENSE(scst_sense_invalid_message));
283                 scst_set_cmd_abnormal_done_state(cmd);
284                 goto active;
285         }
286
287         cmd->state = SCST_CMD_STATE_INIT;
288         /* cmd must be inited here to preserve the order */
289         rc = scst_init_cmd(cmd, &pref_context);
290         if (unlikely(rc < 0))
291                 goto out;
292
293 active:
294         /* Here cmd must not be in any cmd list, no locks */
295         switch (pref_context) {
296         case SCST_CONTEXT_TASKLET:
297                 scst_schedule_tasklet(cmd);
298                 break;
299
300         case SCST_CONTEXT_DIRECT:
301                 scst_process_active_cmd(cmd, false);
302                 /* For *NEED_THREAD wake_up() is already done */
303                 break;
304
305         case SCST_CONTEXT_DIRECT_ATOMIC:
306                 scst_process_active_cmd(cmd, true);
307                 /* For *NEED_THREAD wake_up() is already done */
308                 break;
309
310         default:
311                 PRINT_ERROR("Context %x is undefined, using the thread one",
312                         pref_context);
313                 /* go through */
314         case SCST_CONTEXT_THREAD:
315                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
316                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
317                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
318                         list_add(&cmd->cmd_list_entry,
319                                 &cmd->cmd_lists->active_cmd_list);
320                 else
321                         list_add_tail(&cmd->cmd_list_entry,
322                                 &cmd->cmd_lists->active_cmd_list);
323                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
324                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
325                 break;
326         }
327
328 out:
329         TRACE_EXIT();
330         return;
331 }
332 EXPORT_SYMBOL(scst_cmd_init_done);
333
334 static int scst_pre_parse(struct scst_cmd *cmd)
335 {
336         int res = SCST_CMD_STATE_RES_CONT_SAME;
337         struct scst_device *dev = cmd->dev;
338         int rc;
339
340         TRACE_ENTRY();
341
342         cmd->inc_expected_sn_on_done = dev->handler->exec_sync ||
343              (!dev->has_own_order_mgmt &&
344               (dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER ||
345                cmd->queue_type == SCST_CMD_QUEUE_ORDERED));
346
347         /*
348          * Expected transfer data supplied by the SCSI transport via the
349          * target driver are untrusted, so we prefer to fetch them from CDB.
350          * Additionally, not all transports support supplying the expected
351          * transfer data.
352          */
353
354         rc = scst_get_cdb_info(cmd);
355         if (unlikely(rc != 0)) {
356                 if (rc > 0) {
357                         PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
358                         goto out_xmit;
359                 }
360                 PRINT_ERROR("Unknown opcode 0x%02x for %s. "
361                         "Should you update scst_scsi_op_table?",
362                         cmd->cdb[0], dev->handler->name);
363                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
364 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
365                 if (scst_cmd_is_expected_set(cmd)) {
366                         TRACE(TRACE_SCSI, "Using initiator supplied values: "
367                                 "direction %d, transfer_len %d",
368                                 cmd->expected_data_direction,
369                                 cmd->expected_transfer_len);
370                         cmd->data_direction = cmd->expected_data_direction;
371
372                         cmd->bufflen = cmd->expected_transfer_len;
373                         /* Restore (likely) lost CDB length */
374                         cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
375                         if (cmd->cdb_len == -1) {
376                                 PRINT_ERROR("Unable to get CDB length for "
377                                         "opcode 0x%02x. Returning INVALID "
378                                         "OPCODE", cmd->cdb[0]);
379                                 scst_set_cmd_error(cmd,
380                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
381                                 goto out_xmit;
382                         }
383                 } else {
384                         PRINT_ERROR("Unknown opcode 0x%02x for %s and "
385                              "target %s not supplied expected values",
386                              cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
387                         scst_set_cmd_error(cmd,
388                                    SCST_LOAD_SENSE(scst_sense_invalid_opcode));
389                         goto out_xmit;
390                 }
391 #else
392                 scst_set_cmd_error(cmd,
393                            SCST_LOAD_SENSE(scst_sense_invalid_opcode));
394                 goto out_xmit;
395 #endif
396         } else {
397                 TRACE(TRACE_SCSI, "op_name <%s> (cmd %p), direction=%d "
398                         "(expected %d, set %s), transfer_len=%d (expected "
399                         "len %d), flags=%d", cmd->op_name, cmd,
400                         cmd->data_direction, cmd->expected_data_direction,
401                         scst_cmd_is_expected_set(cmd) ? "yes" : "no",
402                         cmd->bufflen, cmd->expected_transfer_len,
403                         cmd->op_flags);
404
405                 if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
406                         if (scst_cmd_is_expected_set(cmd)) {
407                                 /*
408                                  * Command data length can't be easily
409                                  * determined from the CDB. ToDo, all such
410                                  * commands processing should be fixed. Until
411                                  * it's done, get the length from the supplied
412                                  * expected value, but limit it to some
413                                  * reasonable value (15MB).
414                                  */
415                                 cmd->bufflen = min(cmd->expected_transfer_len,
416                                                         15*1024*1024);
417                                 cmd->op_flags &= ~SCST_UNKNOWN_LENGTH;
418                         } else
419                                 cmd->bufflen = 0;
420                 }
421         }
422
423         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
424                 PRINT_ERROR("NACA bit in control byte CDB is not supported "
425                             "(opcode 0x%02x)", cmd->cdb[0]);
426                 scst_set_cmd_error(cmd,
427                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
428                 goto out_xmit;
429         }
430
431         if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
432                 PRINT_ERROR("Linked commands are not supported "
433                             "(opcode 0x%02x)", cmd->cdb[0]);
434                 scst_set_cmd_error(cmd,
435                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
436                 goto out_xmit;
437         }
438
439         cmd->state = SCST_CMD_STATE_DEV_PARSE;
440
441 out:
442         TRACE_EXIT_RES(res);
443         return res;
444
445 out_xmit:
446         scst_set_cmd_abnormal_done_state(cmd);
447         res = SCST_CMD_STATE_RES_CONT_SAME;
448         goto out;
449 }
450
451 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
452 static bool scst_is_allowed_to_mismatch_cmd(struct scst_cmd *cmd)
453 {
454         bool res = false;
455
456         switch (cmd->cdb[0]) {
457         case TEST_UNIT_READY:
458                 /* Crazy VMware people sometimes do TUR with READ direction */
459                 res = true;
460                 break;
461         case VERIFY:
462         case VERIFY_6:
463         case VERIFY_12:
464         case VERIFY_16:
465                 /* VERIFY commands with BYTCHK unset shouldn't fail here */
466                 if ((cmd->op_flags & SCST_VERIFY_BYTCHK_MISMATCH_ALLOWED) &&
467                     (cmd->cdb[1] & BYTCHK) == 0)
468                         res = true;
469                 break;
470         }
471
472         return res;
473 }
474 #endif
475
476 static int scst_parse_cmd(struct scst_cmd *cmd)
477 {
478         int res = SCST_CMD_STATE_RES_CONT_SAME;
479         int state;
480         struct scst_device *dev = cmd->dev;
481         int orig_bufflen = cmd->bufflen;
482
483         TRACE_ENTRY();
484
485         if (likely(!scst_is_cmd_local(cmd))) {
486                 if (unlikely(!dev->handler->parse_atomic &&
487                              scst_cmd_atomic(cmd))) {
488                         /*
489                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
490                          * optimization.
491                          */
492                         TRACE_DBG("Dev handler %s parse() needs thread "
493                                 "context, rescheduling", dev->handler->name);
494                         res = SCST_CMD_STATE_RES_NEED_THREAD;
495                         goto out;
496                 }
497
498                 TRACE_DBG("Calling dev handler %s parse(%p)",
499                       dev->handler->name, cmd);
500                 TRACE_BUFF_FLAG(TRACE_SND_BOT, "Parsing: ",
501                                 cmd->cdb, cmd->cdb_len);
502                 state = dev->handler->parse(cmd);
503                 /* Caution: cmd can be already dead here */
504                 TRACE_DBG("Dev handler %s parse() returned %d",
505                         dev->handler->name, state);
506
507                 switch (state) {
508                 case SCST_CMD_STATE_NEED_THREAD_CTX:
509                         TRACE_DBG("Dev handler %s parse() requested thread "
510                               "context, rescheduling", dev->handler->name);
511                         res = SCST_CMD_STATE_RES_NEED_THREAD;
512                         goto out;
513
514                 case SCST_CMD_STATE_STOP:
515                         TRACE_DBG("Dev handler %s parse() requested stop "
516                                 "processing", dev->handler->name);
517                         res = SCST_CMD_STATE_RES_CONT_NEXT;
518                         goto out;
519                 }
520
521                 if (state == SCST_CMD_STATE_DEFAULT)
522                         state = SCST_CMD_STATE_PREPARE_SPACE;
523         } else
524                 state = SCST_CMD_STATE_PREPARE_SPACE;
525
526         if (cmd->data_len == -1)
527                 cmd->data_len = cmd->bufflen;
528
529         if (cmd->dh_data_buf_alloced &&
530             unlikely((orig_bufflen > cmd->bufflen))) {
531                 PRINT_ERROR("Dev handler supplied data buffer (size %d), "
532                         "is less, than required (size %d)", cmd->bufflen,
533                         orig_bufflen);
534                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
535                 goto out_error;
536         }
537
538         if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
539                 goto set_res;
540
541         if (unlikely((cmd->bufflen == 0) &&
542                      (cmd->op_flags & SCST_UNKNOWN_LENGTH))) {
543                 PRINT_ERROR("Unknown data transfer length for opcode 0x%x "
544                         "(handler %s, target %s)", cmd->cdb[0],
545                         dev->handler->name, cmd->tgtt->name);
546                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
547                 goto out_error;
548         }
549
550 #ifdef CONFIG_SCST_EXTRACHECKS
551         if ((cmd->bufflen != 0) &&
552             ((cmd->data_direction == SCST_DATA_NONE) ||
553              ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
554                 PRINT_ERROR("Dev handler %s parse() returned "
555                         "invalid cmd data_direction %d, bufflen %d, state %d "
556                         "or sg %p (opcode 0x%x)", dev->handler->name,
557                         cmd->data_direction, cmd->bufflen, state, cmd->sg,
558                         cmd->cdb[0]);
559                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
560                 goto out_error;
561         }
562 #endif
563
564         if (scst_cmd_is_expected_set(cmd)) {
565 #ifdef CONFIG_SCST_USE_EXPECTED_VALUES
566 #       ifdef CONFIG_SCST_EXTRACHECKS
567                 if ((cmd->data_direction != cmd->expected_data_direction) ||
568                     (cmd->bufflen != cmd->expected_transfer_len)) {
569                         PRINT_WARNING("Expected values don't match decoded "
570                                 "ones: data_direction %d, "
571                                 "expected_data_direction %d, "
572                                 "bufflen %d, expected_transfer_len %d",
573                                 cmd->data_direction,
574                                 cmd->expected_data_direction,
575                                 cmd->bufflen, cmd->expected_transfer_len);
576                         PRINT_BUFFER("Suspicious CDB", cmd->cdb, cmd->cdb_len);
577                 }
578 #       endif
579                 cmd->data_direction = cmd->expected_data_direction;
580                 cmd->bufflen = cmd->expected_transfer_len;
581 #else
582                 if (unlikely(cmd->data_direction !=
583                                 cmd->expected_data_direction)) {
584                         if (((cmd->expected_data_direction != SCST_DATA_NONE) ||
585                              (cmd->bufflen != 0)) &&
586                             !scst_is_allowed_to_mismatch_cmd(cmd)) {
587                                 PRINT_ERROR("Expected data direction %d for "
588                                         "opcode 0x%02x (handler %s, target %s) "
589                                         "doesn't match "
590                                         "decoded value %d",
591                                         cmd->expected_data_direction,
592                                         cmd->cdb[0], dev->handler->name,
593                                         cmd->tgtt->name, cmd->data_direction);
594                                 PRINT_BUFFER("Failed CDB",
595                                         cmd->cdb, cmd->cdb_len);
596                                 scst_set_cmd_error(cmd,
597                                    SCST_LOAD_SENSE(scst_sense_invalid_message));
598                                 goto out_dev_done;
599                         }
600                 }
601                 if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
602                         TRACE(TRACE_MGMT_MINOR, "Warning: expected "
603                                 "transfer length %d for opcode 0x%02x "
604                                 "(handler %s, target %s) doesn't match "
605                                 "decoded value %d. Faulty initiator "
606                                 "(e.g. VMware is known to be such) or "
607                                 "scst_scsi_op_table should be updated?",
608                                 cmd->expected_transfer_len, cmd->cdb[0],
609                                 dev->handler->name, cmd->tgtt->name,
610                                 cmd->bufflen);
611                         PRINT_BUFF_FLAG(TRACE_MGMT_MINOR, "Suspicious CDB",
612                                 cmd->cdb, cmd->cdb_len);
613                 }
614 #endif
615         }
616
617         if (unlikely(cmd->data_direction == SCST_DATA_UNKNOWN)) {
618                 PRINT_ERROR("Unknown data direction. Opcode 0x%x, handler %s, "
619                         "target %s", cmd->cdb[0], dev->handler->name,
620                         cmd->tgtt->name);
621                 PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
622                 goto out_error;
623         }
624
625 set_res:
626         switch (state) {
627         case SCST_CMD_STATE_PREPARE_SPACE:
628         case SCST_CMD_STATE_PRE_PARSE:
629         case SCST_CMD_STATE_DEV_PARSE:
630         case SCST_CMD_STATE_RDY_TO_XFER:
631         case SCST_CMD_STATE_TGT_PRE_EXEC:
632         case SCST_CMD_STATE_SEND_FOR_EXEC:
633         case SCST_CMD_STATE_LOCAL_EXEC:
634         case SCST_CMD_STATE_REAL_EXEC:
635         case SCST_CMD_STATE_PRE_DEV_DONE:
636         case SCST_CMD_STATE_DEV_DONE:
637         case SCST_CMD_STATE_PRE_XMIT_RESP:
638         case SCST_CMD_STATE_XMIT_RESP:
639         case SCST_CMD_STATE_FINISHED:
640                 cmd->state = state;
641                 res = SCST_CMD_STATE_RES_CONT_SAME;
642                 break;
643
644         default:
645                 if (state >= 0) {
646                         PRINT_ERROR("Dev handler %s parse() returned "
647                              "invalid cmd state %d (opcode %d)",
648                              dev->handler->name, state, cmd->cdb[0]);
649                 } else {
650                         PRINT_ERROR("Dev handler %s parse() returned "
651                                 "error %d (opcode %d)", dev->handler->name,
652                                 state, cmd->cdb[0]);
653                 }
654                 goto out_error;
655         }
656
657         if (cmd->resp_data_len == -1) {
658                 if (cmd->data_direction == SCST_DATA_READ)
659                         cmd->resp_data_len = cmd->bufflen;
660                 else
661                          cmd->resp_data_len = 0;
662         }
663
664 out:
665         TRACE_EXIT_HRES(res);
666         return res;
667
668 out_error:
669         /* dev_done() will be called as part of the regular cmd's finish */
670         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
671
672 #ifndef CONFIG_SCST_USE_EXPECTED_VALUES
673 out_dev_done:
674 #endif
675         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
676         res = SCST_CMD_STATE_RES_CONT_SAME;
677         goto out;
678 }
679
680 static int scst_prepare_space(struct scst_cmd *cmd)
681 {
682         int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
683
684         TRACE_ENTRY();
685
686         if (cmd->data_direction == SCST_DATA_NONE)
687                 goto prep_done;
688
689         if (cmd->tgt_need_alloc_data_buf) {
690                 int orig_bufflen = cmd->bufflen;
691
692                 TRACE_MEM("Custom tgt data buf allocation requested (cmd %p)",
693                         cmd);
694
695                 r = cmd->tgtt->alloc_data_buf(cmd);
696                 if (r > 0)
697                         goto alloc;
698                 else if (r == 0) {
699                         if (unlikely(cmd->bufflen == 0)) {
700                                 /* See comment in scst_alloc_space() */
701                                 if (cmd->sg == NULL)
702                                         goto alloc;
703                         }
704
705                         cmd->tgt_data_buf_alloced = 1;
706
707                         if (unlikely(orig_bufflen < cmd->bufflen)) {
708                                 PRINT_ERROR("Target driver allocated data "
709                                         "buffer (size %d), is less, than "
710                                         "required (size %d)", orig_bufflen,
711                                         cmd->bufflen);
712                                 goto out_error;
713                         }
714                         TRACE_MEM("tgt_data_buf_alloced (cmd %p)", cmd);
715                 } else
716                         goto check;
717         }
718
719 alloc:
720         if (!cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
721                 r = scst_alloc_space(cmd);
722                 cmd->tgt_sg = cmd->sg;
723                 cmd->tgt_sg_cnt = cmd->sg_cnt;
724         } else if (cmd->dh_data_buf_alloced && !cmd->tgt_data_buf_alloced) {
725                 TRACE_MEM("dh_data_buf_alloced set (cmd %p)", cmd);
726                 cmd->tgt_sg = cmd->sg;
727                 cmd->tgt_sg_cnt = cmd->sg_cnt;
728                 r = 0;
729         } else if (cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
730                 TRACE_MEM("tgt_data_buf_alloced set (cmd %p)", cmd);
731                 cmd->sg = cmd->tgt_sg;
732                 cmd->sg_cnt = cmd->tgt_sg_cnt;
733                 r = 0;
734         } else {
735                 TRACE_MEM("Both *_data_buf_alloced set (cmd %p, sg %p, "
736                         "sg_cnt %d, tgt_sg %p, tgt_sg_cnt %d)", cmd, cmd->sg,
737                         cmd->sg_cnt, cmd->tgt_sg, cmd->tgt_sg_cnt);
738                 r = 0;
739         }
740
741 check:
742         if (r != 0) {
743                 if (scst_cmd_atomic(cmd)) {
744                         TRACE_MEM("%s", "Atomic memory allocation failed, "
745                               "rescheduling to the thread");
746                         res = SCST_CMD_STATE_RES_NEED_THREAD;
747                         goto out;
748                 } else
749                         goto out_no_space;
750         }
751
752 prep_done:
753         if (cmd->preprocessing_only) {
754                 if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
755                         TRACE_MGMT_DBG("ABORTED set, returning ABORTED for "
756                                 "cmd %p", cmd);
757                         scst_set_cmd_abnormal_done_state(cmd);
758                         res = SCST_CMD_STATE_RES_CONT_SAME;
759                         goto out;
760                 }
761
762                 res = SCST_CMD_STATE_RES_CONT_NEXT;
763                 cmd->state = SCST_CMD_STATE_PREPROCESS_DONE;
764
765                 TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
766                 cmd->tgtt->preprocessing_done(cmd);
767                 TRACE_DBG("%s", "preprocessing_done() returned");
768                 goto out;
769
770         }
771
772         switch (cmd->data_direction) {
773         case SCST_DATA_WRITE:
774                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
775                 break;
776
777         default:
778                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
779                 break;
780         }
781
782 out:
783         TRACE_EXIT_HRES(res);
784         return res;
785
786 out_no_space:
787         TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
788                 "(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
789         scst_set_busy(cmd);
790         scst_set_cmd_abnormal_done_state(cmd);
791         res = SCST_CMD_STATE_RES_CONT_SAME;
792         goto out;
793
794 out_error:
795         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
796         scst_set_cmd_abnormal_done_state(cmd);
797         res = SCST_CMD_STATE_RES_CONT_SAME;
798         goto out;
799 }
800
801 void scst_restart_cmd(struct scst_cmd *cmd, int status,
802         enum scst_exec_context pref_context)
803 {
804         TRACE_ENTRY();
805
806         TRACE_DBG("Preferred context: %d", pref_context);
807         TRACE_DBG("tag=%llu, status=%#x",
808                   (long long unsigned int)scst_cmd_get_tag(cmd),
809                   status);
810
811 #ifdef CONFIG_SCST_EXTRACHECKS
812         if ((in_irq() || irqs_disabled()) &&
813             ((pref_context == SCST_CONTEXT_DIRECT) ||
814              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
815                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
816                         "SCST_CONTEXT_THREAD instead\n", pref_context,
817                         cmd->tgtt->name);
818                 pref_context = SCST_CONTEXT_THREAD;
819         }
820 #endif
821
822         switch (status) {
823         case SCST_PREPROCESS_STATUS_SUCCESS:
824                 switch (cmd->data_direction) {
825                 case SCST_DATA_WRITE:
826                         cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
827                         break;
828                 default:
829                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
830                         break;
831                 }
832                 if (cmd->set_sn_on_restart_cmd)
833                         scst_cmd_set_sn(cmd);
834                 /* Small context optimization */
835                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
836                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
837                     ((pref_context == SCST_CONTEXT_SAME) &&
838                      scst_cmd_atomic(cmd))) {
839                         if (cmd->data_direction == SCST_DATA_WRITE) {
840                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
841                                                 &cmd->tgt_dev->tgt_dev_flags))
842                                         pref_context = SCST_CONTEXT_THREAD;
843                         } else {
844                                 if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
845                                                 &cmd->tgt_dev->tgt_dev_flags))
846                                         pref_context = SCST_CONTEXT_THREAD;
847                         }
848                 }
849                 break;
850
851         case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
852                 scst_set_cmd_abnormal_done_state(cmd);
853                 break;
854
855         case SCST_PREPROCESS_STATUS_ERROR_FATAL:
856                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
857                 /* go through */
858         case SCST_PREPROCESS_STATUS_ERROR:
859                 scst_set_cmd_error(cmd,
860                            SCST_LOAD_SENSE(scst_sense_hardw_error));
861                 scst_set_cmd_abnormal_done_state(cmd);
862                 break;
863
864         default:
865                 PRINT_ERROR("%s() received unknown status %x", __func__,
866                         status);
867                 scst_set_cmd_abnormal_done_state(cmd);
868                 break;
869         }
870
871         scst_proccess_redirect_cmd(cmd, pref_context, 1);
872
873         TRACE_EXIT();
874         return;
875 }
876 EXPORT_SYMBOL(scst_restart_cmd);
877
878 /* No locks */
879 static int scst_queue_retry_cmd(struct scst_cmd *cmd, int finished_cmds)
880 {
881         struct scst_tgt *tgt = cmd->sess->tgt;
882         int res = 0;
883         unsigned long flags;
884
885         TRACE_ENTRY();
886
887         spin_lock_irqsave(&tgt->tgt_lock, flags);
888         tgt->retry_cmds++;
889         /*
890          * Memory barrier is needed here, because we need the exact order
891          * between the read and write between retry_cmds and finished_cmds to
892          * not miss the case when a command finished while we queuing it for
893          * retry after the finished_cmds check.
894          */
895         smp_mb();
896         TRACE_RETRY("TGT QUEUE FULL: incrementing retry_cmds %d",
897               tgt->retry_cmds);
898         if (finished_cmds != atomic_read(&tgt->finished_cmds)) {
899                 /* At least one cmd finished, so try again */
900                 tgt->retry_cmds--;
901                 TRACE_RETRY("Some command(s) finished, direct retry "
902                       "(finished_cmds=%d, tgt->finished_cmds=%d, "
903                       "retry_cmds=%d)", finished_cmds,
904                       atomic_read(&tgt->finished_cmds), tgt->retry_cmds);
905                 res = -1;
906                 goto out_unlock_tgt;
907         }
908
909         TRACE_RETRY("Adding cmd %p to retry cmd list", cmd);
910         list_add_tail(&cmd->cmd_list_entry, &tgt->retry_cmd_list);
911
912         if (!tgt->retry_timer_active) {
913                 tgt->retry_timer.expires = jiffies + SCST_TGT_RETRY_TIMEOUT;
914                 add_timer(&tgt->retry_timer);
915                 tgt->retry_timer_active = 1;
916         }
917
918 out_unlock_tgt:
919         spin_unlock_irqrestore(&tgt->tgt_lock, flags);
920
921         TRACE_EXIT_RES(res);
922         return res;
923 }
924
925 static int scst_rdy_to_xfer(struct scst_cmd *cmd)
926 {
927         int res, rc;
928
929         TRACE_ENTRY();
930
931         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
932                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
933                 goto out_dev_done;
934         }
935
936         if ((cmd->tgtt->rdy_to_xfer == NULL) || unlikely(cmd->internal)) {
937                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
938                 res = SCST_CMD_STATE_RES_CONT_SAME;
939                 goto out;
940         }
941
942         if (unlikely(!cmd->tgtt->rdy_to_xfer_atomic && scst_cmd_atomic(cmd))) {
943                 /*
944                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
945                  * optimization.
946                  */
947                 TRACE_DBG("Target driver %s rdy_to_xfer() needs thread "
948                               "context, rescheduling", cmd->tgtt->name);
949                 res = SCST_CMD_STATE_RES_NEED_THREAD;
950                 goto out;
951         }
952
953         while (1) {
954                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
955
956                 res = SCST_CMD_STATE_RES_CONT_NEXT;
957                 cmd->state = SCST_CMD_STATE_DATA_WAIT;
958
959                 TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
960 #ifdef CONFIG_SCST_DEBUG_RETRY
961                 if (((scst_random() % 100) == 75))
962                         rc = SCST_TGT_RES_QUEUE_FULL;
963                 else
964 #endif
965                         rc = cmd->tgtt->rdy_to_xfer(cmd);
966                 TRACE_DBG("rdy_to_xfer() returned %d", rc);
967
968                 if (likely(rc == SCST_TGT_RES_SUCCESS))
969                         goto out;
970
971                 /* Restore the previous state */
972                 cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
973
974                 switch (rc) {
975                 case SCST_TGT_RES_QUEUE_FULL:
976                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
977                                 break;
978                         else
979                                 continue;
980
981                 case SCST_TGT_RES_NEED_THREAD_CTX:
982                         TRACE_DBG("Target driver %s "
983                               "rdy_to_xfer() requested thread "
984                               "context, rescheduling", cmd->tgtt->name);
985                         res = SCST_CMD_STATE_RES_NEED_THREAD;
986                         break;
987
988                 default:
989                         goto out_error_rc;
990                 }
991                 break;
992         }
993
994 out:
995         TRACE_EXIT_HRES(res);
996         return res;
997
998 out_error_rc:
999         if (rc == SCST_TGT_RES_FATAL_ERROR) {
1000                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
1001                      "fatal error", cmd->tgtt->name);
1002         } else {
1003                 PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
1004                             "value %d", cmd->tgtt->name, rc);
1005         }
1006         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1007
1008 out_dev_done:
1009         scst_set_cmd_abnormal_done_state(cmd);
1010         res = SCST_CMD_STATE_RES_CONT_SAME;
1011         goto out;
1012 }
1013
1014 /* No locks, but might be in IRQ */
1015 void scst_proccess_redirect_cmd(struct scst_cmd *cmd,
1016         enum scst_exec_context context, int check_retries)
1017 {
1018         unsigned long flags;
1019
1020         TRACE_ENTRY();
1021
1022         TRACE_DBG("Context: %x", context);
1023
1024         if (context == SCST_CONTEXT_SAME)
1025                 context = scst_cmd_atomic(cmd) ? SCST_CONTEXT_DIRECT_ATOMIC :
1026                                                  SCST_CONTEXT_DIRECT;
1027
1028         switch (context) {
1029         case SCST_CONTEXT_DIRECT_ATOMIC:
1030                 scst_process_active_cmd(cmd, true);
1031                 break;
1032
1033         case SCST_CONTEXT_DIRECT:
1034                 if (check_retries)
1035                         scst_check_retries(cmd->tgt);
1036                 scst_process_active_cmd(cmd, false);
1037                 break;
1038
1039         default:
1040                 PRINT_ERROR("Context %x is unknown, using the thread one",
1041                             context);
1042                 /* go through */
1043         case SCST_CONTEXT_THREAD:
1044                 if (check_retries)
1045                         scst_check_retries(cmd->tgt);
1046                 spin_lock_irqsave(&cmd->cmd_lists->cmd_list_lock, flags);
1047                 TRACE_DBG("Adding cmd %p to active cmd list", cmd);
1048                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
1049                         list_add(&cmd->cmd_list_entry,
1050                                 &cmd->cmd_lists->active_cmd_list);
1051                 else
1052                         list_add_tail(&cmd->cmd_list_entry,
1053                                 &cmd->cmd_lists->active_cmd_list);
1054                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
1055                 spin_unlock_irqrestore(&cmd->cmd_lists->cmd_list_lock, flags);
1056                 break;
1057
1058         case SCST_CONTEXT_TASKLET:
1059                 if (check_retries)
1060                         scst_check_retries(cmd->tgt);
1061                 scst_schedule_tasklet(cmd);
1062                 break;
1063         }
1064
1065         TRACE_EXIT();
1066         return;
1067 }
1068
1069 void scst_rx_data(struct scst_cmd *cmd, int status,
1070         enum scst_exec_context pref_context)
1071 {
1072         TRACE_ENTRY();
1073
1074         TRACE_DBG("Preferred context: %d", pref_context);
1075         TRACE(TRACE_SCSI, "cmd %p, status %#x", cmd, status);
1076
1077 #ifdef CONFIG_SCST_EXTRACHECKS
1078         if ((in_irq() || irqs_disabled()) &&
1079             ((pref_context == SCST_CONTEXT_DIRECT) ||
1080              (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
1081                 PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
1082                         "SCST_CONTEXT_THREAD instead\n", pref_context,
1083                         cmd->tgtt->name);
1084                 pref_context = SCST_CONTEXT_THREAD;
1085         }
1086 #endif
1087
1088         switch (status) {
1089         case SCST_RX_STATUS_SUCCESS:
1090 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
1091                 if (cmd->tgt_sg) {
1092                         int i;
1093                         struct scatterlist *sg = cmd->tgt_sg;
1094                         TRACE_RECV_BOT("RX data for cmd %p "
1095                                 "(sg_cnt %d, sg %p, sg[0].page %p)", cmd,
1096                                 cmd->tgt_sg_cnt, sg, (void *)sg_page(&sg[0]));
1097                         for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
1098                                 PRINT_BUFF_FLAG(TRACE_RCV_BOT, "RX sg",
1099                                         sg_virt(&sg[i]), sg[i].length);
1100                         }
1101                 }
1102 #endif
1103                 cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1104                 /* Small context optimization */
1105                 if ((pref_context == SCST_CONTEXT_TASKLET) ||
1106                     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
1107                     ((pref_context == SCST_CONTEXT_SAME) &&
1108                      scst_cmd_atomic(cmd))) {
1109                         if (!test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC,
1110                                         &cmd->tgt_dev->tgt_dev_flags))
1111                                 pref_context = SCST_CONTEXT_THREAD;
1112                 }
1113                 break;
1114
1115         case SCST_RX_STATUS_ERROR_SENSE_SET:
1116                 scst_set_cmd_abnormal_done_state(cmd);
1117                 break;
1118
1119         case SCST_RX_STATUS_ERROR_FATAL:
1120                 set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1121                 /* go through */
1122         case SCST_RX_STATUS_ERROR:
1123                 scst_set_cmd_error(cmd,
1124                            SCST_LOAD_SENSE(scst_sense_hardw_error));
1125                 scst_set_cmd_abnormal_done_state(cmd);
1126                 break;
1127
1128         default:
1129                 PRINT_ERROR("scst_rx_data() received unknown status %x",
1130                         status);
1131                 scst_set_cmd_abnormal_done_state(cmd);
1132                 break;
1133         }
1134
1135         scst_proccess_redirect_cmd(cmd, pref_context, 1);
1136
1137         TRACE_EXIT();
1138         return;
1139 }
1140 EXPORT_SYMBOL(scst_rx_data);
1141
1142 static int scst_tgt_pre_exec(struct scst_cmd *cmd)
1143 {
1144         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
1145
1146         TRACE_ENTRY();
1147
1148         cmd->state = SCST_CMD_STATE_SEND_FOR_EXEC;
1149
1150         if ((cmd->tgtt->pre_exec == NULL) || unlikely(cmd->internal))
1151                 goto out;
1152
1153         TRACE_DBG("Calling pre_exec(%p)", cmd);
1154         rc = cmd->tgtt->pre_exec(cmd);
1155         TRACE_DBG("pre_exec() returned %d", rc);
1156
1157         if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
1158                 switch (rc) {
1159                 case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
1160                         scst_set_cmd_abnormal_done_state(cmd);
1161                         break;
1162                 case SCST_PREPROCESS_STATUS_ERROR_FATAL:
1163                         set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
1164                         /* go through */
1165                 case SCST_PREPROCESS_STATUS_ERROR:
1166                         scst_set_cmd_error(cmd,
1167                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1168                         scst_set_cmd_abnormal_done_state(cmd);
1169                         break;
1170                 case SCST_PREPROCESS_STATUS_NEED_THREAD:
1171                         TRACE_DBG("Target driver's %s pre_exec() requested "
1172                                 "thread context, rescheduling",
1173                                 cmd->tgtt->name);
1174                         res = SCST_CMD_STATE_RES_NEED_THREAD;
1175                         cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
1176                         break;
1177                 default:
1178                         sBUG();
1179                         break;
1180                 }
1181         }
1182
1183 out:
1184         TRACE_EXIT_RES(res);
1185         return res;
1186 }
1187
1188 static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
1189         const uint8_t *rq_sense, int rq_sense_len, int resid)
1190 {
1191         TRACE_ENTRY();
1192
1193 #ifdef CONFIG_SCST_MEASURE_LATENCY
1194         {
1195                 struct timespec ts;
1196                 getnstimeofday(&ts);
1197                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1198                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1199                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1200                         ts.tv_sec, ts.tv_nsec);
1201         }
1202 #endif
1203
1204         cmd->status = result & 0xff;
1205         cmd->msg_status = msg_byte(result);
1206         cmd->host_status = host_byte(result);
1207         cmd->driver_status = driver_byte(result);
1208         if (unlikely(resid != 0)) {
1209 #ifdef CONFIG_SCST_EXTRACHECKS
1210                 if ((resid < 0) || (resid > cmd->resp_data_len)) {
1211                         PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
1212                                 "op %x)", resid, cmd->resp_data_len,
1213                                 cmd->cdb[0]);
1214                 } else
1215 #endif
1216                         scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
1217         }
1218
1219         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION)) {
1220                 /* We might have double reset UA here */
1221                 cmd->dbl_ua_orig_resp_data_len = cmd->resp_data_len;
1222                 cmd->dbl_ua_orig_data_direction = cmd->data_direction;
1223
1224                 scst_alloc_set_sense(cmd, 1, rq_sense, rq_sense_len);
1225         }
1226
1227         TRACE(TRACE_SCSI, "cmd %p, result=%x, cmd->status=%x, resid=%d, "
1228               "cmd->msg_status=%x, cmd->host_status=%x, "
1229               "cmd->driver_status=%x (cmd %p)", cmd, result, cmd->status, resid,
1230               cmd->msg_status, cmd->host_status, cmd->driver_status, cmd);
1231
1232         cmd->completed = 1;
1233
1234         TRACE_EXIT();
1235         return;
1236 }
1237
1238 /* For small context optimization */
1239 static inline enum scst_exec_context scst_optimize_post_exec_context(
1240         struct scst_cmd *cmd, enum scst_exec_context context)
1241 {
1242         if (((context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd)) ||
1243             (context == SCST_CONTEXT_TASKLET) ||
1244             (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
1245                 if (!test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC,
1246                                 &cmd->tgt_dev->tgt_dev_flags))
1247                         context = SCST_CONTEXT_THREAD;
1248         }
1249         return context;
1250 }
1251
1252 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1253 static inline struct scst_cmd *scst_get_cmd(struct scsi_cmnd *scsi_cmd,
1254                                             struct scsi_request **req)
1255 {
1256         struct scst_cmd *cmd = NULL;
1257
1258         if (scsi_cmd && (*req = scsi_cmd->sc_request))
1259                 cmd = (struct scst_cmd *)(*req)->upper_private_data;
1260
1261         if (cmd == NULL) {
1262                 PRINT_ERROR("%s", "Request with NULL cmd");
1263                 if (*req)
1264                         scsi_release_request(*req);
1265         }
1266
1267         return cmd;
1268 }
1269
1270 static void scst_cmd_done(struct scsi_cmnd *scsi_cmd)
1271 {
1272         struct scsi_request *req = NULL;
1273         struct scst_cmd *cmd;
1274
1275         TRACE_ENTRY();
1276
1277         cmd = scst_get_cmd(scsi_cmd, &req);
1278         if (cmd == NULL)
1279                 goto out;
1280
1281         scst_do_cmd_done(cmd, req->sr_result, req->sr_sense_buffer,
1282                 sizeof(req->sr_sense_buffer), scsi_cmd->resid);
1283
1284         /* Clear out request structure */
1285         req->sr_use_sg = 0;
1286         req->sr_sglist_len = 0;
1287         req->sr_bufflen = 0;
1288         req->sr_buffer = NULL;
1289         req->sr_underflow = 0;
1290         req->sr_request->rq_disk = NULL; /* disown request blk */
1291
1292         scst_release_request(cmd);
1293
1294         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1295
1296         scst_proccess_redirect_cmd(cmd,
1297                 scst_optimize_post_exec_context(cmd, scst_estimate_context()),
1298                                                 0);
1299
1300 out:
1301         TRACE_EXIT();
1302         return;
1303 }
1304 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1305 static void scst_cmd_done(void *data, char *sense, int result, int resid)
1306 {
1307         struct scst_cmd *cmd;
1308
1309         TRACE_ENTRY();
1310
1311         cmd = (struct scst_cmd *)data;
1312         if (cmd == NULL)
1313                 goto out;
1314
1315         scst_do_cmd_done(cmd, result, sense, SCST_SENSE_BUFFERSIZE, resid);
1316
1317         cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
1318
1319         scst_proccess_redirect_cmd(cmd,
1320                 scst_optimize_post_exec_context(cmd, scst_estimate_context()),
1321                                                 0);
1322
1323 out:
1324         TRACE_EXIT();
1325         return;
1326 }
1327 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18) */
1328
1329 static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state,
1330         enum scst_exec_context pref_context)
1331 {
1332         TRACE_ENTRY();
1333
1334 #ifdef CONFIG_SCST_MEASURE_LATENCY
1335         {
1336                 struct timespec ts;
1337                 getnstimeofday(&ts);
1338                 cmd->post_exec_start = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
1339                 TRACE_DBG("cmd %p (sess %p): post_exec_start %lld (tv_sec %ld, "
1340                         "tv_nsec %ld)", cmd, cmd->sess, cmd->post_exec_start,
1341                         ts.tv_sec, ts.tv_nsec);
1342         }
1343 #endif
1344
1345         if (next_state == SCST_CMD_STATE_DEFAULT)
1346                 next_state = SCST_CMD_STATE_PRE_DEV_DONE;
1347
1348 #if defined(CONFIG_SCST_DEBUG)
1349         if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
1350                 if (cmd->sg) {
1351                         int i;
1352                         struct scatterlist *sg = cmd->sg;
1353                         TRACE_RECV_TOP("Exec'd %d S/G(s) at %p sg[0].page at "
1354                                 "%p", cmd->sg_cnt, sg, (void *)sg_page(&sg[0]));
1355                         for (i = 0; i < cmd->sg_cnt; ++i) {
1356                                 TRACE_BUFF_FLAG(TRACE_RCV_TOP,
1357                                         "Exec'd sg", sg_virt(&sg[i]),
1358                                         sg[i].length);
1359                         }
1360                 }
1361         }
1362 #endif
1363
1364         cmd->state = next_state;
1365
1366 #ifdef CONFIG_SCST_EXTRACHECKS
1367         if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
1368             (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
1369             (next_state != SCST_CMD_STATE_FINISHED)) {
1370                 PRINT_ERROR("%s() received invalid cmd state %d (opcode %d)",
1371                         __func__, next_state, cmd->cdb[0]);
1372                 scst_set_cmd_error(cmd,
1373                                    SCST_LOAD_SENSE(scst_sense_hardw_error));
1374                 scst_set_cmd_abnormal_done_state(cmd);
1375         }
1376 #endif
1377         pref_context = scst_optimize_post_exec_context(cmd, pref_context);
1378         scst_proccess_redirect_cmd(cmd, pref_context, 0);
1379
1380         TRACE_EXIT();
1381         return;
1382 }
1383
1384 static int scst_report_luns_local(struct scst_cmd *cmd)
1385 {
1386         int rc;
1387         int dev_cnt = 0;
1388         int buffer_size;
1389         int i;
1390         struct scst_tgt_dev *tgt_dev = NULL;
1391         uint8_t *buffer;
1392         int offs, overflow = 0;
1393
1394         TRACE_ENTRY();
1395
1396         rc = scst_check_local_events(cmd);
1397         if (unlikely(rc != 0))
1398                 goto out_done;
1399
1400         cmd->status = 0;
1401         cmd->msg_status = 0;
1402         cmd->host_status = DID_OK;
1403         cmd->driver_status = 0;
1404
1405         if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
1406                 PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
1407                         "LUNS command", cmd->cdb[2]);
1408                 goto out_err;
1409         }
1410
1411         buffer_size = scst_get_buf_first(cmd, &buffer);
1412         if (unlikely(buffer_size == 0))
1413                 goto out_compl;
1414         else if (unlikely(buffer_size < 0))
1415                 goto out_err;
1416
1417         if (buffer_size < 16)
1418                 goto out_put_err;
1419
1420         memset(buffer, 0, buffer_size);
1421         offs = 8;
1422
1423         /* sess->sess_tgt_dev_list_hash is protected by suspended activity */
1424         for (i = 0; i < TGT_DEV_HASH_SIZE; i++) {
1425                 struct list_head *sess_tgt_dev_list_head =
1426                         &cmd->sess->sess_tgt_dev_list_hash[i];
1427                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
1428                                 sess_tgt_dev_list_entry) {
1429                         if (!overflow) {
1430                                 if (offs >= buffer_size) {
1431                                         scst_put_buf(cmd, buffer);
1432                                         buffer_size = scst_get_buf_next(cmd,
1433                                                                        &buffer);
1434                                         if (buffer_size > 0) {
1435                                                 memset(buffer, 0, buffer_size);
1436                                                 offs = 0;
1437                                         } else {
1438                                                 overflow = 1;
1439                                                 goto inc_dev_cnt;
1440                                         }
1441                                 }
1442                                 if ((buffer_size - offs) < 8) {
1443                                         PRINT_ERROR("Buffer allocated for "
1444                                                 "REPORT LUNS command doesn't "
1445                                                 "allow to fit 8 byte entry "
1446                                                 "(buffer_size=%d)",
1447                                                 buffer_size);
1448                                         goto out_put_hw_err;
1449                                 }
1450                                 buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
1451                                 buffer[offs+1] = tgt_dev->lun & 0xff;
1452                                 offs += 8;
1453                         }
1454 inc_dev_cnt:
1455                         dev_cnt++;
1456                 }
1457         }
1458         if (!overflow)
1459                 scst_put_buf(cmd, buffer);
1460
1461         /* Set the response header */
1462         buffer_size = scst_get_buf_first(cmd, &buffer);
1463         if (unlikely(buffer_size == 0))
1464                 goto out_compl;
1465         else if (unlikely(buffer_size < 0))
1466                 goto out_err;
1467
1468         dev_cnt *= 8;
1469         buffer[0] = (dev_cnt >> 24) & 0xff;
1470         buffer[1] = (dev_cnt >> 16) & 0xff;
1471         buffer[2] = (dev_cnt >> 8) & 0xff;
1472         buffer[3] = dev_cnt & 0xff;
1473
1474         scst_put_buf(cmd, buffer);
1475
1476         dev_cnt += 8;
1477         if (dev_cnt < cmd->resp_data_len)
1478                 scst_set_resp_data_len(cmd, dev_cnt);
1479
1480 out_compl:
1481         cmd->completed = 1;
1482
1483 out_done:
1484         /* Report the result */
1485         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1486
1487         TRACE_EXIT();
1488         return SCST_EXEC_COMPLETED;
1489
1490 out_put_err:
1491         scst_put_buf(cmd, buffer);
1492
1493 out_err:
1494         scst_set_cmd_error(cmd,
1495                    SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1496         goto out_compl;
1497
1498 out_put_hw_err:
1499         scst_put_buf(cmd, buffer);
1500         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1501         goto out_compl;
1502 }
1503
1504 static int scst_pre_select(struct scst_cmd *cmd)
1505 {
1506         int res = SCST_EXEC_NOT_COMPLETED;
1507
1508         TRACE_ENTRY();
1509
1510         if (scst_cmd_atomic(cmd)) {
1511                 res = SCST_EXEC_NEED_THREAD;
1512                 goto out;
1513         }
1514
1515         scst_block_dev_cmd(cmd, 1);
1516
1517         /* Check for local events will be done when cmd will be executed */
1518
1519 out:
1520         TRACE_EXIT_RES(res);
1521         return res;
1522 }
1523
1524 static int scst_reserve_local(struct scst_cmd *cmd)
1525 {
1526         int res = SCST_EXEC_NOT_COMPLETED, rc;
1527         struct scst_device *dev;
1528         struct scst_tgt_dev *tgt_dev_tmp;
1529
1530         TRACE_ENTRY();
1531
1532         if (scst_cmd_atomic(cmd)) {
1533                 res = SCST_EXEC_NEED_THREAD;
1534                 goto out;
1535         }
1536
1537         if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
1538                 PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
1539                      "(lun=%lld)", (long long unsigned int)cmd->lun);
1540                 scst_set_cmd_error(cmd,
1541                         SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
1542                 goto out_done;
1543         }
1544
1545         dev = cmd->dev;
1546
1547         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1548                 scst_block_dev_cmd(cmd, 1);
1549
1550         rc = scst_check_local_events(cmd);
1551         if (unlikely(rc != 0))
1552                 goto out_done;
1553
1554         spin_lock_bh(&dev->dev_lock);
1555
1556         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1557                 spin_unlock_bh(&dev->dev_lock);
1558                 scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
1559                 goto out_done;
1560         }
1561
1562         list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
1563                             dev_tgt_dev_list_entry) {
1564                 if (cmd->tgt_dev != tgt_dev_tmp)
1565                         set_bit(SCST_TGT_DEV_RESERVED,
1566                                 &tgt_dev_tmp->tgt_dev_flags);
1567         }
1568         dev->dev_reserved = 1;
1569
1570         spin_unlock_bh(&dev->dev_lock);
1571
1572 out:
1573         TRACE_EXIT_RES(res);
1574         return res;
1575
1576 out_done:
1577         /* Report the result */
1578         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1579         res = SCST_EXEC_COMPLETED;
1580         goto out;
1581 }
1582
1583 static int scst_release_local(struct scst_cmd *cmd)
1584 {
1585         int res = SCST_EXEC_NOT_COMPLETED, rc;
1586         struct scst_tgt_dev *tgt_dev_tmp;
1587         struct scst_device *dev;
1588
1589         TRACE_ENTRY();
1590
1591         if (scst_cmd_atomic(cmd)) {
1592                 res = SCST_EXEC_NEED_THREAD;
1593                 goto out;
1594         }
1595
1596         dev = cmd->dev;
1597
1598         if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
1599                 scst_block_dev_cmd(cmd, 1);
1600
1601         rc = scst_check_local_events(cmd);
1602         if (unlikely(rc != 0))
1603                 goto out_done;
1604
1605         spin_lock_bh(&dev->dev_lock);
1606
1607         /*
1608          * The device could be RELEASED behind us, if RESERVING session
1609          * is closed (see scst_free_tgt_dev()), but this actually doesn't
1610          * matter, so use lock and no retest for DEV_RESERVED bits again
1611          */
1612         if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
1613                 res = SCST_EXEC_COMPLETED;
1614                 cmd->status = 0;
1615                 cmd->msg_status = 0;
1616                 cmd->host_status = DID_OK;
1617                 cmd->driver_status = 0;
1618                 cmd->completed = 1;
1619         } else {
1620                 list_for_each_entry(tgt_dev_tmp,
1621                                     &dev->dev_tgt_dev_list,
1622                                     dev_tgt_dev_list_entry) {
1623                         clear_bit(SCST_TGT_DEV_RESERVED,
1624                                 &tgt_dev_tmp->tgt_dev_flags);
1625                 }
1626                 dev->dev_reserved = 0;
1627         }
1628
1629         spin_unlock_bh(&dev->dev_lock);
1630
1631         if (res == SCST_EXEC_COMPLETED)
1632                 goto out_done;
1633
1634 out:
1635         TRACE_EXIT_RES(res);
1636         return res;
1637
1638 out_done:
1639         res = SCST_EXEC_COMPLETED;
1640         /* Report the result */
1641         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1642         goto out;
1643 }
1644
1645 /* No locks, no IRQ or IRQ-safe context allowed */
1646 int scst_check_local_events(struct scst_cmd *cmd)
1647 {
1648         int res, rc;
1649         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1650         struct scst_device *dev = cmd->dev;
1651
1652         TRACE_ENTRY();
1653
1654         /*
1655          * There's no race here, because we need to trace commands sent
1656          * *after* dev_double_ua_possible flag was set.
1657          */
1658         if (unlikely(dev->dev_double_ua_possible))
1659                 cmd->double_ua_possible = 1;
1660
1661         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
1662                 TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
1663                 goto out_uncomplete;
1664         }
1665
1666         /* Reserve check before Unit Attention */
1667         if (unlikely(test_bit(SCST_TGT_DEV_RESERVED,
1668                               &tgt_dev->tgt_dev_flags))) {
1669                 if (cmd->cdb[0] != INQUIRY &&
1670                     cmd->cdb[0] != REPORT_LUNS &&
1671                     cmd->cdb[0] != RELEASE &&
1672                     cmd->cdb[0] != RELEASE_10 &&
1673                     cmd->cdb[0] != REPORT_DEVICE_IDENTIFIER &&
1674                     (cmd->cdb[0] != ALLOW_MEDIUM_REMOVAL ||
1675                      (cmd->cdb[4] & 3)) &&
1676                     cmd->cdb[0] != LOG_SENSE &&
1677                     cmd->cdb[0] != REQUEST_SENSE) {
1678                         scst_set_cmd_error_status(cmd,
1679                                 SAM_STAT_RESERVATION_CONFLICT);
1680                         goto out_complete;
1681                 }
1682         }
1683
1684         /* If we had internal bus reset, set the command error unit attention */
1685         if ((dev->scsi_dev != NULL) &&
1686             unlikely(dev->scsi_dev->was_reset)) {
1687                 if (scst_is_ua_command(cmd)) {
1688                         int done = 0;
1689                         /*
1690                          * Prevent more than 1 cmd to be triggered by
1691                          * was_reset.
1692                          */
1693                         spin_lock_bh(&dev->dev_lock);
1694                         if (dev->scsi_dev->was_reset) {
1695                                 TRACE(TRACE_MGMT, "was_reset is %d", 1);
1696                                 scst_set_cmd_error(cmd,
1697                                           SCST_LOAD_SENSE(scst_sense_reset_UA));
1698                                 /*
1699                                  * It looks like it is safe to clear was_reset
1700                                  * here.
1701                                  */
1702                                 dev->scsi_dev->was_reset = 0;
1703                                 done = 1;
1704                         }
1705                         spin_unlock_bh(&dev->dev_lock);
1706
1707                         if (done)
1708                                 goto out_complete;
1709                 }
1710         }
1711
1712         if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING,
1713                         &cmd->tgt_dev->tgt_dev_flags))) {
1714                 if (scst_is_ua_command(cmd)) {
1715                         rc = scst_set_pending_UA(cmd);
1716                         if (rc == 0)
1717                                 goto out_complete;
1718                 }
1719         }
1720
1721         res = 0;
1722
1723 out:
1724         TRACE_EXIT_RES(res);
1725         return res;
1726
1727 out_complete:
1728         res = 1;
1729         sBUG_ON(!cmd->completed);
1730         goto out;
1731
1732 out_uncomplete:
1733         res = -1;
1734         goto out;
1735 }
1736 EXPORT_SYMBOL(scst_check_local_events);
1737
1738 /* No locks */
1739 void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
1740 {
1741         if (slot == NULL)
1742                 goto inc;
1743
1744         /* Optimized for lockless fast path */
1745
1746         TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
1747                 atomic_read(slot));
1748
1749         if (!atomic_dec_and_test(slot))
1750                 goto out;
1751
1752         TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
1753                 tgt_dev->num_free_sn_slots);
1754         if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
1755                 spin_lock_irq(&tgt_dev->sn_lock);
1756                 if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
1757                         if (tgt_dev->num_free_sn_slots < 0)
1758                                 tgt_dev->cur_sn_slot = slot;
1759                         /*
1760                          * To be in-sync with SIMPLE case in scst_cmd_set_sn()
1761                          */
1762                         smp_mb();
1763                         tgt_dev->num_free_sn_slots++;
1764                         TRACE_SN("Incremented num_free_sn_slots (%d)",
1765                                 tgt_dev->num_free_sn_slots);
1766
1767                 }
1768                 spin_unlock_irq(&tgt_dev->sn_lock);
1769         }
1770
1771 inc:
1772         /*
1773          * No protection of expected_sn is needed, because only one thread
1774          * at time can be here (serialized by sn). Also it is supposed that
1775          * there could not be half-incremented halves.
1776          */
1777         tgt_dev->expected_sn++;
1778         /*
1779          * Write must be before def_cmd_count read to be in sync. with
1780          * scst_post_exec_sn(). See comment in scst_send_for_exec().
1781          */
1782         smp_mb(); 
1783         TRACE_SN("Next expected_sn: %ld", tgt_dev->expected_sn);
1784
1785 out:
1786         return;
1787 }
1788
1789 /* No locks */
1790 static struct scst_cmd *scst_post_exec_sn(struct scst_cmd *cmd,
1791         bool make_active)
1792 {
1793         /* For HQ commands SN is not set */
1794         bool inc_expected_sn = !cmd->inc_expected_sn_on_done &&
1795                                cmd->sn_set && !cmd->retry;
1796         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1797         struct scst_cmd *res;
1798
1799         TRACE_ENTRY();
1800
1801         if (inc_expected_sn)
1802                 scst_inc_expected_sn(tgt_dev, cmd->sn_slot);
1803
1804         if (make_active) {
1805                 scst_make_deferred_commands_active(tgt_dev);
1806                 res = NULL;
1807         } else
1808                 res = scst_check_deferred_commands(tgt_dev);
1809
1810         TRACE_EXIT_HRES(res);
1811         return res;
1812 }
1813
1814 /* cmd must be additionally referenced to not die inside */
1815 static int scst_do_real_exec(struct scst_cmd *cmd)
1816 {
1817         int res = SCST_EXEC_NOT_COMPLETED;
1818 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
1819         int rc;
1820 #endif
1821         struct scst_device *dev = cmd->dev;
1822         struct scst_dev_type *handler = dev->handler;
1823
1824         TRACE_ENTRY();
1825
1826         cmd->state = SCST_CMD_STATE_REAL_EXECUTING;
1827
1828         if (handler->exec) {
1829                 if (unlikely(!dev->handler->exec_atomic &&
1830                              scst_cmd_atomic(cmd))) {
1831                         /*
1832                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
1833                          * optimization.
1834                          */
1835                         TRACE_DBG("Dev handler %s exec() needs thread "
1836                                 "context, rescheduling", dev->handler->name);
1837                         res = SCST_EXEC_NEED_THREAD;
1838                         goto out_restore;
1839                 }
1840
1841                 TRACE_DBG("Calling dev handler %s exec(%p)",
1842                       handler->name, cmd);
1843                 TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb,
1844                         cmd->cdb_len);
1845                 res = handler->exec(cmd);
1846                 TRACE_DBG("Dev handler %s exec() returned %d",
1847                       handler->name, res);
1848
1849                 if (res == SCST_EXEC_COMPLETED)
1850                         goto out_complete;
1851                 else if (res == SCST_EXEC_NEED_THREAD)
1852                         goto out_restore;
1853
1854                 sBUG_ON(res != SCST_EXEC_NOT_COMPLETED);
1855         }
1856
1857         TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
1858
1859         if (unlikely(dev->scsi_dev == NULL)) {
1860                 PRINT_ERROR("Command for virtual device must be "
1861                         "processed by device handler (lun %lld)!",
1862                         (long long unsigned int)cmd->lun);
1863                 goto out_error;
1864         }
1865
1866         res = scst_check_local_events(cmd);
1867         if (unlikely(res != 0))
1868                 goto out_done;
1869
1870 #ifndef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
1871         if (unlikely(scst_cmd_atomic(cmd))) {
1872                 TRACE_DBG("Pass-through exec() can not be called in atomic "
1873                         "context, rescheduling to the thread (handler %s)",
1874                         handler->name);
1875                 res = SCST_EXEC_NEED_THREAD;
1876                 goto out_restore;
1877         }
1878 #endif
1879
1880 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1881         if (unlikely(scst_alloc_request(cmd) != 0)) {
1882                 if (scst_cmd_atomic(cmd)) {
1883                         res = SCST_EXEC_NEED_THREAD;
1884                         goto out_restore;
1885                 } else {
1886                         PRINT_INFO("%s", "Unable to allocate request, "
1887                                 "sending BUSY status");
1888                         goto out_busy;
1889                 }
1890         }
1891
1892         scst_do_req(cmd->scsi_req, (void *)cmd->cdb,
1893                     (void *)cmd->scsi_req->sr_buffer,
1894                     cmd->scsi_req->sr_bufflen, scst_cmd_done, cmd->timeout,
1895                     cmd->retries);
1896 #else
1897         rc = scst_exec_req(dev->scsi_dev, cmd->cdb, cmd->cdb_len,
1898                         cmd->data_direction, cmd->sg, cmd->bufflen, cmd->sg_cnt,
1899                         cmd->timeout, cmd->retries, cmd, scst_cmd_done,
1900                         scst_cmd_atomic(cmd) ? GFP_ATOMIC : GFP_KERNEL);
1901         if (unlikely(rc != 0)) {
1902                 if (scst_cmd_atomic(cmd)) {
1903                         res = SCST_EXEC_NEED_THREAD;
1904                         goto out_restore;
1905                 } else {
1906                         PRINT_ERROR("scst_exec_req() failed: %d", res);
1907                         goto out_error;
1908                 }
1909         }
1910 #endif
1911
1912 out_complete:
1913         res = SCST_EXEC_COMPLETED;
1914
1915 out:
1916         TRACE_EXIT();
1917         return res;
1918
1919 out_restore:
1920         /* Restore the state */
1921         cmd->state = SCST_CMD_STATE_REAL_EXEC;
1922         goto out;
1923
1924 out_error:
1925         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
1926         goto out_done;
1927
1928 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
1929 out_busy:
1930         scst_set_busy(cmd);
1931         /* go through */
1932 #endif
1933
1934 out_done:
1935         res = SCST_EXEC_COMPLETED;
1936         /* Report the result */
1937         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
1938         goto out_complete;
1939 }
1940
1941 static inline int scst_real_exec(struct scst_cmd *cmd)
1942 {
1943         int res;
1944
1945         TRACE_ENTRY();
1946
1947         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
1948         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
1949         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
1950
1951         __scst_cmd_get(cmd);
1952
1953         res = scst_do_real_exec(cmd);
1954
1955         if (likely(res == SCST_EXEC_COMPLETED)) {
1956                 scst_post_exec_sn(cmd, true);
1957                 if (cmd->dev->scsi_dev != NULL)
1958                         generic_unplug_device(
1959                                 cmd->dev->scsi_dev->request_queue);
1960         } else
1961                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
1962
1963         __scst_cmd_put(cmd);
1964
1965         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
1966
1967         TRACE_EXIT_RES(res);
1968         return res;
1969 }
1970
1971 static int scst_do_local_exec(struct scst_cmd *cmd)
1972 {
1973         int res;
1974         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
1975
1976         TRACE_ENTRY();
1977
1978         /* Check READ_ONLY device status */
1979         if (((tgt_dev->acg_dev->rd_only_flag) || cmd->dev->swp) &&
1980             (cmd->cdb[0] == WRITE_6 ||  /* ToDo: full list of the modify cmds */
1981              cmd->cdb[0] == WRITE_10 ||
1982              cmd->cdb[0] == WRITE_12 ||
1983              cmd->cdb[0] == WRITE_16 ||
1984              cmd->cdb[0] == WRITE_VERIFY ||
1985              cmd->cdb[0] == WRITE_VERIFY_12 ||
1986              cmd->cdb[0] == WRITE_VERIFY_16 ||
1987              (cmd->dev->handler->type == TYPE_TAPE &&
1988               (cmd->cdb[0] == ERASE || cmd->cdb[0] == WRITE_FILEMARKS)))) {
1989                 scst_set_cmd_error(cmd,
1990                            SCST_LOAD_SENSE(scst_sense_data_protect));
1991                 goto out_done;
1992         }
1993
1994         /*
1995          * Adding new commands here don't forget to update
1996          * scst_is_cmd_local() in scst.h, if necessary
1997          */
1998
1999         switch (cmd->cdb[0]) {
2000         case MODE_SELECT:
2001         case MODE_SELECT_10:
2002         case LOG_SELECT:
2003                 res = scst_pre_select(cmd);
2004                 break;
2005         case RESERVE:
2006         case RESERVE_10:
2007                 res = scst_reserve_local(cmd);
2008                 break;
2009         case RELEASE:
2010         case RELEASE_10:
2011                 res = scst_release_local(cmd);
2012                 break;
2013         case REPORT_LUNS:
2014                 res = scst_report_luns_local(cmd);
2015                 break;
2016         default:
2017                 res = SCST_EXEC_NOT_COMPLETED;
2018                 break;
2019         }
2020
2021 out:
2022         TRACE_EXIT_RES(res);
2023         return res;
2024
2025 out_done:
2026         /* Report the result */
2027         cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
2028         res = SCST_EXEC_COMPLETED;
2029         goto out;
2030 }
2031
2032 static int scst_local_exec(struct scst_cmd *cmd)
2033 {
2034         int res;
2035
2036         TRACE_ENTRY();
2037
2038         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
2039         BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
2040         BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
2041
2042         __scst_cmd_get(cmd);
2043
2044         res = scst_do_local_exec(cmd);
2045         if (likely(res == SCST_EXEC_NOT_COMPLETED))
2046                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2047         else if (res == SCST_EXEC_COMPLETED)
2048                 scst_post_exec_sn(cmd, true);
2049         else
2050                 sBUG_ON(res != SCST_EXEC_NEED_THREAD);
2051
2052         __scst_cmd_put(cmd);
2053
2054         /* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
2055         TRACE_EXIT_RES(res);
2056         return res;
2057 }
2058
2059 static int scst_exec(struct scst_cmd **active_cmd)
2060 {
2061         struct scst_cmd *cmd = *active_cmd;
2062         struct scst_cmd *ref_cmd;
2063         struct scst_device *dev = cmd->dev;
2064         int res = SCST_CMD_STATE_RES_CONT_NEXT, count;
2065
2066         TRACE_ENTRY();
2067
2068         if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2069                 goto out;
2070
2071         /* To protect tgt_dev */
2072         ref_cmd = cmd;
2073         __scst_cmd_get(ref_cmd);
2074
2075         count = 0;
2076         while (1) {
2077                 int rc;
2078
2079                 cmd->sent_for_exec = 1;
2080                 cmd->scst_cmd_done = scst_cmd_done_local;
2081                 cmd->state = SCST_CMD_STATE_LOCAL_EXEC;
2082
2083                 if (cmd->tgt_data_buf_alloced && cmd->dh_data_buf_alloced &&
2084                     (cmd->data_direction == SCST_DATA_WRITE))
2085                         scst_copy_sg(cmd, SCST_SG_COPY_FROM_TARGET);
2086
2087                 rc = scst_do_local_exec(cmd);
2088                 if (likely(rc == SCST_EXEC_NOT_COMPLETED))
2089                         /* Nothing to do */;
2090                 else if (rc == SCST_EXEC_NEED_THREAD) {
2091                         TRACE_DBG("%s", "scst_do_local_exec() requested "
2092                                 "thread context, rescheduling");
2093                         scst_dec_on_dev_cmd(cmd);
2094                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2095                         break;
2096                 } else {
2097                         sBUG_ON(rc != SCST_EXEC_COMPLETED);
2098                         goto done;
2099                 }
2100
2101                 cmd->state = SCST_CMD_STATE_REAL_EXEC;
2102
2103                 rc = scst_do_real_exec(cmd);
2104                 if (likely(rc == SCST_EXEC_COMPLETED))
2105                         /* Nothing to do */;
2106                 else if (rc == SCST_EXEC_NEED_THREAD) {
2107                         TRACE_DBG("scst_real_exec() requested thread "
2108                                 "context, rescheduling (cmd %p)", cmd);
2109                         scst_dec_on_dev_cmd(cmd);
2110                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2111                         break;
2112                 } else
2113                         sBUG();
2114
2115 done:
2116                 count++;
2117
2118                 cmd = scst_post_exec_sn(cmd, false);
2119                 if (cmd == NULL)
2120                         break;
2121
2122                 if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
2123                         break;
2124
2125                 __scst_cmd_put(ref_cmd);
2126                 ref_cmd = cmd;
2127                 __scst_cmd_get(ref_cmd);
2128         }
2129
2130         *active_cmd = cmd;
2131
2132         if (count == 0)
2133                 goto out_put;
2134
2135         if (dev->scsi_dev != NULL)
2136                 generic_unplug_device(dev->scsi_dev->request_queue);
2137
2138 out_put:
2139         __scst_cmd_put(ref_cmd);
2140         /* !! At this point sess, dev and tgt_dev can be already freed !! */
2141
2142 out:
2143         TRACE_EXIT_RES(res);
2144         return res;
2145 }
2146
2147 static int scst_send_for_exec(struct scst_cmd **active_cmd)
2148 {
2149         int res;
2150         struct scst_cmd *cmd = *active_cmd;
2151         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2152         typeof(tgt_dev->expected_sn) expected_sn;
2153
2154         TRACE_ENTRY();
2155
2156 #ifdef CONFIG_SCST_MEASURE_LATENCY
2157         if (cmd->pre_exec_finish == 0) {
2158                 struct timespec ts;
2159                 getnstimeofday(&ts);
2160                 cmd->pre_exec_finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2161                 TRACE_DBG("cmd %p (sess %p): pre_exec_finish %lld (tv_sec %ld, "
2162                         "tv_nsec %ld)", cmd, cmd->sess, cmd->pre_exec_finish,
2163                         ts.tv_sec, ts.tv_nsec);
2164         }
2165 #endif
2166
2167         if (unlikely(cmd->internal))
2168                 goto exec;
2169
2170         if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2171                 goto exec;
2172
2173         sBUG_ON(!cmd->sn_set);
2174
2175         expected_sn = tgt_dev->expected_sn;
2176         /* Optimized for lockless fast path */
2177         if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2178                 spin_lock_irq(&tgt_dev->sn_lock);
2179
2180                 tgt_dev->def_cmd_count++;
2181                 /*
2182                  * Memory barrier is needed here to implement lockless fast
2183                  * path. We need the exact order of read and write between
2184                  * def_cmd_count and expected_sn. Otherwise, we can miss case,
2185                  * when expected_sn was changed to be equal to cmd->sn while
2186                  * we are queuing cmd the deferred list after the expected_sn
2187                  * below. It will lead to a forever stuck command. But with
2188                  * the barrier in such case __scst_check_deferred_commands()
2189                  * will be called and it will take sn_lock, so we will be
2190                  * synchronized.
2191                  */
2192                 smp_mb();
2193
2194                 expected_sn = tgt_dev->expected_sn;
2195                 if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
2196                         if (unlikely(test_bit(SCST_CMD_ABORTED,
2197                                               &cmd->cmd_flags))) {
2198                                 /* Necessary to allow aborting out of sn cmds */
2199                                 TRACE_MGMT_DBG("Aborting out of sn cmd %p "
2200                                         "(tag %llu, sn %lu)", cmd,
2201                                         (long long unsigned)cmd->tag, cmd->sn);
2202                                 tgt_dev->def_cmd_count--;
2203                                 scst_set_cmd_abnormal_done_state(cmd);
2204                                 res = SCST_CMD_STATE_RES_CONT_SAME;
2205                         } else {
2206                                 TRACE_SN("Deferring cmd %p (sn=%ld, set %d, "
2207                                         "expected_sn=%ld)", cmd, cmd->sn,
2208                                         cmd->sn_set, expected_sn);
2209                                 list_add_tail(&cmd->sn_cmd_list_entry,
2210                                               &tgt_dev->deferred_cmd_list);
2211                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2212                         }
2213                         spin_unlock_irq(&tgt_dev->sn_lock);
2214                         goto out;
2215                 } else {
2216                         TRACE_SN("Somebody incremented expected_sn %ld, "
2217                                 "continuing", expected_sn);
2218                         tgt_dev->def_cmd_count--;
2219                         spin_unlock_irq(&tgt_dev->sn_lock);
2220                 }
2221         }
2222
2223 exec:
2224         res = scst_exec(active_cmd);
2225
2226 out:
2227         TRACE_EXIT_HRES(res);
2228         return res;
2229 }
2230
2231 /* No locks supposed to be held */
2232 static int scst_check_sense(struct scst_cmd *cmd)
2233 {
2234         int res = 0;
2235         struct scst_device *dev = cmd->dev;
2236
2237         TRACE_ENTRY();
2238
2239         if (unlikely(cmd->ua_ignore))
2240                 goto out;
2241
2242         /* If we had internal bus reset behind us, set the command error UA */
2243         if ((dev->scsi_dev != NULL) &&
2244             unlikely(cmd->host_status == DID_RESET) &&
2245             scst_is_ua_command(cmd)) {
2246                 TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
2247                       dev->scsi_dev->was_reset, cmd->host_status);
2248                 scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_reset_UA));
2249                 /* It looks like it is safe to clear was_reset here */
2250                 dev->scsi_dev->was_reset = 0;
2251         }
2252
2253         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2254             SCST_SENSE_VALID(cmd->sense)) {
2255                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2256                         SCST_SENSE_BUFFERSIZE);
2257
2258                 /* Check Unit Attention Sense Key */
2259                 if (scst_is_ua_sense(cmd->sense)) {
2260                         if (cmd->sense[12] == SCST_SENSE_ASC_UA_RESET) {
2261                                 if (cmd->double_ua_possible) {
2262                                         TRACE(TRACE_MGMT_MINOR, "Double UA "
2263                                                 "detected for device %p", dev);
2264                                         TRACE(TRACE_MGMT_MINOR, "Retrying cmd"
2265                                                 " %p (tag %llu)", cmd,
2266                                                 (long long unsigned)cmd->tag);
2267
2268                                         cmd->status = 0;
2269                                         cmd->msg_status = 0;
2270                                         cmd->host_status = DID_OK;
2271                                         cmd->driver_status = 0;
2272
2273                                         mempool_free(cmd->sense,
2274                                                      scst_sense_mempool);
2275                                         cmd->sense = NULL;
2276
2277                                         scst_check_restore_sg_buff(cmd);
2278
2279                                         sBUG_ON(cmd->dbl_ua_orig_resp_data_len < 0);
2280                                         cmd->data_direction =
2281                                                 cmd->dbl_ua_orig_data_direction;
2282                                         cmd->resp_data_len =
2283                                                 cmd->dbl_ua_orig_resp_data_len;
2284
2285                                         cmd->state = SCST_CMD_STATE_REAL_EXEC;
2286                                         cmd->retry = 1;
2287                                         res = 1;
2288                                         goto out;
2289                                 }
2290                         }
2291                         scst_dev_check_set_UA(dev, cmd, cmd->sense,
2292                                 SCST_SENSE_BUFFERSIZE);
2293                 }
2294         }
2295
2296         if (unlikely(cmd->double_ua_possible)) {
2297                 if (scst_is_ua_command(cmd)) {
2298                         TRACE_MGMT_DBG("Clearing dbl_ua_possible flag (dev %p, "
2299                                 "cmd %p)", dev, cmd);
2300                         /*
2301                          * Lock used to protect other flags in the bitfield
2302                          * (just in case, actually). Those 2 flags can't be
2303                          * changed in parallel, because the device is
2304                          * serialized.
2305                          */
2306                         spin_lock_bh(&dev->dev_lock);
2307                         dev->dev_double_ua_possible = 0;
2308                         dev->dev_serialized = 0;
2309                         spin_unlock_bh(&dev->dev_lock);
2310                 }
2311         }
2312
2313 out:
2314         TRACE_EXIT_RES(res);
2315         return res;
2316 }
2317
2318 static int scst_check_auto_sense(struct scst_cmd *cmd)
2319 {
2320         int res = 0;
2321
2322         TRACE_ENTRY();
2323
2324         if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
2325             (!SCST_SENSE_VALID(cmd->sense) ||
2326              SCST_NO_SENSE(cmd->sense))) {
2327                 TRACE(TRACE_SCSI|TRACE_MINOR, "CHECK_CONDITION, but no sense: "
2328                       "cmd->status=%x, cmd->msg_status=%x, "
2329                       "cmd->host_status=%x, cmd->driver_status=%x (cmd %p)",
2330                       cmd->status, cmd->msg_status, cmd->host_status,
2331                       cmd->driver_status, cmd);
2332                 res = 1;
2333         } else if (unlikely(cmd->host_status)) {
2334                 if ((cmd->host_status == DID_REQUEUE) ||
2335                     (cmd->host_status == DID_IMM_RETRY) ||
2336                     (cmd->host_status == DID_SOFT_ERROR) ||
2337                     (cmd->host_status == DID_ABORT)) {
2338                         scst_set_busy(cmd);
2339                 } else {
2340                         TRACE(TRACE_SCSI|TRACE_MINOR, "Host status %x "
2341                                 "received, returning HARDWARE ERROR instead "
2342                                 "(cmd %p)", cmd->host_status, cmd);
2343                         scst_set_cmd_error(cmd,
2344                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2345                 }
2346         }
2347
2348         TRACE_EXIT_RES(res);
2349         return res;
2350 }
2351
2352 static int scst_pre_dev_done(struct scst_cmd *cmd)
2353 {
2354         int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
2355
2356         TRACE_ENTRY();
2357
2358         if (unlikely(scst_check_auto_sense(cmd))) {
2359                 PRINT_INFO("Command finished with CHECK CONDITION, but "
2360                             "without sense data (opcode 0x%x), issuing "
2361                             "REQUEST SENSE", cmd->cdb[0]);
2362                 rc = scst_prepare_request_sense(cmd);
2363                 if (rc == 0)
2364                         res = SCST_CMD_STATE_RES_CONT_NEXT;
2365                 else {
2366                         PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
2367                                     "returning HARDWARE ERROR");
2368                         scst_set_cmd_error(cmd,
2369                                 SCST_LOAD_SENSE(scst_sense_hardw_error));
2370                 }
2371                 goto out;
2372         } else if (unlikely(scst_check_sense(cmd)))
2373                 goto out;
2374
2375         if (likely(scsi_status_is_good(cmd->status))) {
2376                 unsigned char type = cmd->dev->handler->type;
2377                 if (unlikely((cmd->cdb[0] == MODE_SENSE ||
2378                               cmd->cdb[0] == MODE_SENSE_10)) &&
2379                     cmd->tgt_dev->acg_dev->rd_only_flag &&
2380                     (type == TYPE_DISK ||
2381                      type == TYPE_WORM ||
2382                      type == TYPE_MOD ||
2383                      type == TYPE_TAPE)) {
2384                         int32_t length;
2385                         uint8_t *address;
2386                         bool err = false;
2387
2388                         length = scst_get_buf_first(cmd, &address);
2389                         if (length < 0) {
2390                                 PRINT_ERROR("%s", "Unable to get "
2391                                         "MODE_SENSE buffer");
2392                                 scst_set_cmd_error(cmd,
2393                                         SCST_LOAD_SENSE(
2394                                                 scst_sense_hardw_error));
2395                                 err = true;
2396                         } else if (length > 2 && cmd->cdb[0] == MODE_SENSE)
2397                                 address[2] |= 0x80;   /* Write Protect*/
2398                         else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
2399                                 address[3] |= 0x80;   /* Write Protect*/
2400                         scst_put_buf(cmd, address);
2401
2402                         if (err)
2403                                 goto out;
2404                 }
2405
2406                 /*
2407                  * Check and clear NormACA option for the device, if necessary,
2408                  * since we don't support ACA
2409                  */
2410                 if (unlikely((cmd->cdb[0] == INQUIRY)) &&
2411                     /* Std INQUIRY data (no EVPD) */
2412                     !(cmd->cdb[1] & SCST_INQ_EVPD) &&
2413                     (cmd->resp_data_len > SCST_INQ_BYTE3)) {
2414                         uint8_t *buffer;
2415                         int buflen;
2416                         bool err = false;
2417
2418                         /* ToDo: all pages ?? */
2419                         buflen = scst_get_buf_first(cmd, &buffer);
2420                         if (buflen > SCST_INQ_BYTE3) {
2421 #ifdef CONFIG_SCST_EXTRACHECKS
2422                                 if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
2423                                         PRINT_INFO("NormACA set for device: "
2424                                             "lun=%lld, type 0x%02x. Clear it, "
2425                                             "since it's unsupported.",
2426                                             (long long unsigned int)cmd->lun,
2427                                             buffer[0]);
2428                                 }
2429 #endif
2430                                 buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
2431                         } else if (buflen != 0) {
2432                                 PRINT_ERROR("%s", "Unable to get INQUIRY "
2433                                     "buffer");
2434                                 scst_set_cmd_error(cmd,
2435                                        SCST_LOAD_SENSE(scst_sense_hardw_error));
2436                                 err = true;
2437                         }
2438                         if (buflen > 0)
2439                                 scst_put_buf(cmd, buffer);
2440
2441                         if (err)
2442                                 goto out;
2443                 }
2444
2445                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2446                     (cmd->cdb[0] == MODE_SELECT_10) ||
2447                     (cmd->cdb[0] == LOG_SELECT))) {
2448                         TRACE(TRACE_SCSI,
2449                                 "MODE/LOG SELECT succeeded (LUN %lld)",
2450                                 (long long unsigned int)cmd->lun);
2451                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2452                         goto out;
2453                 }
2454         } else {
2455                 if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
2456                         if (!test_bit(SCST_TGT_DEV_RESERVED,
2457                                         &cmd->tgt_dev->tgt_dev_flags)) {
2458                                 struct scst_tgt_dev *tgt_dev_tmp;
2459                                 struct scst_device *dev = cmd->dev;
2460
2461                                 TRACE(TRACE_SCSI,
2462                                         "Real RESERVE failed lun=%lld, "
2463                                         "status=%x",
2464                                         (long long unsigned int)cmd->lun,
2465                                         cmd->status);
2466                                 PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
2467                                         SCST_SENSE_BUFFERSIZE);
2468
2469                                 /* Clearing the reservation */
2470                                 spin_lock_bh(&dev->dev_lock);
2471                                 list_for_each_entry(tgt_dev_tmp,
2472                                                     &dev->dev_tgt_dev_list,
2473                                                     dev_tgt_dev_list_entry) {
2474                                         clear_bit(SCST_TGT_DEV_RESERVED,
2475                                                 &tgt_dev_tmp->tgt_dev_flags);
2476                                 }
2477                                 dev->dev_reserved = 0;
2478                                 spin_unlock_bh(&dev->dev_lock);
2479                         }
2480                 }
2481
2482                 /* Check for MODE PARAMETERS CHANGED UA */
2483                 if ((cmd->dev->scsi_dev != NULL) &&
2484                     (cmd->status == SAM_STAT_CHECK_CONDITION) &&
2485                     SCST_SENSE_VALID(cmd->sense) &&
2486                     scst_is_ua_sense(cmd->sense) &&
2487                     (cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) {
2488                         TRACE(TRACE_SCSI,
2489                               "MODE PARAMETERS CHANGED UA (lun %lld)",
2490                               (long long unsigned int)cmd->lun);
2491                         cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
2492                         goto out;
2493                 }
2494         }
2495
2496         cmd->state = SCST_CMD_STATE_DEV_DONE;
2497
2498 out:
2499         TRACE_EXIT_RES(res);
2500         return res;
2501 }
2502
2503 static int scst_mode_select_checks(struct scst_cmd *cmd)
2504 {
2505         int res = SCST_CMD_STATE_RES_CONT_SAME;
2506         int atomic = scst_cmd_atomic(cmd);
2507
2508         TRACE_ENTRY();
2509
2510         if (likely(scsi_status_is_good(cmd->status))) {
2511                 if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
2512                     (cmd->cdb[0] == MODE_SELECT_10) ||
2513                     (cmd->cdb[0] == LOG_SELECT))) {
2514                         struct scst_device *dev = cmd->dev;
2515                         if (atomic && (dev->scsi_dev != NULL)) {
2516                                 TRACE_DBG("%s", "MODE/LOG SELECT: thread "
2517                                         "context required");
2518                                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2519                                 goto out;
2520                         }
2521
2522                         TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
2523                                 "setting the SELECT UA (lun=%lld)",
2524                                 (long long unsigned int)cmd->lun);
2525
2526                         spin_lock_bh(&dev->dev_lock);
2527                         spin_lock(&scst_temp_UA_lock);
2528                         if (cmd->cdb[0] == LOG_SELECT) {
2529                                 scst_set_sense(scst_temp_UA,
2530                                         sizeof(scst_temp_UA),
2531                                         UNIT_ATTENTION, 0x2a, 0x02);
2532                         } else {
2533                                 scst_set_sense(scst_temp_UA,
2534                                         sizeof(scst_temp_UA),
2535                                         UNIT_ATTENTION, 0x2a, 0x01);
2536                         }
2537                         scst_dev_check_set_local_UA(dev, cmd, scst_temp_UA,
2538                                 sizeof(scst_temp_UA));
2539                         spin_unlock(&scst_temp_UA_lock);
2540                         spin_unlock_bh(&dev->dev_lock);
2541
2542                         if (dev->scsi_dev != NULL)
2543                                 scst_obtain_device_parameters(dev);
2544                 }
2545         } else if ((cmd->status == SAM_STAT_CHECK_CONDITION) &&
2546                     SCST_SENSE_VALID(cmd->sense) &&
2547                     scst_is_ua_sense(cmd->sense) &&
2548                     (((cmd->sense[12] == 0x2a) && (cmd->sense[13] == 0x01)) ||
2549                      (cmd->sense[12] == 0x29) /* reset */ ||
2550                      (cmd->sense[12] == 0x28) /* medium changed */ ||
2551                      /* cleared by another ini (just in case) */
2552                      (cmd->sense[12] == 0x2F))) {
2553                 if (atomic) {
2554                         TRACE_DBG("Possible parameters changed UA %x: "
2555                                 "thread context required", cmd->sense[12]);
2556                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2557                         goto out;
2558                 }
2559
2560                 TRACE(TRACE_SCSI, "Possible parameters changed UA %x "
2561                         "(lun %lld): getting new parameters", cmd->sense[12],
2562                         (long long unsigned int)cmd->lun);
2563
2564                 scst_obtain_device_parameters(cmd->dev);
2565         } else
2566                 sBUG();
2567
2568         cmd->state = SCST_CMD_STATE_DEV_DONE;
2569
2570 out:
2571         TRACE_EXIT_HRES(res);
2572         return res;
2573 }
2574
2575 static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
2576 {
2577         if (likely(cmd->sn_set))
2578                 scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
2579
2580         scst_make_deferred_commands_active(cmd->tgt_dev);
2581 }
2582
2583 static int scst_dev_done(struct scst_cmd **pcmd)
2584 {
2585         int res = SCST_CMD_STATE_RES_CONT_SAME;
2586         struct scst_cmd *cmd = *pcmd;
2587         int state;
2588         struct scst_device *dev = cmd->dev;
2589
2590         TRACE_ENTRY();
2591
2592         state = SCST_CMD_STATE_PRE_XMIT_RESP;
2593
2594         if (likely(!scst_is_cmd_local(cmd)) &&
2595             likely(dev->handler->dev_done != NULL)) {
2596                 int rc;
2597
2598                 if (unlikely(!dev->handler->dev_done_atomic &&
2599                              scst_cmd_atomic(cmd))) {
2600                         /*
2601                          * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2602                          * optimization.
2603                          */
2604                         TRACE_DBG("Dev handler %s dev_done() needs thread "
2605                               "context, rescheduling", dev->handler->name);
2606                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2607                         goto out;
2608                 }
2609
2610                 TRACE_DBG("Calling dev handler %s dev_done(%p)",
2611                       dev->handler->name, cmd);
2612                 rc = dev->handler->dev_done(cmd);
2613                 TRACE_DBG("Dev handler %s dev_done() returned %d",
2614                       dev->handler->name, rc);
2615                 if (rc != SCST_CMD_STATE_DEFAULT)
2616                         state = rc;
2617         }
2618
2619         switch (state) {
2620         case SCST_CMD_STATE_PRE_XMIT_RESP:
2621         case SCST_CMD_STATE_DEV_PARSE:
2622         case SCST_CMD_STATE_PRE_PARSE:
2623         case SCST_CMD_STATE_PREPARE_SPACE:
2624         case SCST_CMD_STATE_RDY_TO_XFER:
2625         case SCST_CMD_STATE_TGT_PRE_EXEC:
2626         case SCST_CMD_STATE_SEND_FOR_EXEC:
2627         case SCST_CMD_STATE_LOCAL_EXEC:
2628         case SCST_CMD_STATE_REAL_EXEC:
2629         case SCST_CMD_STATE_PRE_DEV_DONE:
2630         case SCST_CMD_STATE_MODE_SELECT_CHECKS:
2631         case SCST_CMD_STATE_DEV_DONE:
2632         case SCST_CMD_STATE_XMIT_RESP:
2633         case SCST_CMD_STATE_FINISHED:
2634                 cmd->state = state;
2635                 break;
2636
2637         case SCST_CMD_STATE_NEED_THREAD_CTX:
2638                 TRACE_DBG("Dev handler %s dev_done() requested "
2639                       "thread context, rescheduling",
2640                       dev->handler->name);
2641                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2642                 break;
2643
2644         default:
2645                 if (state >= 0) {
2646                         PRINT_ERROR("Dev handler %s dev_done() returned "
2647                                 "invalid cmd state %d",
2648                                 dev->handler->name, state);
2649                 } else {
2650                         PRINT_ERROR("Dev handler %s dev_done() returned "
2651                                 "error %d", dev->handler->name,
2652                                 state);
2653                 }
2654                 scst_set_cmd_error(cmd,
2655                            SCST_LOAD_SENSE(scst_sense_hardw_error));
2656                 scst_set_cmd_abnormal_done_state(cmd);
2657                 break;
2658         }
2659
2660         if (cmd->needs_unblocking)
2661                 scst_unblock_dev_cmd(cmd);
2662
2663         if (likely(cmd->dec_on_dev_needed))
2664                 scst_dec_on_dev_cmd(cmd);
2665
2666         if (cmd->inc_expected_sn_on_done && cmd->sent_for_exec)
2667                 scst_inc_check_expected_sn(cmd);
2668
2669         if (unlikely(cmd->cdb[0] == REQUEST_SENSE) && (cmd->internal))
2670                 *pcmd = scst_complete_request_sense(cmd);
2671
2672 out:
2673         TRACE_EXIT_HRES(res);
2674         return res;
2675 }
2676
2677 static int scst_pre_xmit_response(struct scst_cmd *cmd)
2678 {
2679         int res;
2680
2681         TRACE_ENTRY();
2682
2683 #ifdef CONFIG_SCST_DEBUG_TM
2684         if (cmd->tm_dbg_delayed &&
2685                         !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
2686                 if (scst_cmd_atomic(cmd)) {
2687                         TRACE_MGMT_DBG("%s",
2688                                 "DEBUG_TM delayed cmd needs a thread");
2689                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2690                         return res;
2691                 }
2692                 TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
2693                         cmd, cmd->tag);
2694                 schedule_timeout_uninterruptible(HZ);
2695         }
2696 #endif
2697
2698         if (likely(cmd->tgt_dev != NULL)) {
2699                 atomic_dec(&cmd->tgt_dev->tgt_dev_cmd_count);
2700                 atomic_dec(&cmd->dev->dev_cmd_count);
2701                 /* If expected values not set, expected direction is UNKNOWN */
2702                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
2703                         atomic_dec(&cmd->dev->write_cmd_count);
2704
2705                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
2706                         scst_on_hq_cmd_response(cmd);
2707
2708                 if (unlikely(!cmd->sent_for_exec)) {
2709                         TRACE_SN("cmd %p was not sent to mid-lev"
2710                                 " (sn %ld, set %d)",
2711                                 cmd, cmd->sn, cmd->sn_set);
2712                         scst_unblock_deferred(cmd->tgt_dev, cmd);
2713                         cmd->sent_for_exec = 1;
2714                 }
2715         }
2716
2717         /*
2718          * If we don't remove cmd from the search list here, before
2719          * submitting it for transmittion, we will have a race, when for
2720          * some reason cmd's release is delayed after transmittion and
2721          * initiator sends cmd with the same tag => it is possible that
2722          * a wrong cmd will be found by find() functions.
2723          */
2724         spin_lock_irq(&cmd->sess->sess_list_lock);
2725         list_del(&cmd->search_cmd_list_entry);
2726         spin_unlock_irq(&cmd->sess->sess_list_lock);
2727
2728         cmd->done = 1;
2729         smp_mb(); /* to sync with scst_abort_cmd() */
2730
2731         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
2732                 scst_xmit_process_aborted_cmd(cmd);
2733
2734         if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
2735                 TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu),"
2736                                 " skipping",
2737                                 cmd, (long long unsigned int)cmd->tag);
2738                 cmd->state = SCST_CMD_STATE_FINISHED;
2739                 res = SCST_CMD_STATE_RES_CONT_SAME;
2740                 goto out;
2741         }
2742
2743         if (cmd->tgt_data_buf_alloced && cmd->dh_data_buf_alloced &&
2744             (cmd->data_direction == SCST_DATA_READ))
2745                 scst_copy_sg(cmd, SCST_SG_COPY_TO_TARGET);
2746
2747         cmd->state = SCST_CMD_STATE_XMIT_RESP;
2748         res = SCST_CMD_STATE_RES_CONT_SAME;
2749
2750 out:
2751 #ifdef CONFIG_SCST_MEASURE_LATENCY
2752         {
2753                 struct timespec ts;
2754                 uint64_t finish, scst_time, proc_time;
2755                 struct scst_session *sess = cmd->sess;
2756
2757                 getnstimeofday(&ts);
2758                 finish = scst_sec_to_nsec(ts.tv_sec) + ts.tv_nsec;
2759
2760                 spin_lock_bh(&sess->meas_lock);
2761
2762                 scst_time = cmd->pre_exec_finish - cmd->start;
2763                 scst_time += finish - cmd->post_exec_start;
2764                 proc_time = finish - cmd->start;
2765
2766                 sess->scst_time += scst_time;
2767                 sess->processing_time += proc_time;
2768                 sess->processed_cmds++;
2769
2770                 spin_unlock_bh(&sess->meas_lock);
2771
2772                 TRACE_DBG("cmd %p (sess %p): finish %lld (tv_sec %ld, "
2773                         "tv_nsec %ld), scst_time %lld, proc_time %lld",
2774                         cmd, sess, finish, ts.tv_sec, ts.tv_nsec, scst_time,
2775                         proc_time);
2776         }
2777 #endif
2778         TRACE_EXIT_HRES(res);
2779         return res;
2780 }
2781
2782 static int scst_xmit_response(struct scst_cmd *cmd)
2783 {
2784         int res, rc;
2785
2786         TRACE_ENTRY();
2787
2788         if (unlikely(!cmd->tgtt->xmit_response_atomic &&
2789                      scst_cmd_atomic(cmd))) {
2790                 /*
2791                  * It shouldn't be because of SCST_TGT_DEV_AFTER_*
2792                  * optimization.
2793                  */
2794                 TRACE_DBG("Target driver %s xmit_response() needs thread "
2795                               "context, rescheduling", cmd->tgtt->name);
2796                 res = SCST_CMD_STATE_RES_NEED_THREAD;
2797                 goto out;
2798         }
2799
2800         while (1) {
2801                 int finished_cmds = atomic_read(&cmd->sess->tgt->finished_cmds);
2802
2803                 res = SCST_CMD_STATE_RES_CONT_NEXT;
2804                 cmd->state = SCST_CMD_STATE_XMIT_WAIT;
2805
2806                 TRACE_DBG("Calling xmit_response(%p)", cmd);
2807
2808 #if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
2809                 if (cmd->tgt_sg) {
2810                         int i;
2811                         struct scatterlist *sg = cmd->tgt_sg;
2812                         TRACE(TRACE_SND_BOT, "Xmitting data for cmd %p "
2813                                 "(sg_cnt %d, sg %p, sg[0].page %p)", cmd,
2814                                 cmd->tgt_sg_cnt, sg, (void *)sg_page(&sg[0]));
2815                         for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
2816                                 PRINT_BUFF_FLAG(TRACE_SND_BOT, "Xmitting sg",
2817                                         sg_virt(&sg[i]), sg[i].length);
2818                         }
2819                 }
2820 #endif
2821
2822 #ifdef CONFIG_SCST_DEBUG_RETRY
2823                 if (((scst_random() % 100) == 77))
2824                         rc = SCST_TGT_RES_QUEUE_FULL;
2825                 else
2826 #endif
2827                         rc = cmd->tgtt->xmit_response(cmd);
2828                 TRACE_DBG("xmit_response() returned %d", rc);
2829
2830                 if (likely(rc == SCST_TGT_RES_SUCCESS))
2831                         goto out;
2832
2833                 /* Restore the previous state */
2834                 cmd->state = SCST_CMD_STATE_XMIT_RESP;
2835
2836                 switch (rc) {
2837                 case SCST_TGT_RES_QUEUE_FULL:
2838                         if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
2839                                 break;
2840                         else
2841                                 continue;
2842
2843                 case SCST_TGT_RES_NEED_THREAD_CTX:
2844                         TRACE_DBG("Target driver %s xmit_response() "
2845                               "requested thread context, rescheduling",
2846                               cmd->tgtt->name);
2847                         res = SCST_CMD_STATE_RES_NEED_THREAD;
2848                         break;
2849
2850                 default:
2851                         goto out_error;
2852                 }
2853                 break;
2854         }
2855
2856 out:
2857         /* Caution: cmd can be already dead here */
2858         TRACE_EXIT_HRES(res);
2859         return res;
2860
2861 out_error:
2862         if (rc == SCST_TGT_RES_FATAL_ERROR) {
2863                 PRINT_ERROR("Target driver %s xmit_response() returned "
2864                         "fatal error", cmd->tgtt->name);
2865         } else {
2866                 PRINT_ERROR("Target driver %s xmit_response() returned "
2867                         "invalid value %d", cmd->tgtt->name, rc);
2868         }
2869         scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
2870         cmd->state = SCST_CMD_STATE_FINISHED;
2871         res = SCST_CMD_STATE_RES_CONT_SAME;
2872         goto out;
2873 }
2874
2875 void scst_tgt_cmd_done(struct scst_cmd *cmd,
2876         enum scst_exec_context pref_context)
2877 {
2878         TRACE_ENTRY();
2879
2880         sBUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
2881
2882         cmd->state = SCST_CMD_STATE_FINISHED;
2883         scst_proccess_redirect_cmd(cmd, pref_context, 1);
2884
2885         TRACE_EXIT();
2886         return;
2887 }
2888 EXPORT_SYMBOL(scst_tgt_cmd_done);
2889
2890 static int scst_finish_cmd(struct scst_cmd *cmd)
2891 {
2892         int res;
2893
2894         TRACE_ENTRY();
2895
2896         atomic_dec(&cmd->sess->sess_cmd_count);
2897
2898         cmd->finished = 1;
2899         smp_mb(); /* to sync with scst_abort_cmd() */
2900
2901         if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
2902                 TRACE_MGMT_DBG("Aborted cmd %p finished (cmd_ref %d, "
2903                         "scst_cmd_count %d)", cmd, atomic_read(&cmd->cmd_ref),
2904                         atomic_read(&scst_cmd_count));
2905
2906                 scst_finish_cmd_mgmt(cmd);
2907         }
2908
2909         if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
2910                 if ((cmd->tgt_dev != NULL) &&
2911                     scst_is_ua_sense(cmd->sense)) {
2912                         /* This UA delivery failed, so requeue it */
2913                         TRACE_MGMT_DBG("Requeuing UA for delivery failed cmd "
2914                                 "%p", cmd);
2915                         scst_check_set_UA(cmd->tgt_dev, cmd->sense,
2916                                         SCST_SENSE_BUFFERSIZE, 1);
2917                 }
2918         }
2919
2920         __scst_cmd_put(cmd);
2921
2922         res = SCST_CMD_STATE_RES_CONT_NEXT;
2923
2924         TRACE_EXIT_HRES(res);
2925         return res;
2926 }
2927
2928 /*
2929  * No locks, but it must be externally serialized (see comment for
2930  * scst_cmd_init_done() in scst.h)
2931  */
2932 static void scst_cmd_set_sn(struct scst_cmd *cmd)
2933 {
2934         struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
2935         unsigned long flags;
2936
2937         TRACE_ENTRY();
2938
2939         if (scst_is_implicit_hq(cmd)) {
2940                 TRACE_SN("Implicit HQ cmd %p", cmd);
2941                 cmd->queue_type = SCST_CMD_QUEUE_HEAD_OF_QUEUE;
2942         }
2943
2944         EXTRACHECKS_BUG_ON(cmd->sn_set || cmd->hq_cmd_inced);
2945
2946         /* Optimized for lockless fast path */
2947
2948         scst_check_debug_sn(cmd);
2949
2950         if (cmd->dev->queue_alg ==
2951                         SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER) {
2952                 /*
2953                  * Not the best way, but well enough until there will be a
2954                  * possibility to specify queue type during pass-through
2955                  * commands submission.
2956                  */
2957                 cmd->queue_type = SCST_CMD_QUEUE_ORDERED;
2958         }
2959
2960         switch (cmd->queue_type) {
2961         case SCST_CMD_QUEUE_SIMPLE:
2962         case SCST_CMD_QUEUE_UNTAGGED:
2963 #if 0 /* left for future performance investigations */
2964                 if (scst_cmd_is_expected_set(cmd)) {
2965                         if ((cmd->expected_data_direction == SCST_DATA_READ) &&
2966                             (atomic_read(&cmd->dev->write_cmd_count) == 0))
2967                                 goto ordered;
2968                 } else
2969                         goto ordered;
2970 #endif
2971                 if (likely(tgt_dev->num_free_sn_slots >= 0)) {
2972                         /*
2973                          * atomic_inc_return() implies memory barrier to sync
2974                          * with scst_inc_expected_sn()
2975                          */
2976                         if (atomic_inc_return(tgt_dev->cur_sn_slot) == 1) {
2977                                 tgt_dev->curr_sn++;
2978                                 TRACE_SN("Incremented curr_sn %ld",
2979                                         tgt_dev->curr_sn);
2980                         }
2981                         cmd->sn_slot = tgt_dev->cur_sn_slot;
2982                         cmd->sn = tgt_dev->curr_sn;
2983
2984                         tgt_dev->prev_cmd_ordered = 0;
2985                 } else {
2986                         TRACE(TRACE_MINOR, "***WARNING*** Not enough SN slots "
2987                                 "%zd", ARRAY_SIZE(tgt_dev->sn_slots));
2988                         goto ordered;
2989                 }
2990                 break;
2991
2992         case SCST_CMD_QUEUE_ORDERED:
2993                 TRACE_SN("ORDERED cmd %p (op %x)", cmd, cmd->cdb[0]);
2994 ordered:
2995                 if (!tgt_dev->prev_cmd_ordered) {
2996                         spin_lock_irqsave(&tgt_dev->sn_lock, flags);
2997                         if (tgt_dev->num_free_sn_slots >= 0) {
2998                                 tgt_dev->num_free_sn_slots--;
2999                                 if (tgt_dev->num_free_sn_slots >= 0) {
3000                                         int i = 0;
3001                                         /* Commands can finish in any order, so
3002                                          * we don't know which slot is empty.
3003                                          */
3004                                         while (1) {
3005                                                 tgt_dev->cur_sn_slot++;
3006                                                 if (tgt_dev->cur_sn_slot ==
3007                                                       tgt_dev->sn_slots + ARRAY_SIZE(tgt_dev->sn_slots))
3008                                                         tgt_dev->cur_sn_slot = tgt_dev->sn_slots;
3009
3010                                                 if (atomic_read(tgt_dev->cur_sn_slot) == 0)
3011                                                         break;
3012
3013                                                 i++;
3014                                                 sBUG_ON(i == ARRAY_SIZE(tgt_dev->sn_slots));
3015                                         }
3016                                         TRACE_SN("New cur SN slot %zd",
3017                                                 tgt_dev->cur_sn_slot -
3018                                                 tgt_dev->sn_slots);
3019                                 }
3020                         }
3021                         spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
3022                 }
3023                 tgt_dev->prev_cmd_ordered = 1;
3024                 tgt_dev->curr_sn++;
3025                 cmd->sn = tgt_dev->curr_sn;
3026                 break;
3027
3028         case SCST_CMD_QUEUE_HEAD_OF_QUEUE:
3029                 TRACE_SN("HQ cmd %p (op %x)", cmd, cmd->cdb[0]);
3030                 spin_lock_irqsave(&tgt_dev->sn_lock, flags);
3031                 tgt_dev->hq_cmd_count++;
3032                 spin_unlock_irqrestore(&tgt_dev->sn_lock, flags);
3033                 cmd->hq_cmd_inced = 1;
3034                 goto out;
3035
3036         default:
3037                 sBUG();
3038         }
3039
3040         TRACE_SN("cmd(%p)->sn: %ld (tgt_dev %p, *cur_sn_slot %d, "
3041                 "num_free_sn_slots %d, prev_cmd_ordered %ld, "
3042                 "cur_sn_slot %zd)", cmd, cmd->sn, tgt_dev,
3043                 atomic_read(tgt_dev->cur_sn_slot),
3044                 tgt_dev->num_free_sn_slots, tgt_dev->prev_cmd_ordered,
3045                 tgt_dev->cur_sn_slot-tgt_dev->sn_slots);
3046
3047         cmd->sn_set = 1;
3048
3049 out:
3050         TRACE_EXIT();
3051         return;
3052 }
3053
3054 /*
3055  * Returns 0 on success, > 0 when we need to wait for unblock,
3056  * < 0 if there is no device (lun) or device type handler.
3057  *
3058  * No locks, but might be on IRQ, protection is done by the
3059  * suspended activity.
3060  */
3061 static int scst_translate_lun(struct scst_cmd *cmd)
3062 {
3063         struct scst_tgt_dev *tgt_dev = NULL;
3064         int res;
3065
3066         TRACE_ENTRY();
3067
3068         /* See comment about smp_mb() in scst_suspend_activity() */
3069         __scst_get(1);
3070
3071         if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) {
3072                 struct list_head *sess_tgt_dev_list_head =
3073                         &cmd->sess->sess_tgt_dev_list_hash[HASH_VAL(cmd->lun)];
3074                 TRACE_DBG("Finding tgt_dev for cmd %p (lun %lld)", cmd,
3075                         (long long unsigned int)cmd->lun);
3076                 res = -1;
3077                 list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
3078                                 sess_tgt_dev_list_entry) {
3079                         if (tgt_dev->lun == cmd->lun) {
3080                                 TRACE_DBG("tgt_dev %p found", tgt_dev);
3081
3082                                 if (unlikely(tgt_dev->dev->handler ==
3083                                                 &scst_null_devtype)) {
3084                                         PRINT_INFO("Dev handler for device "
3085                                           "%lld is NULL, the device will not "
3086                                           "be visible remotely",
3087                                            (long long unsigned int)cmd->lun);
3088                                         break;
3089                                 }
3090
3091                                 cmd->cmd_lists = tgt_dev->dev->p_cmd_lists;
3092                                 cmd->tgt_dev = tgt_dev;
3093                                 cmd->dev = tgt_dev->dev;
3094
3095                                 res = 0;
3096                                 break;
3097                         }
3098                 }
3099                 if (res != 0) {
3100                         TRACE(TRACE_MINOR,
3101                                 "tgt_dev for lun %lld not found, command to "
3102                                 "unexisting LU?",
3103                                 (long long unsigned int)cmd->lun);
3104                         __scst_put();
3105                 }
3106         } else {
3107                 TRACE_MGMT_DBG("%s", "FLAG SUSPENDED set, skipping");
3108                 __scst_put();
3109                 res = 1;
3110         }
3111
3112         TRACE_EXIT_RES(res);
3113         return res;
3114 }
3115
3116 /*
3117  * No locks, but might be on IRQ
3118  *
3119  * Returns 0 on success, > 0 when we need to wait for unblock,
3120  * < 0 if there is no device (lun) or device type handler.
3121  */
3122 static int __scst_init_cmd(struct scst_cmd *cmd)
3123 {
3124         int res = 0;
3125
3126         TRACE_ENTRY();
3127
3128         res = scst_translate_lun(cmd);
3129         if (likely(res == 0)) {
3130                 int cnt;
3131                 bool failure = false;
3132
3133                 cmd->state = SCST_CMD_STATE_PRE_PARSE;
3134
3135                 cnt = atomic_inc_return(&cmd->tgt_dev->tgt_dev_cmd_count);
3136                 if (unlikely(cnt > SCST_MAX_TGT_DEV_COMMANDS)) {
3137                         TRACE(TRACE_MGMT_MINOR,
3138                                 "Too many pending commands (%d) in "
3139                                 "session, returning BUSY to initiator \"%s\"",
3140                                 cnt, (cmd->sess->initiator_name[0] == '\0') ?
3141                                   "Anonymous" : cmd->sess->initiator_name);
3142                         failure = true;
3143                 }
3144
3145                 cnt = atomic_inc_return(&cmd->dev->dev_cmd_count);
3146                 if (unlikely(cnt > SCST_MAX_DEV_COMMANDS)) {
3147                         if (!failure) {
3148                                 TRACE(TRACE_MGMT_MINOR,
3149                                         "Too many pending device "
3150                                         "commands (%d), returning BUSY to "
3151                                         "initiator \"%s\"", cnt,
3152                                         (cmd->sess->initiator_name[0] == '\0') ?
3153                                                 "Anonymous" :
3154                                                 cmd->sess->initiator_name);
3155                                 failure = true;
3156                         }
3157                 }
3158
3159                 /* If expected values not set, expected direction is UNKNOWN */
3160                 if (cmd->expected_data_direction == SCST_DATA_WRITE)
3161                         atomic_inc(&cmd->dev->write_cmd_count);
3162
3163                 if (unlikely(failure))
3164                         goto out_busy;
3165
3166                 if (!cmd->set_sn_on_restart_cmd)
3167                         scst_cmd_set_sn(cmd);
3168         } else if (res < 0) {
3169                 TRACE_DBG("Finishing cmd %p", cmd);
3170                 scst_set_cmd_error(cmd,
3171                            SCST_LOAD_SENSE(scst_sense_lun_not_supported));
3172                 scst_set_cmd_abnormal_done_state(cmd);
3173         } else
3174                 goto out;
3175
3176 out:
3177         TRACE_EXIT_RES(res);
3178         return res;
3179
3180 out_busy:
3181         scst_set_busy(cmd);
3182         scst_set_cmd_abnormal_done_state(cmd);
3183         goto out;
3184 }
3185
3186 /* Called under scst_init_lock and IRQs disabled */
3187 static void scst_do_job_init(void)
3188         __releases(&scst_init_lock)
3189         __acquires(&scst_init_lock)
3190 {
3191         struct scst_cmd *cmd;
3192         int susp;
3193
3194         TRACE_ENTRY();
3195
3196 restart:
3197         /*
3198          * There is no need for read barrier here, because we don't care where
3199          * this check will be done.
3200          */
3201         susp = test_bit(SCST_FLAG_SUSPENDED, &scst_flags);
3202         if (scst_init_poll_cnt > 0)
3203                 scst_init_poll_cnt--;
3204
3205         list_for_each_entry(cmd, &scst_init_cmd_list, cmd_list_entry) {
3206                 int rc;
3207                 if (susp && !test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
3208                         continue;
3209                 if (!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
3210                         spin_unlock_irq(&scst_init_lock);
3211                         rc = __scst_init_cmd(cmd);
3212                         spin_lock_irq(&scst_init_lock);
3213                         if (rc > 0) {
3214                                 TRACE_MGMT_DBG("%s",
3215                                         "FLAG SUSPENDED set, restarting");
3216                                 goto restart;
3217                         }
3218                 } else {
3219                         TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)",
3220                                        cmd, (long long unsigned int)cmd->tag);
3221                         scst_set_cmd_abnormal_done_state(cmd);
3222                 }
3223
3224                 /*
3225                  * Deleting cmd from init cmd list after __scst_init_cmd()
3226                  * is necessary to keep the check in scst_init_cmd() correct
3227                  * to preserve the commands order.
3228                  *
3229                  * We don't care about the race, when init cmd list is empty
3230                  * and one command detected that it just was not empty, so
3231                  * it's inserting to it, but another command at the same time
3232                  * seeing init cmd list empty and goes directly, because it
3233                  * could affect only commands from the same initiator to the
3234                  * same tgt_dev, but scst_cmd_init_done*() doesn't guarantee
3235                  * the order in case of simultaneous such calls anyway.
3236                  */
3237                 TRACE_MGMT_DBG("Deleting cmd %p from init cmd list", cmd);
3238                 smp_wmb(); /* enforce the required order */
3239                 list_del(&cmd->cmd_list_entry);
3240                 spin_unlock(&scst_init_lock);
3241
3242                 spin_lock(&cmd->cmd_lists->cmd_list_lock);
3243                 TRACE_MGMT_DBG("Adding cmd %p to active cmd list", cmd);
3244                 if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
3245                         list_add(&cmd->cmd_list_entry,
3246                                 &cmd->cmd_lists->active_cmd_list);
3247                 else
3248                         list_add_tail(&cmd->cmd_list_entry,
3249                                 &cmd->cmd_lists->active_cmd_list);
3250                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3251                 spin_unlock(&cmd->cmd_lists->cmd_list_lock);
3252
3253                 spin_lock(&scst_init_lock);
3254                 goto restart;
3255         }
3256
3257         /* It isn't really needed, but let's keep it */
3258         if (susp != test_bit(SCST_FLAG_SUSPENDED, &scst_flags))
3259                 goto restart;
3260
3261         TRACE_EXIT();
3262         return;
3263 }
3264
3265 static inline int test_init_cmd_list(void)
3266 {
3267         int res = (!list_empty(&scst_init_cmd_list) &&
3268                    !test_bit(SCST_FLAG_SUSPENDED, &scst_flags)) ||
3269                   unlikely(kthread_should_stop()) ||
3270                   (scst_init_poll_cnt > 0);
3271         return res;
3272 }
3273
3274 int scst_init_cmd_thread(void *arg)
3275 {
3276         TRACE_ENTRY();
3277
3278         PRINT_INFO("Init thread started, PID %d", current->pid);
3279
3280         current->flags |= PF_NOFREEZE;
3281
3282         set_user_nice(current, -10);
3283
3284         spin_lock_irq(&scst_init_lock);
3285         while (!kthread_should_stop()) {
3286                 wait_queue_t wait;
3287                 init_waitqueue_entry(&wait, current);
3288
3289                 if (!test_init_cmd_list()) {
3290                         add_wait_queue_exclusive(&scst_init_cmd_list_waitQ,
3291                                                  &wait);
3292                         for (;;) {
3293                                 set_current_state(TASK_INTERRUPTIBLE);
3294                                 if (test_init_cmd_list())
3295                                         break;
3296                                 spin_unlock_irq(&scst_init_lock);
3297                                 schedule();
3298                                 spin_lock_irq(&scst_init_lock);
3299                         }
3300                         set_current_state(TASK_RUNNING);
3301                         remove_wait_queue(&scst_init_cmd_list_waitQ, &wait);
3302                 }
3303                 scst_do_job_init();
3304         }
3305         spin_unlock_irq(&scst_init_lock);
3306
3307         /*
3308          * If kthread_should_stop() is true, we are guaranteed to be
3309          * on the module unload, so scst_init_cmd_list must be empty.
3310          */
3311         sBUG_ON(!list_empty(&scst_init_cmd_list));
3312
3313         PRINT_INFO("Init thread PID %d finished", current->pid);
3314
3315         TRACE_EXIT();
3316         return 0;
3317 }
3318
3319 /* Called with no locks held */
3320 void scst_process_active_cmd(struct scst_cmd *cmd, bool atomic)
3321 {
3322         int res;
3323
3324         TRACE_ENTRY();
3325
3326         EXTRACHECKS_BUG_ON(in_irq() || irqs_disabled());
3327
3328         cmd->atomic = atomic;
3329
3330         TRACE_DBG("cmd %p, atomic %d", cmd, atomic);
3331
3332         do {
3333                 switch (cmd->state) {
3334                 case SCST_CMD_STATE_PRE_PARSE:
3335                         res = scst_pre_parse(cmd);
3336                         EXTRACHECKS_BUG_ON(res ==
3337                                 SCST_CMD_STATE_RES_NEED_THREAD);
3338                         break;
3339
3340                 case SCST_CMD_STATE_DEV_PARSE:
3341                         res = scst_parse_cmd(cmd);
3342                         break;
3343
3344                 case SCST_CMD_STATE_PREPARE_SPACE:
3345                         res = scst_prepare_space(cmd);
3346                         break;
3347
3348                 case SCST_CMD_STATE_RDY_TO_XFER:
3349                         res = scst_rdy_to_xfer(cmd);
3350                         break;
3351
3352                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3353                         res = scst_tgt_pre_exec(cmd);
3354                         break;
3355
3356                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3357                         if (tm_dbg_check_cmd(cmd) != 0) {
3358                                 res = SCST_CMD_STATE_RES_CONT_NEXT;
3359                                 TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), "
3360                                         "because of TM DBG delay", cmd,
3361                                         (long long unsigned int)cmd->tag);
3362                                 break;
3363                         }
3364                         res = scst_send_for_exec(&cmd);
3365                         /*
3366                          * !! At this point cmd, sess & tgt_dev can already be
3367                          * freed !!
3368                          */
3369                         break;
3370
3371                 case SCST_CMD_STATE_LOCAL_EXEC:
3372                         res = scst_local_exec(cmd);
3373                         /*
3374                          * !! At this point cmd, sess & tgt_dev can already be
3375                          * freed !!
3376                          */
3377                         break;
3378
3379                 case SCST_CMD_STATE_REAL_EXEC:
3380                         res = scst_real_exec(cmd);
3381                         /*
3382                          * !! At this point cmd, sess & tgt_dev can already be
3383                          * freed !!
3384                          */
3385                         break;
3386
3387                 case SCST_CMD_STATE_PRE_DEV_DONE:
3388                         res = scst_pre_dev_done(cmd);
3389                         EXTRACHECKS_BUG_ON(res ==
3390                                 SCST_CMD_STATE_RES_NEED_THREAD);
3391                         break;
3392
3393                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3394                         res = scst_mode_select_checks(cmd);
3395                         break;
3396
3397                 case SCST_CMD_STATE_DEV_DONE:
3398                         res = scst_dev_done(&cmd);
3399                         break;
3400
3401                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3402                         res = scst_pre_xmit_response(cmd);
3403                         EXTRACHECKS_BUG_ON(res ==
3404                                 SCST_CMD_STATE_RES_NEED_THREAD);
3405                         break;
3406
3407                 case SCST_CMD_STATE_XMIT_RESP:
3408                         res = scst_xmit_response(cmd);
3409                         break;
3410
3411                 case SCST_CMD_STATE_FINISHED:
3412                         res = scst_finish_cmd(cmd);
3413                         EXTRACHECKS_BUG_ON(res ==
3414                                 SCST_CMD_STATE_RES_NEED_THREAD);
3415                         break;
3416
3417                 default:
3418                         PRINT_CRIT_ERROR("cmd (%p) in state %d, but shouldn't "
3419                                 "be", cmd, cmd->state);
3420                         sBUG();
3421                         res = SCST_CMD_STATE_RES_CONT_NEXT;
3422                         break;
3423                 }
3424         } while (res == SCST_CMD_STATE_RES_CONT_SAME);
3425
3426         if (res == SCST_CMD_STATE_RES_CONT_NEXT) {
3427                 /* None */
3428         } else if (res == SCST_CMD_STATE_RES_NEED_THREAD) {
3429                 spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3430                 switch (cmd->state) {
3431                 case SCST_CMD_STATE_PRE_PARSE:
3432                 case SCST_CMD_STATE_DEV_PARSE:
3433                 case SCST_CMD_STATE_PREPARE_SPACE:
3434                 case SCST_CMD_STATE_RDY_TO_XFER:
3435                 case SCST_CMD_STATE_TGT_PRE_EXEC:
3436                 case SCST_CMD_STATE_SEND_FOR_EXEC:
3437                 case SCST_CMD_STATE_LOCAL_EXEC:
3438                 case SCST_CMD_STATE_REAL_EXEC:
3439                 case SCST_CMD_STATE_PRE_DEV_DONE:
3440                 case SCST_CMD_STATE_MODE_SELECT_CHECKS:
3441                 case SCST_CMD_STATE_DEV_DONE:
3442                 case SCST_CMD_STATE_PRE_XMIT_RESP:
3443                 case SCST_CMD_STATE_XMIT_RESP:
3444                 case SCST_CMD_STATE_FINISHED:
3445                         TRACE_DBG("Adding cmd %p to head of active cmd list",
3446                                   cmd);
3447                         list_add(&cmd->cmd_list_entry,
3448                                 &cmd->cmd_lists->active_cmd_list);
3449                         break;
3450 #ifdef CONFIG_SCST_EXTRACHECKS
3451                 /* not very valid commands */
3452                 case SCST_CMD_STATE_DEFAULT:
3453                 case SCST_CMD_STATE_NEED_THREAD_CTX:
3454                         PRINT_CRIT_ERROR("cmd %p is in invalid state %d)", cmd,
3455                                 cmd->state);
3456                         spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3457                         sBUG();
3458                         spin_lock_irq(&cmd->cmd_lists->cmd_list_lock);
3459                         break;
3460 #endif
3461                 default:
3462                         break;
3463                 }
3464                 wake_up(&cmd->cmd_lists->cmd_list_waitQ);
3465                 spin_unlock_irq(&cmd->cmd_lists->cmd_list_lock);
3466         } else
3467                 sBUG();
3468
3469         TRACE_EXIT();
3470         return;
3471 }
3472 EXPORT_SYMBOL(scst_process_active_cmd);
3473
3474 /* Called under cmd_list_lock and IRQs disabled */
3475 static void scst_do_job_active(struct list_head *cmd_list,
3476         spinlock_t *cmd_list_lock, bool atomic)
3477         __releases(cmd_list_lock)
3478         __acquires(cmd_list_lock)
3479 {
3480         TRACE_ENTRY();
3481
3482         while (!list_empty(cmd_list)) {
3483                 struct scst_cmd *cmd = list_entry(cmd_list->next, typeof(*cmd),
3484                                         cmd_list_entry);
3485                 TRACE_DBG("Deleting cmd %p from active cmd list", cmd);
3486                 list_del(&cmd->cmd_list_entry);
3487                 spin_unlock_irq(cmd_list_lock);
3488                 scst_process_active_cmd(cmd, atomic);
3489                 spin_lock_irq(cmd_list_lock);
3490         }
3491
3492         TRACE_EXIT();
3493         return;
3494 }
3495
3496 static inline int test_cmd_lists(struct scst_cmd_lists *p_cmd_lists)
3497 {
3498         int res = !list_empty(&p_cmd_lists->active_cmd_list) ||
3499             unlikely(kthread_should_stop()) ||
3500             tm_dbg_is_release();
3501         return res;